This Pull Request upgrades to Llama 3.2 11B

And add also two image examples.

Click on _Merge_ to add this feature.

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Example2.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -4,12 +4,10 @@ emoji: 🌘w🌖
 colorFrom: yellow
 colorTo: red
 sdk: gradio
-sdk_version: 5.7.1
+sdk_version: 5.2.0
 app_file: app.py
 pinned: true
 short_description: A retrieval system with chatbot integration
-thumbnail: >-
-  https://cdn-uploads.huggingface.co/production/uploads/6527e89a8808d80ccff88b7a/XVgtQiizeFHIUUj1huwdv.png
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).

app.py

@@ -1,140 +1,97 @@
-import gradio as gr
-from datasets import load_dataset
-
-import os
-import spaces
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
+from transformers import MllamaForConditionalGeneration, AutoProcessor, TextIteratorStreamer
+from PIL import Image
+import requests
 import torch
 from threading import Thread
-from sentence_transformers import SentenceTransformer
-from datasets import load_dataset
+import gradio as gr
+from gradio import FileData
 import time
-
-token = os.environ["HF_TOKEN"]
-ST = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
-
-dataset = load_dataset("not-lain/wikipedia",revision = "embedded")
-
-data = dataset["train"]
-data = data.add_faiss_index("embeddings") # column name that has the embeddings of the dataset
-
-
-model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
-
-# use quantization to lower GPU usage
-bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16
-)
-
-tokenizer = AutoTokenizer.from_pretrained(model_id,token=token)
-model = AutoModelForCausalLM.from_pretrained(
-    model_id,
-    torch_dtype=torch.bfloat16,
-    device_map="auto",
-    quantization_config=bnb_config,
-    token=token
-)
-terminators = [
-    tokenizer.eos_token_id,
-    tokenizer.convert_tokens_to_ids("<|eot_id|>")
-]
-
-SYS_PROMPT = """You are an assistant for answering questions.
-You are given the extracted parts of a long document and a question. Provide a conversational answer.
-If you don't know the answer, just say "I do not know." Don't make up an answer."""
-
-
-
-def search(query: str, k: int = 3 ):
-    """a function that embeds a new query and returns the most probable results"""
-    embedded_query = ST.encode(query) # embed new query
-    scores, retrieved_examples = data.get_nearest_examples( # retrieve results
-        "embeddings", embedded_query, # compare our new embedded query with the dataset embeddings
-        k=k # get only top k results
-    )
-    return scores, retrieved_examples
-
-def format_prompt(prompt,retrieved_documents,k):
-    """using the retrieved documents we will prompt the model to generate our responses"""
-    PROMPT = f"Question:{prompt}\nContext:"
-    for idx in range(k) :
-        PROMPT+= f"{retrieved_documents['text'][idx]}\n"
-    return PROMPT
+import spaces
+import re
+ckpt = "Xkev/Llama-3.2V-11B-cot"
+model = MllamaForConditionalGeneration.from_pretrained(ckpt,
+    torch_dtype=torch.bfloat16).to("cuda")
+processor = AutoProcessor.from_pretrained(ckpt)
 
 
 @spaces.GPU
-def talk(prompt,history=[]):
-    k = 1 # number of retrieved documents
-    scores , retrieved_documents = search(prompt, k)
-    formatted_prompt = format_prompt(prompt,retrieved_documents,k)
-    formatted_prompt = formatted_prompt[:2000] # to avoid GPU OOM
-    messages = [{"role":"system","content":SYS_PROMPT},{"role":"user","content":formatted_prompt}]
-    # tell the model to generate
-    input_ids = tokenizer.apply_chat_template(
-      messages,
-      add_generation_prompt=True,
-      return_tensors="pt"
-    ).to(model.device)
-    outputs = model.generate(
-      input_ids,
-      max_new_tokens=1024,
-      eos_token_id=terminators,
-      do_sample=True,
-      temperature=0.6,
-      top_p=0.9,
-    )
-    streamer = TextIteratorStreamer(
-            tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True
-        )
-    generate_kwargs = dict(
-        input_ids= input_ids,
-        streamer=streamer,
-        max_new_tokens=1024,
-        do_sample=True,
-        top_p=0.95,
-        temperature=0.75,
-        eos_token_id=terminators,
-    )
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
-
-    outputs = []
-    for text in streamer:
-        outputs.append(text)
-        yield "".join(outputs)
-
-
-TITLE = "# RAG"
-
-DESCRIPTION = """
-A rag pipeline with a chatbot feature
-
-Resources used to build this project :
-
-* embedding model : https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1
-* dataset : https://huggingface.co/datasets/not-lain/wikipedia
-* faiss docs : https://huggingface.co/docs/datasets/v2.18.0/en/package_reference/main_classes#datasets.Dataset.add_faiss_index 
-* chatbot : https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct
-* Full documentation : https://huggingface.co/blog/not-lain/rag-chatbot-using-llama3 
-"""
-
+def bot_streaming(message, history, max_new_tokens=250):
+    
+    txt = message["text"]
+    ext_buffer = f"{txt}"
+    
+    messages= [] 
+    images = []
+    
 
-demo = gr.ChatInterface(
-    fn=talk,
-    chatbot=gr.Chatbot(
-        show_label=True,
-        show_share_button=True,
-        show_copy_button=True,
-        layout="bubble",
-        bubble_full_width=False,
-    ),
-    theme="Soft",
-    type="tuples",
-    examples=["what's anarchy ? "],
-    title=TITLE,
-    description=DESCRIPTION,
-    autofocus=False,
-    autoscroll = False,
+    for i, msg in enumerate(history): 
+        if isinstance(msg[0], tuple):
+            messages.append({"role": "user", "content": [{"type": "text", "text": history[i+1][0]}, {"type": "image"}]})
+            messages.append({"role": "assistant", "content": [{"type": "text", "text": history[i+1][1]}]})
+            images.append(Image.open(msg[0][0]).convert("RGB"))
+        elif isinstance(history[i-1], tuple) and isinstance(msg[0], str):
+            # messages are already handled
+            pass
+        elif isinstance(history[i-1][0], str) and isinstance(msg[0], str): # text only turn
+            messages.append({"role": "user", "content": [{"type": "text", "text": msg[0]}]})
+            messages.append({"role": "assistant", "content": [{"type": "text", "text": msg[1]}]})
+
+    # add current message
+    if len(message["files"]) == 1:
+        
+        if isinstance(message["files"][0], str): # examples
+            image = Image.open(message["files"][0]).convert("RGB")
+        else: # regular input
+            image = Image.open(message["files"][0]["path"]).convert("RGB")
+        images.append(image)
+        messages.append({"role": "user", "content": [{"type": "text", "text": txt}, {"type": "image"}]})
+    else:
+        messages.append({"role": "user", "content": [{"type": "text", "text": txt}]})
+
+
+    texts = processor.apply_chat_template(messages, add_generation_prompt=True)
+
+    if images == []:
+        inputs = processor(text=texts, return_tensors="pt").to("cuda")
+    else:
+        inputs = processor(text=texts, images=images, return_tensors="pt").to("cuda")
+        
+    streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True)
+
+    generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_new_tokens, do_sample=True, temperature=0.6, top_p=0.9)
+    generated_text = ""
+    
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
+    buffer = ""
+    
+    for new_text in streamer:
+        buffer += new_text
+        generated_text_without_prompt = buffer
+        time.sleep(0.01)
+    
+        buffer = re.sub(r"<(\w+)>", r"\<\1\>", buffer)  
+        buffer = re.sub(r"</(\w+)>", r"\</\1\>", buffer)    
+    
+        yield buffer
+
+
+demo = gr.ChatInterface(fn=bot_streaming, title="LLaVA-CoT",
+      textbox=gr.MultimodalTextbox(), 
+      additional_inputs = [gr.Slider(
+              minimum=512,
+              maximum=1024,
+              value=512,
+              step=1,
+              label="Maximum number of new tokens to generate",
+          )
+        ],
+      examples=[[{"text": "What is on the flower?", "files": ["./Example1.webp"]},512],
+                [{"text": "How to make this pastry?", "files": ["./Example2.png"]},512]],
+      cache_examples=False,
+      description="Upload an image, and start chatting about it. To learn more about LLaVA-CoT, visit [our GitHub page](https://github.com/PKU-YuanGroup/LLaVA-CoT).",
+      stop_btn="Stop Generation", 
+      fill_height=True,
+    multimodal=True)
     
-)
-demo.launch(debug=True)
+demo.launch(debug=True)

requirements.txt

@@ -1,9 +1,3 @@
-numpy<2
-torch>=2
+torch
 spaces
-transformers
-sentence-transformers
-faiss-gpu
-datasets
-accelerate
-bitsandbytes
+git+https://github.com/huggingface/transformers.git