Update app.py

app.py

@@ -18,42 +18,45 @@ def get_text_from_url(url):
     visible_texts = filter(tag_visible, texts)
     return "\n".join(t.strip() for t in visible_texts)
 
-# Gather text from your homepage (and any extensions)
+# Pre-fetch and truncate homepage text to keep the prompt short
 text_list = []
 homepage_url = "https://sites.google.com/view/abhilashnandy/home/"
 extensions = ["", "pmrf-profile-page"]
 for ext in extensions:
-    url_text = get_text_from_url(homepage_url + ext)
-    text_list.append(url_text)
+    full_text = get_text_from_url(homepage_url + ext)
+    truncated_text = full_text[:1000]  # use only the first 1000 characters
+    text_list.append(truncated_text)
 
-# Build the system message with homepage info.
 SYSTEM_MESSAGE = (
-    "You are a QA chatbot to answer queries (in less than 30 words) on my homepage that has the following information -\n\n"
-    + "\n\n".join(text_list)
-    + "\n\n"
+    "You are a QA chatbot to answer queries (in less than 30 words) on my homepage. "
+    "Context: " + " ".join(text_list)
 )
 
-# Use the GPTQ variant of TinyLlama which includes the tokenizer configuration
+# Use the GPTQ version that includes the tokenizer configuration
 client = InferenceClient("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ")
 
 def respond(message, history: list[tuple[str, str]], system_message=SYSTEM_MESSAGE,
-            max_tokens=140, temperature=0.7, top_p=0.95):
+            max_tokens=100, temperature=0.7, top_p=0.95):
     messages = [{"role": "system", "content": system_message}]
-    for val in history:
-        if len(val) >= 1:
-            messages.append({"role": "user", "content": "Question: " + val[0]})
-        if len(val) >= 2:
-            messages.append({"role": "assistant", "content": "Answer: " + val[1]})
+    for q, a in history:
+        messages.append({"role": "user", "content": "Question: " + q})
+        messages.append({"role": "assistant", "content": "Answer: " + a})
     messages.append({"role": "user", "content": message})
     try:
-        response = client.chat_completion(
+        # Enable streaming mode to start receiving output faster.
+        response_stream = client.chat_completion(
             messages,
             max_tokens=max_tokens,
             temperature=temperature,
             top_p=top_p,
-            # stream=True,  # Uncomment this if you want streaming output for debugging.
+            stream=True,
         )
-        return response.choices[0].message["content"]
+        output = ""
+        for chunk in response_stream:
+            if hasattr(chunk, "choices") and chunk.choices:
+                part = chunk.choices[0].message.get("content", "")
+                output += part
+        return output.strip()
     except Exception as e:
         print(f"An error occurred: {e}")
         return str(e)
@@ -68,7 +71,7 @@ with demo:
         fn=respond,
         # examples=["Yo who dis Abhilash?", "What is Abhilash's most recent publication?"],
         additional_inputs=[
-            # Additional components can be added here if needed.
+            # You can add extra Gradio components here if needed.
         ],
     )