Update app.py

app.py

@@ -3,8 +3,12 @@ import json
 import re
 import gradio as gr
 import pandas as pd
+import requests
+import random
+import urllib.parse
 from tempfile import NamedTemporaryFile
 from typing import List
+from bs4 import BeautifulSoup
 from langchain_core.prompts import ChatPromptTemplate
 from langchain_community.vectorstores import FAISS
 from langchain_community.document_loaders import PyPDFLoader
@@ -119,7 +123,78 @@ def is_related_to_history(question, history, threshold=0.3):
     similarity = get_similarity(question, history_text)
     return similarity > threshold
 
-def ask_question(question, temperature, top_p, repetition_penalty):
+def extract_text_from_webpage(html):
+    soup = BeautifulSoup(html, 'html.parser')
+    for script in soup(["script", "style"]):
+        script.extract()  # Remove scripts and styles
+    text = soup.get_text()
+    lines = (line.strip() for line in text.splitlines())
+    chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
+    text = '\n'.join(chunk for chunk in chunks if chunk)
+    return text
+
+_useragent_list = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.59 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36",
+]
+
+def google_search(term, num_results=5, lang="en", timeout=5, safe="active", ssl_verify=None):
+    escaped_term = urllib.parse.quote_plus(term)
+    start = 0
+    all_results = []
+    max_chars_per_page = 8000  # Limit the number of characters from each webpage to stay under the token limit
+
+    with requests.Session() as session:
+        while start < num_results:
+            try:
+                user_agent = random.choice(_useragent_list)
+                headers = {
+                    'User-Agent': user_agent
+                }
+                resp = session.get(
+                    url="https://www.google.com/search",
+                    headers=headers,
+                    params={
+                        "q": term,
+                        "num": num_results - start,
+                        "hl": lang,
+                        "start": start,
+                        "safe": safe,
+                    },
+                    timeout=timeout,
+                    verify=ssl_verify,
+                )
+                resp.raise_for_status()
+            except requests.exceptions.RequestException as e:
+                break
+
+            soup = BeautifulSoup(resp.text, "html.parser")
+            result_block = soup.find_all("div", attrs={"class": "g"})
+            if not result_block:
+                break
+            for result in result_block:
+                link = result.find("a", href=True)
+                if link:
+                    link = link["href"]
+                    try:
+                        webpage = session.get(link, headers=headers, timeout=timeout)
+                        webpage.raise_for_status()
+                        visible_text = extract_text_from_webpage(webpage.text)
+                        if len(visible_text) > max_chars_per_page:
+                            visible_text = visible_text[:max_chars_per_page] + "..."
+                        all_results.append({"link": link, "text": visible_text})
+                    except requests.exceptions.RequestException as e:
+                        all_results.append({"link": link, "text": None})
+                else:
+                    all_results.append({"link": None, "text": None})
+            start += len(result_block)
+    return all_results
+
+def ask_question(question, temperature, top_p, repetition_penalty, web_search):
     global conversation_history
     
     if not question:
@@ -129,17 +204,21 @@ def ask_question(question, temperature, top_p, repetition_penalty):
         answer = memory_database[question]
     else:
         embed = get_embeddings()
-        database = FAISS.load_local("faiss_database", embed, allow_dangerous_deserialization=True)
         model = get_model(temperature, top_p, repetition_penalty)
         
         history_str = "\n".join([f"Q: {item['question']}\nA: {item['answer']}" for item in conversation_history])
         
-        if is_related_to_history(question, conversation_history):
-            context_str = "No additional context needed. Please refer to the conversation history."
+        if web_search:
+            search_results = google_search(question)
+            context_str = "\n".join([result["text"] for result in search_results if result["text"]])
         else:
-            retriever = database.as_retriever()
-            relevant_docs = retriever.get_relevant_documents(question)
-            context_str = "\n".join([doc.page_content for doc in relevant_docs])
+            database = FAISS.load_local("faiss_database", embed, allow_dangerous_deserialization=True)
+            if is_related_to_history(question, conversation_history):
+                context_str = "No additional context needed. Please refer to the conversation history."
+            else:
+                retriever = database.as_retriever()
+                relevant_docs = retriever.get_relevant_documents(question)
+                context_str = "\n".join([doc.page_content for doc in relevant_docs])
         
         prompt_val = ChatPromptTemplate.from_template(prompt)
         formatted_prompt = prompt_val.format(history=history_str, context=context_str, question=question)
@@ -220,9 +299,10 @@ with gr.Blocks() as demo:
             temperature_slider = gr.Slider(label="Temperature", minimum=0.0, maximum=1.0, value=0.5, step=0.1)
             top_p_slider = gr.Slider(label="Top P", minimum=0.0, maximum=1.0, value=0.9, step=0.1)
             repetition_penalty_slider = gr.Slider(label="Repetition Penalty", minimum=1.0, maximum=2.0, value=1.0, step=0.1)
+            web_search_checkbox = gr.Checkbox(label="Enable Web Search", value=False)
     
     def chat(question, history):
-        answer = ask_question(question, temperature_slider.value, top_p_slider.value, repetition_penalty_slider.value)
+        answer = ask_question(question, temperature_slider.value, top_p_slider.value, repetition_penalty_slider.value, web_search_checkbox.value)
         history.append((question, answer))
         return "", history
     
@@ -241,4 +321,4 @@ with gr.Blocks() as demo:
     clear_button.click(clear_cache, inputs=[], outputs=clear_output)
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()