Update app.py

app.py

@@ -1,46 +1,47 @@
-import os
-import time
-import threading
 import streamlit as st
-from twilio.rest import Client
-from sentence_transformers import SentenceTransformer
-from transformers import AutoTokenizer
+import os
+import tempfile
+import requests
+from PyPDF2 import PdfReader
 import faiss
 import numpy as np
-import docx
+from transformers import AutoTokenizer
+import pdfplumber
 from groq import Groq
-import requests
-from io import StringIO
-from pdfminer.high_level import extract_text_to_fp
-from pdfminer.layout import LAParams
-
-# --- PDF Extraction (Improved for Tables & Paragraphs) ---
-def extract_text_from_pdf(pdf_path):
-    output_string = StringIO()
-    with open(pdf_path, 'rb') as file:
-        extract_text_to_fp(file, output_string, laparams=LAParams(), output_type='text', codec=None)
-    return output_string.getvalue()
-
-def clean_extracted_text(text):
-    lines = text.splitlines()
-    cleaned = []
-    for line in lines:
-        line = line.strip()
-        if line:
-            line = ' '.join(line.split())  # remove extra spaces
-            cleaned.append(line)
-    return '\n'.join(cleaned)
-
-# --- DOCX Extraction ---
-def extract_text_from_docx(docx_path):
-    try:
-        doc = docx.Document(docx_path)
-        return '\n'.join(para.text for para in doc.paragraphs)
-    except:
-        return ""
-
-# --- Chunking & Retrieval ---
-def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32, max_tokens=512):
+import time
+import json
+from twilio.rest import Client
+from flask import Flask, request
+import threading
+
+# --- Configuration ---
+GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+TWILIO_ACCOUNT_SID = os.getenv("TWILIO_ACCOUNT_SID")
+TWILIO_AUTH_TOKEN = os.getenv("TWILIO_AUTH_TOKEN")
+TWILIO_SERVICE_SID = os.getenv("TWILIO_SERVICE_SID")
+
+# --- Initialize Clients ---
+client = Groq(api_key=GROQ_API_KEY)
+twilio_client = Client(TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN)
+tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")
+
+# --- Flask App for Webhook ---
+flask_app = Flask(__name__)
+latest_conversation_sid = None
+conversation_index = None
+chunk_store = {}
+
+# --- Helper Functions ---
+def extract_text_with_pdfplumber(pdf_path):
+    full_text = ""
+    with pdfplumber.open(pdf_path) as pdf:
+        for page in pdf.pages:
+            text = page.extract_text()
+            if text:
+                full_text += text + "\n"
+    return full_text
+
+def chunk_text(text, chunk_size=256, chunk_overlap=64, max_tokens=512):
     tokens = tokenizer.tokenize(text)
     chunks = []
     start = 0
@@ -53,163 +54,87 @@ def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32, max_tokens=512
         start += chunk_size - chunk_overlap
     return chunks
 
-def retrieve_chunks(question, index, embed_model, text_chunks, k=3):
-    question_embedding = embed_model.encode(question)
-    D, I = index.search(np.array([question_embedding]), k)
-    return [text_chunks[i] for i in I[0]]
-
-# --- Groq Answer Generator ---
-def generate_answer_with_groq(question, context):
-    url = "https://api.groq.com/openai/v1/chat/completions"
-    api_key = os.environ.get("GROQ_API_KEY")
-    headers = {
-        "Authorization": f"Bearer {api_key}",
-        "Content-Type": "application/json",
-    }
-    prompt = (
-        f"Customer asked: '{question}'\n\n"
-        f"Here is the relevant product or policy info to help:\n{context}\n\n"
-        f"Respond in a friendly and helpful tone as a toy shop support agent."
-    )
-    payload = {
-        "model": "llama3-8b-8192",
-        "messages": [
-            {
-                "role": "system",
-                "content": (
-                    "You are ToyBot, a friendly and helpful WhatsApp assistant for an online toy shop. "
-                    "Your goal is to politely answer customer questions, help them choose the right toys, "
-                    "provide order or delivery information, explain return policies, and guide them through purchases."
-                )
-            },
-            {"role": "user", "content": prompt},
-        ],
-        "temperature": 0.5,
-        "max_tokens": 300,
-    }
-    response = requests.post(url, headers=headers, json=payload)
-    response.raise_for_status()
-    return response.json()['choices'][0]['message']['content'].strip()
-
-# --- Twilio Functions ---
-def get_whatsapp_conversation_sids(client):
-    sids = []
-    conversations = client.conversations.v1.conversations.list(limit=50)
-    for convo in conversations:
-        try:
-            participants = client.conversations.v1.conversations(convo.sid).participants.list()
-            for p in participants:
-                if (p.identity and p.identity.startswith("whatsapp:")) or (
-                    p.messaging_binding and p.messaging_binding.get("address", "").startswith("whatsapp:")
-                ):
-                    sids.append(convo.sid)
-                    break
-        except:
-            continue
-    return sids
-
-def fetch_latest_incoming_message(client, conversation_sid):
-    messages = client.conversations.v1.conversations(conversation_sid).messages.list(limit=10)
-    for msg in reversed(messages):
-        if msg.author.startswith("whatsapp:"):
-            return {
-                "sid": msg.sid,
-                "body": msg.body,
-                "author": msg.author,
-                "timestamp": msg.date_created,
-            }
-    return None
-
-def send_twilio_message(client, conversation_sid, body):
-    return client.conversations.v1.conversations(conversation_sid).messages.create(
-        author="system", body=body
+def get_embedding(text):
+    # Fake embedding for example purposes
+    return np.random.rand(384).astype("float32")
+
+def build_faiss_index(chunks):
+    embeddings = [get_embedding(chunk) for chunk in chunks]
+    dimension = embeddings[0].shape[0]
+    index = faiss.IndexFlatL2(dimension)
+    index.add(np.array(embeddings))
+    return index, chunks
+
+def search_chunks(query, index, chunks, top_k=5):
+    query_embedding = get_embedding(query)
+    _, I = index.search(np.array([query_embedding]), top_k)
+    return [chunks[i] for i in I[0]]
+
+def generate_answer_with_groq(query, context):
+    prompt = f"""
+    You are a helpful assistant. Use the following context to answer the question:
+    {context}
+    Question: {query}
+    Answer:
+    """
+    response = client.chat.completions.create(
+        messages=[{"role": "user", "content": prompt}],
+        model="llama3-8b-8192"
     )
-
-# --- Load Knowledge Base ---
-def setup_knowledge_base():
-    folder_path = "docs"
-    all_text = ""
-    for file in os.listdir(folder_path):
-        path = os.path.join(folder_path, file)
-        if file.endswith(".pdf"):
-            raw_text = extract_text_from_pdf(path)
-            all_text += clean_extracted_text(raw_text) + "\n"
-        elif file.endswith((".docx", ".doc")):
-            all_text += extract_text_from_docx(path) + "\n"
-
-    tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
-    chunks = chunk_text(all_text, tokenizer)
-    model = SentenceTransformer('all-mpnet-base-v2')
-    embeddings = model.encode(chunks, truncate=True, show_progress_bar=False)
-    dim = embeddings[0].shape[0]
-    index = faiss.IndexFlatL2(dim)
-    index.add(np.array(embeddings).astype('float32'))
-    return index, model, chunks
-
-# --- Monitor Conversations ---
-def start_conversation_monitor(client, index, embed_model, text_chunks):
-    monitored_sids = set()
-
-    def poll_conversation(convo_sid):
-        last_processed_timestamp = None
-        while True:
-            try:
-                latest_msg = fetch_latest_incoming_message(client, convo_sid)
-                if latest_msg:
-                    msg_time = latest_msg["timestamp"]
-                    if last_processed_timestamp is None or msg_time > last_processed_timestamp:
-                        last_processed_timestamp = msg_time
-                        question = latest_msg["body"]
-                        sender = latest_msg["author"]
-                        print(f"\n📥 New message from {sender} in {convo_sid}: {question}")
-                        context = "\n\n".join(retrieve_chunks(question, index, embed_model, text_chunks))
-                        answer = generate_answer_with_groq(question, context)
-                        send_twilio_message(client, convo_sid, answer)
-                        print(f"📤 Replied to {sender}: {answer}")
-                time.sleep(3)
-            except Exception as e:
-                print(f"❌ Error in convo {convo_sid} polling:", e)
-                time.sleep(5)
-
-    def monitor_all_conversations():
-        while True:
-            try:
-                current_sids = set(get_whatsapp_conversation_sids(client))
-                new_sids = current_sids - monitored_sids
-                for sid in new_sids:
-                    print(f"➡️ Monitoring new conversation: {sid}")
-                    monitored_sids.add(sid)
-                    threading.Thread(target=poll_conversation, args=(sid,), daemon=True).start()
-                time.sleep(15)
-            except Exception as e:
-                print("❌ Error in conversation monitoring loop:", e)
-                time.sleep(15)
-
-    threading.Thread(target=monitor_all_conversations, daemon=True).start()
+    return response.choices[0].message.content.strip()
+
+# --- Flask Webhook Endpoint ---
+@flask_app.route("/twilio-webhook", methods=['POST'])
+def whatsapp_webhook():
+    global conversation_index, chunk_store
+    data = request.form
+    from_number = data['From']
+    message = data['Body']
+    conversation_sid = data.get('ConversationSid')
+
+    if conversation_sid and conversation_index:
+        context_chunks = search_chunks(message, conversation_index, chunk_store[conversation_sid])
+        context = "\n".join(context_chunks)
+        answer = generate_answer_with_groq(message, context)
+        twilio_client.conversations.conversations(conversation_sid).messages.create(author="bot", body=answer)
+    return "OK"
 
 # --- Streamlit UI ---
-st.set_page_config(page_title="Quasa – A Smart WhatsApp Chatbot", layout="wide")
-st.title("📱 Quasa – A Smart WhatsApp Chatbot")
-
-account_sid = st.secrets.get("TWILIO_SID")
-auth_token = st.secrets.get("TWILIO_TOKEN")
-GROQ_API_KEY = st.secrets.get("GROQ_API_KEY")
-
-if not all([account_sid, auth_token, GROQ_API_KEY]):
-    st.warning("⚠️ Provide all credentials below:")
-    account_sid = st.text_input("Twilio SID", value=account_sid or "")
-    auth_token = st.text_input("Twilio Token", type="password", value=auth_token or "")
-    GROQ_API_KEY = st.text_input("GROQ API Key", type="password", value=GROQ_API_KEY or "")
-
-if all([account_sid, auth_token, GROQ_API_KEY]):
-    os.environ["GROQ_API_KEY"] = GROQ_API_KEY
-    client = Client(account_sid, auth_token)
-    conversation_sids = get_whatsapp_conversation_sids(client)
-
-    if conversation_sids:
-        st.success(f"✅ {len(conversation_sids)} WhatsApp conversation(s) found. Initializing chatbot...")
-        index, model, chunks = setup_knowledge_base()
-        start_conversation_monitor(client, index, model, chunks)
-        st.success("🟢 Chatbot is running in background and will reply to new messages.")
-    else:
-        st.error("❌ No WhatsApp conversations found.")
+def streamlit_ui():
+    global latest_conversation_sid, conversation_index, chunk_store
+    st.title("Quasa – Smart WhatsApp Chatbot")
+
+    uploaded_file = st.file_uploader("Upload a PDF document", type="pdf")
+    if uploaded_file is not None:
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp_file:
+            tmp_file.write(uploaded_file.read())
+            pdf_path = tmp_file.name
+
+        text = extract_text_with_pdfplumber(pdf_path)
+        chunks = chunk_text(text)
+        index, stored_chunks = build_faiss_index(chunks)
+
+        # Get the last conversation SID from Twilio
+        conversations = twilio_client.conversations.conversations.list(limit=1)
+        if conversations:
+            latest_conversation_sid = conversations[0].sid
+            conversation_index = index
+            chunk_store[latest_conversation_sid] = stored_chunks
+            st.success(f"PDF indexed and ready for conversation. Conversation SID: {latest_conversation_sid}")
+        else:
+            st.error("No active Twilio conversations found.")
+
+    st.markdown("---")
+    st.markdown("**Waiting for incoming WhatsApp messages...**")
+
+# --- Run Flask in Background Thread ---
+def run_flask():
+    flask_app.run(host="0.0.0.0", port=5000)
+
+flask_thread = threading.Thread(target=run_flask)
+flask_thread.daemon = True
+flask_thread.start()
+
+# --- Start Streamlit App ---
+if __name__ == "__main__":
+    streamlit_ui()