Update app.py

app.py

@@ -1,140 +1,213 @@
-import streamlit as st
 import os
-import tempfile
-import requests
-from PyPDF2 import PdfReader
+import time
+import threading
+import streamlit as st
+from twilio.rest import Client
+from sentence_transformers import SentenceTransformer
+from transformers import AutoTokenizer
+
 import faiss
 import numpy as np
-from transformers import AutoTokenizer
-import pdfplumber
+import docx
+
 from groq import Groq
-import time
-import json
-from twilio.rest import Client
-from flask import Flask, request
-import threading
+import requests
+from io import StringIO
+from pdfminer.high_level import extract_text_to_fp
+from pdfminer.layout import LAParams
+
+# --- PDF Extraction (Improved for Tables & Paragraphs) ---
+def extract_text_from_pdf(pdf_path):
+    output_string = StringIO()
+    with open(pdf_path, 'rb') as file:
+        extract_text_to_fp(file, output_string, laparams=LAParams(), output_type='text', codec=None)
+    return output_string.getvalue()
+
+def clean_extracted_text(text):
+    lines = text.splitlines()
+    cleaned = []
+    for line in lines:
+        line = line.strip()
+        if line:
+            line = ' '.join(line.split())  # remove extra spaces
+            cleaned.append(line)
+    return '\n'.join(cleaned)
+
+# --- DOCX Extraction ---
+def extract_text_from_docx(docx_path):
+    try:
+        doc = docx.Document(docx_path)
+        return '\n'.join(para.text for para in doc.paragraphs)
+    except:
+        return ""
+
+# --- Chunking & Retrieval ---
+def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32, max_tokens=512):
+
 
-# --- Configuration ---
-GROQ_API_KEY = os.getenv("GROQ_API_KEY")
-TWILIO_ACCOUNT_SID = os.getenv("TWILIO_ACCOUNT_SID")
-TWILIO_AUTH_TOKEN = os.getenv("TWILIO_AUTH_TOKEN")
-TWILIO_SERVICE_SID = os.getenv("TWILIO_SERVICE_SID")
-
-# --- Initialize Clients ---
-client = Groq(api_key=GROQ_API_KEY)
-twilio_client = Client(TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN)
-tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")
-
-# --- Flask App for Webhook ---
-flask_app = Flask(__name__)
-latest_conversation_sid = None
-conversation_index = None
-chunk_store = {}
-
-# --- Helper Functions ---
-def extract_text_with_pdfplumber(pdf_path):
-    full_text = ""
-    with pdfplumber.open(pdf_path) as pdf:
-        for page in pdf.pages:
-            text = page.extract_text()
-            if text:
-                full_text += text + "\n"
-    return full_text
-
-def chunk_text(text, chunk_size=256, chunk_overlap=64, max_tokens=512):
     tokens = tokenizer.tokenize(text)
     chunks = []
     start = 0
-    while start < len(tokens):
-        end = min(start + chunk_size, len(tokens))
-        chunk_tokens = tokens[start:end]
-        chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
-        if len(tokenizer.encode(chunk_text)) <= max_tokens:
-            chunks.append(chunk_text.strip())
         start += chunk_size - chunk_overlap
     return chunks
 
-def get_embedding(text):
-    # Fake embedding for example purposes
-    return np.random.rand(384).astype("float32")
-
-def build_faiss_index(chunks):
-    embeddings = [get_embedding(chunk) for chunk in chunks]
-    dimension = embeddings[0].shape[0]
-    index = faiss.IndexFlatL2(dimension)
-    index.add(np.array(embeddings))
-    return index, chunks
-
-def search_chunks(query, index, chunks, top_k=5):
-    query_embedding = get_embedding(query)
-    _, I = index.search(np.array([query_embedding]), top_k)
-    return [chunks[i] for i in I[0]]
-
-def generate_answer_with_groq(query, context):
-    prompt = f"""
-    You are a helpful assistant. Use the following context to answer the question:
-    {context}
-    Question: {query}
-    Answer:
-    """
-    response = client.chat.completions.create(
-        messages=[{"role": "user", "content": prompt}],
-        model="llama3-8b-8192"
+def retrieve_chunks(question, index, embed_model, text_chunks, k=3):
+    question_embedding = embed_model.encode(question)
+    D, I = index.search(np.array([question_embedding]), k)
+    return [text_chunks[i] for i in I[0]]
+
+# --- Groq Answer Generator ---
+def generate_answer_with_groq(question, context):
+    url = "https://api.groq.com/openai/v1/chat/completions"
+    api_key = os.environ.get("GROQ_API_KEY")
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+    prompt = (
+        f"Customer asked: '{question}'\n\n"
+        f"Here is the relevant product or policy info to help:\n{context}\n\n"
+        f"Respond in a friendly and helpful tone as a toy shop support agent."
     )
-    return response.choices[0].message.content.strip()
-
-# --- Flask Webhook Endpoint ---
-@flask_app.route("/twilio-webhook", methods=['POST'])
-def whatsapp_webhook():
-    global conversation_index, chunk_store
-    data = request.form
-    from_number = data['From']
-    message = data['Body']
-    conversation_sid = data.get('ConversationSid')
-
-    if conversation_sid and conversation_index:
-        context_chunks = search_chunks(message, conversation_index, chunk_store[conversation_sid])
-        context = "\n".join(context_chunks)
-        answer = generate_answer_with_groq(message, context)
-        twilio_client.conversations.conversations(conversation_sid).messages.create(author="bot", body=answer)
-    return "OK"
+    payload = {
+        "model": "llama3-8b-8192",
+        "messages": [
+            {
+                "role": "system",
+                "content": (
+                    "You are ToyBot, a friendly and helpful WhatsApp assistant for an online toy shop. "
+                    "Your goal is to politely answer customer questions, help them choose the right toys, "
+                    "provide order or delivery information, explain return policies, and guide them through purchases."
+                )
+            },
+            {"role": "user", "content": prompt},
+        ],
+        "temperature": 0.5,
+        "max_tokens": 300,
+    }
+    response = requests.post(url, headers=headers, json=payload)
+    response.raise_for_status()
+    return response.json()['choices'][0]['message']['content'].strip()
+
+# --- Twilio Functions ---
+def get_whatsapp_conversation_sids(client):
+    sids = []
+    conversations = client.conversations.v1.conversations.list(limit=50)
+    for convo in conversations:
+        try:
+            participants = client.conversations.v1.conversations(convo.sid).participants.list()
+            for p in participants:
+                if (p.identity and p.identity.startswith("whatsapp:")) or (
+                    p.messaging_binding and p.messaging_binding.get("address", "").startswith("whatsapp:")
+                ):
+                    sids.append(convo.sid)
+                    break
+        except:
+            continue
+    return sids
+
+def fetch_latest_incoming_message(client, conversation_sid):
+    messages = client.conversations.v1.conversations(conversation_sid).messages.list(limit=10)
+    for msg in reversed(messages):
+        if msg.author.startswith("whatsapp:"):
+            return {
+                "sid": msg.sid,
+                "body": msg.body,
+                "author": msg.author,
+                "timestamp": msg.date_created,
+            }
+    return None
+
+def send_twilio_message(client, conversation_sid, body):
+    return client.conversations.v1.conversations(conversation_sid).messages.create(
+        author="system", body=body
+    )
+
+# --- Load Knowledge Base ---
+def setup_knowledge_base():
+    folder_path = "docs"
+    all_text = ""
+    for file in os.listdir(folder_path):
+        path = os.path.join(folder_path, file)
+        if file.endswith(".pdf"):
+            raw_text = extract_text_from_pdf(path)
+            all_text += clean_extracted_text(raw_text) + "\n"
+        elif file.endswith((".docx", ".doc")):
+            all_text += extract_text_from_docx(path) + "\n"
+
+    tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
+    chunks = chunk_text(all_text, tokenizer)
+    model = SentenceTransformer('all-mpnet-base-v2')
+    embeddings = model.encode(chunks, truncate=True, show_progress_bar=False)
+    dim = embeddings[0].shape[0]
+    index = faiss.IndexFlatL2(dim)
+    index.add(np.array(embeddings).astype('float32'))
+    return index, model, chunks
+
+# --- Monitor Conversations ---
+def start_conversation_monitor(client, index, embed_model, text_chunks):
+    monitored_sids = set()
+
+    def poll_conversation(convo_sid):
+        last_processed_timestamp = None
+        while True:
+            try:
+                latest_msg = fetch_latest_incoming_message(client, convo_sid)
+                if latest_msg:
+                    msg_time = latest_msg["timestamp"]
+                    if last_processed_timestamp is None or msg_time > last_processed_timestamp:
+                        last_processed_timestamp = msg_time
+                        question = latest_msg["body"]
+                        sender = latest_msg["author"]
+                        print(f"\n📥 New message from {sender} in {convo_sid}: {question}")
+                        context = "\n\n".join(retrieve_chunks(question, index, embed_model, text_chunks))
+                        answer = generate_answer_with_groq(question, context)
+                        send_twilio_message(client, convo_sid, answer)
+                        print(f"📤 Replied to {sender}: {answer}")
+                time.sleep(3)
+            except Exception as e:
+                print(f"❌ Error in convo {convo_sid} polling:", e)
+                time.sleep(5)
+
+    def monitor_all_conversations():
+        while True:
+            try:
+                current_sids = set(get_whatsapp_conversation_sids(client))
+                new_sids = current_sids - monitored_sids
+                for sid in new_sids:
+                    print(f"➡️ Monitoring new conversation: {sid}")
+                    monitored_sids.add(sid)
+                    threading.Thread(target=poll_conversation, args=(sid,), daemon=True).start()
+                time.sleep(15)
+            except Exception as e:
+                print("❌ Error in conversation monitoring loop:", e)
+                time.sleep(15)
+
+    threading.Thread(target=monitor_all_conversations, daemon=True).start()
 
 # --- Streamlit UI ---
-def streamlit_ui():
-    global latest_conversation_sid, conversation_index, chunk_store
-    st.title("Quasa – Smart WhatsApp Chatbot")
-
-    uploaded_file = st.file_uploader("Upload a PDF document", type="pdf")
-    if uploaded_file is not None:
-        with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp_file:
-            tmp_file.write(uploaded_file.read())
-            pdf_path = tmp_file.name
-
-        text = extract_text_with_pdfplumber(pdf_path)
-        chunks = chunk_text(text)
-        index, stored_chunks = build_faiss_index(chunks)
-
-        # Get the last conversation SID from Twilio
-        conversations = twilio_client.conversations.conversations.list(limit=1)
-        if conversations:
-            latest_conversation_sid = conversations[0].sid
-            conversation_index = index
-            chunk_store[latest_conversation_sid] = stored_chunks
-            st.success(f"PDF indexed and ready for conversation. Conversation SID: {latest_conversation_sid}")
-        else:
-            st.error("No active Twilio conversations found.")
-
-    st.markdown("---")
-    st.markdown("**Waiting for incoming WhatsApp messages...**")
-
-# --- Run Flask in Background Thread ---
-def run_flask():
-    flask_app.run(host="0.0.0.0", port=5000)
-
-flask_thread = threading.Thread(target=run_flask)
-flask_thread.daemon = True
-flask_thread.start()
-
-# --- Start Streamlit App ---
-if __name__ == "__main__":
-    streamlit_ui()
+st.set_page_config(page_title="Quasa – A Smart WhatsApp Chatbot", layout="wide")
+st.title("📱 Quasa – A Smart WhatsApp Chatbot")
+
+account_sid = st.secrets.get("TWILIO_SID")
+auth_token = st.secrets.get("TWILIO_TOKEN")
+GROQ_API_KEY = st.secrets.get("GROQ_API_KEY")
+
+if not all([account_sid, auth_token, GROQ_API_KEY]):
+    st.warning("⚠️ Provide all credentials below:")
+    account_sid = st.text_input("Twilio SID", value=account_sid or "")
+    auth_token = st.text_input("Twilio Token", type="password", value=auth_token or "")
+    GROQ_API_KEY = st.text_input("GROQ API Key", type="password", value=GROQ_API_KEY or "")
+
+if all([account_sid, auth_token, GROQ_API_KEY]):
+    os.environ["GROQ_API_KEY"] = GROQ_API_KEY
+    client = Client(account_sid, auth_token)
+    conversation_sids = get_whatsapp_conversation_sids(client)
+
+    if conversation_sids:
+        st.success(f"✅ {len(conversation_sids)} WhatsApp conversation(s) found. Initializing chatbot...")
+        index, model, chunks = setup_knowledge_base()
+        start_conversation_monitor(client, index, model, chunks)
+        st.success("🟢 Chatbot is running in background and will reply to new messages.")
+    else:
+        st.error("❌ No WhatsApp conversations found.")