Update app.py

app.py

@@ -3,30 +3,35 @@ import time
 import threading
 import streamlit as st
 from twilio.rest import Client
-from pdfminer.high_level import extract_text
 from sentence_transformers import SentenceTransformer
 from transformers import AutoTokenizer
 import faiss
 import numpy as np
 import docx
 from groq import Groq
-import PyPDF2
 import requests
+from io import StringIO
+from pdfminer.high_level import extract_text_to_fp
+from pdfminer.layout import LAParams
 
-# --- Text Extraction Utilities ---
+# --- PDF Extraction (Improved for Tables & Paragraphs) ---
 def extract_text_from_pdf(pdf_path):
-    try:
-        text = ""
-        with open(pdf_path, 'rb') as file:
-            reader = PyPDF2.PdfReader(file)
-            for page in reader.pages:
-                page_text = page.extract_text()
-                if page_text:
-                    text += page_text
-        return text
-    except:
-        return extract_text(pdf_path)
-
+    output_string = StringIO()
+    with open(pdf_path, 'rb') as file:
+        extract_text_to_fp(file, output_string, laparams=LAParams(), output_type='text', codec=None)
+    return output_string.getvalue()
+
+def clean_extracted_text(text):
+    lines = text.splitlines()
+    cleaned = []
+    for line in lines:
+        line = line.strip()
+        if line:
+            line = ' '.join(line.split())  # remove extra spaces
+            cleaned.append(line)
+    return '\n'.join(cleaned)
+
+# --- DOCX Extraction ---
 def extract_text_from_docx(docx_path):
     try:
         doc = docx.Document(docx_path)
@@ -42,20 +47,16 @@ def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32, max_tokens=512
     while start < len(tokens):
         end = min(start + chunk_size, len(tokens))
         chunk_tokens = tokens[start:end]
-        # Drop chunk if it's too long after detokenization
         chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
-        # Double-check token count with tokenizer to be safe
         if len(tokenizer.encode(chunk_text)) <= max_tokens:
             chunks.append(chunk_text.strip())
         start += chunk_size - chunk_overlap
     return chunks
 
-
 def retrieve_chunks(question, index, embed_model, text_chunks, k=3):
     question_embedding = embed_model.encode(question)
     D, I = index.search(np.array([question_embedding]), k)
-    relevant_chunks = [text_chunks[i] for i in I[0]]
-    return relevant_chunks
+    return [text_chunks[i] for i in I[0]]
 
 # --- Groq Answer Generator ---
 def generate_answer_with_groq(question, context):
@@ -119,7 +120,6 @@ def fetch_latest_incoming_message(client, conversation_sid):
             }
     return None
 
-
 def send_twilio_message(client, conversation_sid, body):
     return client.conversations.v1.conversations(conversation_sid).messages.create(
         author="system", body=body
@@ -132,45 +132,44 @@ def setup_knowledge_base():
     for file in os.listdir(folder_path):
         path = os.path.join(folder_path, file)
         if file.endswith(".pdf"):
-            all_text += extract_text_from_pdf(path) + "\n"
+            raw_text = extract_text_from_pdf(path)
+            all_text += clean_extracted_text(raw_text) + "\n"
         elif file.endswith((".docx", ".doc")):
             all_text += extract_text_from_docx(path) + "\n"
+
     tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
     chunks = chunk_text(all_text, tokenizer)
     model = SentenceTransformer('all-mpnet-base-v2')
-    embeddings = model.encode(chunks)
+    embeddings = model.encode(chunks, truncate=True, show_progress_bar=False)
     dim = embeddings[0].shape[0]
     index = faiss.IndexFlatL2(dim)
     index.add(np.array(embeddings).astype('float32'))
     return index, model, chunks
 
-# --- Monitor All Conversations ---
+# --- Monitor Conversations ---
 def start_conversation_monitor(client, index, embed_model, text_chunks):
-    last_msg_index = {}
     monitored_sids = set()
 
-def poll_conversation(convo_sid):
-    last_processed_timestamp = None
-    while True:
-        try:
-            latest_msg = fetch_latest_incoming_message(client, convo_sid)
-            if latest_msg:
-                msg_time = latest_msg["timestamp"]
-                if last_processed_timestamp is None or msg_time > last_processed_timestamp:
-                    last_processed_timestamp = msg_time
-                    question = latest_msg["body"]
-                    sender = latest_msg["author"]
-                    print(f"\n📥 New message from {sender} in {convo_sid}: {question}")
-                    context = "\n\n".join(retrieve_chunks(question, index, embed_model, text_chunks))
-                    answer = generate_answer_with_groq(question, context)
-                    send_twilio_message(client, convo_sid, answer)
-                    print(f"📤 Replied to {sender}: {answer}")
-            time.sleep(3)
-        except Exception as e:
-            print(f"❌ Error in convo {convo_sid} polling:", e)
-            time.sleep(5)
-
-
+    def poll_conversation(convo_sid):
+        last_processed_timestamp = None
+        while True:
+            try:
+                latest_msg = fetch_latest_incoming_message(client, convo_sid)
+                if latest_msg:
+                    msg_time = latest_msg["timestamp"]
+                    if last_processed_timestamp is None or msg_time > last_processed_timestamp:
+                        last_processed_timestamp = msg_time
+                        question = latest_msg["body"]
+                        sender = latest_msg["author"]
+                        print(f"\n📥 New message from {sender} in {convo_sid}: {question}")
+                        context = "\n\n".join(retrieve_chunks(question, index, embed_model, text_chunks))
+                        answer = generate_answer_with_groq(question, context)
+                        send_twilio_message(client, convo_sid, answer)
+                        print(f"📤 Replied to {sender}: {answer}")
+                time.sleep(3)
+            except Exception as e:
+                print(f"❌ Error in convo {convo_sid} polling:", e)
+                time.sleep(5)
 
     def monitor_all_conversations():
         while True:
@@ -178,18 +177,16 @@ def poll_conversation(convo_sid):
                 current_sids = set(get_whatsapp_conversation_sids(client))
                 new_sids = current_sids - monitored_sids
                 for sid in new_sids:
-                    print(f"➡️ Starting to monitor new conversation: {sid}")
+                    print(f"➡️ Monitoring new conversation: {sid}")
                     monitored_sids.add(sid)
                     threading.Thread(target=poll_conversation, args=(sid,), daemon=True).start()
-                time.sleep(15)  # refresh every 15 seconds or adjust as needed
+                time.sleep(15)
             except Exception as e:
                 print("❌ Error in conversation monitoring loop:", e)
                 time.sleep(15)
 
-    # Start the monitoring loop in a separate thread so it runs in background
     threading.Thread(target=monitor_all_conversations, daemon=True).start()
 
-
 # --- Streamlit UI ---
 st.set_page_config(page_title="Quasa – A Smart WhatsApp Chatbot", layout="wide")
 st.title("📱 Quasa – A Smart WhatsApp Chatbot")
@@ -215,4 +212,4 @@ if all([account_sid, auth_token, GROQ_API_KEY]):
         start_conversation_monitor(client, index, model, chunks)
         st.success("🟢 Chatbot is running in background and will reply to new messages.")
     else:
-        st.error("❌ No WhatsApp conversations found.")
+        st.error("❌ No WhatsApp conversations found.")