Update app.py

app.py

@@ -13,103 +13,108 @@ import requests
 from io import StringIO
 from pdfminer.high_level import extract_text_to_fp
 from pdfminer.layout import LAParams
-from twilio.base.exceptions import TwilioRestException
+from twilio.base.exceptions import TwilioRestException  # Add this at the top
 import pdfplumber
 import datetime
 import csv
-import json
-import re
 
 APP_START_TIME = datetime.datetime.now(datetime.timezone.utc)
-print(f"APP_START_TIME: {APP_START_TIME}")
+
 os.environ["PYTORCH_JIT"] = "0"
 
-# ---------------- PDF & DOCX & JSON Extraction ----------------
+# --- PDF Extraction ---
 def _extract_tables_from_page(page):
+    """Extracts tables from a single page of a PDF."""
+
     tables = page.extract_tables()
+    if not tables:
+        return []
+
     formatted_tables = []
     for table in tables:
         formatted_table = []
         for row in table:
-            formatted_row = [cell if cell is not None else "" for cell in row]
-            formatted_table.append(formatted_row)
+            if row:  # Filter out empty rows
+                formatted_row = [cell if cell is not None else "" for cell in row]  # Replace None with ""
+                formatted_table.append(formatted_row)
+            else:
+                formatted_table.append([""])  # Append an empty row if the row is None
         formatted_tables.append(formatted_table)
     return formatted_tables
-
+    
 def extract_text_from_pdf(pdf_path):
     text_output = StringIO()
     all_tables = []
     try:
         with pdfplumber.open(pdf_path) as pdf:
             for page in pdf.pages:
-                all_tables.extend(_extract_tables_from_page(page))
+                # Extract tables
+                page_tables = _extract_tables_from_page(page)
+                if page_tables:
+                    all_tables.extend(page_tables)
+                # Extract text
                 text = page.extract_text()
                 if text:
                     text_output.write(text + "\n\n")
     except Exception as e:
-        print(f"pdfplumber error: {e}")
+        print(f"Error extracting with pdfplumber: {e}")
+        # Fallback to pdfminer if pdfplumber fails
         with open(pdf_path, 'rb') as file:
-            extract_text_to_fp(file, text_output, laparams=LAParams(), output_type='text')
-    return text_output.getvalue(), all_tables
+            extract_text_to_fp(file, text_output, laparams=LAParams(), output_type='text', codec=None)
+    extracted_text = text_output.getvalue()
+    return extracted_text, all_tables  # Return text and list of tables
+
+def clean_extracted_text(text):
+    lines = text.splitlines()
+    cleaned = []
+    for line in lines:
+        line = line.strip()
+        if line:
+            line = ' '.join(line.split())
+            cleaned.append(line)
+    return '\n'.join(cleaned)
 
 def _format_tables_internal(tables):
+    """Formats extracted tables into a string representation."""
+
     formatted_tables_str = []
     for table in tables:
+        # Use csv writer to handle commas and quotes correctly
         with StringIO() as csvfile:
-            writer = csv.writer(csvfile)
-            writer.writerows(table)
+            csvwriter = csv.writer(csvfile)
+            csvwriter.writerows(table)
             formatted_tables_str.append(csvfile.getvalue())
     return "\n\n".join(formatted_tables_str)
 
-def clean_extracted_text(text):
-    return '\n'.join(' '.join(line.strip().split()) for line in text.splitlines() if line.strip())
-
+# --- DOCX Extraction ---
 def extract_text_from_docx(docx_path):
     try:
         doc = docx.Document(docx_path)
         return '\n'.join(para.text for para in doc.paragraphs)
-    except:
+    except Exception:
         return ""
 
-def load_json_data(json_path):
-    try:
-        with open(json_path, 'r', encoding='utf-8') as f:
-            data = json.load(f)
-        if isinstance(data, dict):
-            # Flatten dictionary values (avoiding nested structures as strings)
-            return "\n".join(f"{key}: {value}" for key, value in data.items() if not isinstance(value, (dict, list)))
-        elif isinstance(data, list):
-            # Flatten list of dictionaries
-            all_items = []
-            for item in data:
-                if isinstance(item, dict):
-                    all_items.append("\n".join(f"{key}: {value}" for key, value in item.items() if not isinstance(value, (dict, list))))
-            return "\n\n".join(all_items)
-        else:
-            return json.dumps(data, ensure_ascii=False, indent=2)
-    except Exception as e:
-        print(f"JSON read error: {e}")
-        return ""
-
-# ---------------- Chunking ----------------
-def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32):
+# --- Chunking ---
+def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32, max_tokens=512):
     tokens = tokenizer.tokenize(text)
     chunks = []
     start = 0
     while start < len(tokens):
         end = min(start + chunk_size, len(tokens))
-        chunk = tokens[start:end]
-        chunks.append(tokenizer.convert_tokens_to_string(chunk))
-        if end == len(tokens): break
+        chunk_tokens = tokens[start:end]
+        chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
+        chunks.append(chunk_text)
+        if end == len(tokens):
+            break
         start += chunk_size - chunk_overlap
     return chunks
 
 def retrieve_chunks(question, index, embed_model, text_chunks, k=3):
-    q_embedding = embed_model.encode(question)
-    D, I = index.search(np.array([q_embedding]), k)
+    question_embedding = embed_model.encode(question)
+    D, I = index.search(np.array([question_embedding]), k)
     return [text_chunks[i] for i in I[0]]
 
-# ---------------- Groq Answer Generator ----------------
+# --- Groq Answer Generator ---
 def generate_answer_with_groq(question, context):
     url = "https://api.groq.com/openai/v1/chat/completions"
     api_key = os.environ.get("GROQ_API_KEY")
@@ -119,9 +124,8 @@ def generate_answer_with_groq(question, context):
     }
     prompt = (
         f"Customer asked: '{question}'\n\n"
-        f"Here is the relevant information to help:\n{context}\n\n"
-        f"Respond in a friendly and helpful tone as a toy shop support agent, "
-        f"addressing the customer by their name if it's available in the context."
+        f"Here is the relevant product or policy info to help:\n{context}\n\n"
+        f"Respond in a friendly and helpful tone as a toy shop support agent."
     )
     payload = {
         "model": "llama3-8b-8192",
@@ -129,11 +133,9 @@ def generate_answer_with_groq(question, context):
             {
                 "role": "system",
                 "content": (
-                    "You are ToyBot, a friendly WhatsApp assistant for an online toy shop. "
-                    "Help customers with toys, delivery, and returns in a helpful tone. "
-                    "When responding, try to find the customer's name in the provided context "
-                    "and address them directly. If the context contains order details and status, "
-                    "include that information in your response."
+                    "You are ToyBot, a friendly and helpful WhatsApp assistant for an online toy shop. "
+                    "Your goal is to politely answer customer questions, help them choose the right toys, "
+                    "provide order or delivery information, explain return policies, and guide them through purchases."
                 )
             },
             {"role": "user", "content": prompt},
@@ -145,10 +147,9 @@ def generate_answer_with_groq(question, context):
     response.raise_for_status()
     return response.json()['choices'][0]['message']['content'].strip()
 
-# ---------------- Twilio Integration ----------------
+# --- Twilio Functions ---
 def fetch_latest_incoming_message(client, conversation_sid):
     try:
-        print(f"fetch_latest_incoming_message Twilio SID : {conversation_sid}")
         messages = client.conversations.v1.conversations(conversation_sid).messages.list()
         for msg in reversed(messages):
             if msg.author.startswith("whatsapp:"):
@@ -159,114 +160,136 @@ def fetch_latest_incoming_message(client, conversation_sid):
                     "timestamp": msg.date_created,
                 }
     except TwilioRestException as e:
-        print(f"❌ Twilio error for SID {conversation_sid}: {e}")
-    return None
+        if e.status == 404:
+            print(f"Conversation {conversation_sid} not found, skipping...")
+        else:
+            print(f"Twilio error fetching messages for {conversation_sid}:", e)
+    except Exception as e:
+        #print(f"Unexpected error in fetch_latest_incoming_message for {conversation_sid}:", e)
+        pass
 
+    return None
 
 def send_twilio_message(client, conversation_sid, body):
     return client.conversations.v1.conversations(conversation_sid).messages.create(
         author="system", body=body
     )
 
-# ---------------- Knowledge Base Setup ----------------
+# --- Load Knowledge Base ---
 def setup_knowledge_base():
     folder_path = "docs"
     all_text = ""
 
-    for filename in os.listdir(folder_path):
-        file_path = os.path.join(folder_path, filename)
-        if filename.endswith(".pdf"):
-            text, tables = extract_text_from_pdf(file_path)
-            all_text += clean_extracted_text(text) + "\n"
-            all_text += _format_tables_internal(tables) + "\n"
-        elif filename.endswith(".docx"):
-            text = extract_text_from_docx(file_path)
-            all_text += clean_extracted_text(text) + "\n"
-        elif filename.endswith(".json"):
-            text = load_json_data(file_path)
-            all_text += text + "\n"
-        elif filename.endswith(".csv"):
-            try:
-                with open(file_path, newline='', encoding='utf-8') as csvfile:
-                    reader = csv.DictReader(csvfile)
-                    for row in reader:
-                        line = ' | '.join(f"{k}: {v}" for k, v in row.items())
-                        all_text += line + "\n"
-            except Exception as e:
-                print(f"CSV read error: {e}")
+    # Process PDFs
+    for filename in ["FAQ.pdf", "ProductReturnPolicy.pdf"]:
+        pdf_path = os.path.join(folder_path, filename)
+        text, tables = extract_text_from_pdf(pdf_path)
+        all_text += clean_extracted_text(text) + "\n"
+        all_text += _format_tables_internal(tables) + "\n"
+
+    # Process CSVs
+    for filename in ["CustomerOrders.csv"]:
+        csv_path = os.path.join(folder_path, filename)
+        try:
+            with open(csv_path, newline='', encoding='utf-8') as csvfile:
+                reader = csv.DictReader(csvfile)
+                for row in reader:
+                    line = f"Order ID: {row.get('OrderID')} | Customer Name: {row.get('CustomerName')} | Order Date: {row.get('OrderDate')} | ProductID: {row.get('ProductID')} | Date: {row.get('OrderDate')} | Quantity: {row.get('Quantity')} | UnitPrice(USD): {row.get('UnitPrice(USD)')} | TotalPrice(USD): {row.get('TotalPrice(USD)')} | ShippingAddress: {row.get('ShippingAddress')} | OrderStatus: {row.get('OrderStatus')}"
+                    all_text += line + "\n"
+        except Exception as e:
+            print(f"❌ Error reading {filename}: {e}")
+
+    for filename in ["Products.csv"]:
+        csv_path = os.path.join(folder_path, filename)
+        try:
+            with open(csv_path, newline='', encoding='utf-8') as csvfile:
+                reader = csv.DictReader(csvfile)
+                for row in reader:
+                    line = f"Product ID: {row.get('ProductID')} | Toy Name: {row.get('ToyName')} | Category: {row.get('Category')} | Price(USD): {row.get('Price(USD)')} | Stock Quantity: {row.get('StockQuantity')} | Description: {row.get('Description')}"
+                    all_text += line + "\n"
+        except Exception as e:
+            print(f"❌ Error reading {filename}: {e}")
 
+    # Tokenization & chunking
     tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
     chunks = chunk_text(all_text, tokenizer)
     model = SentenceTransformer('all-mpnet-base-v2')
-    embeddings = model.encode(chunks, show_progress_bar=False)
+    embeddings = model.encode(chunks, show_progress_bar=False, truncation=True, max_length=512)
     dim = embeddings[0].shape[0]
     index = faiss.IndexFlatL2(dim)
     index.add(np.array(embeddings).astype('float32'))
     return index, model, chunks
 
-# ---------------- Monitor Twilio Conversations ----------------
+
+
+# --- Monitor Conversations ---
 def start_conversation_monitor(client, index, embed_model, text_chunks):
     processed_convos = set()
     last_processed_timestamp = {}
 
-    def poll_convo(convo_sid):
-        print(f"🧵 Started polling for SID: {convo_sid}")
+    def poll_conversation(convo_sid):
         while True:
             try:
                 latest_msg = fetch_latest_incoming_message(client, convo_sid)
                 if latest_msg:
                     msg_time = latest_msg["timestamp"]
-                    prev_time = last_processed_timestamp.get(convo_sid)
-
-                    if prev_time is None or msg_time > prev_time:
+                    if convo_sid not in last_processed_timestamp or msg_time > last_processed_timestamp[convo_sid]:
                         last_processed_timestamp[convo_sid] = msg_time
                         question = latest_msg["body"]
                         sender = latest_msg["author"]
-                        print(f"📩 New message from {sender}: {question}")
+                        print(f"\n📥 New message from {sender} in {convo_sid}: {question}")
                         context = "\n\n".join(retrieve_chunks(question, index, embed_model, text_chunks))
                         answer = generate_answer_with_groq(question, context)
                         send_twilio_message(client, convo_sid, answer)
+                        print(f"📤 Replied to {sender}: {answer}")
+                time.sleep(3)
             except Exception as e:
-                print(f"⚠️ Error in poll_convo: {e}")
-            time.sleep(120)
-
-    # Get all conversations and find the most recent one after APP_START_TIME
-    conversations = client.conversations.v1.conversations.list(limit=20)
-    print("Conversations (date_created) sorted in descending order:")
-    for c in sorted_convos:
-        print(f"Date: {c.date_created}, ID: {c.sid}")
-    sorted_convos = sorted(
-        [c for c in conversations if c.date_created > APP_START_TIME],
-        key=lambda c: c.date_created,
-        reverse=True
-    )
+                print(f"❌ Error in convo {convo_sid} polling:", e)
+                time.sleep(5)
 
-    if not sorted_convos:
-        print("❗ No new Twilio conversations found after app startup.")
-        return
+    def poll_new_conversations():
+        print("➡️ Monitoring for new WhatsApp conversations...")
+        while True:
+            try:
+                conversations = client.conversations.v1.conversations.list(limit=20)
+                for convo in conversations:
+                    convo_full = client.conversations.v1.conversations(convo.sid).fetch()
+                    if convo.sid not in processed_convos and convo_full.date_created > APP_START_TIME:
+                        participants = client.conversations.v1.conversations(convo.sid).participants.list()
+                        for p in participants:
+                            address = p.messaging_binding.get("address", "") if p.messaging_binding else ""
+                            if address.startswith("whatsapp:"):
+                                print(f"🆕 New WhatsApp convo found: {convo.sid}")
+                                processed_convos.add(convo.sid)
+                                threading.Thread(target=poll_conversation, args=(convo.sid,), daemon=True).start()
+            except Exception as e:
+                print("❌ Error polling conversations:", e)
+            time.sleep(5)
 
-    latest_convo = sorted_convos[0]
-    if latest_convo.sid not in processed_convos:
-        processed_convos.add(latest_convo.sid)
-        print(f"✅ Monitoring latest conversation SID: {latest_convo.sid}, Created: {latest_convo.date_created}")
-        threading.Thread(target=poll_convo, args=(latest_convo.sid,), daemon=True).start()
+    # ✅ Launch conversation polling monitor
+    threading.Thread(target=poll_new_conversations, daemon=True).start()
 
 
-# ---------------- Main Entry ----------------
-if __name__ == "__main__":
-    st.title("🤖 ToyBot WhatsApp Assistant")
-    st.write("Initializing knowledge base...")
 
-    index, model, chunks = setup_knowledge_base()
+# --- Streamlit UI ---
+st.set_page_config(page_title="Quasa – A Smart WhatsApp Chatbot", layout="wide")
+st.title("📱 Quasa – A Smart WhatsApp Chatbot")
 
-    st.success("Knowledge base loaded.")
-    st.write("Waiting for WhatsApp messages...")
-
-    account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
-    auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
-    if not account_sid or not auth_token:
-        st.error("❌ Twilio credentials not set.")
-    else:
-        client = Client(account_sid, auth_token)
-        start_conversation_monitor(client, index, model, chunks)
-        st.info("✅ Bot is now monitoring the latest Twilio conversation.")
+account_sid = st.secrets.get("TWILIO_SID")
+auth_token = st.secrets.get("TWILIO_TOKEN")
+GROQ_API_KEY = st.secrets.get("GROQ_API_KEY")
+
+if not all([account_sid, auth_token, GROQ_API_KEY]):
+    st.warning("⚠️ Provide all credentials below:")
+    account_sid = st.text_input("Twilio SID", value=account_sid or "")
+    auth_token = st.text_input("Twilio Token", type="password", value=auth_token or "")
+    GROQ_API_KEY = st.text_input("GROQ API Key", type="password", value=GROQ_API_KEY or "")
+
+if all([account_sid, auth_token, GROQ_API_KEY]):
+    os.environ["GROQ_API_KEY"] = GROQ_API_KEY
+    client = Client(account_sid, auth_token)
+
+    st.success("🟢 Monitoring new WhatsApp conversations...")
+    index, model, chunks = setup_knowledge_base()
+    threading.Thread(target=start_conversation_monitor, args=(client, index, model, chunks), daemon=True).start()
+    st.info("⏳ Waiting for new messages...")