Update app.py

app.py

@@ -13,283 +13,201 @@ import requests
 from io import StringIO
 from pdfminer.high_level import extract_text_to_fp
 from pdfminer.layout import LAParams
-from twilio.base.exceptions import TwilioRestException  # Add this at the top
+from twilio.base.exceptions import TwilioRestException
 import pdfplumber
 import datetime
 import csv
+import json
+import re
 
 APP_START_TIME = datetime.datetime.now(datetime.timezone.utc)
-
 os.environ["PYTORCH_JIT"] = "0"
 
-# --- PDF Extraction ---
-def _extract_tables_from_page(page):
-    """Extracts tables from a single page of a PDF."""
+# Twilio Setup
+TWILIO_ACCOUNT_SID = os.getenv("TWILIO_ACCOUNT_SID")
+TWILIO_AUTH_TOKEN = os.getenv("TWILIO_AUTH_TOKEN")
+twilio_client = Client(TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN)
 
+# ---------------- PDF & DOCX & JSON Extraction ----------------
+def _extract_tables_from_page(page):
     tables = page.extract_tables()
-    if not tables:
-        return []
-
     formatted_tables = []
     for table in tables:
-        formatted_table = []
-        for row in table:
-            if row:  # Filter out empty rows
-                formatted_row = [cell if cell is not None else "" for cell in row]  # Replace None with ""
-                formatted_table.append(formatted_row)
-            else:
-                formatted_table.append([""])  # Append an empty row if the row is None
-        formatted_tables.append(formatted_table)
+        formatted_row = [[cell if cell is not None else "" for cell in row] for row in table]
+        formatted_tables.append(formatted_row)
     return formatted_tables
-    
+
 def extract_text_from_pdf(pdf_path):
     text_output = StringIO()
     all_tables = []
     try:
         with pdfplumber.open(pdf_path) as pdf:
             for page in pdf.pages:
-                # Extract tables
-                page_tables = _extract_tables_from_page(page)
-                if page_tables:
-                    all_tables.extend(page_tables)
-                # Extract text
+                all_tables.extend(_extract_tables_from_page(page))
                 text = page.extract_text()
                 if text:
                     text_output.write(text + "\n\n")
     except Exception as e:
-        print(f"Error extracting with pdfplumber: {e}")
-        # Fallback to pdfminer if pdfplumber fails
         with open(pdf_path, 'rb') as file:
-            extract_text_to_fp(file, text_output, laparams=LAParams(), output_type='text', codec=None)
-    extracted_text = text_output.getvalue()
-    return extracted_text, all_tables  # Return text and list of tables
-
-def clean_extracted_text(text):
-    lines = text.splitlines()
-    cleaned = []
-    for line in lines:
-        line = line.strip()
-        if line:
-            line = ' '.join(line.split())
-            cleaned.append(line)
-    return '\n'.join(cleaned)
+            extract_text_to_fp(file, text_output, laparams=LAParams(), output_type='text')
+    return text_output.getvalue(), all_tables
 
 def _format_tables_internal(tables):
-    """Formats extracted tables into a string representation."""
-
-    formatted_tables_str = []
+    formatted = []
     for table in tables:
-        # Use csv writer to handle commas and quotes correctly
         with StringIO() as csvfile:
-            csvwriter = csv.writer(csvfile)
-            csvwriter.writerows(table)
-            formatted_tables_str.append(csvfile.getvalue())
-    return "\n\n".join(formatted_tables_str)
+            writer = csv.writer(csvfile)
+            writer.writerows(table)
+            formatted.append(csvfile.getvalue())
+    return "\n\n".join(formatted)
+
+def clean_extracted_text(text):
+    return '\n'.join(' '.join(line.strip().split()) for line in text.splitlines() if line.strip())
 
-# --- DOCX Extraction ---
-def extract_text_from_docx(docx_path):
+def extract_text_from_docx(path):
     try:
-        doc = docx.Document(docx_path)
-        return '\n'.join(para.text for para in doc.paragraphs)
-    except Exception:
+        doc = docx.Document(path)
+        return '\n'.join(p.text for p in doc.paragraphs)
+    except:
         return ""
 
-# --- Chunking ---
-def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32, max_tokens=512):
+def load_json_data(path):
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        if isinstance(data, dict):
+            return "\n".join(f"{k}: {v}" for k, v in data.items() if not isinstance(v, (dict, list)))
+        elif isinstance(data, list):
+            return "\n\n".join("\n".join(f"{k}: {v}" for k, v in item.items() if isinstance(item, dict)) for item in data)
+        else:
+            return json.dumps(data, ensure_ascii=False, indent=2)
+    except Exception as e:
+        return ""
+
+# ---------------- Chunking ----------------
+def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32):
     tokens = tokenizer.tokenize(text)
     chunks = []
     start = 0
     while start < len(tokens):
         end = min(start + chunk_size, len(tokens))
-        chunk_tokens = tokens[start:end]
-        chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
-        chunks.append(chunk_text)
-        if end == len(tokens):
-            break
+        chunk = tokens[start:end]
+        chunks.append(tokenizer.convert_tokens_to_string(chunk))
         start += chunk_size - chunk_overlap
     return chunks
 
 def retrieve_chunks(question, index, embed_model, text_chunks, k=3):
-    question_embedding = embed_model.encode(question)
-    D, I = index.search(np.array([question_embedding]), k)
+    q_embedding = embed_model.encode(question)
+    D, I = index.search(np.array([q_embedding]), k)
     return [text_chunks[i] for i in I[0]]
 
-# --- Groq Answer Generator ---
+# ---------------- Groq Answer Generator ----------------
 def generate_answer_with_groq(question, context):
     url = "https://api.groq.com/openai/v1/chat/completions"
-    api_key = os.environ.get("GROQ_API_KEY")
-    headers = {
-        "Authorization": f"Bearer {api_key}",
-        "Content-Type": "application/json",
-    }
-    prompt = (
-        f"Customer asked: '{question}'\n\n"
-        f"Here is the relevant product or policy info to help:\n{context}\n\n"
-        f"Respond in a friendly and helpful tone as a toy shop support agent."
-    )
+    api_key = os.getenv("GROQ_API_KEY")
+    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+    prompt = f"Customer asked: '{question}'\n\nHere is the relevant information to help:\n{context}"
     payload = {
         "model": "llama3-8b-8192",
         "messages": [
-            {
-                "role": "system",
-                "content": (
-                    "You are ToyBot, a friendly and helpful WhatsApp assistant for an online toy shop. "
-                    "Your goal is to politely answer customer questions, help them choose the right toys, "
-                    "provide order or delivery information, explain return policies, and guide them through purchases."
-                )
-            },
+            {"role": "system", "content": "You are ToyBot, a friendly WhatsApp assistant..."},
             {"role": "user", "content": prompt},
         ],
         "temperature": 0.5,
         "max_tokens": 300,
     }
     response = requests.post(url, headers=headers, json=payload)
-    response.raise_for_status()
     return response.json()['choices'][0]['message']['content'].strip()
 
-# --- Twilio Functions ---
+# ---------------- Twilio Integration ----------------
 def fetch_latest_incoming_message(client, conversation_sid):
     try:
         messages = client.conversations.v1.conversations(conversation_sid).messages.list()
         for msg in reversed(messages):
-            if msg.author.startswith("whatsapp:"):
+            if (
+                msg.author.startswith("whatsapp:") and
+                msg.date_created and
+                msg.date_created > APP_START_TIME
+            ):
                 return {
                     "sid": msg.sid,
                     "body": msg.body,
                     "author": msg.author,
                     "timestamp": msg.date_created,
                 }
-    except TwilioRestException as e:
-        if e.status == 404:
-            print(f"Conversation {conversation_sid} not found, skipping...")
-        else:
-            print(f"Twilio error fetching messages for {conversation_sid}:", e)
-    except Exception as e:
-        #print(f"Unexpected error in fetch_latest_incoming_message for {conversation_sid}:", e)
-        pass
-
-    return None
+    except TwilioRestException:
+        return None
 
 def send_twilio_message(client, conversation_sid, body):
-    return client.conversations.v1.conversations(conversation_sid).messages.create(
-        author="system", body=body
-    )
-
-# --- Load Knowledge Base ---
-def setup_knowledge_base():
-    folder_path = "docs"
-    all_text = ""
-
-    # Process PDFs
-    for filename in ["FAQ.pdf", "ProductReturnPolicy.pdf"]:
-        pdf_path = os.path.join(folder_path, filename)
-        text, tables = extract_text_from_pdf(pdf_path)
-        all_text += clean_extracted_text(text) + "\n"
-        all_text += _format_tables_internal(tables) + "\n"
-
-    # Process CSVs
-    for filename in ["CustomerOrders.csv"]:
-        csv_path = os.path.join(folder_path, filename)
-        try:
-            with open(csv_path, newline='', encoding='utf-8') as csvfile:
-                reader = csv.DictReader(csvfile)
-                for row in reader:
-                    line = f"Order ID: {row.get('OrderID')} | Customer Name: {row.get('CustomerName')} | Order Date: {row.get('OrderDate')} | ProductID: {row.get('ProductID')} | Date: {row.get('OrderDate')} | Quantity: {row.get('Quantity')} | UnitPrice(USD): {row.get('UnitPrice(USD)')} | TotalPrice(USD): {row.get('TotalPrice(USD)')} | ShippingAddress: {row.get('ShippingAddress')} | OrderStatus: {row.get('OrderStatus')}"
-                    all_text += line + "\n"
-        except Exception as e:
-            print(f"❌ Error reading {filename}: {e}")
-
-    for filename in ["Products.csv"]:
-        csv_path = os.path.join(folder_path, filename)
-        try:
-            with open(csv_path, newline='', encoding='utf-8') as csvfile:
-                reader = csv.DictReader(csvfile)
-                for row in reader:
-                    line = f"Product ID: {row.get('ProductID')} | Toy Name: {row.get('ToyName')} | Category: {row.get('Category')} | Price(USD): {row.get('Price(USD)')} | Stock Quantity: {row.get('StockQuantity')} | Description: {row.get('Description')}"
-                    all_text += line + "\n"
-        except Exception as e:
-            print(f"❌ Error reading {filename}: {e}")
-
-    # Tokenization & chunking
-    tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
-    chunks = chunk_text(all_text, tokenizer)
-    model = SentenceTransformer('all-mpnet-base-v2')
-    embeddings = model.encode(chunks, show_progress_bar=False, truncation=True, max_length=512)
-    dim = embeddings[0].shape[0]
-    index = faiss.IndexFlatL2(dim)
-    index.add(np.array(embeddings).astype('float32'))
-    return index, model, chunks
-
-
+    return client.conversations.v1.conversations(conversation_sid).messages.create(author="system", body=body)
 
-# --- Monitor Conversations ---
-def start_conversation_monitor(client, index, embed_model, text_chunks):
-    processed_convos = set()
-    last_processed_timestamp = {}
-
-    def poll_conversation(convo_sid):
-        while True:
-            try:
-                latest_msg = fetch_latest_incoming_message(client, convo_sid)
-                if latest_msg:
-                    msg_time = latest_msg["timestamp"]
-                    if convo_sid not in last_processed_timestamp or msg_time > last_processed_timestamp[convo_sid]:
-                        last_processed_timestamp[convo_sid] = msg_time
-                        question = latest_msg["body"]
-                        sender = latest_msg["author"]
-                        print(f"\n📥 New message from {sender} in {convo_sid}: {question}")
-                        context = "\n\n".join(retrieve_chunks(question, index, embed_model, text_chunks))
-                        answer = generate_answer_with_groq(question, context)
-                        send_twilio_message(client, convo_sid, answer)
-                        print(f"📤 Replied to {sender}: {answer}")
-                time.sleep(3)
-            except Exception as e:
-                print(f"❌ Error in convo {convo_sid} polling:", e)
-                time.sleep(5)
+def get_latest_whatsapp_conversation_sid(client):
+    try:
+        conversations = client.conversations.v1.conversations.list(limit=20)
+        # Filter conversations created after app start time
+        filtered = [
+            c for c in conversations
+            if c.date_created and c.date_created > APP_START_TIME
+        ]
+        for convo in sorted(filtered, key=lambda c: c.date_created, reverse=True):
+            messages = convo.messages.list(limit=1)
+            if messages and any(m.author.startswith("whatsapp:") and m.date_created > APP_START_TIME for m in messages):
+                return convo.sid
+    except Exception as e:
+        print("Error fetching valid conversation SID:", e)
+    return None
 
-    def poll_new_conversations():
-        print("➡️ Monitoring for new WhatsApp conversations...")
-        while True:
-            try:
-                conversations = client.conversations.v1.conversations.list(limit=20)
-                for convo in conversations:
-                    convo_full = client.conversations.v1.conversations(convo.sid).fetch()
-                    if convo.sid not in processed_convos and convo_full.date_created > APP_START_TIME:
-                        participants = client.conversations.v1.conversations(convo.sid).participants.list()
-                        for p in participants:
-                            address = p.messaging_binding.get("address", "") if p.messaging_binding else ""
-                            if address.startswith("whatsapp:"):
-                                print(f"🆕 New WhatsApp convo found: {convo.sid}")
-                                processed_convos.add(convo.sid)
-                                threading.Thread(target=poll_conversation, args=(convo.sid,), daemon=True).start()
-            except Exception as e:
-                print("❌ Error polling conversations:", e)
+# ---------------- Knowledge Base Setup ----------------
+def setup_knowledge_base():
+    folder = "docs"
+    text = ""
+    for f in os.listdir(folder):
+        path = os.path.join(folder, f)
+        if f.endswith(".pdf"):
+            t, tables = extract_text_from_pdf(path)
+            text += clean_extracted_text(t) + "\n" + _format_tables_internal(tables) + "\n"
+        elif f.endswith(".docx"):
+            text += clean_extracted_text(extract_text_from_docx(path)) + "\n"
+        elif f.endswith(".json"):
+            text += load_json_data(path) + "\n"
+        elif f.endswith(".csv"):
+            with open(path, newline='', encoding='utf-8') as csvfile:
+                reader = csv.reader(csvfile)
+                text += "\n".join(", ".join(row) for row in reader) + "\n"
+    return text
+
+# ---------------- App Logic ----------------
+def process_messages_loop():
+    embed_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+    tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+    knowledge_text = setup_knowledge_base()
+    text_chunks = chunk_text(knowledge_text, tokenizer)
+    embeddings = embed_model.encode(text_chunks)
+    index = faiss.IndexFlatL2(embeddings.shape[1])
+    index.add(embeddings)
+
+    seen_sids = set()
+
+    while True:
+        conversation_sid = get_latest_whatsapp_conversation_sid(twilio_client)
+        if not conversation_sid:
             time.sleep(5)
+            continue
 
-    # ✅ Launch conversation polling monitor
-    threading.Thread(target=poll_new_conversations, daemon=True).start()
-
-
-
-# --- Streamlit UI ---
-st.set_page_config(page_title="Quasa – A Smart WhatsApp Chatbot", layout="wide")
-st.title("📱 Quasa – A Smart WhatsApp Chatbot")
-
-account_sid = st.secrets.get("TWILIO_SID")
-auth_token = st.secrets.get("TWILIO_TOKEN")
-GROQ_API_KEY = st.secrets.get("GROQ_API_KEY")
+        message = fetch_latest_incoming_message(twilio_client, conversation_sid)
+        if message and message["sid"] not in seen_sids:
+            seen_sids.add(message["sid"])
+            question = message["body"]
+            chunks = retrieve_chunks(question, index, embed_model, text_chunks)
+            answer = generate_answer_with_groq(question, "\n\n".join(chunks))
+            send_twilio_message(twilio_client, conversation_sid, answer)
 
-if not all([account_sid, auth_token, GROQ_API_KEY]):
-    st.warning("⚠️ Provide all credentials below:")
-    account_sid = st.text_input("Twilio SID", value=account_sid or "")
-    auth_token = st.text_input("Twilio Token", type="password", value=auth_token or "")
-    GROQ_API_KEY = st.text_input("GROQ API Key", type="password", value=GROQ_API_KEY or "")
+        time.sleep(5)
 
-if all([account_sid, auth_token, GROQ_API_KEY]):
-    os.environ["GROQ_API_KEY"] = GROQ_API_KEY
-    client = Client(account_sid, auth_token)
+# ---------------- Streamlit UI ----------------
+st.title("ToyShop WhatsApp Assistant (Groq + Twilio)")
 
-    st.success("🟢 Monitoring new WhatsApp conversations...")
-    index, model, chunks = setup_knowledge_base()
-    threading.Thread(target=start_conversation_monitor, args=(client, index, model, chunks), daemon=True).start()
-    st.info("⏳ Waiting for new messages...")
+if st.button("Start WhatsApp Bot"):
+    thread = threading.Thread(target=process_messages_loop)
+    thread.start()
+    st.success("WhatsApp assistant started and monitoring for new messages.")