Update app.py

app.py

@@ -1,62 +1,123 @@
-import streamlit as st
-from PyPDF2 import PdfReader
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain_community.embeddings import HuggingFaceEmbeddings
-from langchain.vectorstores import FAISS
-import pandas as pd
 import os
-import io
+import time
+import threading
+import streamlit as st
+from twilio.rest import Client
+from sentence_transformers import SentenceTransformer
+from transformers import AutoTokenizer
+import faiss
+import numpy as np
+import docx
+from groq import Groq
 import requests
+from io import StringIO
+from pdfminer.high_level import extract_text_to_fp
+from pdfminer.layout import LAParams
+from twilio.base.exceptions import TwilioRestException  # Add this at the top
+import pdfplumber
+import datetime
+import csv
 
-# --- 1. Data Loading and Preprocessing ---
+APP_START_TIME = datetime.datetime.now(datetime.timezone.utc)
 
-@st.cache_data()
-def load_and_process_pdfs_from_folder(docs_folder="docs"):
-    """Loads and processes all PDF files from the specified folder."""
-    all_text = ""
+os.environ["PYTORCH_JIT"] = "0"
+
+# --- PDF Extraction ---
+def _extract_tables_from_page(page):
+    """Extracts tables from a single page of a PDF."""
+
+    tables = page.extract_tables()
+    if not tables:
+        return []
+
+    formatted_tables = []
+    for table in tables:
+        formatted_table = []
+        for row in table:
+            if row:  # Filter out empty rows
+                formatted_row = [cell if cell is not None else "" for cell in row]  # Replace None with ""
+                formatted_table.append(formatted_row)
+            else:
+                formatted_table.append([""])  # Append an empty row if the row is None
+        formatted_tables.append(formatted_table)
+    return formatted_tables
+    
+def extract_text_from_pdf(pdf_path):
+    text_output = StringIO()
     all_tables = []
-    for filename in os.listdir(docs_folder):
-        if filename.endswith(".pdf"):
-            filepath = os.path.join(docs_folder, filename)
-            try:
-                with open(filepath, 'rb') as file:
-                    pdf_reader = PdfReader(file)
-                    for page in pdf_reader.pages:
-                        all_text += page.extract_text() + "\n"
-                        try:
-                            for table in page.extract_tables():
-                                df = pd.DataFrame(table)
-                                all_tables.append(df)
-                        except Exception as e:
-                            print(f"Could not extract tables from page in {filename}. Error: {e}")
-            except Exception as e:
-                st.error(f"Error reading PDF {filename}: {e}")
-    return all_text, all_tables
-
-@st.cache_data()
-def split_text_into_chunks(text):
-    """Splits the text into smaller, manageable chunks."""
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
-    chunks = text_splitter.split_text(text)
-    return chunks
+    try:
+        with pdfplumber.open(pdf_path) as pdf:
+            for page in pdf.pages:
+                # Extract tables
+                page_tables = _extract_tables_from_page(page)
+                if page_tables:
+                    all_tables.extend(page_tables)
+                # Extract text
+                text = page.extract_text()
+                if text:
+                    text_output.write(text + "\n\n")
+    except Exception as e:
+        print(f"Error extracting with pdfplumber: {e}")
+        # Fallback to pdfminer if pdfplumber fails
+        with open(pdf_path, 'rb') as file:
+            extract_text_to_fp(file, text_output, laparams=LAParams(), output_type='text', codec=None)
+    extracted_text = text_output.getvalue()
+    return extracted_text, all_tables  # Return text and list of tables
+
+def clean_extracted_text(text):
+    lines = text.splitlines()
+    cleaned = []
+    for line in lines:
+        line = line.strip()
+        if line:
+            line = ' '.join(line.split())
+            cleaned.append(line)
+    return '\n'.join(cleaned)
 
-@st.cache_data()
-def create_vectorstore(chunks):
-    """Creates a vectorstore from the text chunks using HuggingFace embeddings."""
-    embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
-    vectorstore = FAISS.from_texts(chunks, embeddings)
-    return vectorstore
+def _format_tables_internal(tables):
+    """Formats extracted tables into a string representation."""
 
-# --- 2. Question Answering with Groq ---
+    formatted_tables_str = []
+    for table in tables:
+        # Use csv writer to handle commas and quotes correctly
+        with StringIO() as csvfile:
+            csvwriter = csv.writer(csvfile)
+            csvwriter.writerows(table)
+            formatted_tables_str.append(csvfile.getvalue())
+    return "\n\n".join(formatted_tables_str)
 
+# --- DOCX Extraction ---
+def extract_text_from_docx(docx_path):
+    try:
+        doc = docx.Document(docx_path)
+        return '\n'.join(para.text for para in doc.paragraphs)
+    except Exception:
+        return ""
+
+# --- Chunking ---
+def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32, max_tokens=512):
+    tokens = tokenizer.tokenize(text)
+    chunks = []
+    start = 0
+    while start < len(tokens):
+        end = min(start + chunk_size, len(tokens))
+        chunk_tokens = tokens[start:end]
+        chunk_text = tokenizer.convert_tokens_to_string(chunk_tokens)
+        chunks.append(chunk_text)
+        if end == len(tokens):
+            break
+        start += chunk_size - chunk_overlap
+    return chunks
+
+def retrieve_chunks(question, index, embed_model, text_chunks, k=3):
+    question_embedding = embed_model.encode(question)
+    D, I = index.search(np.array([question_embedding]), k)
+    return [text_chunks[i] for i in I[0]]
+
+# --- Groq Answer Generator ---
 def generate_answer_with_groq(question, context):
-    """Generates an answer using the Groq API."""
     url = "https://api.groq.com/openai/v1/chat/completions"
     api_key = os.environ.get("GROQ_API_KEY")
-    if not api_key:
-        st.error("GROQ_API_KEY environment variable not found. Please set it.")
-        return None  # Indicate failure
-
     headers = {
         "Authorization": f"Bearer {api_key}",
         "Content-Type": "application/json",
@@ -82,59 +143,153 @@ def generate_answer_with_groq(question, context):
         "temperature": 0.5,
         "max_tokens": 300,
     }
+    response = requests.post(url, headers=headers, json=payload)
+    response.raise_for_status()
+    return response.json()['choices'][0]['message']['content'].strip()
+
+# --- Twilio Functions ---
+def fetch_latest_incoming_message(client, conversation_sid):
     try:
-        response = requests.post(url, headers=headers, json=payload)
-        response.raise_for_status()  # Raise an exception for bad status codes
-        return response.json()['choices'][0]['message']['content'].strip()
-    except requests.exceptions.RequestException as e:
-        st.error(f"Error communicating with Groq API: {e}")
-        return "An error occurred while trying to get the answer."
-
-def perform_rag_groq(vectorstore, query):
-    """Performs retrieval and generates an answer using Groq."""
-    retriever = vectorstore.as_retriever()
-    relevant_docs = retriever.get_relevant_documents(query)
-    context = "\n\n".join([doc.page_content for doc in relevant_docs])
-    answer = generate_answer_with_groq(query, context)
-    return {"answer": answer, "sources": [doc.metadata['source'] for doc in relevant_docs] if relevant_docs else []}
-
-# --- 3. Streamlit UI ---
-
-def main():
-    st.title("PDF Q&A with Local Docs (Powered by Groq)")
-    st.info("Make sure you have a 'docs' folder in the same directory as this script containing your PDF files.")
-
-    with st.spinner("Loading and processing PDF(s)..."):
-        all_text, all_tables = load_and_process_pdfs_from_folder()
-
-    if all_text:
-        with st.spinner("Creating knowledge base..."):
-            chunks = split_text_into_chunks(all_text)
-            # We need to add metadata (source) to the chunks for accurate source tracking
-            metadatas = [{"source": f"doc_{i+1}"} for i in range(len(chunks))] # Basic source tracking
-            embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
-            vectorstore = FAISS.from_texts(chunks, embeddings, metadatas=metadatas)
-
-        query = st.text_input("Ask a question about the documents:")
-        if query:
-            with st.spinner("Searching for answer..."):
-                result = perform_rag_groq(vectorstore, query)
-                if result and result.get("answer"):
-                    st.subheader("Answer:")
-                    st.write(result["answer"])
-                    if "sources" in result and result["sources"]:
-                        st.subheader("Source:")
-                        st.write(", ".join(result["sources"]))
-                else:
-                    st.warning("Could not generate an answer.")
-
-    if all_tables:
-        st.subheader("Extracted Tables:")
-        for i, table_df in enumerate(all_tables):
-            st.write(f"Table {i+1}:")
-            st.dataframe(table_df)
-    elif not all_text:
-        st.warning("No PDF files found in the 'docs' folder.")
-
-if __name__ == "__main__":
-    main()
+        messages = client.conversations.v1.conversations(conversation_sid).messages.list()
+        for msg in reversed(messages):
+            if msg.author.startswith("whatsapp:"):
+                return {
+                    "sid": msg.sid,
+                    "body": msg.body,
+                    "author": msg.author,
+                    "timestamp": msg.date_created,
+                }
+    except TwilioRestException as e:
+        if e.status == 404:
+            print(f"Conversation {conversation_sid} not found, skipping...")
+        else:
+            print(f"Twilio error fetching messages for {conversation_sid}:", e)
+    except Exception as e:
+        #print(f"Unexpected error in fetch_latest_incoming_message for {conversation_sid}:", e)
+        pass
+
+    return None
+
+def send_twilio_message(client, conversation_sid, body):
+    return client.conversations.v1.conversations(conversation_sid).messages.create(
+        author="system", body=body
+    )
+
+# --- Load Knowledge Base ---
+def setup_knowledge_base():
+    folder_path = "docs"
+    all_text = ""
+
+    # Process PDFs
+    for filename in ["FAQ.pdf", "ProductReturnPolicy.pdf"]:
+        pdf_path = os.path.join(folder_path, filename)
+        text, tables = extract_text_from_pdf(pdf_path)
+        all_text += clean_extracted_text(text) + "\n"
+        all_text += _format_tables_internal(tables) + "\n"
+
+    # Process CSVs
+    for filename in ["CustomerOrders.csv"]:
+        csv_path = os.path.join(folder_path, filename)
+        try:
+            with open(csv_path, newline='', encoding='utf-8') as csvfile:
+                reader = csv.DictReader(csvfile)
+                for row in reader:
+                    line = f"Order ID: {row.get('OrderID')} | Customer Name: {row.get('CustomerName')} | Order Date: {row.get('OrderDate')} | ProductID: {row.get('ProductID')} | Date: {row.get('OrderDate')} | Quantity: {row.get('Quantity')} | UnitPrice(USD): {row.get('UnitPrice(USD)')} | TotalPrice(USD): {row.get('TotalPrice(USD)')} | ShippingAddress: {row.get('ShippingAddress')} | OrderStatus: {row.get('OrderStatus')}"
+                    all_text += line + "\n"
+        except Exception as e:
+            print(f"❌ Error reading {filename}: {e}")
+
+    for filename in ["Products.csv"]:
+        csv_path = os.path.join(folder_path, filename)
+        try:
+            with open(csv_path, newline='', encoding='utf-8') as csvfile:
+                reader = csv.DictReader(csvfile)
+                for row in reader:
+                    line = f"Product ID: {row.get('ProductID')} | Toy Name: {row.get('ToyName')} | Category: {row.get('Category')} | Price(USD): {row.get('Price(USD)')} | Stock Quantity: {row.get('StockQuantity')} | Description: {row.get('Description')}"
+                    all_text += line + "\n"
+        except Exception as e:
+            print(f"❌ Error reading {filename}: {e}")
+
+    # Tokenization & chunking
+    tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
+    chunks = chunk_text(all_text, tokenizer)
+    model = SentenceTransformer('all-mpnet-base-v2')
+    embeddings = model.encode(chunks, show_progress_bar=False, truncation=True, max_length=512)
+    dim = embeddings[0].shape[0]
+    index = faiss.IndexFlatL2(dim)
+    index.add(np.array(embeddings).astype('float32'))
+    return index, model, chunks
+
+
+
+# --- Monitor Conversations ---
+def start_conversation_monitor(client, index, embed_model, text_chunks):
+    processed_convos = set()
+    last_processed_timestamp = {}
+
+    def poll_conversation(convo_sid):
+        while True:
+            try:
+                latest_msg = fetch_latest_incoming_message(client, convo_sid)
+                if latest_msg:
+                    msg_time = latest_msg["timestamp"]
+                    if convo_sid not in last_processed_timestamp or msg_time > last_processed_timestamp[convo_sid]:
+                        last_processed_timestamp[convo_sid] = msg_time
+                        question = latest_msg["body"]
+                        sender = latest_msg["author"]
+                        print(f"\n📥 New message from {sender} in {convo_sid}: {question}")
+                        context = "\n\n".join(retrieve_chunks(question, index, embed_model, text_chunks))
+                        answer = generate_answer_with_groq(question, context)
+                        send_twilio_message(client, convo_sid, answer)
+                        print(f"📤 Replied to {sender}: {answer}")
+                time.sleep(3)
+            except Exception as e:
+                print(f"❌ Error in convo {convo_sid} polling:", e)
+                time.sleep(5)
+
+    def poll_new_conversations():
+        print("➡️ Monitoring for new WhatsApp conversations...")
+        while True:
+            try:
+                conversations = client.conversations.v1.conversations.list(limit=20)
+                for convo in conversations:
+                    convo_full = client.conversations.v1.conversations(convo.sid).fetch()
+                    if convo.sid not in processed_convos and convo_full.date_created > APP_START_TIME:
+                        participants = client.conversations.v1.conversations(convo.sid).participants.list()
+                        for p in participants:
+                            address = p.messaging_binding.get("address", "") if p.messaging_binding else ""
+                            if address.startswith("whatsapp:"):
+                                print(f"🆕 New WhatsApp convo found: {convo.sid}")
+                                processed_convos.add(convo.sid)
+                                threading.Thread(target=poll_conversation, args=(convo.sid,), daemon=True).start()
+            except Exception as e:
+                print("❌ Error polling conversations:", e)
+            time.sleep(5)
+
+    # ✅ Launch conversation polling monitor
+    threading.Thread(target=poll_new_conversations, daemon=True).start()
+
+
+
+# --- Streamlit UI ---
+st.set_page_config(page_title="Quasa – A Smart WhatsApp Chatbot", layout="wide")
+st.title("📱 Quasa – A Smart WhatsApp Chatbot")
+
+account_sid = st.secrets.get("TWILIO_SID")
+auth_token = st.secrets.get("TWILIO_TOKEN")
+GROQ_API_KEY = st.secrets.get("GROQ_API_KEY")
+
+if not all([account_sid, auth_token, GROQ_API_KEY]):
+    st.warning("⚠️ Provide all credentials below:")
+    account_sid = st.text_input("Twilio SID", value=account_sid or "")
+    auth_token = st.text_input("Twilio Token", type="password", value=auth_token or "")
+    GROQ_API_KEY = st.text_input("GROQ API Key", type="password", value=GROQ_API_KEY or "")
+
+if all([account_sid, auth_token, GROQ_API_KEY]):
+    os.environ["GROQ_API_KEY"] = GROQ_API_KEY
+    client = Client(account_sid, auth_token)
+
+    st.success("🟢 Monitoring new WhatsApp conversations...")
+    index, model, chunks = setup_knowledge_base()
+    threading.Thread(target=start_conversation_monitor, args=(client, index, model, chunks), daemon=True).start()
+    st.info("⏳ Waiting for new messages...")