Update app.py

app.py

@@ -23,18 +23,16 @@ import re
 APP_START_TIME = datetime.datetime.now(datetime.timezone.utc)
 os.environ["PYTORCH_JIT"] = "0"
 
-# Twilio Setup
-TWILIO_ACCOUNT_SID = os.getenv("TWILIO_ACCOUNT_SID")
-TWILIO_AUTH_TOKEN = os.getenv("TWILIO_AUTH_TOKEN")
-twilio_client = Client(TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN)
-
 # ---------------- PDF & DOCX & JSON Extraction ----------------
 def _extract_tables_from_page(page):
     tables = page.extract_tables()
     formatted_tables = []
     for table in tables:
-        formatted_row = [[cell if cell is not None else "" for cell in row] for row in table]
-        formatted_tables.append(formatted_row)
+        formatted_table = []
+        for row in table:
+            formatted_row = [cell if cell is not None else "" for cell in row]
+            formatted_table.append(formatted_row)
+        formatted_tables.append(formatted_table)
     return formatted_tables
 
 def extract_text_from_pdf(pdf_path):
@@ -48,40 +46,48 @@ def extract_text_from_pdf(pdf_path):
                 if text:
                     text_output.write(text + "\n\n")
     except Exception as e:
+        print(f"pdfplumber error: {e}")
         with open(pdf_path, 'rb') as file:
             extract_text_to_fp(file, text_output, laparams=LAParams(), output_type='text')
     return text_output.getvalue(), all_tables
 
 def _format_tables_internal(tables):
-    formatted = []
+    formatted_tables_str = []
     for table in tables:
         with StringIO() as csvfile:
             writer = csv.writer(csvfile)
             writer.writerows(table)
-            formatted.append(csvfile.getvalue())
-    return "\n\n".join(formatted)
+            formatted_tables_str.append(csvfile.getvalue())
+    return "\n\n".join(formatted_tables_str)
 
 def clean_extracted_text(text):
     return '\n'.join(' '.join(line.strip().split()) for line in text.splitlines() if line.strip())
 
-def extract_text_from_docx(path):
+def extract_text_from_docx(docx_path):
     try:
-        doc = docx.Document(path)
-        return '\n'.join(p.text for p in doc.paragraphs)
+        doc = docx.Document(docx_path)
+        return '\n'.join(para.text for para in doc.paragraphs)
     except:
         return ""
 
-def load_json_data(path):
+def load_json_data(json_path):
     try:
-        with open(path, 'r', encoding='utf-8') as f:
+        with open(json_path, 'r', encoding='utf-8') as f:
             data = json.load(f)
         if isinstance(data, dict):
-            return "\n".join(f"{k}: {v}" for k, v in data.items() if not isinstance(v, (dict, list)))
+            # Flatten dictionary values (avoiding nested structures as strings)
+            return "\n".join(f"{key}: {value}" for key, value in data.items() if not isinstance(value, (dict, list)))
         elif isinstance(data, list):
-            return "\n\n".join("\n".join(f"{k}: {v}" for k, v in item.items() if isinstance(item, dict)) for item in data)
+            # Flatten list of dictionaries
+            all_items = []
+            for item in data:
+                if isinstance(item, dict):
+                    all_items.append("\n".join(f"{key}: {value}" for key, value in item.items() if not isinstance(value, (dict, list))))
+            return "\n\n".join(all_items)
         else:
             return json.dumps(data, ensure_ascii=False, indent=2)
     except Exception as e:
+        print(f"JSON read error: {e}")
         return ""
 
 # ---------------- Chunking ----------------
@@ -93,6 +99,7 @@ def chunk_text(text, tokenizer, chunk_size=128, chunk_overlap=32):
         end = min(start + chunk_size, len(tokens))
         chunk = tokens[start:end]
         chunks.append(tokenizer.convert_tokens_to_string(chunk))
+        if end == len(tokens): break
         start += chunk_size - chunk_overlap
     return chunks
 
@@ -104,19 +111,37 @@ def retrieve_chunks(question, index, embed_model, text_chunks, k=3):
 # ---------------- Groq Answer Generator ----------------
 def generate_answer_with_groq(question, context):
     url = "https://api.groq.com/openai/v1/chat/completions"
-    api_key = os.getenv("GROQ_API_KEY")
-    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
-    prompt = f"Customer asked: '{question}'\n\nHere is the relevant information to help:\n{context}"
+    api_key = os.environ.get("GROQ_API_KEY")
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+    prompt = (
+        f"Customer asked: '{question}'\n\n"
+        f"Here is the relevant information to help:\n{context}\n\n"
+        f"Respond in a friendly and helpful tone as a toy shop support agent, "
+        f"addressing the customer by their name if it's available in the context."
+    )
     payload = {
         "model": "llama3-8b-8192",
         "messages": [
-            {"role": "system", "content": "You are ToyBot, a friendly WhatsApp assistant..."},
+            {
+                "role": "system",
+                "content": (
+                    "You are ToyBot, a friendly WhatsApp assistant for an online toy shop. "
+                    "Help customers with toys, delivery, and returns in a helpful tone. "
+                    "When responding, try to find the customer's name in the provided context "
+                    "and address them directly. If the context contains order details and status, "
+                    "include that information in your response."
+                )
+            },
             {"role": "user", "content": prompt},
         ],
         "temperature": 0.5,
         "max_tokens": 300,
     }
     response = requests.post(url, headers=headers, json=payload)
+    response.raise_for_status()
     return response.json()['choices'][0]['message']['content'].strip()
 
 # ---------------- Twilio Integration ----------------
@@ -124,90 +149,98 @@ def fetch_latest_incoming_message(client, conversation_sid):
     try:
         messages = client.conversations.v1.conversations(conversation_sid).messages.list()
         for msg in reversed(messages):
-            if (
-                msg.author.startswith("whatsapp:") and
-                msg.date_created and
-                msg.date_created > APP_START_TIME
-            ):
+            if msg.author.startswith("whatsapp:"):
                 return {
                     "sid": msg.sid,
                     "body": msg.body,
                     "author": msg.author,
                     "timestamp": msg.date_created,
                 }
-    except TwilioRestException:
-        return None
+    except TwilioRestException as e:
+        print(f"Twilio error: {e}")
+    return None
 
 def send_twilio_message(client, conversation_sid, body):
-    return client.conversations.v1.conversations(conversation_sid).messages.create(author="system", body=body)
-
-def get_latest_whatsapp_conversation_sid(client):
-    try:
-        conversations = client.conversations.v1.conversations.list(limit=20)
-        # Filter conversations created after app start time
-        filtered = [
-            c for c in conversations
-            if c.date_created and c.date_created > APP_START_TIME
-        ]
-        for convo in sorted(filtered, key=lambda c: c.date_created, reverse=True):
-            messages = convo.messages.list(limit=1)
-            if messages and any(m.author.startswith("whatsapp:") and m.date_created > APP_START_TIME for m in messages):
-                return convo.sid
-    except Exception as e:
-        print("Error fetching valid conversation SID:", e)
-    return None
+    return client.conversations.v1.conversations(conversation_sid).messages.create(
+        author="system", body=body
+    )
 
 # ---------------- Knowledge Base Setup ----------------
 def setup_knowledge_base():
-    folder = "docs"
-    text = ""
-    for f in os.listdir(folder):
-        path = os.path.join(folder, f)
-        if f.endswith(".pdf"):
-            t, tables = extract_text_from_pdf(path)
-            text += clean_extracted_text(t) + "\n" + _format_tables_internal(tables) + "\n"
-        elif f.endswith(".docx"):
-            text += clean_extracted_text(extract_text_from_docx(path)) + "\n"
-        elif f.endswith(".json"):
-            text += load_json_data(path) + "\n"
-        elif f.endswith(".csv"):
-            with open(path, newline='', encoding='utf-8') as csvfile:
-                reader = csv.reader(csvfile)
-                text += "\n".join(", ".join(row) for row in reader) + "\n"
-    return text
-
-# ---------------- App Logic ----------------
-def process_messages_loop():
-    embed_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
-    tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
-    knowledge_text = setup_knowledge_base()
-    text_chunks = chunk_text(knowledge_text, tokenizer)
-    embeddings = embed_model.encode(text_chunks)
-    index = faiss.IndexFlatL2(embeddings.shape[1])
-    index.add(embeddings)
-
-    seen_sids = set()
-
-    while True:
-        conversation_sid = get_latest_whatsapp_conversation_sid(twilio_client)
-        if not conversation_sid:
+    folder_path = "docs"
+    all_text = ""
+
+    for filename in os.listdir(folder_path):
+        file_path = os.path.join(folder_path, filename)
+        if filename.endswith(".pdf"):
+            text, tables = extract_text_from_pdf(file_path)
+            all_text += clean_extracted_text(text) + "\n"
+            all_text += _format_tables_internal(tables) + "\n"
+        elif filename.endswith(".docx"):
+            text = extract_text_from_docx(file_path)
+            all_text += clean_extracted_text(text) + "\n"
+        elif filename.endswith(".json"):
+            text = load_json_data(file_path)
+            all_text += text + "\n"
+        elif filename.endswith(".csv"):
+            try:
+                with open(file_path, newline='', encoding='utf-8') as csvfile:
+                    reader = csv.DictReader(csvfile)
+                    for row in reader:
+                        line = ' | '.join(f"{k}: {v}" for k, v in row.items())
+                        all_text += line + "\n"
+            except Exception as e:
+                print(f"CSV read error: {e}")
+
+    tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
+    chunks = chunk_text(all_text, tokenizer)
+    model = SentenceTransformer('all-mpnet-base-v2')
+    embeddings = model.encode(chunks, show_progress_bar=False)
+    dim = embeddings[0].shape[0]
+    index = faiss.IndexFlatL2(dim)
+    index.add(np.array(embeddings).astype('float32'))
+    return index, model, chunks
+
+# ---------------- Monitor Twilio Conversations ----------------
+def start_conversation_monitor(client, index, embed_model, text_chunks):
+    processed_convos = set()
+    last_processed_timestamp = {}
+
+    def poll_convo(convo_sid):
+        while True:
+            latest_msg = fetch_latest_incoming_message(client, convo_sid)
+            if latest_msg:
+                msg_time = latest_msg["timestamp"]
+                if convo_sid not in last_processed_timestamp or msg_time > last_processed_timestamp[convo_sid]:
+                    last_processed_timestamp[convo_sid] = msg_time
+                    question = latest_msg["body"]
+                    sender = latest_msg["author"]
+                    print(f"📩 New message from {sender}: {question}")
+                    context = "\n\n".join(retrieve_chunks(question, index, embed_model, text_chunks))
+                    answer = generate_answer_with_groq(question, context)
+                    send_twilio_message(client, convo_sid, answer)
             time.sleep(5)
-            continue
-
-        message = fetch_latest_incoming_message(twilio_client, conversation_sid)
-        if message and message["sid"] not in seen_sids:
-            seen_sids.add(message["sid"])
-            question = message["body"]
-            chunks = retrieve_chunks(question, index, embed_model, text_chunks)
-            answer = generate_answer_with_groq(question, "\n\n".join(chunks))
-            send_twilio_message(twilio_client, conversation_sid, answer)
-
-        time.sleep(5)
-
-# ---------------- Streamlit UI ----------------
-st.title("ToyShop WhatsApp Assistant (Groq + Twilio)")
 
-if st.button("Start WhatsApp Bot"):
-    thread = threading.Thread(target=process_messages_loop)
-    thread.start()
-    st.success("WhatsApp assistant started and monitoring for new messages.")
+    for convo in client.conversations.v1.conversations.list():
+        if convo.sid not in processed_convos:
+            processed_convos.add(convo.sid)
+            threading.Thread(target=poll_convo, args=(convo.sid,), daemon=True).start()
+
+# ---------------- Main Entry ----------------
+if __name__ == "__main__":
+    st.title("🤖 ToyBot WhatsApp Assistant")
+    st.write("Initializing knowledge base...")
+
+    index, model, chunks = setup_knowledge_base()
+
+    st.success("Knowledge base loaded.")
+    st.write("Waiting for WhatsApp messages...")
+
+    account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
+    auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
+    if not account_sid or not auth_token:
+        st.error("❌ Twilio credentials not set.")
+    else:
+        client = Client(account_sid, auth_token)
+        start_conversation_monitor(client, index, model, chunks)
+        st.info("✅ Bot is now monitoring Twilio conversations.")