Update app.py

app.py

@@ -18,8 +18,6 @@ st.set_page_config(page_title="RAG Customer Support Chatbot", layout="wide")
 DEFAULT_TWILIO_ACCOUNT_SID_FALLBACK = "" 
 DEFAULT_TWILIO_AUTH_TOKEN_FALLBACK = "" 
 DEFAULT_GROQ_API_KEY_FALLBACK = "" 
-
-#DEFAULT_TWILIO_CONVERSATION_SERVICE_SID = ""
 DEFAULT_TWILIO_BOT_WHATSAPP_IDENTITY = st.secrets.get("TWILIO_PHONE_NUMBER", "whatsapp:+14155238886") 
 DEFAULT_EMBEDDING_MODEL_NAME = "sentence-transformers/all-MiniLM-L6-v2"
 DEFAULT_POLLING_INTERVAL_S = 30
@@ -33,11 +31,8 @@ FAQ_PDF_FILE = os.path.join(DOCS_FOLDER, "FAQ.pdf")
 APP_TWILIO_ACCOUNT_SID = st.secrets.get("TWILIO_ACCOUNT_SID")
 APP_TWILIO_AUTH_TOKEN = st.secrets.get("TWILIO_AUTH_TOKEN")
 APP_GROQ_API_KEY = st.secrets.get("GROQ_API_KEY")
-
-#APP_TWILIO_CONVERSATION_SERVICE_SID_SECRET = st.secrets.get("TWILIO_CONVERSATION_SERVICE_SID")
 APP_TWILIO_BOT_WHATSAPP_IDENTITY_SECRET = st.secrets.get("TWILIO_BOT_WHATSAPP_IDENTITY")
 
-
 # --- RAG Processing Utilities ---
 def load_json_data(file_path):
     try:
@@ -202,7 +197,6 @@ def generate_response_groq(_groq_client, query, context, model="llama3-8b-8192",
         
         item_description = item_name if item_name else "the ordered item(s)"
         
-        # Construct the core information string that the LLM needs to build upon
         core_info_parts = [
             f"your order {order_id}",
             f"for {item_description}",
@@ -220,10 +214,9 @@ def generate_response_groq(_groq_client, query, context, model="llama3-8b-8192",
                 core_info_parts.append("(delivery date not specified)")
         
         core_information_to_include = ", ".join(core_info_parts[:-1]) + (f" {core_info_parts[-1]}" if len(core_info_parts) > 1 else "")
-        if not order_status.lower() == "delivered" and len(core_info_parts) > 1 : # for non-delivered, avoid 'and' before status
+        if not order_status.lower() == "delivered" and len(core_info_parts) > 1:
              core_information_to_include = f"your order {order_id} for {item_description} has a status of '{order_status}'"
 
-
         user_prompt = (
             f"Customer: {customer_name}\n"
             f"Order ID: {order_id}\n"
@@ -244,10 +237,8 @@ def generate_response_groq(_groq_client, query, context, model="llama3-8b-8192",
             f"Ensure the sentence flows naturally and uses the details you've been given.\n"
             f"Respond now with ONLY that single sentence."
         )
-        # For LLM's deeper reference, though the primary instruction is above:
-        # user_prompt += f"\n\nFull database context for your reference if needed: {context}"
 
-    else: # Default prompt structure for other intents or if details are missing
+    else:
         system_message = "You are a helpful customer support assistant."
         user_prompt = f"""Use the following context to answer the user's question.
 If the context doesn't contain the answer, state that you don't have enough information or ask clarifying questions.
@@ -267,17 +258,16 @@ Assistant Answer:
                 {"role": "user", "content": user_prompt}
             ],
             model=model, 
-            temperature=0.5, # Slightly lower temperature might help with stricter adherence
+            temperature=0.5,
             max_tokens=1024, 
             top_p=1
         )
-        response = chat_completion.choices[0].message.content.strip() # Added strip()
+        response = chat_completion.choices[0].message.content.strip()
         return response
     except Exception as e:
         st.error(f"Error calling GROQ API: {e}")
         return "Sorry, I encountered an error while trying to generate a response."
 
-
 def initialize_groq_client(api_key_val):
     if not api_key_val:
         st.warning("GROQ API Key is missing.")
@@ -301,62 +291,53 @@ def initialize_twilio_client(acc_sid, auth_tkn):
         st.error(f"Failed to initialize Twilio client: {e}")
         return None
 
-def get_new_whatsapp_messages(twilio_client, conversation_service_sid_val, bot_start_time_utc,
-                              processed_message_sids, bot_whatsapp_identity_val):
+def get_new_whatsapp_messages(twilio_client, bot_start_time_utc, processed_message_sids, bot_whatsapp_identity_val):
     if not twilio_client:
         st.warning("Twilio client not initialized.")
         return []
-    if not conversation_service_sid_val:
-        st.warning("Twilio Conversation Service SID not provided.")
-        return []
     if not bot_whatsapp_identity_val: 
         st.warning("Twilio Bot WhatsApp Identity not provided.")
         return []
 
     new_messages_to_process = []
     try:
-        conversations = twilio_client.conversations.v1 \
-            .services(conversation_service_sid_val) \
-            .conversations \
-            .list(limit=50) 
+        # Get all conversations (not limited to a specific service)
+        conversations = twilio_client.conversations.v1.conversations.list(limit=50)
 
         for conv in conversations:
             if conv.date_updated and conv.date_updated > bot_start_time_utc:
                 messages = twilio_client.conversations.v1 \
-                    .services(conversation_service_sid_val) \
                     .conversations(conv.sid) \
                     .messages \
-                    .list(order='desc', limit=10) 
+                    .list(order='desc', limit=10)
 
                 for msg in messages:
                     if msg.sid in processed_message_sids:
-                        continue 
+                        continue
 
+                    # Check if message is from WhatsApp and not from the bot
                     if msg.author and msg.author.lower() != bot_whatsapp_identity_val.lower() and \
-                       msg.date_created and msg.date_created > bot_start_time_utc:
+                       msg.date_created and msg.date_created > bot_start_time_utc and \
+                       msg.author.startswith('whatsapp:'):
                         new_messages_to_process.append({
                             "conversation_sid": conv.sid, "message_sid": msg.sid,
                             "author_identity": msg.author, "message_body": msg.body,
                             "timestamp_utc": msg.date_created
                         })
-                        break 
+                        break
     except Exception as e:
         st.error(f"Error fetching Twilio messages: {e}")
-    return sorted(new_messages_to_process, key=lambda m: m['timestamp_utc']) 
+    return sorted(new_messages_to_process, key=lambda m: m['timestamp_utc'])
 
-def send_whatsapp_message(twilio_client, conversation_service_sid_val, conversation_sid, message_body, bot_identity_val):
+def send_whatsapp_message(twilio_client, conversation_sid, message_body, bot_identity_val):
     if not twilio_client:
         st.error("Twilio client not initialized for sending message.")
         return False
-    if not conversation_service_sid_val:
-        st.error("Twilio Conversation Service SID not provided for sending message.")
-        return False
     if not bot_identity_val:
         st.error("Bot identity not provided for sending message.")
         return False
     try:
         twilio_client.conversations.v1 \
-            .services(conversation_service_sid_val) \
             .conversations(conversation_sid) \
             .messages \
             .create(author=bot_identity_val, body=message_body)
@@ -366,6 +347,27 @@ def send_whatsapp_message(twilio_client, conversation_service_sid_val, conversat
         st.error(f"Error sending Twilio message to {conversation_sid}: {e}")
         return False
 
+# --- Simple Intent Classifier ---
+def simple_intent_classifier(query):
+    query_lower = query.lower()
+    order_keywords = ["order", "status", "track", "delivery"]
+    order_id_match = re.search(r'\b(ord\d{3,})\b', query_lower, re.IGNORECASE)
+
+    if any(k in query_lower for k in order_keywords):
+        if order_id_match:
+            return "ORDER_STATUS", order_id_match.group(1).upper() 
+        return "ORDER_STATUS", None 
+
+    product_keywords = ["product", "item", "buy", "price", "feature", "stock"]
+    product_id_match = re.search(r'\b(prd\d{3,})\b', query_lower, re.IGNORECASE) 
+    if any(k in query_lower for k in product_keywords) or product_id_match:
+        return "PRODUCT_INFO", None 
+
+    if any(k in query_lower for k in ["return", "policy", "refund", "exchange", "faq", "question", "how to", "support"]):
+        return "GENERAL_POLICY_FAQ", None
+        
+    return "UNKNOWN", None
+
 # --- Main Application Logic & UI ---
 st.title("🤖 RAG-Based Customer Support Chatbot")
 st.markdown("Powered by Streamlit, Twilio, GROQ LLaMA3, and FAISS.")
@@ -394,16 +396,10 @@ else:
     st.sidebar.warning("Secret 'GROQ_API_KEY' not found.")
     groq_api_key_to_use = st.sidebar.text_input("GROQ API Key (Enter Manually)", value=DEFAULT_GROQ_API_KEY_FALLBACK, type="password")
 
-# twilio_conversation_service_sid_to_use = st.sidebar.text_input(
-    # "Twilio Conversation Service SID (IS...)",
-    # value=APP_TWILIO_CONVERSATION_SERVICE_SID_SECRET or DEFAULT_TWILIO_CONVERSATION_SERVICE_SID,
-    # type="password", 
-    # help="The SID of your Twilio Conversations Service. Can be set by 'TWILIO_CONVERSATION_SERVICE_SID' secret."
-# )
 twilio_bot_whatsapp_identity_to_use = st.sidebar.text_input(
     "Twilio Bot WhatsApp Identity",
     value=APP_TWILIO_BOT_WHATSAPP_IDENTITY_SECRET or DEFAULT_TWILIO_BOT_WHATSAPP_IDENTITY,
-    help="e.g., 'whatsapp:+1234567890'. Can be set by 'TWILIO_BOT_WHATSAPP_IDENTITY' secret."
+    help="e.g., 'whatsapp:+1234567890'"
 )
 embedding_model_name_to_use = st.sidebar.text_input(
     "Embedding Model Name",
@@ -425,27 +421,6 @@ if "bot_start_time_utc" not in st.session_state: st.session_state.bot_start_time
 if "processed_message_sids" not in st.session_state: st.session_state.processed_message_sids = set()
 if "manual_chat_history" not in st.session_state: st.session_state.manual_chat_history = []
 
-# --- Helper: Simple Intent Classifier ---
-def simple_intent_classifier(query):
-    query_lower = query.lower()
-    order_keywords = ["order", "status", "track", "delivery"]
-    order_id_match = re.search(r'\b(ord\d{3,})\b', query_lower, re.IGNORECASE)
-
-    if any(k in query_lower for k in order_keywords):
-        if order_id_match:
-            return "ORDER_STATUS", order_id_match.group(1).upper() 
-        return "ORDER_STATUS", None 
-
-    product_keywords = ["product", "item", "buy", "price", "feature", "stock"]
-    product_id_match = re.search(r'\b(prd\d{3,})\b', query_lower, re.IGNORECASE) 
-    if any(k in query_lower for k in product_keywords) or product_id_match:
-        return "PRODUCT_INFO", None 
-
-    if any(k in query_lower for k in ["return", "policy", "refund", "exchange", "faq", "question", "how to", "support"]):
-        return "GENERAL_POLICY_FAQ", None
-        
-    return "UNKNOWN", None
-
 # --- Main Application Controls ---
 col1, col2, col3, col4 = st.columns(4)
 with col1:
@@ -506,8 +481,8 @@ with col2:
         st.rerun()
 with col3:
     if st.button("💬 Start WhatsApp Bot", disabled=not st.session_state.app_started or st.session_state.bot_started, use_container_width=True):
-        if not all([twilio_account_sid_to_use, twilio_auth_token_to_use, twilio_conversation_service_sid_to_use, twilio_bot_whatsapp_identity_to_use]):
-            st.error("Twilio Account SID, Auth Token, Conversation Service SID, and Bot WhatsApp Identity are all required.")
+        if not all([twilio_account_sid_to_use, twilio_auth_token_to_use, twilio_bot_whatsapp_identity_to_use]):
+            st.error("Twilio Account SID, Auth Token, and Bot WhatsApp Identity are all required.")
         else:
             st.session_state.twilio_client = initialize_twilio_client(twilio_account_sid_to_use, twilio_auth_token_to_use)
             if st.session_state.twilio_client:
@@ -560,7 +535,6 @@ if st.session_state.get("app_started") and st.session_state.get("rag_pipeline_re
             extracted_customer_name, extracted_item_name, extracted_shipping_address, \
             extracted_delivery_date, extracted_order_id, extracted_order_status = [None] * 6
 
-
             if intent == "ORDER_STATUS":
                 order_id_to_check = None
                 if potential_oid_from_intent: 
@@ -572,9 +546,6 @@ if st.session_state.get("app_started") and st.session_state.get("rag_pipeline_re
                 
                 if order_id_to_check:
                     raw_context_data = get_order_details(order_id_to_check, st.session_state.customer_orders_data)
-                    # context_for_llm will be used as the 'context' parameter in generate_response_groq
-                    # For ORDER_STATUS, this raw_context_data (JSON string) is still useful for LLM's reference,
-                    # even though specific fields are extracted for the specialized prompt.
                     context_for_llm = raw_context_data 
                     
                     if isinstance(raw_context_data, str) and not raw_context_data.startswith("No order found") and not raw_context_data.startswith("Customer order data is not loaded"):
@@ -585,23 +556,23 @@ if st.session_state.get("app_started") and st.session_state.get("rag_pipeline_re
                             if items and len(items) > 0 and isinstance(items[0], dict):
                                 extracted_item_name = items[0].get("name", "your item(s)")
                             else:
-                                extracted_item_name = "your item(s)" # Fallback
+                                extracted_item_name = "your item(s)"
                             extracted_shipping_address = order_data_dict.get("shipping_address")
                             extracted_delivery_date = order_data_dict.get("delivered_on")
                             extracted_order_status = order_data_dict.get("status")
-                            extracted_order_id = order_data_dict.get("order_id") # Should be same as order_id_to_check
+                            extracted_order_id = order_data_dict.get("order_id")
                         except json.JSONDecodeError:
                             st.warning(f"Could not parse order details JSON for {order_id_to_check} for personalization.")
                             context_for_llm = f"Error parsing order details for {order_id_to_check}. Raw data: {raw_context_data}"
-                    elif isinstance(raw_context_data, str): # Handle "No order found" or "data not loaded"
-                         context_for_llm = raw_context_data # LLM will state this
+                    elif isinstance(raw_context_data, str):
+                         context_for_llm = raw_context_data
                 else:
                     context_for_llm = "To check an order status, please provide a valid Order ID (e.g., ORD123)."
                     raw_context_data = {"message": "Order ID needed or not found in query."}
 
             elif intent == "PRODUCT_INFO":
                 raw_context_data = get_product_info(user_query_manual, st.session_state.products_data)
-                context_for_llm = raw_context_data # Product info is directly used as context
+                context_for_llm = raw_context_data
             
             elif intent == "GENERAL_POLICY_FAQ" or intent == "UNKNOWN": 
                 if st.session_state.faiss_index_pdfs and st.session_state.embedding_model and st.session_state.indexed_pdf_chunks:
@@ -627,7 +598,7 @@ if st.session_state.get("app_started") and st.session_state.get("rag_pipeline_re
                 item_name=extracted_item_name,
                 shipping_address=extracted_shipping_address,
                 delivery_date=extracted_delivery_date,
-                order_id=extracted_order_id, # This will be the specific order ID from user query
+                order_id=extracted_order_id,
                 order_status=extracted_order_status
             )
             
@@ -646,7 +617,7 @@ if st.session_state.get("app_started") and st.session_state.get("rag_pipeline_re
                         except (json.JSONDecodeError, TypeError):
                             st.text(str(raw_context_data))
             st.session_state.manual_chat_history.append({"role": "assistant", "content": llm_response, "context": raw_context_data})
-            st.rerun() 
+            st.rerun()
 
 # --- Twilio Bot Polling Logic ---
 if st.session_state.get("bot_started") and st.session_state.get("rag_pipeline_ready"):
@@ -657,18 +628,17 @@ if st.session_state.get("bot_started") and st.session_state.get("rag_pipeline_re
     if (current_time - st.session_state.last_twilio_poll_time) > polling_interval_to_use:
         st.session_state.last_twilio_poll_time = current_time
         
-        if not st.session_state.get("twilio_client") or \
-           not twilio_conversation_service_sid_to_use or \
-           not twilio_bot_whatsapp_identity_to_use or \
-           not st.session_state.get("bot_start_time_utc"):
-            st.warning("Twilio client/config missing for polling. Ensure bot is started and SIDs are set.")
+        if not st.session_state.get("twilio_client") or not twilio_bot_whatsapp_identity_to_use or not st.session_state.get("bot_start_time_utc"):
+            st.warning("Twilio client/config missing for polling. Ensure bot is started and WhatsApp identity is set.")
         else:
             with st.spinner(f"Checking WhatsApp messages (last poll: {datetime.fromtimestamp(st.session_state.last_twilio_poll_time).strftime('%H:%M:%S')})..."):
-                new_messages = get_new_whatsapp_messages(st.session_state.twilio_client, 
-                                                         twilio_conversation_service_sid_to_use,
-                                                         st.session_state.bot_start_time_utc, 
-                                                         st.session_state.processed_message_sids,
-                                                         twilio_bot_whatsapp_identity_to_use)
+                new_messages = get_new_whatsapp_messages(
+                    st.session_state.twilio_client,
+                    st.session_state.bot_start_time_utc, 
+                    st.session_state.processed_message_sids,
+                    twilio_bot_whatsapp_identity_to_use
+                )
+                
                 if new_messages:
                     st.info(f"Found {len(new_messages)} new WhatsApp message(s) to process.")
                     for msg_data in new_messages:
@@ -688,7 +658,6 @@ if st.session_state.get("bot_started") and st.session_state.get("rag_pipeline_re
                         wa_customer_name, wa_item_name, wa_shipping_address, \
                         wa_delivery_date, wa_order_id, wa_order_status = [None] * 6
 
-
                         if intent_whatsapp == "ORDER_STATUS":
                             order_id_to_check_whatsapp = None
                             if potential_oid_whatsapp:
@@ -700,7 +669,7 @@ if st.session_state.get("bot_started") and st.session_state.get("rag_pipeline_re
 
                             if order_id_to_check_whatsapp:
                                 raw_context_data_whatsapp = get_order_details(order_id_to_check_whatsapp, st.session_state.customer_orders_data)
-                                context_for_llm_whatsapp = raw_context_data_whatsapp # Full JSON string as context
+                                context_for_llm_whatsapp = raw_context_data_whatsapp
 
                                 if isinstance(raw_context_data_whatsapp, str) and not raw_context_data_whatsapp.startswith("No order found") and not raw_context_data_whatsapp.startswith("Customer order data is not loaded"):
                                     try:
@@ -718,63 +687,4 @@ if st.session_state.get("bot_started") and st.session_state.get("rag_pipeline_re
                                     except json.JSONDecodeError:
                                          st.warning(f"Could not parse order details JSON for {order_id_to_check_whatsapp} (WhatsApp) for personalization.")
                                          context_for_llm_whatsapp = f"Error parsing order details for {order_id_to_check_whatsapp}. Raw data: {raw_context_data_whatsapp}"
-                                elif isinstance(raw_context_data_whatsapp, str):
-                                     context_for_llm_whatsapp = raw_context_data_whatsapp
-                            else:
-                                context_for_llm_whatsapp = "To check an order status, please provide a valid Order ID (e.g., ORD123)."
-                                raw_context_data_whatsapp = {"message": "Order ID needed or not found in query."}
-
-
-                        elif intent_whatsapp == "PRODUCT_INFO":
-                            raw_context_data_whatsapp = get_product_info(user_query_whatsapp, st.session_state.products_data)
-                            context_for_llm_whatsapp = raw_context_data_whatsapp
-                        
-                        elif intent_whatsapp == "GENERAL_POLICY_FAQ" or intent_whatsapp == "UNKNOWN":
-                            if st.session_state.faiss_index_pdfs and st.session_state.embedding_model and st.session_state.indexed_pdf_chunks:
-                                k_val_whatsapp = 3 if intent_whatsapp == "GENERAL_POLICY_FAQ" else 2
-                                chunks_whatsapp = search_faiss_index(st.session_state.faiss_index_pdfs, user_query_whatsapp, 
-                                                                     st.session_state.embedding_model, st.session_state.indexed_pdf_chunks, k=k_val_whatsapp)
-                                if chunks_whatsapp:
-                                    context_for_llm_whatsapp = "Relevant information from documents:\n\n" + "\n\n---\n\n".join(chunks_whatsapp)
-                                    raw_context_data_whatsapp = chunks_whatsapp
-                                else:
-                                    context_for_llm_whatsapp = "I couldn't find specific information in our policy or FAQ documents regarding your query."
-                                    raw_context_data_whatsapp = {"message": "No relevant PDF chunks found."}
-                            else:
-                                context_for_llm_whatsapp = "Our policy and FAQ documents are currently unavailable for search."
-                                raw_context_data_whatsapp = {"message": "PDF index or embedding model not ready."}
-
-                        response_whatsapp = generate_response_groq(
-                            _groq_client=st.session_state.groq_client, 
-                            query=user_query_whatsapp, 
-                            context=context_for_llm_whatsapp,
-                            intent=intent_whatsapp,
-                            customer_name=wa_customer_name,
-                            item_name=wa_item_name,
-                            shipping_address=wa_shipping_address,
-                            delivery_date=wa_delivery_date,
-                            order_id=wa_order_id,
-                            order_status=wa_order_status
-                        )
-                        
-                        if send_whatsapp_message(st.session_state.twilio_client, twilio_conversation_service_sid_to_use, 
-                                                 conv_sid, response_whatsapp, twilio_bot_whatsapp_identity_to_use):
-                            st.session_state.processed_message_sids.add(msg_sid)
-                            st.success(f"Successfully responded to WhatsApp message SID {msg_sid} from {author_id}.")
-                        else:
-                            st.error(f"Failed to send WhatsApp response for message SID {msg_sid} from {author_id}.")
-                    st.experimental_rerun() 
-
-
-# --- Footer & Status ---
-st.sidebar.markdown("---")
-st.sidebar.info("Ensure all keys and SIDs are correctly configured. Primary API keys (Twilio SID/Token, GROQ Key) are loaded from secrets if available.")
-if st.session_state.get("app_started"):
-    status_color = "green" if st.session_state.get("rag_pipeline_ready") else "orange"
-    app_status_text = "App RUNNING" if st.session_state.get("rag_pipeline_ready") else "App Initializing/Error"
-    bot_status_text = "WhatsApp Bot RUNNING" if st.session_state.get("bot_started") else "WhatsApp Bot STOPPED"
-    st.sidebar.markdown(f"<span style='color:{status_color};'>{app_status_text}</span>. {bot_status_text}.", unsafe_allow_html=True)
-
-else:
-    st.sidebar.warning("App is STOPPED.")
-
+                                elif isinstance(raw_context_data_whats