Update app.py

app.py

@@ -289,19 +289,19 @@ async def transcribe_voice_with_openai(file_path: str) -> str:
         
         logger.info(f"[Transcribe] Transcribing file: {file_path} (size: {file_size} bytes)")
         
-        # Comprehensive system prompt for veterinary WhatsApp assistant
+        # Comprehensive system prompt for veterinary WhatsApp assistant - ENGLISH AND URDU ONLY
         system_prompt = """
 You are transcribing voice messages for Apex Biotical Veterinary WhatsApp Assistant. This is a professional veterinary products chatbot.
 
-CRITICAL: TRANSCRIBE ONLY ENGLISH OR URDU SPEECH - NOTHING ELSE
+CRITICAL: TRANSCRIBE ONLY ENGLISH OR URDU SPEECH - REJECT ALL OTHER LANGUAGES
 
 IMPORTANT RULES:
 1. ONLY transcribe English or Urdu speech
-2. If you hear unclear audio, transcribe as English
-3. If you hear mixed languages, transcribe as English
+2. If you hear any other language, transcribe as "unclear audio"
+3. If you hear unclear audio, transcribe as "unclear audio"
 4. Never transcribe gibberish or random characters
-5. If audio is unclear, transcribe as "unclear audio"
-6. Keep transcriptions simple and clean
+5. Keep transcriptions simple and clean
+6. Reject non-English/Urdu languages completely
 
 PRODUCT NAMES (exact spelling required):
 - Hydropex, Respira Aid Plus, Heposel, Bromacid, Hexatox
@@ -320,12 +320,12 @@ GREETINGS:
 - Urdu: salam, assalamu alaikum, adaab, namaste, khuda hafiz
 
 TRANSCRIPTION RULES:
-1. Transcribe exactly what you hear in English or Urdu
+1. Transcribe exactly what you hear in English or Urdu ONLY
 2. Convert numbers to digits (one->1, two->2, etc.)
 3. Preserve product names exactly
-4. If unclear, transcribe as "unclear audio"
+4. If unclear or non-English/Urdu, transcribe as "unclear audio"
 5. Keep it simple and clean
-6. No random characters or mixed languages
+6. Reject all other languages
 
 EXAMPLES:
 - "hydropex" -> "hydropex"
@@ -334,6 +334,7 @@ EXAMPLES:
 - "main menu" -> "main"
 - "salam" -> "salam"
 - "search products" -> "search products"
+- Non-English/Urdu speech -> "unclear audio"
 - Unclear audio -> "unclear audio"
 """
         
@@ -475,6 +476,53 @@ def process_voice_input(text: str) -> str:
     # Remove extra whitespace
     processed_text = re.sub(r'\s+', ' ', processed_text)
     
+    # Fix special characters and encoding issues
+    processed_text = clean_special_characters(processed_text)
+    
+    return processed_text
+
+def clean_special_characters(text: str) -> str:
+    """Clean special characters and fix encoding issues"""
+    if not text:
+        return text
+    
+    # Fix common encoding issues
+    replacements = {
+        'â€"': '–',  # Fix en dash
+        'â€"': '—',  # Fix em dash
+        '’': "'",   # Fix apostrophe
+        '“': '"',   # Fix opening quote
+        'â€': '"',    # Fix closing quote
+        '…': '...', # Fix ellipsis
+        '•': '•',   # Fix bullet
+        '‰': '°',   # Fix degree symbol
+        '′': '′',   # Fix prime
+        '″': '″',   # Fix double prime
+        '‼': '™',   # Fix trademark
+        '‮': '®',   # Fix registered
+        '
': '©',   # Fix copyright
+        'â€': '–',    # Generic fix for en dash
+        'â€"': '—',   # Generic fix for em dash
+        '’': "'",    # Generic fix for apostrophe
+        '“': '"',    # Generic fix for opening quote
+        'â€': '"',     # Generic fix for closing quote
+        '…': '...',  # Generic fix for ellipsis
+        '•': '•',    # Generic fix for bullet
+        '‰': '°',    # Generic fix for degree symbol
+    }
+    
+    # Apply replacements
+    for old_char, new_char in replacements.items():
+        text = text.replace(old_char, new_char)
+    
+    # Remove any remaining problematic characters
+    text = re.sub(r'â€[^"]*', '', text)  # Remove any remaining †patterns
+    
+    # Clean up multiple spaces
+    text = re.sub(r'\s+', ' ', text)
+    
+    return text.strip()
+    
     # Basic punctuation cleanup
     processed_text = processed_text.replace(' ,', ',').replace(' .', '.')
     
@@ -832,7 +880,10 @@ def generate_veterinary_product_response(product_info: Dict[str, Any], user_cont
     def clean_text(text):
         if pd.isna(text) or text is None:
             return "Not specified"
-        return str(text).strip()
+        cleaned = str(text).strip()
+        # Apply special character cleaning
+        cleaned = clean_special_characters(cleaned)
+        return cleaned
     
     # Extract product details
     product_name = clean_text(product_info.get('Product Name', ''))
@@ -881,6 +932,8 @@ def clean_text_for_pdf(text: str) -> str:
         return "N/A"
     
     cleaned = str(text)
+    # Apply special character cleaning first
+    cleaned = clean_special_characters(cleaned)
     # Remove or replace problematic characters for PDF
     cleaned = cleaned.replace('â€"', '-').replace('â€"', '"').replace('’', "'")
     cleaned = cleaned.replace('“', '"').replace('â€', '"').replace('…', '...')
@@ -1783,7 +1836,7 @@ async def process_incoming_message(from_number: str, msg: dict):
             )
             return
         
-        # 🎯 PRIORITY 2: State-specific handling (contact_request, availability_request, ai_chat_mode)
+        # 🎯 PRIORITY 2: State-specific handling (contact_request, availability_request, ai_chat_mode, clarification)
         if current_state == 'contact_request':
             await handle_contact_request_response(from_number, message_body)
             return
@@ -1793,8 +1846,29 @@ async def process_incoming_message(from_number: str, msg: dict):
         elif current_state == 'ai_chat_mode':
             await handle_ai_chat_mode(from_number, message_body, reply_language)
             return
+        elif user_context.get('awaiting_clarification', False):
+            # Handle clarification responses
+            await handle_clarification_response(from_number, message_body, user_context)
+            return
         
-        # 🎯 PRIORITY 3: Menu selections - check if this is a valid menu selection for current state
+        # 🎯 PRIORITY 3: Intelligent product queries from any menu state
+        # Check if the message is about a product from CSV, regardless of current menu
+        products = get_veterinary_product_matches(message_body)
+        
+        if products:
+            logger.info(f"[Process] Product query detected from menu state '{current_state}': '{message_body}' -> Found {len(products)} products")
+            
+            # If user is in a specific menu but asks about a product, handle it intelligently
+            if current_state in ['main_menu', 'category_selection_menu', 'category_products_menu', 'all_products_menu', 'product_inquiry', 'intelligent_products_menu']:
+                # Use intelligent product inquiry to handle the product query
+                await handle_intelligent_product_inquiry(from_number, message_body, user_context, reply_language)
+                return
+            else:
+                # For other menu states, still handle product queries but remind about menu context
+                await handle_intelligent_product_inquiry(from_number, message_body, user_context, reply_language)
+                return
+        
+        # 🎯 PRIORITY 4: Menu selections - check if this is a valid menu selection for current state
         if current_state in ['main_menu', 'category_selection_menu', 'category_products_menu', 'all_products_menu', 'product_inquiry', 'intelligent_products_menu']:
             # Validate menu selection
             is_valid, error_msg = validate_menu_selection(message_body, current_state, user_context)
@@ -1897,8 +1971,16 @@ async def process_incoming_message(from_number: str, msg: dict):
                         await send_product_image_with_caption(from_number, selected_product, user_context)
                         return
                     else:
-                        send_whatsjet_message(from_number, get_menu_validation_message(current_state, user_context))
-                        return
+                        # Check if the invalid input might be a product query
+                        products = get_veterinary_product_matches(message_body)
+                        if products:
+                            logger.info(f"[Process] Invalid menu selection but product found: '{message_body}' -> Handling as product query")
+                            await handle_intelligent_product_inquiry(from_number, message_body, user_context, reply_language)
+                            return
+                        else:
+                            # Show menu validation message with guidance
+                            send_whatsjet_message(from_number, get_menu_validation_message(current_state, user_context))
+                            return
                 return  # Exit after handling menu selection
         
         # 🎯 PRIORITY 4: Product names - works from ANY menu state
@@ -2074,10 +2156,11 @@ Response Guidelines:
             else:
                 ai_response += "\n\n💬 *Please type 'main' to go to main menu*"
         
-        # Translate response if needed
+        # Translate response if needed (ENGLISH/URDU ONLY)
         if reply_language == 'ur':
             try:
-                translated_response = GoogleTranslator(source='auto', target='ur').translate(ai_response)
+                # Only translate from English to Urdu - no other languages
+                translated_response = GoogleTranslator(source='en', target='ur').translate(ai_response)
                 send_whatsjet_message(from_number, translated_response)
             except Exception as e:
                 logger.error(f"[AI] Translation error: {e}")
@@ -2104,6 +2187,54 @@ Response Guidelines:
             current_menu_options=list(MENU_CONFIG['main_menu']['option_descriptions'].values())
         )
 
+async def handle_clarification_response(from_number: str, response: str, user_context: dict):
+    """Handle user response to clarification questions"""
+    try:
+        clean_response = response.strip().lower()
+        
+        if clean_response in ['yes', 'y', 'apex', 'apex biotical', 'apex biotical solution']:
+            # User confirmed they want Apex Biotical
+            clarification_product = user_context.get('clarification_product', 'apex biotical')
+            
+            # Search for the product
+            products = get_veterinary_product_matches(clarification_product)
+            
+            if products:
+                # Found the product - show detailed information
+                product = products[0]
+                await send_product_with_image(from_number, product, user_context)
+                
+                # Update context
+                user_context['current_product'] = product
+                user_context['current_state'] = 'product_detail'
+                user_context['awaiting_clarification'] = False
+                user_context.pop('clarification_product', None)
+                context_manager.update_context(from_number, **user_context)
+            else:
+                # Product not found even after clarification
+                send_whatsjet_message(from_number, "❌ *Product not found. Please type 'main' to go to main menu.*")
+                user_context['awaiting_clarification'] = False
+                user_context.pop('clarification_product', None)
+                context_manager.update_context(from_number, **user_context)
+        else:
+            # User didn't confirm - use OpenAI for intelligent response
+            if OPENAI_API_KEY:
+                await handle_ai_chat_mode(from_number, response, 'en')
+            else:
+                send_whatsjet_message(from_number, "💬 *Type 'main' to go to main menu or ask another question.*")
+            
+            # Clear clarification context
+            user_context['awaiting_clarification'] = False
+            user_context.pop('clarification_product', None)
+            context_manager.update_context(from_number, **user_context)
+            
+    except Exception as e:
+        logger.error(f"[Clarification] Error: {str(e)}")
+        send_whatsjet_message(from_number, "❌ *An error occurred. Please type 'main' to go to main menu.*")
+        user_context['awaiting_clarification'] = False
+        user_context.pop('clarification_product', None)
+        context_manager.update_context(from_number, **user_context)
+
 async def handle_contact_request(from_number: str):
     """Handle contact request"""
     try:
@@ -2613,13 +2744,14 @@ Response:
         else:
             ai_response += "\n\n💬 *Type 'main' to return to main menu*"
         
-        # Translate response if needed
+        # Translate response if needed (ENGLISH/URDU ONLY)
         if reply_language == 'ur':
             try:
                 # Get all product and category names
                 product_names = [str(p.get('Product Name', '')) for p in all_products if p.get('Product Name')]
                 category_names = list(set([str(p.get('Category', '')) for p in all_products if p.get('Category')]))
-                translated_response = GoogleTranslator(source='auto', target='ur').translate(ai_response)
+                # Only translate from English to Urdu - no other languages
+                translated_response = GoogleTranslator(source='en', target='ur').translate(ai_response)
                 # Restore English terms
                 translated_response = restore_english_terms(translated_response, ai_response, product_names, category_names)
                 send_whatsjet_message(from_number, translated_response)
@@ -2836,7 +2968,10 @@ def generate_veterinary_product_response_with_media(product_info: Dict[str, Any]
     def clean_text(text):
         if pd.isna(text) or text is None:
             return "Not specified"
-        return str(text).strip()
+        cleaned = str(text).strip()
+        # Apply special character cleaning
+        cleaned = clean_special_characters(cleaned)
+        return cleaned
     
     product_name = clean_text(product_info.get('Product Name', ''))
     product_type = clean_text(product_info.get('Type', ''))
@@ -3487,14 +3622,17 @@ async def handle_voice_message_complete(from_number: str, msg: dict):
             detected_lang = detect(transcribed_text)
             logger.info(f"[Voice] Raw detected language: {detected_lang}")
             
-            # STRICTLY ENGLISH OR URDU ONLY - NO OTHER LANGUAGES
-            # Only allow English and Urdu, reject everything else
+            # STRICTLY ENGLISH OR URDU ONLY - REJECT ALL OTHER LANGUAGES
             if detected_lang in ['en', 'ur']:
                 reply_language = detected_lang
+                logger.info(f"[Voice] Valid language detected: {detected_lang}")
             else:
-                # Force any other language to English
+                # Reject any other language and force to English
                 reply_language = 'en'
-                logger.warning(f"[Voice] Detected language '{detected_lang}' is not English or Urdu, forcing to English")
+                logger.warning(f"[Voice] Invalid language '{detected_lang}' detected - forcing to English")
+                # If it's clearly not English/Urdu, mark as unclear
+                if detected_lang not in ['en', 'ur', 'unknown']:
+                    logger.warning(f"[Voice] Non-English/Urdu language detected: {detected_lang}")
             
             # Check if text contains Urdu/Arabic characters or Islamic greetings
             urdu_arabic_pattern = re.compile(r'[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF]')
@@ -3514,11 +3652,12 @@ async def handle_voice_message_complete(from_number: str, msg: dict):
             logger.warning(f"[Voice] Language detection failed: {e}, defaulting to English")
             reply_language = 'en'
         
-        # For Urdu voice notes, translate to English for processing
+        # For Urdu voice notes, translate to English for processing (ENGLISH/URDU ONLY)
         processing_text = transcribed_text
         if reply_language == 'ur' and detected_lang == 'ur':
             try:
                 logger.info(f"[Voice] Translating Urdu voice note to English for processing")
+                # Only translate from Urdu to English - no other languages
                 translated_text = GoogleTranslator(source='ur', target='en').translate(transcribed_text)
                 processing_text = translated_text
                 logger.info(f"[Voice] Translated to English: {translated_text}")
@@ -3526,6 +3665,17 @@ async def handle_voice_message_complete(from_number: str, msg: dict):
                 logger.error(f"[Voice] Translation failed: {e}")
                 # If translation fails, use original text
                 processing_text = transcribed_text
+        elif detected_lang not in ['en', 'ur']:
+            # If language is not English or Urdu, reject it
+            logger.warning(f"[Voice] Non-English/Urdu language detected: {detected_lang}")
+            send_whatsjet_message(from_number, 
+                "🎤 *Voice Message Issue*\n\n"
+                "I can only process voice messages in English or Urdu.\n\n"
+                "💡 *Please:*\n"
+                "• Speak in English or Urdu only\n"
+                "• Send a text message instead\n"
+                "• Type 'main' to see menu options")
+            return
         
         # Determine reply language - always respond in English or Urdu
         if detected_lang == 'ur':
@@ -3556,13 +3706,14 @@ async def handle_voice_message_complete(from_number: str, msg: dict):
                 context_manager.update_context(from_number, current_state='main_menu', current_menu='main_menu', current_menu_options=list(MENU_CONFIG['main_menu']['option_descriptions'].values()))
                 return
         
-        # Process the translated text using the same strict state-based logic as text messages
-        # This ensures voice messages follow the same menu and state rules as text messages
+        # Process the transcribed text using the same intelligent system as text messages
+        # This ensures voice messages get the same intelligent clarification and product search
         await process_incoming_message(from_number, {
             'body': processing_text,  # Use translated text for processing
             'type': 'text',
             'reply_language': reply_language,
-            'original_transcription': transcribed_text  # Keep original for context
+            'original_transcription': transcribed_text,  # Keep original for context
+            'is_voice_message': True  # Flag to indicate this came from voice
         })
         
     except Exception as e:
@@ -3892,10 +4043,11 @@ Format your response professionally with emojis and clear structure. Keep it con
                         
                         full_response = ai_response + selection_instructions
                         
-                        # Translate response if needed
+                        # Translate response if needed (ENGLISH/URDU ONLY)
                         if reply_language == 'ur':
                             try:
-                                translated_response = GoogleTranslator(source='auto', target='ur').translate(full_response)
+                                # Only translate from English to Urdu - no other languages
+                                translated_response = GoogleTranslator(source='en', target='ur').translate(full_response)
                                 send_whatsjet_message(from_number, translated_response)
                             except Exception as e:
                                 logger.error(f"[AI] Translation error: {e}")
@@ -3937,10 +4089,11 @@ Format your response professionally with emojis and clear structure. Keep it con
                     "💬 *Type 'main' to return to the main menu*"
                 )
                 
-                # Translate response if needed
+                # Translate response if needed (ENGLISH/URDU ONLY)
                 if reply_language == 'ur':
                     try:
-                        translated_message = GoogleTranslator(source='auto', target='ur').translate(message)
+                        # Only translate from English to Urdu - no other languages
+                        translated_message = GoogleTranslator(source='en', target='ur').translate(message)
                         send_whatsjet_message(from_number, translated_message)
                     except Exception as e:
                         logger.error(f"[AI] Translation error: {e}")
@@ -3974,9 +4127,9 @@ Format your response professionally with emojis and clear structure. Keep it con
                 
                 # If it's a mode of action query, provide detailed mechanism information
                 if is_mode_of_action_query:
-                    product_name = selected_product.get('Product Name', 'Unknown')
-                    composition = selected_product.get('Composition', 'Not specified')
-                    indications = selected_product.get('Indications', 'Not specified')
+                    product_name = clean_special_characters(selected_product.get('Product Name', 'Unknown'))
+                    composition = clean_special_characters(selected_product.get('Composition', 'Not specified'))
+                    indications = clean_special_characters(selected_product.get('Indications', 'Not specified'))
                     
                     mode_of_action_response = (
                         f"🧪 *{product_name} - Mode of Action*\n\n"
@@ -3997,13 +4150,52 @@ Format your response professionally with emojis and clear structure. Keep it con
                 context_manager.add_to_history(from_number, query, f"Product inquiry for {selected_product.get('Product Name', 'Unknown')}")
             
         else:
-            # Generic, professional "not found" response
-            if reply_language == 'ur':
-                message = "❌ *سوال درست نہیں ہے*\n\n💬 *براہ کرم اپنا سوال درست کریں یا 'main' لکھ کر مین مینو پر جائیں*"
+            # No products found - check if it's an ambiguous query that needs clarification
+            if any(keyword in clean_query for keyword in ['what is', 'apex', 'immune', 'hydro', 'hepo', 'brom', 'trib', 'symo']):
+                # Check if it might be about a product but needs clarification
+                if 'apex' in clean_query:
+                    clarification_message = (
+                        "🤔 *I found a similar product in our database.*\n\n"
+                        "Are you asking about:\n"
+                        "• Apex Biotical Solution\n"
+                        "• Or something else?\n\n"
+                        "💬 *Type 'yes' to continue with Apex Biotical, or specify what you mean.*"
+                    )
+                    send_whatsjet_message(from_number, clarification_message)
+                    user_context['awaiting_clarification'] = True
+                    user_context['clarification_product'] = 'apex biotical'
+                    context_manager.update_context(from_number, **user_context)
+                    return
+                elif any(product_hint in clean_query for product_hint in ['immune', 'hydro', 'hepo', 'brom', 'trib', 'symo']):
+                    # Use OpenAI for intelligent response about veterinary topics
+                    if OPENAI_API_KEY:
+                        await handle_ai_chat_mode(from_number, query, reply_language)
+                    else:
+                        # Fallback to generic response
+                        if reply_language == 'ur':
+                            message = "❌ *سوال درست نہیں ہے*\n\n💬 *براہ کرم اپنا سوال درست کریں یا 'main' لکھ کر مین مینو پر جائیں*"
+                        else:
+                            message = "❌ *Please correct your question*\n\n💬 *Type 'main' to go to main menu*"
+                        send_whatsjet_message(from_number, message)
+                else:
+                    # Use OpenAI for general veterinary questions
+                    if OPENAI_API_KEY:
+                        await handle_ai_chat_mode(from_number, query, reply_language)
+                    else:
+                        # Fallback to generic response
+                        if reply_language == 'ur':
+                            message = "❌ *سوال درست نہیں ہے*\n\n💬 *براہ کرم اپنا سوال درست کریں یا 'main' لکھ کر مین مینو پر جائیں*"
+                        else:
+                            message = "❌ *Please correct your question*\n\n💬 *Type 'main' to go to main menu*"
+                        send_whatsjet_message(from_number, message)
             else:
-                message = "❌ *Please correct your question*\n\n💬 *Type 'main' to go to main menu*"
-            
-            send_whatsjet_message(from_number, message)
+                # Generic, professional "not found" response
+                if reply_language == 'ur':
+                    message = "❌ *سوال درست نہیں ہے*\n\n💬 *براہ کرم اپنا سوال درست کریں یا 'main' لکھ کر مین مینو پر جائیں*"
+                else:
+                    message = "❌ *Please correct your question*\n\n💬 *Type 'main' to go to main menu*"
+                
+                send_whatsjet_message(from_number, message)
             
     except Exception as e:
         logger.error(f"Error in product inquiry: {e}")