app.py

import gradio as gr
import torch
from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM
import re

# Global variables to store the models
atlas_pipe = None
transliteration_tokenizer = None
transliteration_model = None

def load_models():
    """Load both Atlas-Chat and Transliteration models"""
    global atlas_pipe, transliteration_tokenizer, transliteration_model
    
    # Load Atlas-Chat model
    if atlas_pipe is None:
        print("🏔️ Loading Atlas-Chat-2B model...")
        atlas_pipe = pipeline(
            "text-generation",
            model="MBZUAI-Paris/Atlas-Chat-2B",
            model_kwargs={"torch_dtype": torch.bfloat16},
            device="cuda" if torch.cuda.is_available() else "cpu"
        )
        print("✅ Atlas-Chat model loaded!")
    
    # Load Transliteration model
    if transliteration_tokenizer is None or transliteration_model is None:
        print("🔄 Loading Transliteration model...")
        transliteration_tokenizer = AutoTokenizer.from_pretrained("atlasia/Transliteration-Moroccan-Darija")
        transliteration_model = AutoModelForSeq2SeqLM.from_pretrained("atlasia/Transliteration-Moroccan-Darija")
        print("✅ Transliteration model loaded!")
    
    return atlas_pipe, transliteration_tokenizer, transliteration_model

def detect_arabizi(text):
    """
    Detect if input text is written in Arabizi (Latin script with numbers)
    Returns True if Arabizi is detected
    """
    if not text or len(text.strip()) < 2:
        return False
    
    # Check for Arabic script - if present, it's NOT Arabizi
    arabic_pattern = r'[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF]'
    if re.search(arabic_pattern, text):
        return False
    
    # Arabizi indicators - numbers used as letters
    arabizi_numbers = ['2', '3', '7', '9', '5', '6', '8']
    has_arabizi_numbers = any(num in text for num in arabizi_numbers)
    
    # Common Arabizi words and patterns
    arabizi_patterns = [
        'wach', 'wash', 'ach', 'achno', 'chno', 'shno', 'shkoun', 'chkoun',
        'kif', 'kifash', 'ki', 'kayf', 'kien', 'kima',
        'feen', 'fin', 'fen', 'fain', 'mnin',
        'imta', 'meta', 'waqt', 'mata', 'emta',
        'hna', 'ahna', 'ana', 'nta', 'nti', 'ntuma', 'ntouma',
        'howa', 'hiya', 'huma', 'houma', 'hoa', 'hia',
        'had', 'hadchi', 'hada', 'hadi', 'hadou', 'hadouk',
        'bghit', 'bghiti', 'bgha', 'bghina', 'bghitiou',
        'galt', 'galti', 'gal', 'galet', 'galou',
        'rah', 'raha', 'rahi', 'rahom', 'rahin',
        'kan', 'kanu', 'kana', 'kanet', 'kano',
        'ghadi', 'ghad', 'gha', 'ghadia', 'ghadiyin',
        'daba', 'dak', 'dakchi', 'dik', 'dok',
        'bzf', 'bzzaf', 'bezzaf', 'bzaaaaf',
        'chway', 'chwiya', 'shwiya', 'chwia',
        'khoya', 'khuya', 'akhi', 'kho',
        'khti', 'khtiya', 'ukhti', 'kht',
        'mama', 'baba', 'lwaldin', 'lwalidin',
        'salam', 'salamu aleikum', 'slm',
        'yallah', 'yalla', 'hya', 'aji',
        'mabghitsh', 'mabghach', 'makansh', 'machi',
        'walakin', 'walaken', 'ama', 'mais',
        'kayn', 'makaynsh', 'chi', 'tayi'
    ]
    
    text_lower = text.lower()
    has_arabizi_words = any(pattern in text_lower for pattern in arabizi_patterns)
    
    # Decision logic
    if has_arabizi_numbers and has_arabizi_words:
        return True
    if has_arabizi_numbers and len([c for c in text if c.isalpha()]) > len(text) * 0.6:
        return True
    if has_arabizi_words and len([c for c in text if c.isalpha()]) > len(text) * 0.7:
        return True
    
    return False

def arabizi_to_arabic_ai(arabizi_text):
    """
    Convert Arabizi text to Arabic using the specialized AI model
    """
    try:
        _, tokenizer, model = load_models()
        
        # Tokenize the input text
        input_tokens = tokenizer(arabizi_text, return_tensors="pt", padding=True, truncation=True, max_length=512)
        
        # Perform transliteration
        with torch.no_grad():
            output_tokens = model.generate(
                **input_tokens,
                max_length=512,
                num_beams=4,
                early_stopping=True,
                no_repeat_ngram_size=2
            )
        
        # Decode the output tokens
        arabic_text = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
        
        return arabic_text.strip()
        
    except Exception as e:
        print(f"❌ Error in Arabizi→Arabic conversion: {e}")
        # Fallback to original text if conversion fails
        return arabizi_text

def arabic_to_arabizi(arabic_text):
    """
    Convert Arabic script to Arabizi using character mappings
    (Keeping this as backup since no reverse model available)
    """
    if not arabic_text:
        return arabic_text
    
    # COMPREHENSIVE WORD MAPPINGS (Arabic → Arabizi)
    word_mappings = {
        # Common words first (most likely to appear)
        'أنا': 'ana', 'نتا': 'nta', 'نتي': 'nti', 'هوا': 'howa', 'هيا': 'hiya',
        'حنا': 'hna', 'أحنا': 'ahna', 'نتوما': 'ntuma', 'هوما': 'huma',
        'شكون': 'shkoun', 'أشنو': 'achno', 'شنو': 'chno', 'واش': 'wach',
        'كيفاش': 'kifash', 'كيف': 'kif', 'فين': 'feen', 'منين': 'mnin',
        'إمتا': 'imta', 'متا': 'meta', 'علاش': '3lach', 'أش': 'ach',
        'بغيت': 'bghit', 'بغيتي': 'bghiti', 'بغا': 'bgha', 'بغينا': 'bghina',
        'كان': 'kan', 'كانا': 'kana', 'كانت': 'kanet', 'كانو': 'kanu',
        'قلت': 'galt', 'قلتي': 'galti', 'قال': 'gal', 'قالت': 'galet',
        'راح': 'rah', 'راها': 'raha', 'راهي': 'rahi', 'راهم': 'rahom',
        'غادي': 'ghadi', 'غاد': 'ghad', 'غا': 'gha',
        'هاد': 'had', 'هادا': 'hada', 'هادي': 'hadi', 'هادشي': 'hadchi',
        'داك': 'dak', 'ديك': 'dik', 'داكشي': 'dakchi',
        'بزاف': 'bzzaf', 'شوياة': 'chwiya', 'كولشي': 'kolchi',
        'ماشي': 'machi', 'مابغيتش': 'mabghitsh', 'ماكاينش': 'makainch',
        'دابا': 'daba', 'توا': 'tawa', 'غدا': 'ghda',
        'ماما': 'mama', 'بابا': 'baba', 'خويا': 'khoya', 'ختي': 'khti',
        'سلام': 'salam', 'يالاه': 'yallah', 'هيا': 'hya',
        'المغرب': 'lmaghrib', 'مغرب': 'maghrib',
        'طاجين': 'tajine', 'أتاي': 'atay', 'خوبز': 'khobz',
        'كاين': 'kayn', 'ماكاينش': 'makaynsh', 'شي': 'chi',
        'زوين': 'zwin', 'زوينا': 'zwina', 'مزيان': 'mzyan', 'مزيانا': 'mzyana',
        'درت': 'dert', 'درتي': 'derti', 'دار': 'dar', 'درات': 'derat',
        'مشيت': 'mchit', 'مشيتي': 'mchiti', 'مشا': 'mcha', 'مشات': 'mchat',
        'جيت': 'jit', 'جيتي': 'jiti', 'جا': 'ja', 'جات': 'jat',
        'شفت': 'cheft', 'شفتي': 'chefti', 'شاف': 'chaf', 'شافت': 'chafat',
        'سمعت': 'sme3t', 'سمعتي': 'sme3ti', 'سمع': 'sma3', 'سمعات': 'sma3at',
        'أكلت': 'klit', 'أكلتي': 'kliti', 'كلا': 'kla', 'كلات': 'klat',
        'شربت': 'chrebt', 'شربتي': 'chrebti', 'شرب': 'chreb', 'شربات': 'chrebat',
        'نعست': 'ne3st', 'نعستي': 'ne3sti', 'نعس': 'ne3s', 'نعسات': 'ne3sat',
        'خرجت': 'khrjt', 'خرجتي': 'khrjti', 'خرج': 'khrj', 'خرجات': 'khrjat',
        'دخلت': 'dkhlt', 'دخلتي': 'dkhlti', 'دخل': 'dkhl', 'دخلات': 'dkhlat',
        'قريت': 'qrit', 'قريتي': 'qriti', 'قرا': 'qra', 'قرات': 'qrat',
        'كتبت': 'ktebt', 'كتبتي': 'ktebti', 'كتب': 'kteb', 'كتبات': 'ktebat',
        'لعبت': 'l3ebt', 'لعبتي': 'l3ebti', 'لعب': 'l3eb', 'لعبات': 'l3ebat',
        'خدمت': 'khdmt', 'خدمتي': 'khdmti', 'خدم': 'khdm', 'خدمات': 'khdmat',
        'صليت': 'sllit', 'صليتي': 'slliti', 'صلا': 'slla', 'صلات': 'sllat',
        'طبخت': '6bkht', 'طبختي': '6bkhti', 'طبخ': '6bekh', 'طبخات': '6bekhat',
        'واحد': 'wa7ed', 'جوج': 'joj', 'تلاتا': 'tlata', 'ربعا': 'reb3a',
        'خمسا': 'khamsa', 'ستا': 'setta', 'سبعا': 'seb3a', 'تمنيا': 'tmnya',
        'تسعا': 'tes3a', 'عشرا': '3echra', 'حداش': '7dach', 'طناش': '6nach',
        'نهار': 'nhar', 'ليلا': 'lila', 'صباح': 'sba7', 'عشيا': '3echiya',
        'أمس': 'ems', 'البارح': 'lbare7', 'غدا': 'ghda', 'بعد غدا': 'b3d ghda',
        'دار': 'dar', 'بيت': 'bit', 'شارع': 'char3', 'مدينا': 'mdina',
        'كرهوبا': 'karhouba', 'طوموبيل': 'tomobil', 'قطار': 'q6ar', 'باص': 'bas',
        'ماكلا': 'makla', 'شراب': 'chrab', 'لما': 'lma', 'عطش': '36ch',
        'جوع': 'jo3', 'شبعان': 'cheb3an', 'عيان': '3yyan', 'صحيح': 's7i7',
        'مريض': 'mrid', 'دكتور': 'doktor', 'سبيطار': 'sbitar', 'دوا': 'dwa',
        'فلوس': 'flous', 'درهم': 'derhem', 'ريال': 'riyal', 'اليورو': 'lyoro',
        'خدما': 'khedma', 'معلم': 'mo3alim', 'طالب': 'talib', 'أستاذ': 'ostaz',
        'كتاب': 'ktab', 'قلم': 'qalam', 'كاغط': 'kaghet', 'طاولا': 'tabla'
    }
    
    # CHARACTER MAPPINGS (Arabic → Arabizi)
    char_mappings = {
        'ا': 'a', 'ب': 'b', 'ت': 't', 'ث': 'th', 'ج': 'j', 'ح': '7',
        'خ': 'kh', 'د': 'd', 'ذ': 'dh', 'ر': 'r', 'ز': 'z', 'س': 's',
        'ش': 'sh', 'ص': 's', 'ض': 'd', 'ط': '6', 'ظ': 'z', 'ع': '3',
        'غ': 'gh', 'ف': 'f', 'ق': '9', 'ك': 'k', 'ل': 'l', 'م': 'm',
        'ن': 'n', 'ه': 'h', 'و': 'w', 'ي': 'y', 'ء': '2',
        'آ': 'aa', 'أ': 'a', 'إ': 'i', 'ة': 'a', 'ى': 'a',
        '؟': '?', '،': ',', '؛': ';', '：': ':', '！': '!',
        'َ': 'a', 'ُ': 'o', 'ِ': 'i', 'ً': 'an', 'ٌ': 'on', 'ٍ': 'in'
    }
    
    result = arabic_text
    
    # Step 1: Apply word mappings
    for arabic_word, arabizi_word in word_mappings.items():
        # Use word boundaries to avoid partial matches
        result = re.sub(r'\b' + re.escape(arabic_word) + r'\b', arabizi_word, result)
    
    # Step 2: Apply character mappings
    for arabic_char, arabizi_char in char_mappings.items():
        result = result.replace(arabic_char, arabizi_char)
    
    return result.strip()

def chat_with_atlas(message, history):
    """Generate response from Atlas-Chat model with AI-powered Arabizi conversion"""
    if not message.strip():
        return "ahlan wa sahlan! kifash n9der n3awnek? / مرحبا! كيفاش نقدر نعاونك؟"
    
    try:
        # Load models
        atlas_model, _, _ = load_models()
        
        # Detect if input is Arabizi
        is_arabizi_input = detect_arabizi(message)
        
        print("\n" + "="*60)
        print("🔍 DEBUG LOG - FULL CONVERSION PIPELINE")
        print("="*60)
        print(f"📥 ORIGINAL INPUT: '{message}'")
        print(f"🤖 ARABIZI DETECTED: {is_arabizi_input}")
        
        # Prepare input for the model
        if is_arabizi_input:
            # Convert Arabizi to Arabic using AI model
            print(f"\n🔄 STEP 1: Converting Arabizi to Arabic...")
            arabic_input = arabizi_to_arabic_ai(message)
            print(f"✅ ARABIC CONVERSION: '{arabic_input}'")
            model_input = arabic_input
        else:
            # Use original input (Arabic or English)
            print(f"\n➡️  NO CONVERSION NEEDED - Using original input")
            model_input = message
        
        print(f"\n🤖 STEP 2: Sending to Atlas-Chat model...")
        print(f"📤 MODEL INPUT: '{model_input}'")
        
        # Generate response using Arabic input
        messages = [{"role": "user", "content": model_input}]
        
        outputs = atlas_model(
            messages, 
            max_new_tokens=256, 
            temperature=0.1,
            do_sample=True,
            pad_token_id=atlas_model.tokenizer.eos_token_id
        )
        
        # Extract the response
        response = outputs[0]["generated_text"][-1]["content"].strip()
        print(f"✅ MODEL RESPONSE (Arabic): '{response}'")
        
        # Convert response back to Arabizi if input was Arabizi
        if is_arabizi_input:
            print(f"\n🔄 STEP 3: Converting response back to Arabizi...")
            arabizi_response = arabic_to_arabizi(response)
            print(f"✅ FINAL ARABIZI RESPONSE: '{arabizi_response}'")
            print("="*60)
            print("🎯 FINAL OUTPUT TO USER:", arabizi_response)
            print("="*60 + "\n")
            return arabizi_response
        else:
            # Return original response for Arabic/English
            print(f"\n➡️  NO BACK-CONVERSION NEEDED")
            print("="*60)
            print("🎯 FINAL OUTPUT TO USER:", response)
            print("="*60 + "\n")
            return response
        
    except Exception as e:
        print(f"\n❌ ERROR OCCURRED: {str(e)}")
        print("="*60 + "\n")
        # Return error in appropriate language
        if detect_arabizi(message):
            return f"sorry, kan chi mochkil: {str(e)}. 3awd jar'b!"
        else:
            return f"عذراً، واجهت خطأ: {str(e)}. جرب مرة أخرى! / Sorry, error occurred: {str(e)}. Try again!"

# Create the Gradio interface
demo = gr.ChatInterface(
    fn=chat_with_atlas,
    title="🏔️ Atlas-Chat: Advanced Moroccan Arabic AI",
    description="""
    **مرحبا بك في أطلس شات المطور!** Welcome to Advanced Atlas-Chat! 🇲🇦
    
    **🧠 AI-Powered Language Detection & Conversion:**
    - **Arabic Script (العربية)** → AI responds in Arabic
    - **Arabizi (3arabi bi 7oruf latin)** → AI-powered conversion → Arabizi response  
    - **English** → AI responds in English
    
    **⚡ Professional Arabizi Conversion**
    - Uses specialized AI model trained on Moroccan Darija
    - Perfect understanding of context: "kayn chi" → "كاين شي"
    - Handles complex phrases accurately
    
    **جرب هذه الأسئلة / Try these questions:**
    """,
    examples=[
        "شكون لي صنعك؟",
        "shkoun li sna3ek?",
        "اشنو هو الطاجين؟",
        "achno howa tajine?", 
        "شنو كيتسمى المنتخب المغربي؟",
        "chno kaytsma lmontakhab lmaghribi?",
        "What is Morocco famous for?",
        "كيفاش نقدر نتعلم الدارجة؟",
        "kifash n9der nt3elem darija?",
        "wach kayn atay f lmaghrib?",
        "3lach lmaghrib zwien bzzaf?",
        "kifash nsali tajine?",
        "chno homa l2aklat lmaghribiya?",
        "kayn chi restaurants zwinin f casa?",
        "mr7ba! kif dayr?"
    ],
    cache_examples=False
)

# Launch the app
if __name__ == "__main__":
    demo.launch()