Update app.py

app.py

@@ -15,6 +15,51 @@ load_dotenv()
 
 DEFAULT_FAST_TEXT_MODEL = "laurievb/OpenLID"
 
+# Language code mapping - feel free to expand this
+LANGUAGE_MAPPING = {
+    "spa_Latn": {"name": "Spanish", "iso_639_1": "es", "full_code": "es_ES"},
+    "eng_Latn": {"name": "English", "iso_639_1": "en", "full_code": "en_US"},
+    "fra_Latn": {"name": "French", "iso_639_1": "fr", "full_code": "fr_FR"},
+    "deu_Latn": {"name": "German", "iso_639_1": "de", "full_code": "de_DE"},
+    "ita_Latn": {"name": "Italian", "iso_639_1": "it", "full_code": "it_IT"},
+    "por_Latn": {"name": "Portuguese", "iso_639_1": "pt", "full_code": "pt_PT"},
+    "rus_Cyrl": {"name": "Russian", "iso_639_1": "ru", "full_code": "ru_RU"},
+    "zho_Hans": {"name": "Chinese (Simplified)", "iso_639_1": "zh", "full_code": "zh_CN"},
+    "zho_Hant": {"name": "Chinese (Traditional)", "iso_639_1": "zh", "full_code": "zh_TW"},
+    "jpn_Jpan": {"name": "Japanese", "iso_639_1": "ja", "full_code": "ja_JP"},
+    "kor_Hang": {"name": "Korean", "iso_639_1": "ko", "full_code": "ko_KR"},
+    "ara_Arab": {"name": "Arabic", "iso_639_1": "ar", "full_code": "ar_SA"},
+    "hin_Deva": {"name": "Hindi", "iso_639_1": "hi", "full_code": "hi_IN"},
+    "cat_Latn": {"name": "Catalan", "iso_639_1": "ca", "full_code": "ca_ES"},
+    "glg_Latn": {"name": "Galician", "iso_639_1": "gl", "full_code": "gl_ES"},
+    "nld_Latn": {"name": "Dutch", "iso_639_1": "nl", "full_code": "nl_NL"},
+    "swe_Latn": {"name": "Swedish", "iso_639_1": "sv", "full_code": "sv_SE"},
+    "nor_Latn": {"name": "Norwegian", "iso_639_1": "no", "full_code": "no_NO"},
+    "dan_Latn": {"name": "Danish", "iso_639_1": "da", "full_code": "da_DK"},
+    "fin_Latn": {"name": "Finnish", "iso_639_1": "fi", "full_code": "fi_FI"},
+    "pol_Latn": {"name": "Polish", "iso_639_1": "pl", "full_code": "pl_PL"},
+    "ces_Latn": {"name": "Czech", "iso_639_1": "cs", "full_code": "cs_CZ"},
+    "hun_Latn": {"name": "Hungarian", "iso_639_1": "hu", "full_code": "hu_HU"},
+    "tur_Latn": {"name": "Turkish", "iso_639_1": "tr", "full_code": "tr_TR"},
+    "heb_Hebr": {"name": "Hebrew", "iso_639_1": "he", "full_code": "he_IL"},
+    "tha_Thai": {"name": "Thai", "iso_639_1": "th", "full_code": "th_TH"},
+    "vie_Latn": {"name": "Vietnamese", "iso_639_1": "vi", "full_code": "vi_VN"},
+    "ukr_Cyrl": {"name": "Ukrainian", "iso_639_1": "uk", "full_code": "uk_UA"},
+    "ell_Grek": {"name": "Greek", "iso_639_1": "el", "full_code": "el_GR"},
+    "bul_Cyrl": {"name": "Bulgarian", "iso_639_1": "bg", "full_code": "bg_BG"},
+    "ron_Latn": {"name": "Romanian", "iso_639_1": "ro", "full_code": "ro_RO"},
+    "hrv_Latn": {"name": "Croatian", "iso_639_1": "hr", "full_code": "hr_HR"},
+    "srp_Cyrl": {"name": "Serbian", "iso_639_1": "sr", "full_code": "sr_RS"},
+    "slv_Latn": {"name": "Slovenian", "iso_639_1": "sl", "full_code": "sl_SI"},
+    "slk_Latn": {"name": "Slovak", "iso_639_1": "sk", "full_code": "sk_SK"},
+    "est_Latn": {"name": "Estonian", "iso_639_1": "et", "full_code": "et_EE"},
+    "lav_Latn": {"name": "Latvian", "iso_639_1": "lv", "full_code": "lv_LV"},
+    "lit_Latn": {"name": "Lithuanian", "iso_639_1": "lt", "full_code": "lt_LT"},
+    "msa_Latn": {"name": "Malay", "iso_639_1": "ms", "full_code": "ms_MY"},
+    "ind_Latn": {"name": "Indonesian", "iso_639_1": "id", "full_code": "id_ID"},
+    "tgl_Latn": {"name": "Filipino", "iso_639_1": "tl", "full_code": "tl_PH"},
+}
+
 def load_model(repo_id: str) -> fasttext.FastText._FastText:
     model_path = hf_hub_download(repo_id, filename="model.bin")
     return fasttext.load_model(model_path)
@@ -39,6 +84,62 @@ def yield_clean_rows(rows: Union[list[str], str], min_length: int = 3) -> Iterat
 
 FASTTEXT_PREFIX_LENGTH = 9  # fasttext labels are formatted like "__label__eng_Latn"
 
+def format_language_info(fasttext_code):
+    """Convert FastText language code to human readable format"""
+    if fasttext_code in LANGUAGE_MAPPING:
+        lang_info = LANGUAGE_MAPPING[fasttext_code]
+        return {
+            "name": lang_info["name"],
+            "iso_code": lang_info["iso_639_1"], 
+            "full_code": lang_info["full_code"],
+            "fasttext_code": fasttext_code
+        }
+    else:
+        # Graceful fallback for unmapped languages
+        return {
+            "name": fasttext_code,
+            "iso_code": "unknown",
+            "full_code": "unknown",
+            "fasttext_code": fasttext_code
+        }
+
+def detect_language_segments(text, confidence_threshold=0.3):
+    """Detect language changes in text segments"""
+    # Split text into logical segments (sentences, clauses)
+    import re
+    
+    # More sophisticated splitting on common separators
+    segments = re.split(r'[.!?;/|]\s+|\s+/\s+|\s+\|\s+', text.strip())
+    segments = [seg.strip() for seg in segments if seg.strip() and len(seg.strip()) > 10]
+    
+    if len(segments) < 2:
+        return None
+    
+    segment_results = []
+    for i, segment in enumerate(segments):
+        predictions = model_predict(segment, k=1)
+        if predictions and predictions[0]['score'] > confidence_threshold:
+            lang_info = format_language_info(predictions[0]['label'])
+            segment_results.append({
+                "segment_number": i + 1,
+                "text": segment,
+                "language": lang_info,
+                "confidence": predictions[0]['score']
+            })
+    
+    # Check if we found different languages
+    languages_found = set(result['language']['fasttext_code'] for result in segment_results)
+    
+    if len(languages_found) > 1:
+        return {
+            "is_multilingual": True,
+            "languages_detected": list(languages_found),
+            "segments": segment_results
+        }
+    
+    return None
+
+
 # Load the model
 Path("code/models").mkdir(parents=True, exist_ok=True)
 model = fasttext.load_model(
@@ -97,19 +198,37 @@ def simple_predict(text, num_predictions=3):
         # Calculate average scores for each language
         language_scores = valmap(get_mean_score, predictions_by_lang)
         
+        # Format results
+        # Format with human-readable language info
+        formatted_languages = {}
+        for fasttext_code, score in language_scores.items():
+            lang_info = format_language_info(fasttext_code)
+            formatted_languages[fasttext_code] = {
+                "score": score,
+                "language_info": lang_info
+            }
+        
+        # Check for multilingual segments
+        segment_analysis = detect_language_segments(text)
+        
         # Format results
         results = {
-            "detected_languages": dict(language_scores),
+            "detected_languages": formatted_languages,
             "language_counts": dict(language_counts),
             "total_predictions": len(all_predictions),
             "text_lines_analyzed": len(cleaned_lines)
         }
         
-        return results
+        # Add segment analysis if multilingual
+        if segment_analysis:
+            results["segment_analysis"] = segment_analysis
         
+        return results
+
     except Exception as e:
         return {"error": f"Error during prediction: {str(e)}"}
 
+        
 def batch_predict(text, threshold_percent=0.2):
     """More advanced prediction with filtering"""
     if not text or not text.strip():
@@ -135,18 +254,35 @@ def batch_predict(text, threshold_percent=0.2):
         keys_to_keep = filter_by_frequency(language_counts, threshold_percent=threshold_percent)
         filtered_dict = {k: v for k, v in predictions_by_lang.items() if k in keys_to_keep}
         
+        # Format with human-readable language info
+        formatted_predictions = {}
+        for fasttext_code, score in valmap(get_mean_score, filtered_dict).items():
+            lang_info = format_language_info(fasttext_code)
+            formatted_predictions[fasttext_code] = {
+                "score": score,
+                "language_info": lang_info
+            }
+        
+        # Check for multilingual segments
+        segment_analysis = detect_language_segments(text)
+        
         results = {
-            "predictions": dict(valmap(get_mean_score, filtered_dict)),
+            "predictions": formatted_predictions,
             "all_language_counts": dict(language_counts),
             "filtered_languages": list(keys_to_keep),
             "threshold_used": threshold_percent
         }
         
+        # Add segment analysis if multilingual
+        if segment_analysis:
+            results["segment_analysis"] = segment_analysis
+        
         return results
         
     except Exception as e:
         return {"error": f"Error during prediction: {str(e)}"}
 
+        
 def build_demo_interface():
     app_title = "Language Detection Tool" 
     with gr.Blocks(title=app_title) as demo: