d

app.py

@@ -1,10 +1,12 @@
 import gradio as gr
-from transformers import AutoTokenizer
+from transformers import AutoTokenizer, T5Tokenizer
 
 # Fixed list of custom tokenizers (left)
 TOKENIZER_CUSTOM = {
     "T5 Extended": "alakxender/dhivehi-T5-tokenizer-extended",
-    "RoBERTa Extended": "alakxender/dhivehi-roberta-tokenizer-extended"
+    "RoBERTa Extended": "alakxender/dhivehi-roberta-tokenizer-extended",
+    "Google mT5": "google/mt5-base",
+    "DeBERTa": "alakxender/deberta-dhivehi-tokenizer-extended"
 }
 
 # Suggested stock model paths for the right input
@@ -15,13 +17,23 @@ SUGGESTED_STOCK_PATHS = [
     "t5-large",
     "google/mt5-base",
     "microsoft/trocr-base-handwritten",
-    "microsoft/trocr-base-printed"
+    "microsoft/trocr-base-printed",
+    "microsoft/deberta-v3-base"
 ]
 
+# Load tokenizer with fallback to slow T5
+def load_tokenizer(tokenizer_path):
+    try:
+        return AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True)
+    except Exception:
+        if "t5" in tokenizer_path.lower() or "mt5" in tokenizer_path.lower():
+            return T5Tokenizer.from_pretrained(tokenizer_path)
+        raise
+
 # Tokenize and decode with error handling
 def tokenize_display(text, tokenizer_path):
     try:
-        tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
+        tokenizer = load_tokenizer(tokenizer_path)
         encoding = tokenizer(text, return_offsets_mapping=False, add_special_tokens=True)
         tokens = tokenizer.convert_ids_to_tokens(encoding.input_ids)
         ids = encoding.input_ids

requirements.txt

@@ -1 +1,2 @@
-transformers
+transformers
+sentencepiece