to

app.py

@@ -1,7 +1,101 @@
 import gradio as gr
+from transformers import AutoTokenizer
 
-def greet(name):
-    return "Hello " + name + "!!"
+# Tokenizer options
+TOKENIZER_PATHS = {
+    "Custom Tokenizer (alakxender/flan-t5-dhivehi-tokenizer)": "alakxender/flan-t5-dhivehi-tokenizer",
+    "Stock Tokenizer (google/flan-t5-base)": "google/flan-t5-base",
+    "T5 Small (t5-small)": "t5-small"
+}
+
+def tokenize_display(text, tokenizer_path):
+    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
+    encoding = tokenizer(text, return_offsets_mapping=False, add_special_tokens=True)
+    tokens = tokenizer.convert_ids_to_tokens(encoding.input_ids)
+    ids = encoding.input_ids
+    decoded = tokenizer.decode(ids, skip_special_tokens=False)
+    return tokens, ids, decoded
+
+def compare_side_by_side(dv_text, en_text, custom_label, stock_label):
+    def format_block(title, tokenizer_path):
+        # Dhivehi
+        dv_tokens, dv_ids, dv_decoded = tokenize_display(dv_text, tokenizer_path)
+        # English
+        en_tokens, en_ids, en_decoded = tokenize_display(en_text, tokenizer_path)
+
+        return f"""\
+### 🔤 {title}
+
+#### 🈁 Dhivehi Text  
+`{dv_text}`
+
+**Tokenized:**  
+{' '.join(dv_tokens)}  
+
+**Number of tokens:** {len(dv_tokens)}  
+**IDs:** {dv_ids}  
+**Decoded:** `{dv_decoded}`
+
+---
+
+#### 🇬🇧 English Text  
+`{en_text}`
+
+**Tokenized:**  
+{' '.join(en_tokens)}  
+
+**Number of tokens:** {len(en_tokens)}  
+**IDs:** {en_ids}  
+**Decoded:** `{en_decoded}`
+"""
+
+    custom_path = TOKENIZER_PATHS[custom_label]
+    stock_path = TOKENIZER_PATHS[stock_label]
+
+    return (
+        format_block("Custom Tokenizer", custom_path),
+        format_block("Stock Tokenizer", stock_path)
+    )
+
+# Gradio app
+with gr.Blocks(title="Tokenizer Comparison Tool") as demo:
+    gr.Markdown("## 🧠 Tokenizer Comparison (Custom vs Stock)")
+    gr.Markdown("Compare how different tokenizers handle Dhivehi and English input text.")
+
+    with gr.Row():
+        dhivehi_text = gr.Textbox(
+            label="Dhivehi Text",
+            lines=2,
+            value="އީދުގެ ހަރަކާތްތައް ފެށުމަށް މިރޭ ހުޅުމާލޭގައި އީދު މަޅި ރޯކުރަނީ"
+        )
+        english_text = gr.Textbox(
+            label="English Text",
+            lines=2,
+            value="The quick brown fox jumps over the lazy dog"
+        )
+
+    with gr.Row():
+        tokenizer_a = gr.Dropdown(
+            label="Select Custom Tokenizer",
+            choices=list(TOKENIZER_PATHS.keys()),
+            value="Custom Tokenizer (alakxender/flan-t5-dhivehi-tokenizer)"
+        )
+        tokenizer_b = gr.Dropdown(
+            label="Select Stock Tokenizer",
+            choices=list(TOKENIZER_PATHS.keys()),
+            value="Stock Tokenizer (google/flan-t5-base)"
+        )
+
+    compare_button = gr.Button("Compare Tokenizers")
+
+    with gr.Row():
+        output_custom = gr.Markdown(label="Custom Tokenizer Output")
+        output_stock = gr.Markdown(label="Stock Tokenizer Output")
+
+    compare_button.click(
+        compare_side_by_side,
+        inputs=[dhivehi_text, english_text, tokenizer_a, tokenizer_b],
+        outputs=[output_custom, output_stock]
+    )
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
 demo.launch()

requirements.txt

@@ -0,0 +1 @@
+transformers