Update app.py

app.py

@@ -3,48 +3,21 @@ import gradio as gr
 
 
 def tokenize(input_text):
-    llama_tokens = len(
-        llama_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    llama3_tokens = len(
-        llama3_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    mistral_tokens = len(
-        mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    gpt2_tokens = len(
-        gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    gpt_neox_tokens = len(
-        gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    falcon_tokens = len(
-        falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    phi2_tokens = len(
-        phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    phi3_tokens = len(
-        phi3_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    t5_tokens = len(
-        t5_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    gemma_tokens = len(
-        gemma_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    command_r_tokens = len(
-        command_r_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    qwen_tokens = len(
-        qwen_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    codeqwen_tokens = len(
-        codeqwen_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
-    rwkv_tokens = len(
-        rwkv_tokenizer(input_text, add_special_tokens=True)["input_ids"]
-    )
+    llama_tokens = len(llama_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    llama3_tokens = len(llama3_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    mistral_tokens = len(mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    gpt2_tokens = len(gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    gpt_neox_tokens = len(gpt_neox_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    falcon_tokens = len(falcon_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    phi2_tokens = len(phi2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    phi3_tokens = len(phi3_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    t5_tokens = len(t5_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    gemma_tokens = len(gemma_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    command_r_tokens = len(command_r_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    qwen_tokens = len(qwen_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    codeqwen_tokens = len(codeqwen_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    rwkv4_tokens = len(rwkv4_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    rwkv5_tokens = len(rwkv5_tokenizer(input_text, add_special_tokens=True)["input_ids"])
 
     results = {
         "LLaMa-1/LLaMa-2": llama_tokens,
@@ -60,7 +33,8 @@ def tokenize(input_text):
         "Command-R": command_r_tokens,
         "Qwen/Qwen1.5": qwen_tokens,
         "CodeQwen": codeqwen_tokens,
-        "v5-RWKV": rwkv_tokens
+        "RWKV-v4": rwkv4_tokens,
+        "RWKV-v5/RWKV-v6": rwkv5_tokens
     }
 
     # Sort the results in descending order based on token length
@@ -70,51 +44,21 @@ def tokenize(input_text):
 
 
 if __name__ == "__main__":
-    llama_tokenizer = AutoTokenizer.from_pretrained(
-        "TheBloke/Llama-2-7B-fp16"
-    )
-    llama3_tokenizer = AutoTokenizer.from_pretrained(
-        "unsloth/llama-3-8b"
-    )
-    mistral_tokenizer = AutoTokenizer.from_pretrained(
-        "mistral-community/Mistral-7B-v0.2"
-    )
-    gpt2_tokenizer = AutoTokenizer.from_pretrained(
-        "gpt2"
-    )
-    gpt_neox_tokenizer = AutoTokenizer.from_pretrained(
-        "EleutherAI/gpt-neox-20b"
-    )
-    falcon_tokenizer = AutoTokenizer.from_pretrained(
-        "tiiuae/falcon-7b"
-    )
-    phi2_tokenizer = AutoTokenizer.from_pretrained(
-        "microsoft/phi-2"
-    )
-    phi3_tokenizer = AutoTokenizer.from_pretrained(
-        "microsoft/Phi-3-mini-4k-instruct"
-    )
-    t5_tokenizer = AutoTokenizer.from_pretrained(
-        "google/flan-t5-xxl"
-    )
-    gemma_tokenizer = AutoTokenizer.from_pretrained(
-        "alpindale/gemma-2b"
-    )
-    command_r_tokenizer = AutoTokenizer.from_pretrained(
-        "CohereForAI/c4ai-command-r-plus"
-    )
-    qwen_tokenizer = AutoTokenizer.from_pretrained(
-        "Qwen/Qwen1.5-7B"
-    )
-    codeqwen_tokenizer = AutoTokenizer.from_pretrained(
-        "Qwen/CodeQwen1.5-7B"
-    )
-    rwkv_tokenizer = AutoTokenizer.from_pretrained(
-        "RWKV/v5-EagleX-v2-7B-HF",
-        trust_remote_code=True
-    )
+    llama_tokenizer = AutoTokenizer.from_pretrained("TheBloke/Llama-2-7B-fp16")
+    llama3_tokenizer = AutoTokenizer.from_pretrained("unsloth/llama-3-8b")
+    mistral_tokenizer = AutoTokenizer.from_pretrained("mistral-community/Mistral-7B-v0.2")
+    gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
+    gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
+    falcon_tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b")
+    phi2_tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2")
+    phi3_tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
+    t5_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xxl")
+    gemma_tokenizer = AutoTokenizer.from_pretrained("alpindale/gemma-2b")
+    command_r_tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-plus")
+    qwen_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen1.5-7B")
+    codeqwen_tokenizer = AutoTokenizer.from_pretrained("Qwen/CodeQwen1.5-7B")
+    rwkv4_tokenizer = AutoTokenizer.from_pretrained("RWKV/rwkv-4-14b-pile", trust_remote_code=True)
+    rwkv5_tokenizer = AutoTokenizer.from_pretrained("RWKV/v5-EagleX-v2-7B-HF", trust_remote_code=True)
 
-    iface = gr.Interface(
-        fn=tokenize, inputs=gr.Textbox(label="Input Text", lines=14), outputs="text"
-    )
+    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(label="Input Text", lines=15), outputs="text")
     iface.launch()