Update app.py

app.py

@@ -6,6 +6,9 @@ def tokenize(input_text):
     llama_tokens = len(
         llama_tokenizer(input_text, add_special_tokens=True)["input_ids"]
     )
+    llama3_tokens = len(
+        llama3_tokenizer(input_text, add_special_tokens=True)["input_ids"]
+    )
     mistral_tokens = len(
         mistral_tokenizer(input_text, add_special_tokens=True)["input_ids"]
     )
@@ -22,6 +25,7 @@ def tokenize(input_text):
 
     results = {
         "LLaMa": llama_tokens,
+        "LLaMa-3": llama_tokens,
         "Mistral": mistral_tokens,
         "GPT-2/GPT-J": gpt2_tokens,
         "GPT-NeoX": gpt_neox_tokens,
@@ -39,6 +43,7 @@ def tokenize(input_text):
 
 if __name__ == "__main__":
     llama_tokenizer = AutoTokenizer.from_pretrained("TheBloke/Llama-2-7B-fp16")
+    llama3_tokenizer = AutoTokenizer.from_pretraines("unsloth/llama-3-8b")
     mistral_tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
     gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
     gpt_neox_tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neox-20b")
@@ -47,5 +52,5 @@ if __name__ == "__main__":
     t5_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-xxl")
     gemma_tokenizer = AutoTokenizer.from_pretrained("alpindale/gemma-2b")
 
-    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=8), outputs="text")
+    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(lines=9), outputs="text")
     iface.launch()