Make tokenizer more robuts

app.py

@@ -36,9 +36,13 @@ def count_string_tokens(string: str, model: str) -> int:
     try:
         encoding = tiktoken.encoding_for_model(model.split('/')[-1])
     except:
-        try:
-            encoding = tiktoken.encoding_for_model(model.split('/')[-2] + '/' + model.split('/')[-1])
-        except KeyError:
+        if len(model.split('/')) > 1:
+            try:
+                    encoding = tiktoken.encoding_for_model(model.split('/')[-2] + '/' + model.split('/')[-1])
+            except KeyError:
+                print(f"Model {model} not found. Using cl100k_base encoding.")
+                encoding = tiktoken.get_encoding("cl100k_base")
+        else:
             print(f"Model {model} not found. Using cl100k_base encoding.")
             encoding = tiktoken.get_encoding("cl100k_base")
     return len(encoding.encode(string))
@@ -179,7 +183,7 @@ with gr.Blocks(css="""
                     max_price = gr.Slider(label="Max Price per Input Token", minimum=0, maximum=0.001, step=0.00001, value=0.001)
                 litellm_provider = gr.Dropdown(label="Inference Provider", choices=["Any"] + TOKEN_COSTS['litellm_provider'].unique().tolist(), value="Any")
         
-            model = gr.Dropdown(label="Models (at least 1)", choices=TOKEN_COSTS['model'].tolist(), value="anyscale/meta-llama/Meta-Llama-3-8B-Instruct", multiselect=True)
+            model = gr.Dropdown(label="Models (at least 1)", choices=TOKEN_COSTS['model'].tolist(), value=["anyscale/meta-llama/Meta-Llama-3-8B-Instruct", "gpt-4o", "claude-3-sonnet-20240229"], multiselect=True)
         
     gr.Markdown("## Resulting Costs 👇")