Spaces:
Sleeping
Sleeping
prasanna kumar
commited on
Commit
·
b9a925b
1
Parent(s):
3c6d585
add default values for models and shown some examples for easy try
Browse files
app.py
CHANGED
|
@@ -67,16 +67,26 @@ def process_input(input_type, input_value, model_name):
|
|
| 67 |
|
| 68 |
return analysis, " ".join(tokens), str(token_ids), words_hist, special_chars_hist, numbers_hist
|
| 69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
with gr.Blocks() as iface:
|
| 71 |
gr.Markdown("# LLM Tokenization - Convert Text to tokens and vice versa!")
|
| 72 |
gr.Markdown("Enter text or token IDs and select a model to see the results, including word count, token analysis, and histograms of most common elements.")
|
| 73 |
|
| 74 |
with gr.Row():
|
| 75 |
input_type = gr.Radio(["Text", "Token IDs"], label="Input Type", value="Text")
|
| 76 |
-
model_name = gr.Dropdown(choices=MODELS, label="Select Model")
|
| 77 |
|
| 78 |
input_text = gr.Textbox(lines=5, label="Input")
|
| 79 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
submit_button = gr.Button("Process")
|
| 81 |
|
| 82 |
analysis_output = gr.Textbox(label="Analysis", lines=6)
|
|
@@ -88,6 +98,16 @@ with gr.Blocks() as iface:
|
|
| 88 |
special_chars_plot = gr.Plot(label="Most Common Special Characters")
|
| 89 |
numbers_plot = gr.Plot(label="Most Common Numbers")
|
| 90 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
submit_button.click(
|
| 92 |
process_input,
|
| 93 |
inputs=[input_type, input_text, model_name],
|
|
|
|
| 67 |
|
| 68 |
return analysis, " ".join(tokens), str(token_ids), words_hist, special_chars_hist, numbers_hist
|
| 69 |
|
| 70 |
+
def text_example():
|
| 71 |
+
return "Hello, world! This is an example text input for tokenization."
|
| 72 |
+
|
| 73 |
+
def token_ids_example():
|
| 74 |
+
return "[128000, 9906, 11, 1917, 0, 1115, 374, 459, 3187, 1495, 1988, 369, 4037, 2065, 13]"
|
| 75 |
+
|
| 76 |
with gr.Blocks() as iface:
|
| 77 |
gr.Markdown("# LLM Tokenization - Convert Text to tokens and vice versa!")
|
| 78 |
gr.Markdown("Enter text or token IDs and select a model to see the results, including word count, token analysis, and histograms of most common elements.")
|
| 79 |
|
| 80 |
with gr.Row():
|
| 81 |
input_type = gr.Radio(["Text", "Token IDs"], label="Input Type", value="Text")
|
| 82 |
+
model_name = gr.Dropdown(choices=MODELS, label="Select Model",value=MODELS[0])
|
| 83 |
|
| 84 |
input_text = gr.Textbox(lines=5, label="Input")
|
| 85 |
|
| 86 |
+
with gr.Row():
|
| 87 |
+
text_example_button = gr.Button("Load Text Example")
|
| 88 |
+
token_ids_example_button = gr.Button("Load Token IDs Example")
|
| 89 |
+
|
| 90 |
submit_button = gr.Button("Process")
|
| 91 |
|
| 92 |
analysis_output = gr.Textbox(label="Analysis", lines=6)
|
|
|
|
| 98 |
special_chars_plot = gr.Plot(label="Most Common Special Characters")
|
| 99 |
numbers_plot = gr.Plot(label="Most Common Numbers")
|
| 100 |
|
| 101 |
+
text_example_button.click(
|
| 102 |
+
lambda: (text_example(), "Text"),
|
| 103 |
+
outputs=[input_text, input_type]
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
token_ids_example_button.click(
|
| 107 |
+
lambda: (token_ids_example(), "Token IDs"),
|
| 108 |
+
outputs=[input_text, input_type]
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
submit_button.click(
|
| 112 |
process_input,
|
| 113 |
inputs=[input_type, input_text, model_name],
|