+
+
+Textbox tokenizer
+
+## Installation
+
+```bash
+pip install gradio_tokenizertextbox
+```
+
+## Usage
+
+```python
+#
+# demo/app.py
+#
+import gradio as gr
+from gradio_tokenizertextbox import TokenizerTextBox
+import json
+
+
+
+TOKENIZER_OPTIONS = {
+ "Xenova/clip-vit-large-patch14": "CLIP ViT-L/14",
+ "Xenova/gpt-4": "gpt-4 / gpt-3.5-turbo / text-embedding-ada-002",
+ "Xenova/text-davinci-003": "text-davinci-003 / text-davinci-002",
+ "Xenova/gpt-3": "gpt-3",
+ "Xenova/grok-1-tokenizer": "Grok-1",
+ "Xenova/claude-tokenizer": "Claude",
+ "Xenova/mistral-tokenizer-v3": "Mistral v3",
+ "Xenova/mistral-tokenizer-v1": "Mistral v1",
+ "Xenova/gemma-tokenizer": "Gemma",
+ "Xenova/llama-3-tokenizer": "Llama 3",
+ "Xenova/llama-tokenizer": "LLaMA / Llama 2",
+ "Xenova/c4ai-command-r-v01-tokenizer": "Cohere Command-R",
+ "Xenova/t5-small": "T5",
+ "Xenova/bert-base-cased": "bert-base-cased",
+
+}
+
+# 2. Prepare the choices for the gr.Dropdown component
+# The format is a list of tuples: [(display_name, internal_value)]
+dropdown_choices = [
+ (display_name, model_name)
+ for model_name, display_name in TOKENIZER_OPTIONS.items()
+]
+
+def process_output(tokenization_data):
+ """
+ This function receives the full dictionary from the component.
+ """
+ if not tokenization_data:
+ return {"status": "Waiting for input..."}
+ return tokenization_data
+
+# --- Gradio Application ---
+with gr.Blocks() as demo:
+ gr.Markdown("# TokenizerTextBox Component Demo")
+ gr.Markdown("# Component idea taken from the original example application on [Xenova Tokenizer Playground](https://github.com/huggingface/transformers.js-examples/tree/main/the-tokenizer-playground) ")
+ gr.Markdown("## Select a tokenizer from the dropdown menu to see how it processes your text in real-time.")
+ gr.Markdown("## For more models, check out the [Xenova Transformers Models](https://huggingface.co/Xenova/models) page.")
+
+ with gr.Row():
+ # 3. Create the Dropdown for model selection
+ model_selector = gr.Dropdown(
+ label="Select a Tokenizer",
+ choices=dropdown_choices,
+ value="Xenova/clip-vit-large-patch14", # Set a default value
+ )
+
+ display_mode_radio = gr.Radio(
+ ["text", "token_ids", "hidden"],
+ label="Display Mode",
+ value="text"
+ )
+
+ # 4. Initialize the component with a default model
+ tokenizer_input = TokenizerTextBox(
+ label="Type your text here",
+ value="Gradio is an awesome tool for building ML demos!",
+ model="Xenova/clip-vit-large-patch14", # Must match the dropdown's default value
+ display_mode="text",
+ )
+
+ output_info = gr.JSON(label="Component Output (from preprocess)")
+
+ # --- Event Listeners ---
+
+ # A. When the tokenizer component changes, update the JSON output
+ tokenizer_input.change(
+ fn=process_output,
+ inputs=tokenizer_input,
+ outputs=output_info
+ )
+
+ # B. When the dropdown value changes, update the 'model' prop of our component
+ def update_tokenizer_model(selected_model):
+ return gr.update(model=selected_model)
+
+ model_selector.change(
+ fn=update_tokenizer_model,
+ inputs=model_selector,
+ outputs=tokenizer_input
+ )
+
+ # C. When the radio button value changes, update the 'display_mode' prop
+ def update_display_mode(mode):
+ return gr.update(display_mode=mode)
+
+ display_mode_radio.change(
+ fn=update_display_mode,
+ inputs=display_mode_radio,
+ outputs=tokenizer_input
+ )
+
+if __name__ == '__main__':
+ demo.launch()
+
+```
+
+## `TokenizerTextBox`
+
+### Initialization
+
+| name | +type | +default | +description | +
|---|---|---|---|
value |
++ +```python +typing.Union[str, dict, typing.Callable, NoneType][ + str, dict, Callable, None +] +``` + + | +None |
+The initial value. Can be a string to initialize the text, or a dictionary for full state. If a function is provided, it will be called when the app loads to set the initial value. | +
model |
++ +```python +str +``` + + | +"Xenova/gpt-3" |
+The name of a Hugging Face tokenizer to use (must be compatible with Transformers.js). Defaults to "Xenova/gpt-2". | +
display_mode |
++ +```python +"text" | "token_ids" | "hidden" +``` + + | +"text" |
+Controls the content of the token visualization panel. Can be 'text' (default), 'token_ids', or 'hidden'. | +
lines |
++ +```python +int +``` + + | +2 |
+The minimum number of line rows for the textarea. | +
max_lines |
++ +```python +int | None +``` + + | +None |
+The maximum number of line rows for the textarea. | +
placeholder |
++ +```python +str | None +``` + + | +None |
+A placeholder hint to display in the textarea when it is empty. | +
autofocus |
++ +```python +bool +``` + + | +False |
+If True, will focus on the textbox when the page loads. | +
autoscroll |
++ +```python +bool +``` + + | +True |
+If True, will automatically scroll to the bottom of the textbox when the value changes. | +
text_align |
++ +```python +typing.Optional[typing.Literal["left", "right"]][ + "left" | "right", None +] +``` + + | +None |
+How to align the text in the textbox, can be: "left" or "right". | +
rtl |
++ +```python +bool +``` + + | +False |
+If True, sets the direction of the text to right-to-left. | +
show_copy_button |
++ +```python +bool +``` + + | +False |
+If True, a copy button will be shown. | +
max_length |
++ +```python +int | None +``` + + | +None |
+The maximum number of characters allowed in the textbox. | +
label |
++ +```python +str | None +``` + + | +None |
+The label for this component, displayed above the component. | +
info |
++ +```python +str | None +``` + + | +None |
+Additional component description, displayed below the label. | +
every |
++ +```python +float | None +``` + + | +None |
+If `value` is a callable, this sets a timer to run the function repeatedly. | +
show_label |
++ +```python +bool +``` + + | +True |
+If False, the label is not displayed. | +
container |
++ +```python +bool +``` + + | +True |
+If False, the component will not be wrapped in a container. | +
scale |
++ +```python +int | None +``` + + | +None |
+The relative size of the component compared to others in a `gr.Row` or `gr.Column`. | +
min_width |
++ +```python +int +``` + + | +160 |
+The minimum-width of the component in pixels. | +
interactive |
++ +```python +bool | None +``` + + | +None |
+If False, the user will not be able to edit the text. | +
visible |
++ +```python +bool +``` + + | +True |
+If False, the component will be hidden. | +
elem_id |
++ +```python +str | None +``` + + | +None |
+An optional string that is assigned as the id of this component in the HTML DOM. | +
elem_classes |
++ +```python +list[str] | str | None +``` + + | +None |
+An optional list of strings that are assigned as the classes of this component in the HTML DOM. | +
+
+
+Textbox tokenizer
+
+## Installation
+
+```bash
+pip install gradio_tokenizertextbox
+```
+
+## Usage
+
+```python
+#
+# demo/app.py
+#
+import gradio as gr
+from gradio_tokenizertextbox import TokenizerTextBox
+import json
+
+
+
+TOKENIZER_OPTIONS = {
+ "Xenova/clip-vit-large-patch14": "CLIP ViT-L/14",
+ "Xenova/gpt-4": "gpt-4 / gpt-3.5-turbo / text-embedding-ada-002",
+ "Xenova/text-davinci-003": "text-davinci-003 / text-davinci-002",
+ "Xenova/gpt-3": "gpt-3",
+ "Xenova/grok-1-tokenizer": "Grok-1",
+ "Xenova/claude-tokenizer": "Claude",
+ "Xenova/mistral-tokenizer-v3": "Mistral v3",
+ "Xenova/mistral-tokenizer-v1": "Mistral v1",
+ "Xenova/gemma-tokenizer": "Gemma",
+ "Xenova/llama-3-tokenizer": "Llama 3",
+ "Xenova/llama-tokenizer": "LLaMA / Llama 2",
+ "Xenova/c4ai-command-r-v01-tokenizer": "Cohere Command-R",
+ "Xenova/t5-small": "T5",
+ "Xenova/bert-base-cased": "bert-base-cased",
+
+}
+
+# 2. Prepare the choices for the gr.Dropdown component
+# The format is a list of tuples: [(display_name, internal_value)]
+dropdown_choices = [
+ (display_name, model_name)
+ for model_name, display_name in TOKENIZER_OPTIONS.items()
+]
+
+def process_output(tokenization_data):
+ """
+ This function receives the full dictionary from the component.
+ """
+ if not tokenization_data:
+ return {"status": "Waiting for input..."}
+ return tokenization_data
+
+# --- Gradio Application ---
+with gr.Blocks() as demo:
+ gr.Markdown("# TokenizerTextBox Component Demo")
+ gr.Markdown("# Component idea taken from the original example application on [Xenova Tokenizer Playground](https://github.com/huggingface/transformers.js-examples/tree/main/the-tokenizer-playground) ")
+ gr.Markdown("## Select a tokenizer from the dropdown menu to see how it processes your text in real-time.")
+ gr.Markdown("## For more models, check out the [Xenova Transformers Models](https://huggingface.co/Xenova/models) page.")
+
+ with gr.Row():
+ # 3. Create the Dropdown for model selection
+ model_selector = gr.Dropdown(
+ label="Select a Tokenizer",
+ choices=dropdown_choices,
+ value="Xenova/clip-vit-large-patch14", # Set a default value
+ )
+
+ display_mode_radio = gr.Radio(
+ ["text", "token_ids", "hidden"],
+ label="Display Mode",
+ value="text"
+ )
+
+ # 4. Initialize the component with a default model
+ tokenizer_input = TokenizerTextBox(
+ label="Type your text here",
+ value="Gradio is an awesome tool for building ML demos!",
+ model="Xenova/clip-vit-large-patch14", # Must match the dropdown's default value
+ display_mode="text",
+ )
+
+ output_info = gr.JSON(label="Component Output (from preprocess)")
+
+ # --- Event Listeners ---
+
+ # A. When the tokenizer component changes, update the JSON output
+ tokenizer_input.change(
+ fn=process_output,
+ inputs=tokenizer_input,
+ outputs=output_info
+ )
+
+ # B. When the dropdown value changes, update the 'model' prop of our component
+ def update_tokenizer_model(selected_model):
+ return gr.update(model=selected_model)
+
+ model_selector.change(
+ fn=update_tokenizer_model,
+ inputs=model_selector,
+ outputs=tokenizer_input
+ )
+
+ # C. When the radio button value changes, update the 'display_mode' prop
+ def update_display_mode(mode):
+ return gr.update(display_mode=mode)
+
+ display_mode_radio.change(
+ fn=update_display_mode,
+ inputs=display_mode_radio,
+ outputs=tokenizer_input
+ )
+
+if __name__ == '__main__':
+ demo.launch()
+
+```
+
+## `TokenizerTextBox`
+
+### Initialization
+
+| name | +type | +default | +description | +
|---|---|---|---|
value |
++ +```python +typing.Union[str, dict, typing.Callable, NoneType][ + str, dict, Callable, None +] +``` + + | +None |
+The initial value. Can be a string to initialize the text, or a dictionary for full state. If a function is provided, it will be called when the app loads to set the initial value. | +
model |
++ +```python +str +``` + + | +"Xenova/gpt-3" |
+The name of a Hugging Face tokenizer to use (must be compatible with Transformers.js). Defaults to "Xenova/gpt-2". | +
display_mode |
++ +```python +"text" | "token_ids" | "hidden" +``` + + | +"text" |
+Controls the content of the token visualization panel. Can be 'text' (default), 'token_ids', or 'hidden'. | +
lines |
++ +```python +int +``` + + | +2 |
+The minimum number of line rows for the textarea. | +
max_lines |
++ +```python +int | None +``` + + | +None |
+The maximum number of line rows for the textarea. | +
placeholder |
++ +```python +str | None +``` + + | +None |
+A placeholder hint to display in the textarea when it is empty. | +
autofocus |
++ +```python +bool +``` + + | +False |
+If True, will focus on the textbox when the page loads. | +
autoscroll |
++ +```python +bool +``` + + | +True |
+If True, will automatically scroll to the bottom of the textbox when the value changes. | +
text_align |
++ +```python +typing.Optional[typing.Literal["left", "right"]][ + "left" | "right", None +] +``` + + | +None |
+How to align the text in the textbox, can be: "left" or "right". | +
rtl |
++ +```python +bool +``` + + | +False |
+If True, sets the direction of the text to right-to-left. | +
show_copy_button |
++ +```python +bool +``` + + | +False |
+If True, a copy button will be shown. | +
max_length |
++ +```python +int | None +``` + + | +None |
+The maximum number of characters allowed in the textbox. | +
label |
++ +```python +str | None +``` + + | +None |
+The label for this component, displayed above the component. | +
info |
++ +```python +str | None +``` + + | +None |
+Additional component description, displayed below the label. | +
every |
++ +```python +float | None +``` + + | +None |
+If `value` is a callable, this sets a timer to run the function repeatedly. | +
show_label |
++ +```python +bool +``` + + | +True |
+If False, the label is not displayed. | +
container |
++ +```python +bool +``` + + | +True |
+If False, the component will not be wrapped in a container. | +
scale |
++ +```python +int | None +``` + + | +None |
+The relative size of the component compared to others in a `gr.Row` or `gr.Column`. | +
min_width |
++ +```python +int +``` + + | +160 |
+The minimum-width of the component in pixels. | +
interactive |
++ +```python +bool | None +``` + + | +None |
+If False, the user will not be able to edit the text. | +
visible |
++ +```python +bool +``` + + | +True |
+If False, the component will be hidden. | +
elem_id |
++ +```python +str | None +``` + + | +None |
+An optional string that is assigned as the id of this component in the HTML DOM. | +
elem_classes |
++ +```python +list[str] | str | None +``` + + | +None |
+An optional list of strings that are assigned as the classes of this component in the HTML DOM. | +
' + (r ? e : escape$1$1(e, !0)) + `" + (r ? e : escape$1$1(e, !0)) + `+${e}+`; + } + html(e, t) { + return e; + } + heading(e, t, r) { + return `
${e}
+`; + } + table(e, t) { + return t && (t = `${t}`), `${e}`;
+ }
+ br() {
+ return "An error occurred:
" + escape$1$1(r.message + "", !0) + ""; + return t ? Promise.resolve(s) : s; + } + if (t) + return Promise.reject(r); + throw r; + }; +}; +const markedInstance = new Marked(); +function marked(o, e) { + return markedInstance.parse(o, e); +} +marked.options = marked.setOptions = function(o) { + return markedInstance.setOptions(o), marked.defaults = markedInstance.defaults, changeDefaults(marked.defaults), marked; +}; +marked.getDefaults = _getDefaults; +marked.defaults = _defaults; +marked.use = function(...o) { + return markedInstance.use(...o), marked.defaults = markedInstance.defaults, changeDefaults(marked.defaults), marked; +}; +marked.walkTokens = function(o, e) { + return markedInstance.walkTokens(o, e); +}; +marked.parseInline = markedInstance.parseInline; +marked.Parser = _Parser; +marked.parser = _Parser.parse; +marked.Renderer = _Renderer; +marked.TextRenderer = _TextRenderer; +marked.Lexer = _Lexer; +marked.lexer = _Lexer.lex; +marked.Tokenizer = _Tokenizer; +marked.Hooks = _Hooks; +marked.parse = marked; +marked.options; +marked.setOptions; +marked.use; +marked.walkTokens; +marked.parseInline; +_Parser.parse; +_Lexer.lex; +function markedHighlight(o) { + if (typeof o == "function" && (o = { + highlight: o + }), !o || typeof o.highlight != "function") + throw new Error("Must provide highlight function"); + return typeof o.langPrefix != "string" && (o.langPrefix = "language-"), typeof o.emptyLangClass != "string" && (o.emptyLangClass = ""), { + async: !!o.async, + walkTokens(e) { + if (e.type !== "code") + return; + const t = getLang(e.lang); + if (o.async) + return Promise.resolve(o.highlight(e.text, t, e.lang || "")).then(updateToken(e)); + const r = o.highlight(e.text, t, e.lang || ""); + if (r instanceof Promise) + throw new Error("markedHighlight is not set to async but the highlight function is async. Set the async option to true on markedHighlight to await the async highlight function."); + updateToken(e)(r); + }, + useNewRenderer: !0, + renderer: { + code(e, t, r) { + typeof e == "object" && (r = e.escaped, t = e.lang, e = e.text); + const s = getLang(t), c = s ? o.langPrefix + escape$1(s) : o.emptyLangClass, l = c ? ` class="${c}"` : ""; + return e = e.replace(/\n$/, ""), `