File size: 1,036 Bytes
b4a8aea 28c8817 b4a8aea 1fad4a0 b4a8aea e03f966 b4a8aea e03f966 b4a8aea e03f966 7846d6e b4a8aea |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 |
from transformers import AutoTokenizer
def tokenize(input_text):
tokens = tokenizer(input_text)["input_ids"]
return f"Number of tokens: {len(tokens)}"
tokenize_tool = gr.Interface(
fn=tokenize,
inputs=gr.Textbox(lines=7, label="Input Text"),
outputs=gr.Textbox(label="Tokenization Result"),
live=True,
title="GPT-2 Tokenizer",
description="This tool tokenizes input text using the lgaalves/gpt2-dolly model.",
)
tokenize_tool.launch()
import os
from transformers import pipeline
from transformers import Tool
class TokenCounterTool(Tool):
name = "text_generator"
description = "This is a tool for counting token used by a prompt. It takes a prompt as input and returns the generated text."
inputs = ["text"]
outputs = ["text"]
def __call__(self, prompt: str):
token = os.environ['hf']
tokenizer = AutoTokenizer.from_pretrained("lgaalves/gpt2-dolly")
tokens = tokenizer(input_text)["input_ids"]
return f"Number of tokens: {len(tokens)}"
|