Update app.py
Browse files
app.py
CHANGED
@@ -15,9 +15,9 @@ def get_distinct_colors(n):
|
|
15 |
colors.append(color)
|
16 |
return colors
|
17 |
|
18 |
-
def tokenize_text(hf_model_id, text,
|
19 |
try:
|
20 |
-
tokenizer = AutoTokenizer.from_pretrained(hf_model_id, token
|
21 |
tokens = tokenizer.tokenize(text)
|
22 |
token_count = len(tokens)
|
23 |
colors = get_distinct_colors(token_count)
|
|
|
15 |
colors.append(color)
|
16 |
return colors
|
17 |
|
18 |
+
def tokenize_text(hf_model_id, text, token=None):
|
19 |
try:
|
20 |
+
tokenizer = AutoTokenizer.from_pretrained(hf_model_id, access_token=token)
|
21 |
tokens = tokenizer.tokenize(text)
|
22 |
token_count = len(tokens)
|
23 |
colors = get_distinct_colors(token_count)
|