lucyknada commited on
Commit
72f967c
·
verified ·
1 Parent(s): b1f0cdb

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -15,9 +15,9 @@ def get_distinct_colors(n):
15
  colors.append(color)
16
  return colors
17
 
18
- def tokenize_text(hf_model_id, text, hf_token=None):
19
  try:
20
- tokenizer = AutoTokenizer.from_pretrained(hf_model_id, token=hf_token)
21
  tokens = tokenizer.tokenize(text)
22
  token_count = len(tokens)
23
  colors = get_distinct_colors(token_count)
 
15
  colors.append(color)
16
  return colors
17
 
18
+ def tokenize_text(hf_model_id, text, token=None):
19
  try:
20
+ tokenizer = AutoTokenizer.from_pretrained(hf_model_id, access_token=token)
21
  tokens = tokenizer.tokenize(text)
22
  token_count = len(tokens)
23
  colors = get_distinct_colors(token_count)