Hugging Face's logo

Spaces:
alakxender
/
dhivehi-tokenizers
Running

App Files Community
alakxender commited on
Commit
11f1c3c
·
1 Parent(s): 3a6f187

f

Browse files
Files changed (1) hide show
  1. app.py +1 -1
app.py CHANGED
@@ -41,7 +41,7 @@ def load_tokenizer(tokenizer_path):
41
  return tokenizer_cache[tokenizer_path]
42
 
43
  try:
44
- tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=True)
45
  tokenizer_cache[tokenizer_path] = tokenizer
46
  return tokenizer
47
  except Exception:
 
41
  return tokenizer_cache[tokenizer_path]
42
 
43
  try:
44
+ tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
45
  tokenizer_cache[tokenizer_path] = tokenizer
46
  return tokenizer
47
  except Exception: