Update app.py
Browse files
app.py
CHANGED
@@ -41,7 +41,7 @@ config = AutoConfig.from_pretrained(model_name)
|
|
41 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
42 |
|
43 |
tokenizer_info = xgr.TokenizerInfo.from_huggingface(
|
44 |
-
tokenizer, vocab_size=
|
45 |
)
|
46 |
|
47 |
grammar_compiler = xgr.GrammarCompiler(tokenizer_info)
|
|
|
41 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
42 |
|
43 |
tokenizer_info = xgr.TokenizerInfo.from_huggingface(
|
44 |
+
tokenizer, vocab_size=len(tokenizer)
|
45 |
)
|
46 |
|
47 |
grammar_compiler = xgr.GrammarCompiler(tokenizer_info)
|