Update models/transformer/utils.py

models/transformer/utils.py

@@ -1,13 +1,13 @@
 from transformers import GPT2Tokenizer
 from pathlib import Path
-from .constants import CHECKPOINT_PATH
+from .constants import CHECKPOINT_PATH, HF_SECRET
 
 
 def modified_tokenizer(model_name="ai-forever/rugpt3small_based_on_gpt2", cache_dir="model_cache", data_path=Path(CHECKPOINT_PATH)):
     if cache_dir:
-        tokenizer = GPT2Tokenizer.from_pretrained(model_name, cache_dir=str(data_path / cache_dir))
+        tokenizer = GPT2Tokenizer.from_pretrained(model_name, cache_dir=str(data_path / cache_dir), use_auth_token=HF_SECRET)
     else:
-        tokenizer = GPT2Tokenizer.from_pretrained(model_name)
+        tokenizer = GPT2Tokenizer.from_pretrained(model_name, use_auth_token=HF_SECRET)
     special_tokens_dict = {
         "additional_special_tokens": [
             "<user>",