TeePoat commited on
Commit
25749b4
·
verified ·
1 Parent(s): 3cb4326

Update models/transformer/utils.py

Browse files
Files changed (1) hide show
  1. models/transformer/utils.py +1 -1
models/transformer/utils.py CHANGED
@@ -7,7 +7,7 @@ def modified_tokenizer(model_name="ai-forever/rugpt3small_based_on_gpt2", cache_
7
  if cache_dir:
8
  tokenizer = GPT2Tokenizer.from_pretrained(model_name, cache_dir=str(data_path / cache_dir))
9
  else:
10
- tokenizer = GPT2Tokenizer.from_pretrained(model_name, use_auth_token=HF_SECRET)
11
  special_tokens_dict = {
12
  "additional_special_tokens": [
13
  "<user>",
 
7
  if cache_dir:
8
  tokenizer = GPT2Tokenizer.from_pretrained(model_name, cache_dir=str(data_path / cache_dir))
9
  else:
10
+ tokenizer = GPT2Tokenizer.from_pretrained(model_name)
11
  special_tokens_dict = {
12
  "additional_special_tokens": [
13
  "<user>",