Spaces:
Running
Running
Update models/transformer/utils.py
Browse files
models/transformer/utils.py
CHANGED
@@ -7,7 +7,7 @@ def modified_tokenizer(model_name="ai-forever/rugpt3small_based_on_gpt2", cache_
|
|
7 |
if cache_dir:
|
8 |
tokenizer = GPT2Tokenizer.from_pretrained(model_name, cache_dir=str(data_path / cache_dir))
|
9 |
else:
|
10 |
-
tokenizer = GPT2Tokenizer.from_pretrained(model_name
|
11 |
special_tokens_dict = {
|
12 |
"additional_special_tokens": [
|
13 |
"<user>",
|
|
|
7 |
if cache_dir:
|
8 |
tokenizer = GPT2Tokenizer.from_pretrained(model_name, cache_dir=str(data_path / cache_dir))
|
9 |
else:
|
10 |
+
tokenizer = GPT2Tokenizer.from_pretrained(model_name)
|
11 |
special_tokens_dict = {
|
12 |
"additional_special_tokens": [
|
13 |
"<user>",
|