Spaces:
Sleeping
Sleeping
Commit
·
c563cac
1
Parent(s):
ead38a1
Updated load method
Browse files- language_bpe/base.py +1 -1
language_bpe/base.py
CHANGED
|
@@ -119,7 +119,7 @@ class Tokenizer:
|
|
| 119 |
# read the model file
|
| 120 |
merges = {}
|
| 121 |
special_tokens = {}
|
| 122 |
-
idx =
|
| 123 |
with open(model_file, 'r', encoding="utf-8") as f:
|
| 124 |
# read the version
|
| 125 |
version = f.readline().strip()
|
|
|
|
| 119 |
# read the model file
|
| 120 |
merges = {}
|
| 121 |
special_tokens = {}
|
| 122 |
+
idx = 384
|
| 123 |
with open(model_file, 'r', encoding="utf-8") as f:
|
| 124 |
# read the version
|
| 125 |
version = f.readline().strip()
|