Commit
·
7708da3
1
Parent(s):
636c094
hotfix
Browse files- NeuralTextGenerator.py +1 -1
NeuralTextGenerator.py
CHANGED
@@ -47,7 +47,7 @@ class BertTextGenerator:
|
|
47 |
self.model, optimizer = amp.initialize(self.model, optimizer, opt_level="O2", keep_batchnorm_fp32=True,
|
48 |
loss_scale="dynamic")
|
49 |
|
50 |
-
self.tokenizer = AutoTokenizer.from_pretrained(
|
51 |
use_fast=use_fast,
|
52 |
do_basic_tokenize=do_basic_tokenize) # added to avoid splitting of unused tokens
|
53 |
self.num_attention_masks = len(self.model.base_model.base_model.encoder.layer)
|
|
|
47 |
self.model, optimizer = amp.initialize(self.model, optimizer, opt_level="O2", keep_batchnorm_fp32=True,
|
48 |
loss_scale="dynamic")
|
49 |
|
50 |
+
self.tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased', do_lower_case="uncased" in model_version,
|
51 |
use_fast=use_fast,
|
52 |
do_basic_tokenize=do_basic_tokenize) # added to avoid splitting of unused tokens
|
53 |
self.num_attention_masks = len(self.model.base_model.base_model.encoder.layer)
|