test_dynamic_model_with_tokenizer / tokenization_fast.py
radames's picture
Upload 6 files
3fe0c05
from transformers import BertTokenizerFast
from .tokenization import NewTokenizer
class NewTokenizerFast(BertTokenizerFast):
slow_tokenizer_class = NewTokenizer
special_attribute_present =True