add tokenizer

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "do_lower_case": false, "do_normalize": true, "return_attention_mask": false, "word_delimiter_token": "|", "special_tokens_map_file": "/home/souradeep_saha/.cache/huggingface/transformers/208086b2429fa2ba5b196810c1bcd7d61e2c8d4afd65d05d0670096d735fd5bb.9d6cd81ef646692fb1c169a880161ea1cb95f49694f220aced9b704b457e51dd", "tokenizer_file": null, "name_or_path": "facebook/wav2vec2-base-960h", "tokenizer_class": "Wav2Vec2Tokenizer"}
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -1 +1 @@
-{"<pad>": 0, "<s>": 1, "</s>": 2, "<unk>": 3, "|": 4, "E": 5, "T": 6, "A": 7, "O": 8, "N": 9, "I": 10, "H": 11, "S": 12, "R": 13, "D": 14, "L": 15, "U": 16, "M": 17, "W": 18, "C": 19, "F": 20, "G": 21, "Y": 22, "P": 23, "B": 24, "V": 25, "K": 26, "'": 27, "X": 28, "J": 29, "Q": 30, "Z": 31}
+{"m": 0, "a": 1, "f": 2, "s": 3, "k": 4, "r": 5, "v": 6, "b": 7, "'": 9, "e": 10, "o": 11, "n": 12, "q": 13, "j": 14, "y": 15, "d": 16, "w": 17, "x": 18, "t": 19, "u": 20, "l": 21, "g": 22, "h": 23, "c": 24, "i": 25, "z": 26, "p": 27, "|": 8, "[UNK]": 28, "[PAD]": 29}