add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<s>": 57, "</s>": 58}

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "tokenizer_class": "Wav2Vec2CTCTokenizer"}
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "special_tokens_map_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -1 +1 @@
-{"k": 0, "ь": 1, "ъ": 2, "э": 3, "г": 4, "у": 5, "к": 6, "'": 7, "ч": 8, "r": 9, "s": 10, "я": 11, "е": 12, "б": 13, "ф": 14, "x": 15, "g": 16, "о": 17, "e": 18, "ы": 19, "з": 20, "ж": 21, "b": 22, "в": 23, "х": 24, "ю": 25, "m": 26, "i": 28, "а": 29, "с": 30, "—": 31, "h": 32, "p": 33, "ё": 34, "д": 35, "щ": 36, "o": 37, "п": 38, "n": 39, "р": 40, "f": 41, "c": 42, "t": 43, "л": 44, "a": 45, "и": 46, "l": 47, "т": 48, "»": 49, "ш": 50, "й": 51, "н": 52, "ц": 53, "м": 54, "|": 27, "[UNK]": 55, "[PAD]": 56}
+{"h": 0, "ю": 1, "ы": 2, "x": 3, "b": 4, "ё": 5, "р": 6, "f": 7, "э": 8, "n": 9, "д": 10, "m": 11, "c": 12, "ш": 13, "ч": 14, "g": 15, "а": 16, "г": 17, "ф": 18, "'": 19, "в": 20, "ж": 21, "p": 22, "k": 23, "a": 25, "м": 26, "у": 27, "о": 28, "r": 29, "ъ": 30, "з": 31, "я": 32, "н": 33, "e": 34, "ц": 35, "щ": 36, "o": 37, "l": 38, "б": 39, "е": 40, "i": 41, "с": 42, "л": 43, "ь": 44, "к": 45, "t": 46, "»": 47, "п": 48, "т": 49, "—": 50, "х": 51, "и": 52, "й": 53, "s": 54, "|": 24, "[UNK]": 55, "[PAD]": 56}