add tokenizer

vocab.json

@@ -1 +1 @@
-{"ф": 0, "е": 1, "s": 2, "c": 3, "б": 4, "t": 5, "i": 6, "l": 7, "ж": 8, "й": 9, "g": 10, "в": 11, "o": 12, "х": 13, "а": 14, "ч": 15, "з": 16, "ц": 17, "a": 18, "h": 19, "у": 20, "e": 21, "г": 22, "с": 23, "b": 24, "ы": 25, "т": 26, "п": 27, "r": 28, "щ": 29, "я": 30, "—": 31, "н": 32, "ё": 33, "m": 34, "ю": 35, "ь": 36, "р": 37, "ъ": 38, "м": 39, "о": 40, "э": 41, "д": 42, "x": 43, "k": 44, "и": 46, "n": 47, "p": 48, "ш": 49, "»": 50, "'": 51, "л": 52, "f": 53, "к": 54, "|": 45, "[UNK]": 55, "[PAD]": 56}
+{"k": 0, "ь": 1, "ъ": 2, "э": 3, "г": 4, "у": 5, "к": 6, "'": 7, "ч": 8, "r": 9, "s": 10, "я": 11, "е": 12, "б": 13, "ф": 14, "x": 15, "g": 16, "о": 17, "e": 18, "ы": 19, "з": 20, "ж": 21, "b": 22, "в": 23, "х": 24, "ю": 25, "m": 26, "i": 28, "а": 29, "с": 30, "—": 31, "h": 32, "p": 33, "ё": 34, "д": 35, "щ": 36, "o": 37, "п": 38, "n": 39, "р": 40, "f": 41, "c": 42, "t": 43, "л": 44, "a": 45, "и": 46, "l": 47, "т": 48, "»": 49, "ш": 50, "й": 51, "н": 52, "ц": 53, "м": 54, "|": 27, "[UNK]": 55, "[PAD]": 56}