add tokenizer

vocab.json

@@ -1 +1 @@
-{"1": 0, "4": 1, "x": 2, "à": 3, "â": 4, "c": 5, "e": 6, "o": 7, "5": 8, "v": 9, "m": 10, "ç": 11, "n": 12, "w": 13, "h": 14, "ê": 15, "b": 16, "d": 17, "0": 18, "8": 19, "ü": 20, "3": 21, "j": 22, "g": 23, "k": 24, "t": 25, "u": 26, "y": 27, "a": 28, "è": 29, "ö": 30, "7": 31, "f": 32, "r": 33, "9": 34, "ä": 35, "z": 36, "2": 37, "é": 39, "s": 40, "i": 41, "q": 42, "6": 43, "p": 44, "l": 45, "|": 38, "[UNK]": 46, "[PAD]": 47}
+{"m": 0, "g": 1, "1": 2, "è": 3, "2": 4, "n": 5, "b": 6, "d": 7, "p": 8, "ç": 9, "j": 10, "h": 11, "4": 12, "ä": 14, "s": 15, "ß": 16, "ü": 17, "c": 18, "w": 19, "q": 20, "5": 21, "t": 22, "a": 23, "z": 24, "â": 25, "f": 26, "6": 27, "k": 28, "8": 29, "x": 30, "l": 31, "e": 32, "r": 33, "9": 34, "u": 35, "é": 36, "ö": 37, "y": 38, "0": 39, "à": 40, "7": 41, "o": 42, "3": 43, "v": 44, "i": 45, "ê": 46, "|": 13, "[UNK]": 47, "[PAD]": 48}