add tokenizer

runs/May19_19-54-19_LAPTOP-5QD1OBBG/1652970328.7191496/events.out.tfevents.1652970328.LAPTOP-5QD1OBBG.22936.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d9138fe09d9f210d5d57d451d432afc95e1342bc4f3c015958b9b40578d01ca
+size 4819

runs/May19_19-54-19_LAPTOP-5QD1OBBG/events.out.tfevents.1652970328.LAPTOP-5QD1OBBG.22936.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:562c63f45fa9171a1e60bab452297b60edd49526f0166c7e69a937c0cdeed0da
+size 5035

vocab.json

@@ -1 +1 @@
-{"l": 0, "o": 1, "b": 2, "j": 3, "h": 4, "c": 5, ".": 6, "p": 7, "u": 8, "v": 10, "i": 11, "e": 12, "a": 13, "m": 14, "r": 15, "q": 16, "n": 17, "f": 18, "k": 19, "w": 20, "t": 21, "d": 22, "g": 23, "z": 24, "'": 25, "y": 26, "s": 27, "x": 28, "|": 9, "[UNK]": 31, "[PAD]": 31}
+{"q": 0, "p": 1, "r": 2, "i": 3, ".": 4, "m": 5, "l": 6, "e": 7, "h": 8, "v": 9, "w": 10, "o": 11, "y": 12, "'": 13, "n": 14, "j": 15, "d": 16, "u": 17, "c": 18, "t": 19, "s": 21, "z": 22, "k": 23, "x": 24, "a": 25, "f": 26, "b": 27, "g": 28, "|": 20, "[UNK]": 29, "[PAD]": 30}