add tokenizer

runs/May18_15-18-07_LAPTOP-5QD1OBBG/1652867351.5852191/events.out.tfevents.1652867351.LAPTOP-5QD1OBBG.13204.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32fe0132e1573d3ceb27de6eac6469f8dc9b614431f92a3ad6f1a983f42a1db5
+size 4819

runs/May18_15-18-07_LAPTOP-5QD1OBBG/events.out.tfevents.1652867351.LAPTOP-5QD1OBBG.13204.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed8fbcfd6d521f514cd912287689c9624130365fbb9940508a3e64ab5d696959
+size 5192

vocab.json

@@ -1 +1 @@
-{"q": 0, "p": 1, "j": 2, "b": 3, "v": 4, "e": 5, "t": 6, "d": 7, "r": 8, "g": 9, "x": 10, "z": 11, "c": 12, "k": 13, "n": 14, "i": 15, "o": 16, "s": 17, "a": 18, "'": 19, "m": 21, "h": 22, "u": 23, "f": 24, "l": 25, "y": 26, "w": 27, "|": 20, "[UNK]": 28, "[PAD]": 29}
+{"j": 0, "y": 1, "o": 2, "k": 3, "b": 4, "w": 5, "g": 6, "v": 7, "l": 8, "c": 9, "i": 10, "d": 11, "r": 12, "m": 13, "x": 14, "t": 16, "u": 17, "e": 18, "s": 19, "a": 20, "h": 21, "n": 22, "q": 23, "z": 24, "'": 25, "f": 26, "p": 27, "|": 15, "[UNK]": 28, "[PAD]": 29}