add tokenizer

runs/May19_22-00-22_LAPTOP-5QD1OBBG/1652977893.9118137/events.out.tfevents.1652977893.LAPTOP-5QD1OBBG.21520.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc5a039424c8d5071882f0d386585640495de7e6abfab8bafb255d36aa270949
+size 4819

runs/May19_22-00-22_LAPTOP-5QD1OBBG/events.out.tfevents.1652977893.LAPTOP-5QD1OBBG.21520.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7372d2da52dcbc42d6679b0a0a191d25f91a39af33c7b85ba92e338574b68a18
+size 5035

vocab.json

@@ -1 +1 @@
-{"j": 0, ".": 1, "b": 2, "n": 3, "a": 4, "v": 5, "z": 6, "s": 7, "y": 8, "d": 9, "'": 10, "x": 11, "u": 12, "r": 13, "p": 14, "o": 15, "g": 16, "l": 17, "k": 18, "w": 20, "h": 21, "i": 22, "c": 23, "m": 24, "f": 25, "q": 26, "e": 27, "t": 28, "|": 19, "[UNK]": 29, "[PAD]": 30}
+{"f": 0, "k": 1, "j": 2, "d": 3, ".": 4, "b": 5, "c": 6, "r": 7, "p": 8, "h": 10, "i": 11, "v": 12, "n": 13, "a": 14, "q": 15, "e": 16, "t": 17, "w": 18, "o": 19, "x": 20, "z": 21, "s": 22, "u": 23, "m": 24, "l": 25, "y": 26, "g": 27, "'": 28, "|": 9, "[UNK]": 29, "[PAD]": 30}