add tokenizer

runs/Apr05_21-34-37_TUF/1649219740.2601585/events.out.tfevents.1649219740.TUF.9696.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6247085833b0358695e7eefb4d672e584d9cbebdb14ec0d652793a346f24e27f
+size 4778

runs/Apr05_21-34-37_TUF/events.out.tfevents.1649219740.TUF.9696.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22a4e4db87b63f0f4c7ac10051fd674a56d204055a34ea604c0e24efdea1bdb8
+size 4994

vocab.json

@@ -1 +1 @@
-{"n": 0, "y": 2, "w": 3, "v": 4, "e": 5, "k": 6, "m": 7, "p": 8, "o": 9, "f": 10, "j": 11, "c": 12, "r": 13, "x": 14, "l": 15, "h": 16, "q": 17, "b": 18, "d": 19, "g": 20, "i": 21, "'": 22, "u": 23, "t": 24, "z": 25, "a": 26, "s": 27, "[UNK]": 28, "[PAD]": 29, "|": 1}
+{"v": 0, "a": 1, "k": 2, "e": 3, "b": 4, "g": 5, "y": 6, "o": 7, "i": 8, "h": 9, "'": 10, "f": 11, "z": 12, "p": 13, "d": 14, "t": 15, "c": 16, "l": 17, "r": 18, "j": 19, "m": 20, "x": 22, "s": 23, "w": 24, "q": 25, "n": 26, "u": 27, "[UNK]": 28, "[PAD]": 29, "|": 21}