add tokenizer

runs/Apr05_21-15-14_TUF/1649218579.1139233/events.out.tfevents.1649218579.TUF.25280.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:decfefb48731799cbaa40f49edfd5262ad1ff0e062ec3e7ec488827997b6061a
+size 4779

runs/Apr05_21-15-14_TUF/events.out.tfevents.1649218579.TUF.25280.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21d807e24dd773e5a0f7d66e3c1fcc66381f5176a4853647798680ba297a276a
+size 4995

vocab.json

@@ -1 +1 @@
-{"'": 0, "g": 1, "s": 2, "k": 3, "p": 5, "b": 6, "t": 7, "j": 8, "c": 9, "a": 10, "e": 11, "y": 12, "o": 13, "d": 14, "v": 15, "z": 16, "f": 17, "x": 18, "h": 19, "r": 20, "i": 21, "w": 22, "l": 23, "q": 24, "u": 25, "m": 26, "n": 27, "[UNK]": 28, "[PAD]": 29, "|": 4}
+{"h": 0, "d": 1, "g": 2, "f": 3, "b": 4, "v": 5, "m": 6, "j": 7, "z": 8, "r": 9, "t": 10, "u": 11, "p": 12, "l": 13, "s": 14, "a": 15, "w": 16, "x": 17, "c": 18, "o": 19, "'": 20, "n": 21, "k": 22, "y": 23, "e": 25, "q": 26, "i": 27, "[UNK]": 28, "[PAD]": 29, "|": 24}