add tokenizer

runs/Apr08_12-51-49_TUF/1649447573.362371/events.out.tfevents.1649447573.TUF.3856.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3eabeff090abafa8313671de084ad7bdfe2725c3a4f27f1e8f85dbe566f0453e
+size 4782

runs/Apr08_12-51-49_TUF/events.out.tfevents.1649447573.TUF.3856.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70eb39288f966a768d6d8459fb17aa4a761e27e40fc2a45fdbf0cf6ad6af5431
+size 5013

vocab.json

@@ -1 +1 @@
-{"x": 0, "p": 1, "o": 2, "y": 3, "e": 4, "c": 5, "n": 6, "t": 7, "i": 8, "b": 9, "q": 10, "z": 11, "a": 12, "d": 13, "'": 14, "g": 15, "k": 16, "l": 17, "j": 18, "m": 19, "f": 20, "r": 21, "v": 22, "h": 24, "u": 25, "w": 26, "s": 27, "[UNK]": 28, "[PAD]": 29, "|": 23}
+{"s": 0, "v": 1, "l": 2, "o": 3, "u": 4, "n": 5, "w": 6, "g": 7, "'": 8, "t": 9, "a": 10, "z": 11, "d": 12, "e": 14, "i": 15, "j": 16, "h": 17, "q": 18, "f": 19, "k": 20, "c": 21, "m": 22, "x": 23, "y": 24, "b": 25, "r": 26, "p": 27, "[UNK]": 28, "[PAD]": 29, "|": 13}