add tokenizer

runs/Jun12_10-21-38_bca09897739c/1655029401.5457413/events.out.tfevents.1655029401.bca09897739c.574.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9febae3e9c10b81a0bff2227a886d82f4664804e99efbf33f4b8e010fbfa4d4
+size 5210

runs/Jun12_10-21-38_bca09897739c/events.out.tfevents.1655029401.bca09897739c.574.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3028b48860b6653d7ecd665a5fdadc5e0f49f9a8f3cb428fe6a4810925730419
+size 5707

vocab.json

@@ -1 +1 @@
-{"n": 0, "c": 1, "t": 2, "b": 3, "v": 4, "u": 5, "'": 6, "o": 7, "d": 8, "f": 9, "y": 11, "m": 12, "l": 13, "s": 14, "x": 15, "q": 16, "e": 17, "w": 18, "h": 19, "k": 20, "r": 21, "a": 22, "j": 23, "z": 24, "p": 25, "g": 26, "i": 27, "|": 10, "[UNK]": 28, "[PAD]": 29}
+{"'": 0, "g": 1, "s": 2, "w": 3, "b": 4, "k": 5, "d": 6, "c": 7, "p": 8, "m": 9, "i": 10, "x": 11, "n": 12, "l": 13, "t": 14, "j": 15, "f": 16, "u": 17, "a": 18, "e": 19, "y": 20, "r": 21, "q": 22, "h": 24, "z": 25, "v": 26, "o": 27, "|": 23, "[UNK]": 28, "[PAD]": 29}