Trained tokenizer using syssec-utd/segmentation-py310-pylingual-v1

tokenizer.json

@@ -256,67 +256,67 @@
       "##a": 99,
       "##s": 100,
       "##k": 101,
-      "##9": 102,
-      "##4": 103,
-      "##5": 104,
-      "##>": 105,
-      "##8": 106,
-      "##2": 107,
-      "##3": 108,
-      "##7": 109,
-      "##1": 110,
-      "##r": 111,
-      "##e": 112,
-      "##t": 113,
-      "##u": 114,
-      "##n": 115,
-      "##6": 116,
-      "##)": 117,
-      "##o": 118,
-      "##M": 119,
-      "##K": 120,
-      "##E": 121,
-      "##F": 122,
+      "##1": 102,
+      "##7": 103,
+      "##0": 104,
+      "##3": 105,
+      "##>": 106,
+      "##4": 107,
+      "##r": 108,
+      "##e": 109,
+      "##t": 110,
+      "##u": 111,
+      "##n": 112,
+      "##6": 113,
+      "##)": 114,
+      "##^": 115,
+      "##~": 116,
+      "##2": 117,
+      "##8": 118,
+      "##F": 119,
+      "##M": 120,
+      "##K": 121,
+      "##E": 122,
       "##U": 123,
       "##I": 124,
-      "##d": 125,
-      "##f": 126,
-      "##l": 127,
-      "##g": 128,
-      "##:": 129,
-      "##[": 130,
-      "##0": 131,
-      "##]": 132,
-      "##i": 133,
-      "##-": 134,
-      "##R": 135,
-      "##W": 136,
-      "##G": 137,
-      "##P": 138,
-      "##v": 139,
-      "##*": 140,
-      "##^": 141,
-      "##~": 142,
-      "##B": 143,
-      "##Y": 144,
-      "##c": 145,
-      "##w": 146,
-      "##y": 147,
-      "##J": 148,
-      "##H": 149,
-      "##b": 150,
-      "##j": 151,
-      "##=": 152,
+      "##g": 125,
+      "##:": 126,
+      "##[": 127,
+      "##5": 128,
+      "##9": 129,
+      "##]": 130,
+      "##d": 131,
+      "##f": 132,
+      "##l": 133,
+      "##o": 134,
+      "##i": 135,
+      "##R": 136,
+      "##v": 137,
+      "##*": 138,
+      "##W": 139,
+      "##G": 140,
+      "##P": 141,
+      "##c": 142,
+      "##b": 143,
+      "##j": 144,
+      "##B": 145,
+      "##-": 146,
+      "##w": 147,
+      "##y": 148,
+      "##Y": 149,
+      "##H": 150,
+      "##p": 151,
+      "##J": 152,
       "##{": 153,
       "##}": 154,
-      "##p": 155,
-      "##X": 156,
-      "##x": 157,
-      "##V": 158,
-      "##Q": 159,
-      "##q": 160,
-      "##!": 161,
-      "##h": 162,
+      "##X": 155,
+      "##V": 156,
+      "##q": 157,
+      "##Q": 158,
+      "##=": 159,
+      "##x": 160,
+      "##h": 161,
+      "##!": 162,
       "##, ": 163,
       "## , ": 164,
       "##as": 165,
@@ -1113,7 +1113,7 @@
       " STORE_NAME , <mask_9": 956,
       "##8 LOAD_FAST , '<mask_": 957,
       " LOAD_METHOD , <mask_2": 958,
-      "##oc": 959,
+      "##do": 959,
       "##doc": 960,
       " LOAD_CONST , '<mask_26": 961,
       "##doc__ ": 962,
@@ -6635,8 +6635,8 @@
       " LOAD_METHOD , <mask_282> ": 6478,
       " JUMP_ABSOLUTE to 10 ^~> ": 6479,
       " LOAD_FAST , '<mask_545>' ": 6480,
-      "##97>]) ": 6481,
-      "##4>], kwonly: [<mask_": 6482,
+      "##4>], kwonly: [<mask_": 6481,
+      "##97>]) ": 6482,
       " LOAD_ATTR , <mask_378> ": 6483,
       " LOAD_NAME , <mask_187> ": 6484,
       " LOAD_CONST , '<mask_483>' ": 6485,
@@ -7839,8 +7839,8 @@
       " LOAD_CONST , <mask_97> ": 7682,
       " LOAD_METHOD , <mask_367> ": 7683,
       "##192> ": 7684,
-      "##9>-DOT ": 7685,
-      "##64>') ": 7686,
+      "##64>') ": 7685,
+      "##9>-DOT ": 7686,
       " LOAD_CONST , ('<mask_18>', '<mask_19": 7687,
       " STORE_NAME , <mask_231> ": 7688,
       " STORE_ATTR , <mask_49": 7689,
@@ -8311,13 +8311,13 @@
       " LOAD_CONST , '<mask_596>' ": 8154,
       "##02 v~> ": 8155,
       "##SYNC": 8156,
-      "##EF": 8157,
-      "##ORE_ASYNC": 8158,
+      "##FORE_": 8157,
+      "##EFORE_": 8158,
       "##_WITH ": 8159,
       " SETUP_ASYNC": 8160,
-      "##EFORE_ASYNC": 8161,
-      " SETUP_ASYNC_WITH to ": 8162,
-      "##EFORE_ASYNC_WITH ": 8163,
+      "##ASYNC_WITH ": 8161,
+      "##EFORE_ASYNC_WITH ": 8162,
+      " SETUP_ASYNC_WITH to ": 8163,
       " LOAD_NAME , <mask_279> ": 8164,
       " IMPORT_FROM , <mask_50> ": 8165,
       " LOAD_NAME , <mask_302> ": 8166,
@@ -8356,8 +8356,8 @@
       " LOAD_CONST , <codeobj:<mask_187>> ": 8199,
       " LOAD_CONST , '<mask_598>' ": 8200,
       " LOAD_CONST , <mask_134> ": 8201,
-      "##81>> ": 8202,
-      "##71>', '<mask_7": 8203,
+      "##71>', '<mask_7": 8202,
+      "##81>> ": 8203,
       " STORE_NAME , <mask_255> ": 8204,
       "##7>], kwonly: [<mask_": 8205,
       "##24>',) ": 8206,
@@ -13850,8 +13850,8 @@
       "##346> ": 13693,
       " LOAD_CONST , '<mask_1014>' ": 13694,
       " LOAD_CONST , <mask_380> ": 13695,
-      "##50 ^~> ": 13696,
-      "##727": 13697,
+      "##727": 13696,
+      "##50 ^~> ": 13697,
       " LOAD_METHOD , <mask_532> ": 13698,
       "LOAD_GLOBAL , <mask_152> ": 13699,
       " STORE_FAST , '<mask_520>' ": 13700,
@@ -13959,8 +13959,8 @@
       " LOAD_CONST , <mask_385> ": 13802,
       " LOAD_CONST , <mask_439> ": 13803,
       " LOAD_GLOBAL , <mask_393> ": 13804,
-      "##63>, <mask_": 13805,
-      "##06>]) ": 13806,
+      "##06>]) ": 13805,
+      "##63>, <mask_": 13806,
       " MAKE_FUNCTION , 4 (annotations, args: [<mask_55": 13807,
       " LOAD_FAST , '<mask_930>' ": 13808,
       " LOAD_CONST , '<mask_1017>' ": 13809,
@@ -16051,8 +16051,8 @@
       " ^~> 514 RERAISE , 0 ": 15894,
       " MAKE_FUNCTION , 0 (args: [<mask_76>, <mask_": 15895,
       "##>, <mask_14>) ": 15896,
-      "##8 LOAD_NAME , <mask_": 15897,
-      "##00>]) ": 15898,
+      "##00>]) ": 15897,
+      "##8 LOAD_NAME , <mask_": 15898,
       "##20>', '<mask_5": 15899,
       "##352> ": 15900,
       " ^~> 376 LOAD_FAST , '<mask_": 15901,
@@ -16367,8 +16367,8 @@
       "##>, <mask_53>]) ": 16210,
       "##>, <mask_27>, <mask_28>, <mask_29": 16211,
       " LOAD_CONST , '<mask_1338>' ": 16212,
-      "##89>',) ": 16213,
-      "##1>'), ('<mask_": 16214,
+      "##1>'), ('<mask_": 16213,
+      "##89>',) ": 16214,
       " IMPORT_FROM , <mask_94> ": 16215,
       "##40>', '<mask_44": 16216,
       "##57>', '<mask_6": 16217,
@@ -18248,8 +18248,8 @@
       " LOAD_CONST , <mask_587> ": 18091,
       " LOAD_FAST , '<mask_1259>' ": 18092,
       "##>', '<mask_28>', '<mask_29>', '<mask_30>') ": 18093,
-      "##51>',) ": 18094,
-      "##2 LOAD_BUILD_CLASS ": 18095,
+      "##2 LOAD_BUILD_CLASS ": 18094,
+      "##51>',) ": 18095,
       " MAKE_FUNCTION , 0 (args: [<mask_111>, <mask_": 18096,
       " LOAD_METHOD , <mask_76": 18097,
       " LOAD_METHOD , <mask_787> ": 18098,
@@ -19185,9 +19185,9 @@
       " STORE_NAME , <mask_649> ": 19028,
       "##4>, <mask_58": 19029,
       " LOAD_GLOBAL , <mask_463> ": 19030,
-      "##90 ^~> ": 19031,
-      "##416> ": 19032,
-      "##54>',) ": 19033,
+      "##416> ": 19031,
+      "##54>',) ": 19032,
+      "##90 ^~> ": 19033,
       "##20>', '<mask_7": 19034,
       " LOAD_ATTR , <mask_820> ": 19035,
       " LOAD_CONST , '<mask_1770>' ": 19036,
@@ -20650,8 +20650,8 @@
       "##219>') ": 20493,
       " STORE_ATTR , <mask_264> ": 20494,
       "##>, <mask_13>, <mask_14>, <mask_15>]) ": 20495,
-      "##510": 20496,
-      "##8>,) ": 20497,
+      "##8>,) ": 20496,
+      "##510": 20497,
       "##486> ": 20498,
       " LOAD_CONST , ('<mask_296>',) ": 20499,
       "##48>', '<mask_5": 20500,
@@ -20859,8 +20859,8 @@
       " LOAD_CONST , <mask_684> ": 20702,
       " ^~> 296 LOAD_GLOBAL , <mask_": 20703,
       " POP_JUMP_IF_FALSE to 554 v~> ": 20704,
-      "##548>' ": 20705,
-      "##216 ^~> ": 20706,
+      "##216 ^~> ": 20705,
+      "##548>' ": 20706,
       "##406 v~> ": 20707,
       "##390 v~> ": 20708,
       " LOAD_ATTR , <mask_983> ": 20709,
@@ -21189,8 +21189,8 @@
       " LOAD_FAST , '<mask_1287>' ": 21032,
       "##>, <mask_50>, <mask_51>, <mask_": 21033,
       " POP_JUMP_IF_TRUE to 298 v~> ": 21034,
-      "##523>' ": 21035,
-      "##63>',) ": 21036,
+      "##63>',) ": 21035,
+      "##523>' ": 21036,
       " LOAD_CONST , '<mask_1673>' ": 21037,
       " LOAD_CONST , '<mask_1709>' ": 21038,
       " LOAD_CONST , '<mask_1815>' ": 21039,
@@ -21875,8 +21875,8 @@
       " LOAD_GLOBAL , <mask_587> ": 21718,
       " LOAD_FAST , '<mask_2447>' ": 21719,
       "##3>', '<mask_274>') ": 21720,
-      "##520>' ": 21721,
-      "##7>'), ('<mask_": 21722,
+      "##7>'), ('<mask_": 21721,
+      "##520>' ": 21722,
       "##22>, <mask_5": 21723,
       "##34>', '<mask_7": 21724,
       " LOAD_FAST , '<mask_1739>' ": 21725,
@@ -21969,8 +21969,8 @@
       " LOAD_CONST , ('<mask_60>', '<mask_61>') ": 21812,
       " LOAD_CONST , ('<mask_49>', '<mask_50>') ": 21813,
       " MAKE_FUNCTION , 0 (args: [<mask_7>, <mask_8>, <mask_9>]) ": 21814,
-      "##9>'), ('<mask_": 21815,
-      "##511> ": 21816,
+      "##511> ": 21815,
+      "##9>'), ('<mask_": 21816,
       " LOAD_FAST , '<mask_1526>' ": 21817,
       " LOAD_GLOBAL , <mask_513> ": 21818,
       "##7>', '<mask_80": 21819,
@@ -22733,8 +22733,8 @@
       "LOAD_GLOBAL , <mask_397> ": 22576,
       "##9>', '<mask_260>') ": 22577,
       "##DICT_UPDATE , 1 ": 22578,
-      "##513>' ": 22579,
-      "##8 LOAD_CONST , True ": 22580,
+      "##8 LOAD_CONST , True ": 22579,
+      "##513>' ": 22580,
       " LOAD_CONST , '<mask_1931>' ": 22581,
       "##4>, <mask_115": 22582,
       " MAKE_FUNCTION , 1 (defaults, args: [<mask_60>, <mask_": 22583,
@@ -23167,8 +23167,8 @@
       "LOAD_NAME , <mask_130> ": 23010,
       " LOAD_FAST , '<mask_6914>' ": 23011,
       "##5>', '<mask_356>', '<mask_35": 23012,
-      "##531>' ": 23013,
-      "##2>), (<mask_": 23014,
+      "##2>), (<mask_": 23013,
+      "##531>' ": 23014,
       " LOAD_CONST , '<mask_2208>' ": 23015,
       " MAKE_FUNCTION , 4 (annotations, args: [<mask_4>]) ": 23016,
       " STORE_FAST , '<mask_756>' ": 23017,
@@ -23527,8 +23527,8 @@
       "LOAD_GLOBAL , <mask_265> ": 23370,
       " ^~> 68 LOAD_DEREF , <mask_": 23371,
       " LOAD_CONST , '<mask_3591>' ": 23372,
-      "##538>' ": 23373,
-      "##78>',) ": 23374,
+      "##78>',) ": 23373,
+      "##538>' ": 23374,
       " POP_JUMP_IF_FALSE to 116": 23375,
       " LOAD_NAME , <mask_705> ": 23376,
       " LOAD_METHOD , <mask_841> ": 23377,
@@ -24944,8 +24944,8 @@
       "LOAD_FAST , '<mask_279>' ": 24787,
       "##3>, <mask_124>, <mask_12": 24788,
       "##5>', '<mask_376>', '<mask_37": 24789,
-      "##53>) ": 24790,
-      "##626>' ": 24791,
+      "##626>' ": 24790,
+      "##53>) ": 24791,
       " LOAD_CONST , '<mask_3327>' ": 24792,
       " LOAD_FAST , '<mask_1437>' ": 24793,
       " LOAD_ATTR , <mask_992> ": 24794,
@@ -26434,8 +26434,8 @@
       " ^~> 558 LOAD_FAST , '<mask_": 26277,
       " LOAD_CONST , ('<mask_41>', '<mask_42>', '<mask_43>') ": 26278,
       "##24>', '<mask_325>') ": 26279,
-      "##82>',) ": 26280,
-      "##644>' ": 26281,
+      "##644>' ": 26280,
+      "##82>',) ": 26281,
       " LOAD_CONST , '<mask_2345>' ": 26282,
       "##18>', '<mask_13": 26283,
       "##17>', '<mask_9": 26284,
@@ -26675,8 +26675,8 @@
       " LOAD_METHOD , <mask_1171> ": 26518,
       " LOAD_METHOD , <mask_11611> ": 26519,
       "##38>', '<mask_539>', '<mask_5": 26520,
-      "##>), <mask_": 26521,
-      "##730>' ": 26522,
+      "##730>' ": 26521,
+      "##>), <mask_": 26522,
       "LOAD_FAST , '<mask_50": 26523,
       " LOAD_GLOBAL , <mask_1128> ": 26524,
       " LOAD_FAST , '<mask_1582>' ": 26525,
@@ -26839,8 +26839,8 @@
       " STORE_NAME , <mask_1125> ": 26682,
       " LOAD_CONST , '<mask_2392>' ": 26683,
       " MAKE_FUNCTION , 4 (annotations, args: [<mask_465>]) ": 26684,
-      "##522> ": 26685,
-      "##3>, None) ": 26686,
+      "##3>, None) ": 26685,
+      "##522> ": 26686,
       " LOAD_FAST , '<mask_1472>' ": 26687,
       " LOAD_CONST , (<KWARG_PAD>, '<mask_36>') ": 26688,
       " LOAD_NAME , <mask_764> ": 26689,
@@ -27357,8 +27357,8 @@
       " LOAD_FAST , '<mask_6688>' ": 27200,
       " LOAD_CONST , ('<mask_90>', '<mask_91>', '<mask_92>', '<mask_": 27201,
       "##7>', '<mask_408>', '<mask_40": 27202,
-      "##51>) ": 27203,
-      "##7>, '<mask_": 27204,
+      "##7>, '<mask_": 27203,
+      "##51>) ": 27204,
       "##140 ^~> ": 27205,
       "##186>]) ": 27206,
       " LOAD_FAST , '<mask_1668>' ": 27207,
@@ -27789,8 +27789,8 @@
       " MAKE_FUNCTION , 0 (args: [<mask_276>]) ": 27632,
       "##2>, <mask_103>]) ": 27633,
       " LOAD_CONST , '<mask_2372>' ": 27634,
-      "##2 positional arguments ": 27635,
-      "##6>, '<mask_": 27636,
+      "##6>, '<mask_": 27635,
+      "##2 positional arguments ": 27636,
       " LOAD_CONST , '<mask_2179>' ": 27637,
       " LOAD_CONST , '<mask_3299>' ": 27638,
       "##26>) ": 27639,
@@ -28194,8 +28194,8 @@
       " ^~> 418 LOAD_GLOBAL , <mask_30> ": 28037,
       " MAKE_FUNCTION , 4 (annotations, args: [<mask_589>]) ": 28038,
       " MAKE_FUNCTION , 3 (defaults, kwonly_default": 28039,
-      "##925>]) ": 28040,
-      "##624>' ": 28041,
+      "##624>' ": 28040,
+      "##925>]) ": 28041,
       " STORE_ATTR , <mask_210": 28042,
       "##0 POP_EXCEPT ": 28043,
       " LOAD_FAST , '<mask_1528>' ": 28044,
@@ -28269,8 +28269,8 @@
       " STORE_ATTR , <mask_564> ": 28112,
       " LOAD_FAST , '<mask_5526>' ": 28113,
       " MAKE_FUNCTION , 4 (annotations, args: [<mask_691>]) ": 28114,
-      "##806>' ": 28115,
-      "##612>' ": 28116,
+      "##612>' ": 28115,
+      "##806>' ": 28116,
       " LOAD_METHOD , <mask_838> ": 28117,
       "##0>, <mask_90": 28118,
       " ^~> 850 LOAD_GLOBAL , <mask_": 28119,
@@ -28773,8 +28773,8 @@
       "##7>, <mask_158>]) ": 28616,
       " LOAD_CONST , '<mask_2485>' ": 28617,
       " LOAD_CONST , ('<mask_101>', '<mask_102>') ": 28618,
-      "##527> ": 28619,
-      "##634>' ": 28620,
+      "##634>' ": 28619,
+      "##527> ": 28620,
       " LOAD_FAST , '<mask_2458>' ": 28621,
       "##142>]) ": 28622,
       "##26>', '<mask_10": 28623,
@@ -29940,9 +29940,9 @@
       " STORE_ATTR , <mask_448> ": 29783,
       " LOAD_DEREF , <mask_325> ": 29784,
       "## LOAD_CONST , () ": 29785,
-      "##91>', '<mask_10": 29786,
+      "##668>' ": 29786,
       "##801>' ": 29787,
-      "##668>' ": 29788,
+      "##91>', '<mask_10": 29788,
       " LOAD_CONST , '<mask_3364>' ": 29789,
       " LOAD_FAST , '<mask_1498>' ": 29790,
       " LOAD_METHOD , <mask_934> ": 29791,
@@ -30072,8 +30072,8 @@
       "##3>', '<mask_464>', '<mask_46": 29915,
       " LOAD_CONST , ('<mask_48>', '<mask_49>', '<mask_50>') ": 29916,
       "##>, <mask_19796>, <mask_198": 29917,
-      "##95>) ": 29918,
-      "##321": 29919,
+      "##321": 29918,
+      "##95>) ": 29919,
       " LOAD_CONST , '<mask_3336>' ": 29920,
       " IMPORT_FROM , <mask_15": 29921,
       "LOAD_FAST , '<mask_442>' ": 29922,