Fix decoder

added_tokens.json

@@ -1,7 +1,6 @@
 {
-  "<|endofprompt|>": 180021,
   "<|endoftext|>": 180000,
-  "<|extra_id_0|>": 180002,
+  "<|extra_id_0|>": 180010,
   "<|extra_id_10|>": 180020,
   "<|extra_id_1|>": 180011,
   "<|extra_id_2|>": 180012,
@@ -12,13 +11,13 @@
   "<|extra_id_7|>": 180017,
   "<|extra_id_8|>": 180018,
   "<|extra_id_9|>": 180019,
-  "<|fim_middle|>": 180004,
-  "<|fim_prefix|>": 180003,
-  "<|fim_suffix|>": 180005,
-  "<|im_end|>": 180010,
-  "<|im_start|>": 180009,
+  "<|fim_middle|>": 180003,
+  "<|fim_prefix|>": 180002,
+  "<|fim_suffix|>": 180004,
+  "<|im_end|>": 180009,
+  "<|im_start|>": 180008,
   "<|pad|>": 180001,
-  "|||EMAIL_ADDRESS|||": 180007,
-  "|||IP_ADDRESS|||": 180008,
-  "|||PHONE_NUMBER|||": 180006
+  "|||EMAIL_ADDRESS|||": 180006,
+  "|||IP_ADDRESS|||": 180007,
+  "|||PHONE_NUMBER|||": 180005
 }

special_tokens_map.json

@@ -1,57 +1,51 @@
 {
+  "additional_special_tokens": [
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "|||PHONE_NUMBER|||",
+    "|||EMAIL_ADDRESS|||",
+    "|||IP_ADDRESS|||",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|extra_id_0|>",
+    "<|extra_id_1|>",
+    "<|extra_id_2|>",
+    "<|extra_id_3|>",
+    "<|extra_id_4|>",
+    "<|extra_id_5|>",
+    "<|extra_id_6|>",
+    "<|extra_id_7|>",
+    "<|extra_id_8|>",
+    "<|extra_id_9|>",
+    "<|extra_id_10|>"
+  ],
   "bos_token": {
     "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,
-    "single_word": false,
-    "special": true
+    "rstrip": false,
+    "single_word": false
   },
   "eos_token": {
     "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
-    "single_word": false,
-    "special": true
+    "single_word": false
   },
   "pad_token": {
     "content": "<|pad|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
-    "single_word": false,
-    "special": true
+    "single_word": false
   },
   "unk_token": {
     "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
-    "single_word": false,
-    "special": true
-  },
-  "additional_special_tokens": [
-    "<|extra_id_0|>",
-    "<|endoftext|>",
-    "<|fim_prefix|>",
-    "<|fim_middle|>",
-    "<|fim_suffix|>",
-    "|||PHONE_NUMBER|||",
-    "|||EMAIL_ADDRESS|||",
-    "|||IP_ADDRESS|||",
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|extra_id_1|>",
-    "<|extra_id_2|>",
-    "<|extra_id_3|>",
-    "<|extra_id_4|>",
-    "<|extra_id_5|>",
-    "<|extra_id_6|>",
-    "<|extra_id_7|>",
-    "<|extra_id_8|>",
-    "<|extra_id_9|>",
-    "<|extra_id_10|>",
-    "<|endofprompt|>",
-    "<|pad|>" 
-  ]
+    "single_word": false
+  }
 }

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be7041eb9d5f95ffaf1ce82ffa97c9f8bd85b0883127b182a6ca16a595a2738b
-size 13487219
+oid sha256:19f403d793a7ba69e4f49a40e7d592a5d4d1ac4f4e7224f18bc608ef268db68b
+size 13491334

tokenizer_config.json

@@ -18,7 +18,7 @@
       "special": true
     },
     "180002": {
-      "content": "<|extra_id_0|>",
+      "content": "<|fim_prefix|>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -26,7 +26,7 @@
       "special": false
     },
     "180003": {
-      "content": "<|fim_prefix|>",
+      "content": "<|fim_middle|>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -34,7 +34,7 @@
       "special": false
     },
     "180004": {
-      "content": "<|fim_middle|>",
+      "content": "<|fim_suffix|>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -42,7 +42,7 @@
       "special": false
     },
     "180005": {
-      "content": "<|fim_suffix|>",
+      "content": "|||PHONE_NUMBER|||",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -50,7 +50,7 @@
       "special": false
     },
     "180006": {
-      "content": "|||PHONE_NUMBER|||",
+      "content": "|||EMAIL_ADDRESS|||",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -58,7 +58,7 @@
       "special": false
     },
     "180007": {
-      "content": "|||EMAIL_ADDRESS|||",
+      "content": "|||IP_ADDRESS|||",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -66,7 +66,7 @@
       "special": false
     },
     "180008": {
-      "content": "|||IP_ADDRESS|||",
+      "content": "<|im_start|>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -74,7 +74,7 @@
       "special": false
     },
     "180009": {
-      "content": "<|im_start|>",
+      "content": "<|im_end|>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -82,7 +82,7 @@
       "special": false
     },
     "180010": {
-      "content": "<|im_end|>",
+      "content": "<|extra_id_0|>",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -168,19 +168,9 @@
       "rstrip": false,
       "single_word": false,
       "special": false
-    },
-    "180021": {
-      "content": "<|endofprompt|>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
     }
   },
   "additional_special_tokens": [
-    "<|extra_id_0|>",
-    "<|endoftext|>",
     "<|fim_prefix|>",
     "<|fim_middle|>",
     "<|fim_suffix|>",
@@ -189,6 +179,7 @@
     "|||IP_ADDRESS|||",
     "<|im_start|>",
     "<|im_end|>",
+    "<|extra_id_0|>",
     "<|extra_id_1|>",
     "<|extra_id_2|>",
     "<|extra_id_3|>",
@@ -198,9 +189,7 @@
     "<|extra_id_7|>",
     "<|extra_id_8|>",
     "<|extra_id_9|>",
-    "<|extra_id_10|>",
-    "<|endofprompt|>",
-    "<|pad|>"
+    "<|extra_id_10|>"
   ],
   "bos_token": "<|endoftext|>",
   "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",