Training in progress, step 84000

config.json

@@ -16,7 +16,6 @@
   "initializer_range": 0.02,
   "intermediate_size": 1536,
   "max_position_embeddings": 1024,
-  "mlp_bias": false,
   "model_type": "bit_llama",
   "n_ctx": 128,
   "num_attention_heads": 12,
@@ -28,7 +27,7 @@
   "rope_theta": 10000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
-  "transformers_version": "4.43.1",
+  "transformers_version": "4.38.2",
   "use_cache": true,
   "vocab_size": 43176
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7b54db5c0bd7c16b81d205546b0424371ca6c7f3921aff82b21fe49bc3a0f7a2
+oid sha256:8c0708c3f673e330c78b5930215d181a543796f5a4b18429cc9750498fe6e297
 size 510960712

tokenizer.json

@@ -36,13 +36,23 @@
       "special": true
     }
   ],
-  "normalizer": null,
-  "pre_tokenizer": {
-    "type": "Metaspace",
-    "replacement": "▁",
-    "prepend_scheme": "first",
-    "split": false
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Prepend",
+        "prepend": "▁"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "String": " "
+        },
+        "content": "▁"
+      }
+    ]
   },
+  "pre_tokenizer": null,
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [
@@ -129,7 +139,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
-    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b424b2667b6ff1f95ac264726f10142e36ba9a36e85dd23b84f435ef94b9cc29
-size 5176
+oid sha256:9d1245ad5795253debe58bcbfdd83b24f66ab3036e63cd3b66d342f5ad8f590c
+size 4856