Training in progress, step 16000

config.json

@@ -16,6 +16,7 @@
   "initializer_range": 0.02,
   "intermediate_size": 1536,
   "max_position_embeddings": 1024,
+  "mlp_bias": false,
   "model_type": "bit_llama",
   "n_ctx": 128,
   "num_attention_heads": 12,
@@ -27,7 +28,7 @@
   "rope_theta": 10000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
-  "transformers_version": "4.38.2",
+  "transformers_version": "4.43.1",
   "use_cache": true,
   "vocab_size": 43176
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9bfe194e198fef3b80b9aad762a114d4a07d86af744fd7100f4700f0b77eedbe
+oid sha256:7b54db5c0bd7c16b81d205546b0424371ca6c7f3921aff82b21fe49bc3a0f7a2
 size 510960712

tokenizer.json

@@ -36,23 +36,13 @@
       "special": true
     }
   ],
-  "normalizer": {
-    "type": "Sequence",
-    "normalizers": [
-      {
-        "type": "Prepend",
-        "prepend": "▁"
-      },
-      {
-        "type": "Replace",
-        "pattern": {
-          "String": " "
-        },
-        "content": "▁"
-      }
-    ]
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "prepend_scheme": "first",
+    "split": false
   },
-  "pre_tokenizer": null,
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [
@@ -139,6 +129,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": true,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d1245ad5795253debe58bcbfdd83b24f66ab3036e63cd3b66d342f5ad8f590c
-size 4856
+oid sha256:b424b2667b6ff1f95ac264726f10142e36ba9a36e85dd23b84f435ef94b9cc29
+size 5176