Training in progress, step 40, checkpoint

last-checkpoint/added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

last-checkpoint/config.json

@@ -1,31 +1,28 @@
 {
-  "_name_or_path": "fxmarty/small-llama-testing",
+  "_name_or_path": "peft-internal-testing/tiny-dummy-qwen2",
   "architectures": [
-    "LlamaForCausalLM"
+    "Qwen2ForCausalLM"
   ],
-  "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": 0,
-  "eos_token_id": 2,
-  "head_dim": 64,
+  "eos_token_id": 151643,
   "hidden_act": "silu",
-  "hidden_size": 256,
+  "hidden_size": 8,
   "initializer_range": 0.02,
-  "intermediate_size": 128,
-  "max_position_embeddings": 2048,
-  "mlp_bias": false,
-  "model_type": "llama",
+  "intermediate_size": 32,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
   "num_attention_heads": 4,
   "num_hidden_layers": 2,
-  "num_key_value_heads": 4,
-  "pad_token_id": -1,
-  "pretraining_tp": 1,
+  "num_key_value_heads": 2,
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
-  "rope_theta": 10000.0,
-  "tie_word_embeddings": false,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.46.0",
+  "transformers_version": "4.48.1",
   "use_cache": false,
-  "vocab_size": 32000
+  "use_sliding_window": false,
+  "vocab_size": 151646
 }

last-checkpoint/generation_config.json

@@ -1,8 +1,7 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 0,
+  "bos_token_id": 151643,
   "do_sample": true,
-  "eos_token_id": 1,
-  "pad_token_id": 0,
-  "transformers_version": "4.46.0"
+  "eos_token_id": 151643,
+  "transformers_version": "4.48.1"
 }

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3981e030b93d87183bf27dee61297bc699920b1d1063e2a14a6d844ffe232f8b
-size 34214640
+oid sha256:fe161f5ecbc782395222bf2f386584a1d7eea980d6f5af667b405b40dc54ba49
+size 2433024

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9bbf306e4b6a97f05c3a91de8c636a98b058cb119c3bdee85b0cb98f796a16d
-size 34779282
+oid sha256:5eaaef8fa6904a05d7b00660e5c89634cc0fc7514af15647f3b2197f47ba53c2
+size 2498406

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:491f63818c64ff6648de457327747921b2babcb32039fca8bf9c3307b28a7558
+oid sha256:0df10be429c2412198f2c4c684866a8c921cbd0d9ee4c865077476da07bda410
 size 15984

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ffddba77a8223104775eb3a1c056a7d75edc6bf860f477efa9250d5724119c2a
+oid sha256:b14d8363b71c1e824421bf14a513f0f951f1c6c9b9494dcedc75b3fa1fecea91
 size 15984

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd946489a63144cac84a6b6a74fc737d9b6756458719a76e2f70310c4ca7f8f9
+oid sha256:f0d27a8e9aed69fdd3b729f8cbf6300af5e0e26e9226d0e2307b0cf80aff9030
 size 15984

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7b9d2ba99876c3fefd1687e4c0947b9787ec544cab12a64bdb6814ccefbfbfd3
+oid sha256:23de9e35ad029c7a59ff383920fe892474d8f295c2e2f82ec3e6f109f3f96960
 size 15984

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc2121f36b82fd0af2305e5cdccd3b5d0ed7e9605006dca6684c3e6d513aeddb
+oid sha256:50724f43561b5c511c4e37bb1b8ec7e620ff22332c4f39576452e56a9e02d18d
 size 15984

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95bd7e70c78cf607370cf86fe76ce57495142b48bf65473af517bedfd74dc36d
+oid sha256:2f8d108db37b78249e3b0beb22f6120e06fd369dd9fbaa19526994e040db5f9d
 size 15984

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78dee00f79cb684b52a789dfffc13b50359ff9b6a646306b391a800ef6536778
+oid sha256:6e62597393912992a3de2b36aa1663274b75b96f6525f5504e64377c5049c51e
 size 15984

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b8b731c550646d4b266437a563c6210a81eba271f5c46c3dc0af7053d7c4bd3
+oid sha256:3b1f0a5c61bcbceb46a2a335e00a59c951081a94126a8ae4cc9a9102e012eb65
 size 15984

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20e46998e3e088e3aec913e9faf45e0989d546f2ea882278b706c60253407ebc
+oid sha256:ea54818dbc15bd81d4941b71d5e2806c555d1a733f768c00c5f1a670e0a08fb1
 size 1064

last-checkpoint/special_tokens_map.json

@@ -1,30 +1,20 @@
 {
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
   "eos_token": {
-    "content": "</s>",
+    "content": "<|endoftext|>",
     "lstrip": false,
-    "normalized": true,
+    "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "</s>",
+    "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
   }
 }

last-checkpoint/tokenizer_config.json

@@ -1,26 +1,24 @@
 {
-  "add_bos_token": true,
-  "add_eos_token": false,
-  "add_prefix_space": null,
+  "add_prefix_space": false,
   "added_tokens_decoder": {
-    "0": {
-      "content": "<unk>",
+    "151643": {
+      "content": "<|endoftext|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "1": {
-      "content": "<s>",
+    "151644": {
+      "content": "<|im_start|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "2": {
-      "content": "</s>",
+    "151645": {
+      "content": "<|im_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -28,16 +26,20 @@
       "special": true
     }
   },
-  "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '### Instruction: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ '### Response: ' + message['content'] + eos_token}}{% endif %}{% endfor %}",
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "</s>",
-  "legacy": true,
-  "model_max_length": 2048,
-  "pad_token": "</s>",
-  "sp_model_kwargs": {},
-  "tokenizer_class": "LlamaTokenizer",
-  "unk_token": "<unk>",
-  "use_default_system_prompt": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
   "use_fast": true
 }

last-checkpoint/trainer_state.json

@@ -1,1001 +1,70 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 10.0,
+  "epoch": 0.047337278106508875,
   "eval_steps": 20,
-  "global_step": 890,
+  "global_step": 40,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.011235955056179775,
-      "eval_loss": 10.422812461853027,
-      "eval_runtime": 1.3009,
-      "eval_samples_per_second": 1154.612,
-      "eval_steps_per_second": 7.687,
+      "epoch": 0.001183431952662722,
+      "eval_loss": 11.929322242736816,
+      "eval_runtime": 3.7308,
+      "eval_samples_per_second": 402.598,
+      "eval_steps_per_second": 25.196,
       "step": 1
     },
     {
-      "epoch": 0.11235955056179775,
-      "grad_norm": 1.5390625,
-      "learning_rate": 7.692307692307693e-05,
-      "loss": 10.4033,
+      "epoch": 0.011834319526627219,
+      "grad_norm": 0.19140625,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 11.9299,
       "step": 10
     },
     {
-      "epoch": 0.2247191011235955,
-      "grad_norm": 1.46875,
-      "learning_rate": 0.00015384615384615385,
-      "loss": 10.127,
+      "epoch": 0.023668639053254437,
+      "grad_norm": 0.12158203125,
+      "learning_rate": 3.2000000000000005e-05,
+      "loss": 11.9289,
       "step": 20
     },
     {
-      "epoch": 0.2247191011235955,
-      "eval_loss": 9.863175392150879,
-      "eval_runtime": 1.4324,
-      "eval_samples_per_second": 1048.618,
-      "eval_steps_per_second": 6.981,
+      "epoch": 0.023668639053254437,
+      "eval_loss": 11.92916488647461,
+      "eval_runtime": 3.6696,
+      "eval_samples_per_second": 409.305,
+      "eval_steps_per_second": 25.616,
       "step": 20
     },
     {
-      "epoch": 0.33707865168539325,
-      "grad_norm": 1.5234375,
-      "learning_rate": 0.00019998942319271077,
-      "loss": 9.616,
+      "epoch": 0.03550295857988166,
+      "grad_norm": 0.251953125,
+      "learning_rate": 4.8e-05,
+      "loss": 11.9288,
       "step": 30
     },
     {
-      "epoch": 0.449438202247191,
-      "grad_norm": 1.3046875,
-      "learning_rate": 0.00019987045980408906,
-      "loss": 9.0393,
+      "epoch": 0.047337278106508875,
+      "grad_norm": 0.1435546875,
+      "learning_rate": 6.400000000000001e-05,
+      "loss": 11.9293,
       "step": 40
     },
     {
-      "epoch": 0.449438202247191,
-      "eval_loss": 8.740312576293945,
-      "eval_runtime": 1.4689,
-      "eval_samples_per_second": 1022.538,
-      "eval_steps_per_second": 6.808,
+      "epoch": 0.047337278106508875,
+      "eval_loss": 11.928914070129395,
+      "eval_runtime": 3.5621,
+      "eval_samples_per_second": 421.656,
+      "eval_steps_per_second": 26.389,
       "step": 40
-    },
-    {
-      "epoch": 0.5617977528089888,
-      "grad_norm": 1.9296875,
-      "learning_rate": 0.00019961946980917456,
-      "loss": 8.5271,
-      "step": 50
-    },
-    {
-      "epoch": 0.6741573033707865,
-      "grad_norm": 1.15625,
-      "learning_rate": 0.00019923678501135848,
-      "loss": 8.1127,
-      "step": 60
-    },
-    {
-      "epoch": 0.6741573033707865,
-      "eval_loss": 7.918918132781982,
-      "eval_runtime": 1.4635,
-      "eval_samples_per_second": 1026.272,
-      "eval_steps_per_second": 6.833,
-      "step": 60
-    },
-    {
-      "epoch": 0.7865168539325843,
-      "grad_norm": 1.015625,
-      "learning_rate": 0.00019872291131173742,
-      "loss": 7.7733,
-      "step": 70
-    },
-    {
-      "epoch": 0.898876404494382,
-      "grad_norm": 0.9765625,
-      "learning_rate": 0.00019807852804032305,
-      "loss": 7.5513,
-      "step": 80
-    },
-    {
-      "epoch": 0.898876404494382,
-      "eval_loss": 7.457934856414795,
-      "eval_runtime": 1.5378,
-      "eval_samples_per_second": 976.69,
-      "eval_steps_per_second": 6.503,
-      "step": 80
-    },
-    {
-      "epoch": 1.0112359550561798,
-      "grad_norm": 0.625,
-      "learning_rate": 0.00019730448705798239,
-      "loss": 7.4017,
-      "step": 90
-    },
-    {
-      "epoch": 1.1235955056179776,
-      "grad_norm": 0.515625,
-      "learning_rate": 0.00019640181163029595,
-      "loss": 7.2769,
-      "step": 100
-    },
-    {
-      "epoch": 1.1235955056179776,
-      "eval_loss": 7.276979446411133,
-      "eval_runtime": 1.6975,
-      "eval_samples_per_second": 884.842,
-      "eval_steps_per_second": 5.891,
-      "step": 100
-    },
-    {
-      "epoch": 1.2359550561797752,
-      "grad_norm": 0.470703125,
-      "learning_rate": 0.0001953716950748227,
-      "loss": 7.2175,
-      "step": 110
-    },
-    {
-      "epoch": 1.348314606741573,
-      "grad_norm": 0.953125,
-      "learning_rate": 0.00019421549918355966,
-      "loss": 7.1384,
-      "step": 120
-    },
-    {
-      "epoch": 1.348314606741573,
-      "eval_loss": 7.176700115203857,
-      "eval_runtime": 1.4059,
-      "eval_samples_per_second": 1068.345,
-      "eval_steps_per_second": 7.113,
-      "step": 120
-    },
-    {
-      "epoch": 1.4606741573033708,
-      "grad_norm": 0.52734375,
-      "learning_rate": 0.00019293475242268223,
-      "loss": 7.1637,
-      "step": 130
-    },
-    {
-      "epoch": 1.5730337078651684,
-      "grad_norm": 0.49609375,
-      "learning_rate": 0.00019153114791194473,
-      "loss": 7.0576,
-      "step": 140
-    },
-    {
-      "epoch": 1.5730337078651684,
-      "eval_loss": 7.057478904724121,
-      "eval_runtime": 1.5081,
-      "eval_samples_per_second": 995.943,
-      "eval_steps_per_second": 6.631,
-      "step": 140
-    },
-    {
-      "epoch": 1.6853932584269664,
-      "grad_norm": 0.7265625,
-      "learning_rate": 0.00019000654118641211,
-      "loss": 7.0339,
-      "step": 150
-    },
-    {
-      "epoch": 1.797752808988764,
-      "grad_norm": 0.494140625,
-      "learning_rate": 0.00018836294774348278,
-      "loss": 6.9564,
-      "step": 160
-    },
-    {
-      "epoch": 1.797752808988764,
-      "eval_loss": 6.937857627868652,
-      "eval_runtime": 1.4808,
-      "eval_samples_per_second": 1014.296,
-      "eval_steps_per_second": 6.753,
-      "step": 160
-    },
-    {
-      "epoch": 1.9101123595505618,
-      "grad_norm": 1.1171875,
-      "learning_rate": 0.00018660254037844388,
-      "loss": 6.8863,
-      "step": 170
-    },
-    {
-      "epoch": 2.0224719101123596,
-      "grad_norm": 0.78125,
-      "learning_rate": 0.0001847276463120828,
-      "loss": 6.8785,
-      "step": 180
-    },
-    {
-      "epoch": 2.0224719101123596,
-      "eval_loss": 6.820836067199707,
-      "eval_runtime": 1.5264,
-      "eval_samples_per_second": 984.014,
-      "eval_steps_per_second": 6.551,
-      "step": 180
-    },
-    {
-      "epoch": 2.134831460674157,
-      "grad_norm": 1.2265625,
-      "learning_rate": 0.00018274074411415105,
-      "loss": 6.7019,
-      "step": 190
-    },
-    {
-      "epoch": 2.247191011235955,
-      "grad_norm": 0.78515625,
-      "learning_rate": 0.00018064446042674828,
-      "loss": 6.7027,
-      "step": 200
-    },
-    {
-      "epoch": 2.247191011235955,
-      "eval_loss": 6.721198558807373,
-      "eval_runtime": 1.4329,
-      "eval_samples_per_second": 1048.203,
-      "eval_steps_per_second": 6.979,
-      "step": 200
-    },
-    {
-      "epoch": 2.359550561797753,
-      "grad_norm": 1.0078125,
-      "learning_rate": 0.00017844156649195759,
-      "loss": 6.6148,
-      "step": 210
-    },
-    {
-      "epoch": 2.4719101123595504,
-      "grad_norm": 0.7109375,
-      "learning_rate": 0.00017613497448832312,
-      "loss": 6.5913,
-      "step": 220
-    },
-    {
-      "epoch": 2.4719101123595504,
-      "eval_loss": 6.636184215545654,
-      "eval_runtime": 1.5174,
-      "eval_samples_per_second": 989.862,
-      "eval_steps_per_second": 6.59,
-      "step": 220
-    },
-    {
-      "epoch": 2.5842696629213484,
-      "grad_norm": 0.734375,
-      "learning_rate": 0.0001737277336810124,
-      "loss": 6.5713,
-      "step": 230
-    },
-    {
-      "epoch": 2.696629213483146,
-      "grad_norm": 1.1875,
-      "learning_rate": 0.0001712230263907531,
-      "loss": 6.498,
-      "step": 240
-    },
-    {
-      "epoch": 2.696629213483146,
-      "eval_loss": 6.557174205780029,
-      "eval_runtime": 1.526,
-      "eval_samples_per_second": 984.272,
-      "eval_steps_per_second": 6.553,
-      "step": 240
-    },
-    {
-      "epoch": 2.808988764044944,
-      "grad_norm": 0.76953125,
-      "learning_rate": 0.0001686241637868734,
-      "loss": 6.4981,
-      "step": 250
-    },
-    {
-      "epoch": 2.9213483146067416,
-      "grad_norm": 1.34375,
-      "learning_rate": 0.00016593458151000688,
-      "loss": 6.4453,
-      "step": 260
-    },
-    {
-      "epoch": 2.9213483146067416,
-      "eval_loss": 6.472111225128174,
-      "eval_runtime": 1.4329,
-      "eval_samples_per_second": 1048.191,
-      "eval_steps_per_second": 6.979,
-      "step": 260
-    },
-    {
-      "epoch": 3.033707865168539,
-      "grad_norm": 0.96484375,
-      "learning_rate": 0.00016315783513024977,
-      "loss": 6.4341,
-      "step": 270
-    },
-    {
-      "epoch": 3.146067415730337,
-      "grad_norm": 1.34375,
-      "learning_rate": 0.00016029759544677297,
-      "loss": 6.2635,
-      "step": 280
-    },
-    {
-      "epoch": 3.146067415730337,
-      "eval_loss": 6.41255521774292,
-      "eval_runtime": 1.7049,
-      "eval_samples_per_second": 880.983,
-      "eval_steps_per_second": 5.865,
-      "step": 280
-    },
-    {
-      "epoch": 3.258426966292135,
-      "grad_norm": 0.96484375,
-      "learning_rate": 0.0001573576436351046,
-      "loss": 6.3071,
-      "step": 290
-    },
-    {
-      "epoch": 3.370786516853933,
-      "grad_norm": 0.83984375,
-      "learning_rate": 0.000154341866248497,
-      "loss": 6.236,
-      "step": 300
-    },
-    {
-      "epoch": 3.370786516853933,
-      "eval_loss": 6.365837574005127,
-      "eval_runtime": 1.6113,
-      "eval_samples_per_second": 932.168,
-      "eval_steps_per_second": 6.206,
-      "step": 300
-    },
-    {
-      "epoch": 3.4831460674157304,
-      "grad_norm": 1.2265625,
-      "learning_rate": 0.00015125425007998653,
-      "loss": 6.2083,
-      "step": 310
-    },
-    {
-      "epoch": 3.595505617977528,
-      "grad_norm": 0.7890625,
-      "learning_rate": 0.00014809887689193877,
-      "loss": 6.2733,
-      "step": 320
-    },
-    {
-      "epoch": 3.595505617977528,
-      "eval_loss": 6.316156387329102,
-      "eval_runtime": 1.4293,
-      "eval_samples_per_second": 1050.857,
-      "eval_steps_per_second": 6.996,
-      "step": 320
-    },
-    {
-      "epoch": 3.7078651685393256,
-      "grad_norm": 1.46875,
-      "learning_rate": 0.00014487991802004623,
-      "loss": 6.1737,
-      "step": 330
-    },
-    {
-      "epoch": 3.8202247191011236,
-      "grad_norm": 1.15625,
-      "learning_rate": 0.00014160162885891193,
-      "loss": 6.2472,
-      "step": 340
-    },
-    {
-      "epoch": 3.8202247191011236,
-      "eval_loss": 6.286980628967285,
-      "eval_runtime": 1.566,
-      "eval_samples_per_second": 959.153,
-      "eval_steps_per_second": 6.386,
-      "step": 340
-    },
-    {
-      "epoch": 3.932584269662921,
-      "grad_norm": 1.03125,
-      "learning_rate": 0.000138268343236509,
-      "loss": 6.1259,
-      "step": 350
-    },
-    {
-      "epoch": 4.044943820224719,
-      "grad_norm": 0.92578125,
-      "learning_rate": 0.0001348844676849531,
-      "loss": 6.1738,
-      "step": 360
-    },
-    {
-      "epoch": 4.044943820224719,
-      "eval_loss": 6.240147113800049,
-      "eval_runtime": 1.7,
-      "eval_samples_per_second": 883.529,
-      "eval_steps_per_second": 5.882,
-      "step": 360
-    },
-    {
-      "epoch": 4.157303370786517,
-      "grad_norm": 0.96484375,
-      "learning_rate": 0.00013145447561516138,
-      "loss": 6.058,
-      "step": 370
-    },
-    {
-      "epoch": 4.269662921348314,
-      "grad_norm": 1.2421875,
-      "learning_rate": 0.00012798290140309923,
-      "loss": 6.0509,
-      "step": 380
-    },
-    {
-      "epoch": 4.269662921348314,
-      "eval_loss": 6.218361854553223,
-      "eval_runtime": 1.4692,
-      "eval_samples_per_second": 1022.354,
-      "eval_steps_per_second": 6.807,
-      "step": 380
-    },
-    {
-      "epoch": 4.382022471910112,
-      "grad_norm": 0.69921875,
-      "learning_rate": 0.0001244743343954324,
-      "loss": 6.0345,
-      "step": 390
-    },
-    {
-      "epoch": 4.49438202247191,
-      "grad_norm": 1.5078125,
-      "learning_rate": 0.0001209334128425092,
-      "loss": 6.0158,
-      "step": 400
-    },
-    {
-      "epoch": 4.49438202247191,
-      "eval_loss": 6.195931434631348,
-      "eval_runtime": 1.5795,
-      "eval_samples_per_second": 950.928,
-      "eval_steps_per_second": 6.331,
-      "step": 400
-    },
-    {
-      "epoch": 4.606741573033708,
-      "grad_norm": 0.86328125,
-      "learning_rate": 0.00011736481776669306,
-      "loss": 6.0611,
-      "step": 410
-    },
-    {
-      "epoch": 4.719101123595506,
-      "grad_norm": 0.9609375,
-      "learning_rate": 0.00011377326677415108,
-      "loss": 6.0043,
-      "step": 420
-    },
-    {
-      "epoch": 4.719101123595506,
-      "eval_loss": 6.1770148277282715,
-      "eval_runtime": 1.4629,
-      "eval_samples_per_second": 1026.746,
-      "eval_steps_per_second": 6.836,
-      "step": 420
-    },
-    {
-      "epoch": 4.831460674157303,
-      "grad_norm": 1.171875,
-      "learning_rate": 0.00011016350781828019,
-      "loss": 6.0156,
-      "step": 430
-    },
-    {
-      "epoch": 4.943820224719101,
-      "grad_norm": 0.95703125,
-      "learning_rate": 0.00010654031292301432,
-      "loss": 6.0249,
-      "step": 440
-    },
-    {
-      "epoch": 4.943820224719101,
-      "eval_loss": 6.157017230987549,
-      "eval_runtime": 1.5184,
-      "eval_samples_per_second": 989.202,
-      "eval_steps_per_second": 6.586,
-      "step": 440
-    },
-    {
-      "epoch": 5.056179775280899,
-      "grad_norm": 1.296875,
-      "learning_rate": 0.00010290847187431113,
-      "loss": 6.0049,
-      "step": 450
-    },
-    {
-      "epoch": 5.168539325842697,
-      "grad_norm": 1.0703125,
-      "learning_rate": 9.927278588815786e-05,
-      "loss": 5.9625,
-      "step": 460
-    },
-    {
-      "epoch": 5.168539325842697,
-      "eval_loss": 6.147063732147217,
-      "eval_runtime": 1.5949,
-      "eval_samples_per_second": 941.776,
-      "eval_steps_per_second": 6.27,
-      "step": 460
-    },
-    {
-      "epoch": 5.280898876404494,
-      "grad_norm": 1.5859375,
-      "learning_rate": 9.563806126346642e-05,
-      "loss": 5.8988,
-      "step": 470
-    },
-    {
-      "epoch": 5.393258426966292,
-      "grad_norm": 0.74609375,
-      "learning_rate": 9.200910302824963e-05,
-      "loss": 6.0231,
-      "step": 480
-    },
-    {
-      "epoch": 5.393258426966292,
-      "eval_loss": 6.130283355712891,
-      "eval_runtime": 1.482,
-      "eval_samples_per_second": 1013.508,
-      "eval_steps_per_second": 6.748,
-      "step": 480
-    },
-    {
-      "epoch": 5.50561797752809,
-      "grad_norm": 0.78125,
-      "learning_rate": 8.839070858747697e-05,
-      "loss": 5.9,
-      "step": 490
-    },
-    {
-      "epoch": 5.617977528089888,
-      "grad_norm": 0.87890625,
-      "learning_rate": 8.478766138100834e-05,
-      "loss": 5.9395,
-      "step": 500
-    },
-    {
-      "epoch": 5.617977528089888,
-      "eval_loss": 6.124143123626709,
-      "eval_runtime": 1.4263,
-      "eval_samples_per_second": 1053.058,
-      "eval_steps_per_second": 7.011,
-      "step": 500
-    },
-    {
-      "epoch": 5.730337078651686,
-      "grad_norm": 0.75,
-      "learning_rate": 8.120472455998882e-05,
-      "loss": 5.9263,
-      "step": 510
-    },
-    {
-      "epoch": 5.842696629213483,
-      "grad_norm": 1.8046875,
-      "learning_rate": 7.764663469006526e-05,
-      "loss": 5.8278,
-      "step": 520
-    },
-    {
-      "epoch": 5.842696629213483,
-      "eval_loss": 6.109380722045898,
-      "eval_runtime": 1.57,
-      "eval_samples_per_second": 956.718,
-      "eval_steps_per_second": 6.37,
-      "step": 520
-    },
-    {
-      "epoch": 5.955056179775281,
-      "grad_norm": 0.8671875,
-      "learning_rate": 7.411809548974792e-05,
-      "loss": 5.9788,
-      "step": 530
-    },
-    {
-      "epoch": 6.067415730337078,
-      "grad_norm": 1.2109375,
-      "learning_rate": 7.062377161219556e-05,
-      "loss": 5.8774,
-      "step": 540
-    },
-    {
-      "epoch": 6.067415730337078,
-      "eval_loss": 6.107788562774658,
-      "eval_runtime": 1.6988,
-      "eval_samples_per_second": 884.172,
-      "eval_steps_per_second": 5.887,
-      "step": 540
-    },
-    {
-      "epoch": 6.179775280898877,
-      "grad_norm": 0.6640625,
-      "learning_rate": 6.71682824786439e-05,
-      "loss": 5.9356,
-      "step": 550
-    },
-    {
-      "epoch": 6.292134831460674,
-      "grad_norm": 0.57421875,
-      "learning_rate": 6.375619617162985e-05,
-      "loss": 5.8393,
-      "step": 560
-    },
-    {
-      "epoch": 6.292134831460674,
-      "eval_loss": 6.102546691894531,
-      "eval_runtime": 1.5621,
-      "eval_samples_per_second": 961.551,
-      "eval_steps_per_second": 6.402,
-      "step": 560
-    },
-    {
-      "epoch": 6.404494382022472,
-      "grad_norm": 0.84375,
-      "learning_rate": 6.039202339608432e-05,
-      "loss": 5.9184,
-      "step": 570
-    },
-    {
-      "epoch": 6.51685393258427,
-      "grad_norm": 0.66796875,
-      "learning_rate": 5.708021151627712e-05,
-      "loss": 5.8534,
-      "step": 580
-    },
-    {
-      "epoch": 6.51685393258427,
-      "eval_loss": 6.098301410675049,
-      "eval_runtime": 1.5809,
-      "eval_samples_per_second": 950.072,
-      "eval_steps_per_second": 6.325,
-      "step": 580
-    },
-    {
-      "epoch": 6.629213483146067,
-      "grad_norm": 1.4609375,
-      "learning_rate": 5.382513867649663e-05,
-      "loss": 5.8461,
-      "step": 590
-    },
-    {
-      "epoch": 6.741573033707866,
-      "grad_norm": 0.65234375,
-      "learning_rate": 5.063110801323697e-05,
-      "loss": 5.9313,
-      "step": 600
-    },
-    {
-      "epoch": 6.741573033707866,
-      "eval_loss": 6.101324558258057,
-      "eval_runtime": 1.5383,
-      "eval_samples_per_second": 976.413,
-      "eval_steps_per_second": 6.501,
-      "step": 600
-    },
-    {
-      "epoch": 6.853932584269663,
-      "grad_norm": 0.58203125,
-      "learning_rate": 4.7502341966544e-05,
-      "loss": 5.8228,
-      "step": 610
-    },
-    {
-      "epoch": 6.966292134831461,
-      "grad_norm": 0.78125,
-      "learning_rate": 4.444297669803981e-05,
-      "loss": 5.8947,
-      "step": 620
-    },
-    {
-      "epoch": 6.966292134831461,
-      "eval_loss": 6.098939895629883,
-      "eval_runtime": 1.5482,
-      "eval_samples_per_second": 970.169,
-      "eval_steps_per_second": 6.459,
-      "step": 620
-    },
-    {
-      "epoch": 7.078651685393258,
-      "grad_norm": 0.83203125,
-      "learning_rate": 4.145705662300595e-05,
-      "loss": 5.8575,
-      "step": 630
-    },
-    {
-      "epoch": 7.191011235955056,
-      "grad_norm": 0.88671875,
-      "learning_rate": 3.854852906375326e-05,
-      "loss": 5.8936,
-      "step": 640
-    },
-    {
-      "epoch": 7.191011235955056,
-      "eval_loss": 6.097050189971924,
-      "eval_runtime": 1.5395,
-      "eval_samples_per_second": 975.65,
-      "eval_steps_per_second": 6.496,
-      "step": 640
-    },
-    {
-      "epoch": 7.303370786516854,
-      "grad_norm": 0.58203125,
-      "learning_rate": 3.5721239031346066e-05,
-      "loss": 5.8294,
-      "step": 650
-    },
-    {
-      "epoch": 7.415730337078652,
-      "grad_norm": 1.1796875,
-      "learning_rate": 3.297892414258043e-05,
-      "loss": 5.8275,
-      "step": 660
-    },
-    {
-      "epoch": 7.415730337078652,
-      "eval_loss": 6.095025539398193,
-      "eval_runtime": 1.562,
-      "eval_samples_per_second": 961.579,
-      "eval_steps_per_second": 6.402,
-      "step": 660
-    },
-    {
-      "epoch": 7.52808988764045,
-      "grad_norm": 0.71484375,
-      "learning_rate": 3.032520967893453e-05,
-      "loss": 5.88,
-      "step": 670
-    },
-    {
-      "epoch": 7.640449438202247,
-      "grad_norm": 0.8125,
-      "learning_rate": 2.776360379402445e-05,
-      "loss": 5.822,
-      "step": 680
-    },
-    {
-      "epoch": 7.640449438202247,
-      "eval_loss": 6.089855670928955,
-      "eval_runtime": 1.5643,
-      "eval_samples_per_second": 960.159,
-      "eval_steps_per_second": 6.393,
-      "step": 680
-    },
-    {
-      "epoch": 7.752808988764045,
-      "grad_norm": 0.7734375,
-      "learning_rate": 2.529749287590042e-05,
-      "loss": 5.8741,
-      "step": 690
-    },
-    {
-      "epoch": 7.865168539325842,
-      "grad_norm": 0.62890625,
-      "learning_rate": 2.2930137070314194e-05,
-      "loss": 5.8637,
-      "step": 700
-    },
-    {
-      "epoch": 7.865168539325842,
-      "eval_loss": 6.0882697105407715,
-      "eval_runtime": 1.4971,
-      "eval_samples_per_second": 1003.241,
-      "eval_steps_per_second": 6.679,
-      "step": 700
-    },
-    {
-      "epoch": 7.97752808988764,
-      "grad_norm": 1.25,
-      "learning_rate": 2.0664665970876496e-05,
-      "loss": 5.8311,
-      "step": 710
-    },
-    {
-      "epoch": 8.089887640449438,
-      "grad_norm": 0.6484375,
-      "learning_rate": 1.8504074481801238e-05,
-      "loss": 5.8951,
-      "step": 720
-    },
-    {
-      "epoch": 8.089887640449438,
-      "eval_loss": 6.095778465270996,
-      "eval_runtime": 1.6558,
-      "eval_samples_per_second": 907.088,
-      "eval_steps_per_second": 6.039,
-      "step": 720
-    },
-    {
-      "epoch": 8.202247191011235,
-      "grad_norm": 1.015625,
-      "learning_rate": 1.6451218858706374e-05,
-      "loss": 5.8359,
-      "step": 730
-    },
-    {
-      "epoch": 8.314606741573034,
-      "grad_norm": 0.640625,
-      "learning_rate": 1.4508812932705363e-05,
-      "loss": 5.8697,
-      "step": 740
-    },
-    {
-      "epoch": 8.314606741573034,
-      "eval_loss": 6.090635776519775,
-      "eval_runtime": 1.4815,
-      "eval_samples_per_second": 1013.856,
-      "eval_steps_per_second": 6.75,
-      "step": 740
-    },
-    {
-      "epoch": 8.426966292134832,
-      "grad_norm": 1.1328125,
-      "learning_rate": 1.2679424522780426e-05,
-      "loss": 5.8225,
-      "step": 750
-    },
-    {
-      "epoch": 8.539325842696629,
-      "grad_norm": 0.71875,
-      "learning_rate": 1.0965472041181102e-05,
-      "loss": 5.9076,
-      "step": 760
-    },
-    {
-      "epoch": 8.539325842696629,
-      "eval_loss": 6.088898181915283,
-      "eval_runtime": 1.5533,
-      "eval_samples_per_second": 966.97,
-      "eval_steps_per_second": 6.438,
-      "step": 760
-    },
-    {
-      "epoch": 8.651685393258427,
-      "grad_norm": 0.58984375,
-      "learning_rate": 9.369221296335006e-06,
-      "loss": 5.8285,
-      "step": 770
-    },
-    {
-      "epoch": 8.764044943820224,
-      "grad_norm": 1.0703125,
-      "learning_rate": 7.892782497497642e-06,
-      "loss": 5.8149,
-      "step": 780
-    },
-    {
-      "epoch": 8.764044943820224,
-      "eval_loss": 6.089446544647217,
-      "eval_runtime": 1.5112,
-      "eval_samples_per_second": 993.94,
-      "eval_steps_per_second": 6.617,
-      "step": 780
-    },
-    {
-      "epoch": 8.876404494382022,
-      "grad_norm": 0.625,
-      "learning_rate": 6.538107465101162e-06,
-      "loss": 5.8661,
-      "step": 790
-    },
-    {
-      "epoch": 8.98876404494382,
-      "grad_norm": 1.125,
-      "learning_rate": 5.306987050489442e-06,
-      "loss": 5.7888,
-      "step": 800
-    },
-    {
-      "epoch": 8.98876404494382,
-      "eval_loss": 6.0915751457214355,
-      "eval_runtime": 1.5317,
-      "eval_samples_per_second": 980.616,
-      "eval_steps_per_second": 6.529,
-      "step": 800
-    },
-    {
-      "epoch": 9.101123595505618,
-      "grad_norm": 0.63671875,
-      "learning_rate": 4.20104876845111e-06,
-      "loss": 5.9183,
-      "step": 810
-    },
-    {
-      "epoch": 9.213483146067416,
-      "grad_norm": 1.421875,
-      "learning_rate": 3.2217546456799086e-06,
-      "loss": 5.8096,
-      "step": 820
-    },
-    {
-      "epoch": 9.213483146067416,
-      "eval_loss": 6.09375,
-      "eval_runtime": 1.5233,
-      "eval_samples_per_second": 986.02,
-      "eval_steps_per_second": 6.565,
-      "step": 820
-    },
-    {
-      "epoch": 9.325842696629213,
-      "grad_norm": 0.7109375,
-      "learning_rate": 2.3703992880066638e-06,
-      "loss": 5.9093,
-      "step": 830
-    },
-    {
-      "epoch": 9.438202247191011,
-      "grad_norm": 0.578125,
-      "learning_rate": 1.648108168958229e-06,
-      "loss": 5.8319,
-      "step": 840
-    },
-    {
-      "epoch": 9.438202247191011,
-      "eval_loss": 6.085657119750977,
-      "eval_runtime": 1.441,
-      "eval_samples_per_second": 1042.336,
-      "eval_steps_per_second": 6.94,
-      "step": 840
-    },
-    {
-      "epoch": 9.55056179775281,
-      "grad_norm": 1.0234375,
-      "learning_rate": 1.055836141905553e-06,
-      "loss": 5.8358,
-      "step": 850
-    },
-    {
-      "epoch": 9.662921348314606,
-      "grad_norm": 0.58203125,
-      "learning_rate": 5.943661777680354e-07,
-      "loss": 5.8508,
-      "step": 860
-    },
-    {
-      "epoch": 9.662921348314606,
-      "eval_loss": 6.0900702476501465,
-      "eval_runtime": 1.4106,
-      "eval_samples_per_second": 1064.779,
-      "eval_steps_per_second": 7.089,
-      "step": 860
-    },
-    {
-      "epoch": 9.775280898876405,
-      "grad_norm": 1.328125,
-      "learning_rate": 2.643083299427751e-07,
-      "loss": 5.8201,
-      "step": 870
-    },
-    {
-      "epoch": 9.887640449438202,
-      "grad_norm": 0.6875,
-      "learning_rate": 6.609892782699633e-08,
-      "loss": 5.8517,
-      "step": 880
-    },
-    {
-      "epoch": 9.887640449438202,
-      "eval_loss": 6.0848469734191895,
-      "eval_runtime": 1.5852,
-      "eval_samples_per_second": 947.512,
-      "eval_steps_per_second": 6.308,
-      "step": 880
-    },
-    {
-      "epoch": 10.0,
-      "grad_norm": 0.8203125,
-      "learning_rate": 0.0,
-      "loss": 5.7949,
-      "step": 890
     }
   ],
   "logging_steps": 10,
-  "max_steps": 890,
+  "max_steps": 2500,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 10,
+  "num_train_epochs": 3,
   "save_steps": 40,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -1004,13 +73,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 1.5024257713045504e+16,
-  "train_batch_size": 19,
+  "total_flos": 15714680832.0,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88983683f78772e6fd62c9a97937ef6f408b0ee554cb1a96936c237405b3f78a
-size 6520
+oid sha256:d4ab1aa49f52e7f57216c42b69ed41e061894107885c3e00824c758931ff09d8
+size 6648