Training in progress, step 40, checkpoint

last-checkpoint/config.json

@@ -1,31 +1,31 @@
 {
-  "_name_or_path": "Xenova/tiny-random-Phi3ForCausalLM",
+  "_name_or_path": "JackFram/llama-68m",
   "architectures": [
-    "Phi3ForCausalLM"
+    "LlamaForCausalLM"
   ],
+  "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": 1,
-  "embd_pdrop": 0.0,
-  "eos_token_id": 32000,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "head_dim": 64,
   "hidden_act": "silu",
-  "hidden_size": 32,
+  "hidden_size": 768,
   "initializer_range": 0.02,
-  "intermediate_size": 64,
-  "max_position_embeddings": 4096,
-  "model_type": "phi3",
-  "num_attention_heads": 4,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 12,
   "num_hidden_layers": 2,
-  "num_key_value_heads": 4,
-  "original_max_position_embeddings": 4096,
-  "pad_token_id": 32000,
-  "resid_pdrop": 0.0,
-  "rms_norm_eps": 1e-05,
+  "num_key_value_heads": 12,
+  "pad_token_id": 1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
   "rope_scaling": null,
   "rope_theta": 10000.0,
-  "sliding_window": 2047,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.48.1",
   "use_cache": false,
-  "vocab_size": 32011
+  "vocab_size": 32000
 }

last-checkpoint/generation_config.json

@@ -1,8 +1,8 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 1,
+  "bos_token_id": 0,
   "do_sample": true,
-  "eos_token_id": 32000,
-  "pad_token_id": 32000,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
   "transformers_version": "4.48.1"
 }

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1985749813398ab678fa16f868bbbedc4997e1118b6ce01f98888b06f4bc92c0
-size 4140280
+oid sha256:bb869319ab00023388d1e988acef0010027b3678790ccae9050ce5b80348b1f4
+size 136062744

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8084acbc3ed18c5405e1dae91524a8683c3de78b48f0c9f04ed0d349a7c73286
-size 36192998
+oid sha256:cc3c66682da77ffab9c413076f3b17afc624d6825c4dac80307f4ad103e040b7
+size 272133748

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6184eeaad9b63225dc77ef2ebed37b3393eb958bf1b0036e48dd0dd4dce8d3e8
+oid sha256:2e87af3203606e0bd212d0720508716542b16ace065167ca500c79080102e901
 size 15984

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b023e2a6d269c0076aa102ee08abf897f3861f92ad6f8e6c141a2add64f168fc
+oid sha256:a54818910295f3e603bf553aaaad45da6124c88f04cf1b2dc460c5f51f1e31aa
 size 15984

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0cc15daee6eb8334e4ef77378d3160328ff03a12360233ca932fdc7422e74a5c
+oid sha256:859fab48bc4a8b0e143d55bf56530d10e2f3a05817c672c76810d5bbeff713c6
 size 15984

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1799d707236b96f35b0a65f363960fd5940a5fdfa4742f09a2e63a97bbd9dc57
+oid sha256:ef1e3070a4e9003f48f81647995cb67a2bb43aeca20dae5ff1b28e862e9ece94
 size 15984

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b12d1bdab4d9b46cb08e585841e7243af006aa3bab548a59d48e6556aeddac40
+oid sha256:5db80576a4a3f0a67791683c470270d702c1053ee59ca6c748f869f2a12aec02
 size 15984

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6bf839b2526d77010bd7aa679f8e366e5d97bb7f4d317d2d1da257b8178f8e4
+oid sha256:85269b28aa9e3df2ea51cc614858de45ed722dad5054cab73eff8aabd6ddff08
 size 15984

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:55273bfecbe6270bcc0713428d0aef85b6f753e4a96acc6a4ba319abd3048438
+oid sha256:2057c4a62d7e5382a52efe2b7db7f3a010767758bccf53fbfe956f58b713059e
 size 15984

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:212b99b6d3d9b2cedd149518a5819f7b601ae13b9e1660a64dbe679fa815027c
+oid sha256:99fe452d5446af1441f5b4102b5c5c984ca4eacd8c770f8cd502e07b90ab7438
 size 15984

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8546ece63abe39ed6d5eb52d3760f17bec10518134f71da680520ffc565f9e9f
+oid sha256:fde4abb1ba5c270405a21e3f71ef7e0ec6bd9f941b0624669b81510eb3186ff6
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 6.796208530805687,
+  "epoch": 0.7582938388625592,
   "eval_steps": 200,
-  "global_step": 360,
+  "global_step": 40,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -11,270 +11,38 @@
     {
       "epoch": 0.018957345971563982,
       "eval_loss": 4.226168632507324,
-      "eval_runtime": 1.4697,
-      "eval_samples_per_second": 1021.29,
-      "eval_steps_per_second": 127.916,
+      "eval_runtime": 1.1061,
+      "eval_samples_per_second": 1357.025,
+      "eval_steps_per_second": 169.967,
       "step": 1
     },
     {
       "epoch": 0.1895734597156398,
-      "grad_norm": 1.3090990781784058,
+      "grad_norm": 3.125,
       "learning_rate": 2.6666666666666667e-05,
-      "loss": 3.5627,
+      "loss": 3.5678,
       "step": 10
     },
     {
       "epoch": 0.3791469194312796,
-      "grad_norm": 0.8479055762290955,
+      "grad_norm": 1.65625,
       "learning_rate": 5.333333333333333e-05,
-      "loss": 3.5438,
+      "loss": 3.5539,
       "step": 20
     },
     {
       "epoch": 0.5687203791469194,
-      "grad_norm": 1.6812483072280884,
+      "grad_norm": 3.328125,
       "learning_rate": 8e-05,
-      "loss": 3.3698,
+      "loss": 3.3496,
       "step": 30
     },
     {
       "epoch": 0.7582938388625592,
-      "grad_norm": 0.815318763256073,
+      "grad_norm": 1.6640625,
       "learning_rate": 0.00010666666666666667,
-      "loss": 3.2253,
+      "loss": 3.1688,
       "step": 40
-    },
-    {
-      "epoch": 0.9478672985781991,
-      "grad_norm": 4.510071754455566,
-      "learning_rate": 0.00013333333333333334,
-      "loss": 3.1158,
-      "step": 50
-    },
-    {
-      "epoch": 1.132701421800948,
-      "grad_norm": 2.076462507247925,
-      "learning_rate": 0.00016,
-      "loss": 3.0493,
-      "step": 60
-    },
-    {
-      "epoch": 1.3222748815165877,
-      "grad_norm": 0.9293704032897949,
-      "learning_rate": 0.0001866666666666667,
-      "loss": 2.9581,
-      "step": 70
-    },
-    {
-      "epoch": 1.5118483412322274,
-      "grad_norm": 2.360853910446167,
-      "learning_rate": 0.00019999790210013988,
-      "loss": 2.9473,
-      "step": 80
-    },
-    {
-      "epoch": 1.7014218009478674,
-      "grad_norm": 2.270008087158203,
-      "learning_rate": 0.0001999811194293973,
-      "loss": 2.8436,
-      "step": 90
-    },
-    {
-      "epoch": 1.890995260663507,
-      "grad_norm": 0.6728243231773376,
-      "learning_rate": 0.00019994755690455152,
-      "loss": 2.868,
-      "step": 100
-    },
-    {
-      "epoch": 2.075829383886256,
-      "grad_norm": 1.5473170280456543,
-      "learning_rate": 0.0001998972201584088,
-      "loss": 2.8562,
-      "step": 110
-    },
-    {
-      "epoch": 2.265402843601896,
-      "grad_norm": 0.5135859251022339,
-      "learning_rate": 0.00019983011763899673,
-      "loss": 2.8013,
-      "step": 120
-    },
-    {
-      "epoch": 2.4549763033175354,
-      "grad_norm": 2.5838098526000977,
-      "learning_rate": 0.00019974626060814647,
-      "loss": 2.7669,
-      "step": 130
-    },
-    {
-      "epoch": 2.6445497630331753,
-      "grad_norm": 0.6559956073760986,
-      "learning_rate": 0.00019964566313960264,
-      "loss": 2.8232,
-      "step": 140
-    },
-    {
-      "epoch": 2.834123222748815,
-      "grad_norm": 0.6770644187927246,
-      "learning_rate": 0.0001995283421166614,
-      "loss": 2.7517,
-      "step": 150
-    },
-    {
-      "epoch": 3.018957345971564,
-      "grad_norm": 0.5535929203033447,
-      "learning_rate": 0.0001993943172293368,
-      "loss": 2.8265,
-      "step": 160
-    },
-    {
-      "epoch": 3.2085308056872037,
-      "grad_norm": 0.8914756774902344,
-      "learning_rate": 0.00019924361097105623,
-      "loss": 2.6735,
-      "step": 170
-    },
-    {
-      "epoch": 3.3981042654028437,
-      "grad_norm": 2.5695221424102783,
-      "learning_rate": 0.0001990762486348855,
-      "loss": 2.7361,
-      "step": 180
-    },
-    {
-      "epoch": 3.5876777251184833,
-      "grad_norm": 2.0282135009765625,
-      "learning_rate": 0.00019889225830928365,
-      "loss": 2.6677,
-      "step": 190
-    },
-    {
-      "epoch": 3.7772511848341233,
-      "grad_norm": 0.7966509461402893,
-      "learning_rate": 0.00019869167087338907,
-      "loss": 2.7418,
-      "step": 200
-    },
-    {
-      "epoch": 3.7772511848341233,
-      "eval_loss": 2.798069953918457,
-      "eval_runtime": 1.5133,
-      "eval_samples_per_second": 991.865,
-      "eval_steps_per_second": 124.231,
-      "step": 200
-    },
-    {
-      "epoch": 3.966824644549763,
-      "grad_norm": 0.5286687612533569,
-      "learning_rate": 0.00019847451999183694,
-      "loss": 2.6955,
-      "step": 210
-    },
-    {
-      "epoch": 4.151658767772512,
-      "grad_norm": 1.0543094873428345,
-      "learning_rate": 0.00019824084210910925,
-      "loss": 2.6716,
-      "step": 220
-    },
-    {
-      "epoch": 4.341232227488152,
-      "grad_norm": 1.5443323850631714,
-      "learning_rate": 0.00019799067644341844,
-      "loss": 2.5233,
-      "step": 230
-    },
-    {
-      "epoch": 4.530805687203792,
-      "grad_norm": 1.3000928163528442,
-      "learning_rate": 0.0001977240649801253,
-      "loss": 2.678,
-      "step": 240
-    },
-    {
-      "epoch": 4.720379146919432,
-      "grad_norm": 0.7852599620819092,
-      "learning_rate": 0.00019744105246469263,
-      "loss": 2.5693,
-      "step": 250
-    },
-    {
-      "epoch": 4.909952606635071,
-      "grad_norm": 0.8456063866615295,
-      "learning_rate": 0.00019714168639517544,
-      "loss": 2.6705,
-      "step": 260
-    },
-    {
-      "epoch": 5.0947867298578196,
-      "grad_norm": 1.3332568407058716,
-      "learning_rate": 0.0001968260170142496,
-      "loss": 2.5806,
-      "step": 270
-    },
-    {
-      "epoch": 5.2843601895734595,
-      "grad_norm": 0.7185400724411011,
-      "learning_rate": 0.00019649409730077935,
-      "loss": 2.5836,
-      "step": 280
-    },
-    {
-      "epoch": 5.4739336492890995,
-      "grad_norm": 4.849304676055908,
-      "learning_rate": 0.000196145982960926,
-      "loss": 2.4851,
-      "step": 290
-    },
-    {
-      "epoch": 5.6635071090047395,
-      "grad_norm": 0.7679084539413452,
-      "learning_rate": 0.00019578173241879872,
-      "loss": 2.5896,
-      "step": 300
-    },
-    {
-      "epoch": 5.8530805687203795,
-      "grad_norm": 0.6689910888671875,
-      "learning_rate": 0.00019540140680664913,
-      "loss": 2.5587,
-      "step": 310
-    },
-    {
-      "epoch": 6.037914691943128,
-      "grad_norm": 0.5706362128257751,
-      "learning_rate": 0.0001950050699546116,
-      "loss": 2.5373,
-      "step": 320
-    },
-    {
-      "epoch": 6.2274881516587675,
-      "grad_norm": 2.4172613620758057,
-      "learning_rate": 0.00019459278837999046,
-      "loss": 2.3316,
-      "step": 330
-    },
-    {
-      "epoch": 6.4170616113744074,
-      "grad_norm": 0.724682092666626,
-      "learning_rate": 0.00019416463127609656,
-      "loss": 2.5669,
-      "step": 340
-    },
-    {
-      "epoch": 6.606635071090047,
-      "grad_norm": 0.4259670674800873,
-      "learning_rate": 0.00019372067050063438,
-      "loss": 2.4148,
-      "step": 350
-    },
-    {
-      "epoch": 6.796208530805687,
-      "grad_norm": 1.5209062099456787,
-      "learning_rate": 0.00019326098056364222,
-      "loss": 2.4746,
-      "step": 360
     }
   ],
   "logging_steps": 10,
@@ -294,7 +62,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 5.410483850969088e+16,
+  "total_flos": 5467804752936960.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab35bb512bd0815bfb48651f4d2ed30517d244444ffa68a91ce2ff049faa81b0
+oid sha256:5414fad3ccf622c5fff1f84e82069b56b14de9a52dbbb9ddf1d853ff6aff2a29
 size 6840