Model save

README.md

@@ -2,16 +2,11 @@
 license: mit
 base_model: gpt2
 tags:
-- alignment-handbook
 - trl
 - sft
 - generated_from_trainer
-- trl
-- sft
-- alignment-handbook
-- generated_from_trainer
 datasets:
-- appliedai-qx/sample-dataset-ah
+- generator
 model-index:
 - name: gpt2
   results: []
@@ -22,7 +17,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # gpt2
 
-This model is a fine-tuned version of [gpt2](https://huggingface.co/gpt2) on the appliedai-qx/sample-dataset-ah dataset.
+This model is a fine-tuned version of [gpt2](https://huggingface.co/gpt2) on the generator dataset.
 
 ## Model description
 
@@ -62,5 +57,5 @@ The following hyperparameters were used during training:
 
 - Transformers 4.41.2
 - Pytorch 2.3.1+cu121
-- Datasets 2.19.2
+- Datasets 2.20.0
 - Tokenizers 0.19.1

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 7357983621120000.0,
-    "train_loss": 0.11611378409645774,
-    "train_runtime": 16.2542,
+    "train_loss": 1.3832195509563794,
+    "train_runtime": 51.7657,
     "train_samples": 10000,
-    "train_samples_per_second": 864.147,
-    "train_steps_per_second": 13.535
+    "train_samples_per_second": 271.338,
+    "train_steps_per_second": 4.25
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45be5d0365d26e836319f5b468c755d28110be195d1c62b8b3f5858bfd60a158
+oid sha256:e754af9e9e09e482b62fce2b2ea79047dbb99a79bacaff80227b59eabaeaab00
 size 248894656

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 7357983621120000.0,
-    "train_loss": 0.11611378409645774,
-    "train_runtime": 16.2542,
+    "train_loss": 1.3832195509563794,
+    "train_runtime": 51.7657,
     "train_samples": 10000,
-    "train_samples_per_second": 864.147,
-    "train_steps_per_second": 13.535
+    "train_samples_per_second": 271.338,
+    "train_steps_per_second": 4.25
 }

trainer_state.json

@@ -10,327 +10,327 @@
   "log_history": [
     {
       "epoch": 0.004545454545454545,
-      "grad_norm": 3.78125,
+      "grad_norm": 3.71875,
       "learning_rate": 9.090909090909091e-06,
-      "loss": 1.8611,
+      "loss": 1.8651,
       "step": 1
     },
     {
       "epoch": 0.022727272727272728,
-      "grad_norm": 3.21875,
+      "grad_norm": 3.296875,
       "learning_rate": 4.545454545454546e-05,
-      "loss": 1.8713,
+      "loss": 1.8719,
       "step": 5
     },
     {
       "epoch": 0.045454545454545456,
-      "grad_norm": 1.96875,
+      "grad_norm": 1.7578125,
       "learning_rate": 9.090909090909092e-05,
-      "loss": 1.8057,
+      "loss": 1.8046,
       "step": 10
     },
     {
       "epoch": 0.06818181818181818,
-      "grad_norm": 1.2890625,
+      "grad_norm": 1.421875,
       "learning_rate": 0.00013636363636363637,
-      "loss": 1.7424,
+      "loss": 1.7402,
       "step": 15
     },
     {
       "epoch": 0.09090909090909091,
-      "grad_norm": 1.796875,
+      "grad_norm": 2.046875,
       "learning_rate": 0.00018181818181818183,
-      "loss": 1.6841,
+      "loss": 1.6831,
       "step": 20
     },
     {
       "epoch": 0.11363636363636363,
-      "grad_norm": 1.9375,
+      "grad_norm": 1.6875,
       "learning_rate": 0.0001998867339183008,
-      "loss": 1.611,
+      "loss": 1.607,
       "step": 25
     },
     {
       "epoch": 0.13636363636363635,
-      "grad_norm": 1.84375,
+      "grad_norm": 1.4296875,
       "learning_rate": 0.00019919548128307954,
-      "loss": 1.5838,
+      "loss": 1.582,
       "step": 30
     },
     {
       "epoch": 0.1590909090909091,
-      "grad_norm": 1.46875,
+      "grad_norm": 1.484375,
       "learning_rate": 0.00019788024462147788,
-      "loss": 1.5374,
+      "loss": 1.5335,
       "step": 35
     },
     {
       "epoch": 0.18181818181818182,
-      "grad_norm": 0.76953125,
+      "grad_norm": 0.921875,
       "learning_rate": 0.00019594929736144976,
-      "loss": 1.5049,
+      "loss": 1.5002,
       "step": 40
     },
     {
       "epoch": 0.20454545454545456,
-      "grad_norm": 0.9609375,
+      "grad_norm": 0.85546875,
       "learning_rate": 0.00019341478602651069,
-      "loss": 1.4867,
+      "loss": 1.4868,
       "step": 45
     },
     {
       "epoch": 0.22727272727272727,
-      "grad_norm": 0.83203125,
+      "grad_norm": 0.86328125,
       "learning_rate": 0.00019029265382866214,
-      "loss": 1.4711,
+      "loss": 1.47,
       "step": 50
     },
     {
       "epoch": 0.25,
-      "grad_norm": 0.9453125,
+      "grad_norm": 0.83203125,
       "learning_rate": 0.00018660254037844388,
-      "loss": 1.4442,
+      "loss": 1.4441,
       "step": 55
     },
     {
       "epoch": 0.2727272727272727,
-      "grad_norm": 0.95703125,
+      "grad_norm": 0.88671875,
       "learning_rate": 0.0001823676581429833,
-      "loss": 1.4328,
+      "loss": 1.4332,
       "step": 60
     },
     {
       "epoch": 0.29545454545454547,
-      "grad_norm": 0.875,
+      "grad_norm": 0.765625,
       "learning_rate": 0.0001776146464291757,
-      "loss": 1.4086,
+      "loss": 1.4061,
       "step": 65
     },
     {
       "epoch": 0.3181818181818182,
-      "grad_norm": 1.046875,
+      "grad_norm": 0.85546875,
       "learning_rate": 0.00017237340381050703,
-      "loss": 1.3785,
+      "loss": 1.3746,
       "step": 70
     },
     {
       "epoch": 0.3409090909090909,
-      "grad_norm": 0.90625,
+      "grad_norm": 0.98828125,
       "learning_rate": 0.00016667690005162916,
-      "loss": 1.3806,
+      "loss": 1.3776,
       "step": 75
     },
     {
       "epoch": 0.36363636363636365,
-      "grad_norm": 0.7734375,
+      "grad_norm": 0.72265625,
       "learning_rate": 0.00016056096871376667,
-      "loss": 1.3681,
+      "loss": 1.3633,
       "step": 80
     },
     {
       "epoch": 0.38636363636363635,
-      "grad_norm": 0.66015625,
+      "grad_norm": 0.61328125,
       "learning_rate": 0.00015406408174555976,
-      "loss": 1.3576,
+      "loss": 1.3569,
       "step": 85
     },
     {
       "epoch": 0.4090909090909091,
-      "grad_norm": 0.6328125,
+      "grad_norm": 0.73046875,
       "learning_rate": 0.0001472271074772683,
-      "loss": 1.3267,
+      "loss": 1.3216,
       "step": 90
     },
     {
       "epoch": 0.4318181818181818,
-      "grad_norm": 1.015625,
+      "grad_norm": 0.8046875,
       "learning_rate": 0.00014009305354066137,
-      "loss": 1.3442,
+      "loss": 1.3382,
       "step": 95
     },
     {
       "epoch": 0.45454545454545453,
-      "grad_norm": 0.8984375,
+      "grad_norm": 0.91796875,
       "learning_rate": 0.00013270679633174218,
-      "loss": 1.3321,
+      "loss": 1.3274,
       "step": 100
     },
     {
       "epoch": 0.4772727272727273,
-      "grad_norm": 0.9765625,
+      "grad_norm": 0.7890625,
       "learning_rate": 0.0001251147987181079,
-      "loss": 1.3126,
+      "loss": 1.307,
       "step": 105
     },
     {
       "epoch": 0.5,
-      "grad_norm": 0.84765625,
+      "grad_norm": 1.0234375,
       "learning_rate": 0.00011736481776669306,
-      "loss": 1.3108,
+      "loss": 1.3087,
       "step": 110
     },
     {
       "epoch": 0.5227272727272727,
-      "grad_norm": 0.6640625,
+      "grad_norm": 0.59765625,
       "learning_rate": 0.00010950560433041826,
-      "loss": 1.2999,
+      "loss": 1.2977,
       "step": 115
     },
     {
       "epoch": 0.5454545454545454,
-      "grad_norm": 0.52734375,
+      "grad_norm": 0.66796875,
       "learning_rate": 0.00010158659638348081,
-      "loss": 1.3105,
+      "loss": 1.3071,
       "step": 120
     },
     {
       "epoch": 0.5681818181818182,
-      "grad_norm": 0.69140625,
+      "grad_norm": 0.58203125,
       "learning_rate": 9.365760803434355e-05,
-      "loss": 1.3079,
+      "loss": 1.3046,
       "step": 125
     },
     {
       "epoch": 0.5909090909090909,
-      "grad_norm": 0.58984375,
+      "grad_norm": 0.5390625,
       "learning_rate": 8.57685161726715e-05,
-      "loss": 1.2927,
+      "loss": 1.2901,
       "step": 130
     },
     {
       "epoch": 0.6136363636363636,
-      "grad_norm": 0.51953125,
+      "grad_norm": 0.515625,
       "learning_rate": 7.796894672134594e-05,
-      "loss": 1.2819,
+      "loss": 1.2811,
       "step": 135
     },
     {
       "epoch": 0.6363636363636364,
-      "grad_norm": 0.76171875,
+      "grad_norm": 0.6484375,
       "learning_rate": 7.030796246717255e-05,
-      "loss": 1.2973,
+      "loss": 1.2961,
       "step": 140
     },
     {
       "epoch": 0.6590909090909091,
-      "grad_norm": 0.64453125,
+      "grad_norm": 0.65234375,
       "learning_rate": 6.283375443396726e-05,
-      "loss": 1.2926,
+      "loss": 1.2884,
       "step": 145
     },
     {
       "epoch": 0.6818181818181818,
-      "grad_norm": 0.462890625,
+      "grad_norm": 0.59375,
       "learning_rate": 5.559333873942259e-05,
-      "loss": 1.2886,
+      "loss": 1.2864,
       "step": 150
     },
     {
       "epoch": 0.7045454545454546,
-      "grad_norm": 0.671875,
+      "grad_norm": 0.72265625,
       "learning_rate": 4.8632260842659393e-05,
-      "loss": 1.2927,
+      "loss": 1.2918,
       "step": 155
     },
     {
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.51953125,
+      "grad_norm": 0.5,
       "learning_rate": 4.19943090428802e-05,
-      "loss": 1.2802,
+      "loss": 1.2773,
       "step": 160
     },
     {
       "epoch": 0.75,
-      "grad_norm": 0.447265625,
+      "grad_norm": 0.453125,
       "learning_rate": 3.5721239031346066e-05,
-      "loss": 1.2582,
+      "loss": 1.2545,
       "step": 165
     },
     {
       "epoch": 0.7727272727272727,
-      "grad_norm": 0.5625,
+      "grad_norm": 0.51171875,
       "learning_rate": 2.9852511229367865e-05,
-      "loss": 1.2829,
+      "loss": 1.2819,
       "step": 170
     },
     {
       "epoch": 0.7954545454545454,
-      "grad_norm": 0.41796875,
+      "grad_norm": 0.4453125,
       "learning_rate": 2.4425042564574184e-05,
-      "loss": 1.2812,
+      "loss": 1.2799,
       "step": 175
     },
     {
       "epoch": 0.8181818181818182,
-      "grad_norm": 0.4375,
+      "grad_norm": 0.458984375,
       "learning_rate": 1.947297424689414e-05,
-      "loss": 1.2832,
+      "loss": 1.2809,
       "step": 180
     },
     {
       "epoch": 0.8409090909090909,
-      "grad_norm": 0.4609375,
+      "grad_norm": 0.447265625,
       "learning_rate": 1.5027457005048573e-05,
-      "loss": 1.277,
+      "loss": 1.2754,
       "step": 185
     },
     {
       "epoch": 0.8636363636363636,
-      "grad_norm": 0.5234375,
+      "grad_norm": 0.671875,
       "learning_rate": 1.1116455134507664e-05,
-      "loss": 1.2788,
+      "loss": 1.2745,
       "step": 190
     },
     {
       "epoch": 0.8863636363636364,
-      "grad_norm": 0.53125,
+      "grad_norm": 0.447265625,
       "learning_rate": 7.764570589541875e-06,
-      "loss": 1.2764,
+      "loss": 1.2754,
       "step": 195
     },
     {
       "epoch": 0.9090909090909091,
-      "grad_norm": 0.5078125,
+      "grad_norm": 0.47265625,
       "learning_rate": 4.992888225905468e-06,
-      "loss": 1.2836,
+      "loss": 1.2819,
       "step": 200
     },
     {
       "epoch": 0.9318181818181818,
-      "grad_norm": 0.65625,
+      "grad_norm": 0.54296875,
       "learning_rate": 2.818843167645835e-06,
-      "loss": 1.2792,
+      "loss": 1.2782,
       "step": 205
     },
     {
       "epoch": 0.9545454545454546,
-      "grad_norm": 0.5546875,
+      "grad_norm": 0.470703125,
       "learning_rate": 1.2561111323605712e-06,
-      "loss": 1.2774,
+      "loss": 1.2758,
       "step": 210
     },
     {
       "epoch": 0.9772727272727273,
-      "grad_norm": 0.47265625,
+      "grad_norm": 0.453125,
       "learning_rate": 3.145224048057727e-07,
-      "loss": 1.2793,
+      "loss": 1.2759,
       "step": 215
     },
     {
       "epoch": 1.0,
-      "grad_norm": 0.51171875,
+      "grad_norm": 0.470703125,
       "learning_rate": 0.0,
-      "loss": 1.2731,
+      "loss": 1.2702,
       "step": 220
     },
     {
       "epoch": 1.0,
       "step": 220,
       "total_flos": 7357983621120000.0,
-      "train_loss": 0.11611378409645774,
-      "train_runtime": 16.2542,
-      "train_samples_per_second": 864.147,
-      "train_steps_per_second": 13.535
+      "train_loss": 1.3832195509563794,
+      "train_runtime": 51.7657,
+      "train_samples_per_second": 271.338,
+      "train_steps_per_second": 4.25
     }
   ],
   "logging_steps": 5,