Training in progress, step 78, checkpoint

checkpoint-78/config.json

@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "attention_bias": true,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "classifier_dropout": 0.1,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "partial_rotary_factor": 0.25,
+  "rope_scaling": null,
+  "rope_theta": 10000,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.50.3",
+  "use_cache": true,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}

checkpoint-78/generation_config.json

@@ -0,0 +1,5 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "transformers_version": "4.50.3"
+}

checkpoint-78/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f10eea320b494e319765db8a6494dfe01173e8946c138a9f30eff6ceaedda49
+size 1621370224

checkpoint-78/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8112ad21f844c4ac1b12843b017f35e216fa5b59acf2551e5f939ba0851e2a3
+size 3242916026

checkpoint-78/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25bbb175dd092246fddc37931c9c9f0b6f45ba03fc1b2a8c0e82fad787929c85
+size 14244

checkpoint-78/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c59324c910c4107d840546a8827f113227dd95a2e9af86704b4d2372782dff8
+size 1064

checkpoint-78/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|padding|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-78/tokenizer_config.json

@@ -0,0 +1,220 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|padding|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50254": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50255": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50256": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50257": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "max_length": 580,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|padding|>",
+  "padding_side": "left",
+  "stride": 0,
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-78/trainer_state.json

@@ -0,0 +1,1595 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "episode": 39936,
+  "epoch": 0.1710760795065113,
+  "eval_steps": 500,
+  "global_step": 78,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "episode": 512,
+      "epoch": 0.0021932830705962986,
+      "eps": 6,
+      "loss/policy_avg": 0.12750720977783203,
+      "lr": 3e-06,
+      "objective/entropy": -47.1529541015625,
+      "objective/kl": 0.3561969995498657,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 4.533186912536621,
+      "objective/scores": 4.533186912536621,
+      "policy/approxkl_avg": 0.12109499424695969,
+      "policy/clipfrac_avg": 0.359375,
+      "policy/entropy_avg": 0.934253454208374,
+      "step": 0,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2466,
+      "val/ratio": 1.000074863433838,
+      "val/ratio_var": 1.5309906302718446e-06
+    },
+    {
+      "episode": 1024,
+      "epoch": 0.004386566141192597,
+      "eps": 6,
+      "loss/policy_avg": 0.14252598583698273,
+      "lr": 2.9882812500000002e-06,
+      "objective/entropy": -45.39274597167969,
+      "objective/kl": 0.6608734130859375,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 4.582604885101318,
+      "objective/scores": 4.582604885101318,
+      "policy/approxkl_avg": 0.12793150544166565,
+      "policy/clipfrac_avg": 0.34765625,
+      "policy/entropy_avg": 0.9144482612609863,
+      "step": 1,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2017,
+      "val/ratio": 0.9999631643295288,
+      "val/ratio_var": 2.3689563022344373e-06
+    },
+    {
+      "episode": 1536,
+      "epoch": 0.006579849211788897,
+      "eps": 6,
+      "loss/policy_avg": 0.15036526322364807,
+      "lr": 2.9765625e-06,
+      "objective/entropy": -45.15424728393555,
+      "objective/kl": 1.1336889266967773,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 5.002047538757324,
+      "objective/scores": 5.002047538757324,
+      "policy/approxkl_avg": 0.12406677007675171,
+      "policy/clipfrac_avg": 0.3125,
+      "policy/entropy_avg": 0.9146890640258789,
+      "step": 2,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1670,
+      "val/ratio": 0.9997891783714294,
+      "val/ratio_var": 1.4586486258849618e-06
+    },
+    {
+      "episode": 2048,
+      "epoch": 0.008773132282385195,
+      "eps": 6,
+      "loss/policy_avg": 0.1377829760313034,
+      "lr": 2.96484375e-06,
+      "objective/entropy": -42.19343948364258,
+      "objective/kl": 1.5318495035171509,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 4.917952537536621,
+      "objective/scores": 4.917952537536621,
+      "policy/approxkl_avg": 0.11953485012054443,
+      "policy/clipfrac_avg": 0.306640625,
+      "policy/entropy_avg": 0.8580580949783325,
+      "step": 3,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1605,
+      "val/ratio": 0.9999769926071167,
+      "val/ratio_var": 1.8137678807761404e-06
+    },
+    {
+      "episode": 2560,
+      "epoch": 0.010966415352981495,
+      "eps": 6,
+      "loss/policy_avg": 0.16134658455848694,
+      "lr": 2.953125e-06,
+      "objective/entropy": -47.00198745727539,
+      "objective/kl": 2.190272808074951,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 5.18130350112915,
+      "objective/scores": 5.18130350112915,
+      "policy/approxkl_avg": 0.12579435110092163,
+      "policy/clipfrac_avg": 0.310546875,
+      "policy/entropy_avg": 0.768105149269104,
+      "step": 4,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1625,
+      "val/ratio": 0.9999212026596069,
+      "val/ratio_var": 1.7392967492924072e-06
+    },
+    {
+      "episode": 3072,
+      "epoch": 0.013159698423577794,
+      "eps": 6,
+      "loss/policy_avg": 0.13819235563278198,
+      "lr": 2.94140625e-06,
+      "objective/entropy": -49.404972076416016,
+      "objective/kl": 2.7731494903564453,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 5.2876763343811035,
+      "objective/scores": 5.2876763343811035,
+      "policy/approxkl_avg": 0.1179528534412384,
+      "policy/clipfrac_avg": 0.3359375,
+      "policy/entropy_avg": 0.7313921451568604,
+      "step": 5,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1740,
+      "val/ratio": 1.0000691413879395,
+      "val/ratio_var": 1.3418108437690535e-06
+    },
+    {
+      "episode": 3584,
+      "epoch": 0.015352981494174092,
+      "eps": 6,
+      "loss/policy_avg": 0.21166321635246277,
+      "lr": 2.9296875e-06,
+      "objective/entropy": -50.94463348388672,
+      "objective/kl": 3.567718982696533,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 5.437840461730957,
+      "objective/scores": 5.437840461730957,
+      "policy/approxkl_avg": 0.1175212487578392,
+      "policy/clipfrac_avg": 0.3046875,
+      "policy/entropy_avg": 0.6900524497032166,
+      "step": 6,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1894,
+      "val/ratio": 1.000088095664978,
+      "val/ratio_var": 1.8867278868128778e-06
+    },
+    {
+      "episode": 4096,
+      "epoch": 0.01754626456477039,
+      "eps": 6,
+      "loss/policy_avg": 0.23977062106132507,
+      "lr": 2.91796875e-06,
+      "objective/entropy": -55.779144287109375,
+      "objective/kl": 4.140018939971924,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 5.744781494140625,
+      "objective/scores": 5.744781494140625,
+      "policy/approxkl_avg": 0.1309545934200287,
+      "policy/clipfrac_avg": 0.333984375,
+      "policy/entropy_avg": 0.633447527885437,
+      "step": 7,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2119,
+      "val/ratio": 0.999936580657959,
+      "val/ratio_var": 1.55381121658138e-06
+    },
+    {
+      "episode": 4608,
+      "epoch": 0.01973954763536669,
+      "eps": 6,
+      "loss/policy_avg": 0.2528630793094635,
+      "lr": 2.90625e-06,
+      "objective/entropy": -61.04898452758789,
+      "objective/kl": 4.573873519897461,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 5.735022068023682,
+      "objective/scores": 5.735022068023682,
+      "policy/approxkl_avg": 0.11911365389823914,
+      "policy/clipfrac_avg": 0.28515625,
+      "policy/entropy_avg": 0.5363410711288452,
+      "step": 8,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2481,
+      "val/ratio": 1.000119686126709,
+      "val/ratio_var": 1.5850573618081398e-06
+    },
+    {
+      "episode": 5120,
+      "epoch": 0.02193283070596299,
+      "eps": 6,
+      "loss/policy_avg": 0.14096947014331818,
+      "lr": 2.89453125e-06,
+      "objective/entropy": -62.552093505859375,
+      "objective/kl": 5.485900402069092,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 5.751631259918213,
+      "objective/scores": 5.751631259918213,
+      "policy/approxkl_avg": 0.11631271243095398,
+      "policy/clipfrac_avg": 0.31640625,
+      "policy/entropy_avg": 0.5053629875183105,
+      "step": 9,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2779,
+      "val/ratio": 0.9999102354049683,
+      "val/ratio_var": 1.997669187403517e-06
+    },
+    {
+      "episode": 5632,
+      "epoch": 0.024126113776559287,
+      "eps": 6,
+      "loss/policy_avg": 0.1477496325969696,
+      "lr": 2.8828125e-06,
+      "objective/entropy": -66.58514404296875,
+      "objective/kl": 6.1831889152526855,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 6.104897499084473,
+      "objective/scores": 6.104897499084473,
+      "policy/approxkl_avg": 0.11205172538757324,
+      "policy/clipfrac_avg": 0.2890625,
+      "policy/entropy_avg": 0.472697913646698,
+      "step": 10,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2989,
+      "val/ratio": 0.9999105334281921,
+      "val/ratio_var": 1.112983909479226e-06
+    },
+    {
+      "episode": 6144,
+      "epoch": 0.026319396847155587,
+      "eps": 6,
+      "loss/policy_avg": 0.1740282028913498,
+      "lr": 2.87109375e-06,
+      "objective/entropy": -68.2206039428711,
+      "objective/kl": 6.888599395751953,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 5.872122764587402,
+      "objective/scores": 5.872122764587402,
+      "policy/approxkl_avg": 0.1110803633928299,
+      "policy/clipfrac_avg": 0.279296875,
+      "policy/entropy_avg": 0.4416949152946472,
+      "step": 11,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3219,
+      "val/ratio": 1.0001301765441895,
+      "val/ratio_var": 1.5931813095448888e-06
+    },
+    {
+      "episode": 6656,
+      "epoch": 0.028512679917751884,
+      "eps": 6,
+      "loss/policy_avg": 0.17489734292030334,
+      "lr": 2.859375e-06,
+      "objective/entropy": -65.95451354980469,
+      "objective/kl": 7.667189121246338,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 6.356113433837891,
+      "objective/scores": 6.356113433837891,
+      "policy/approxkl_avg": 0.11979297548532486,
+      "policy/clipfrac_avg": 0.2890625,
+      "policy/entropy_avg": 0.4357627332210541,
+      "step": 12,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3045,
+      "val/ratio": 0.9999862313270569,
+      "val/ratio_var": 2.047661610049545e-06
+    },
+    {
+      "episode": 7168,
+      "epoch": 0.030705962988348184,
+      "eps": 6,
+      "loss/policy_avg": 0.175493061542511,
+      "lr": 2.84765625e-06,
+      "objective/entropy": -67.00045776367188,
+      "objective/kl": 8.392281532287598,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 6.625285625457764,
+      "objective/scores": 6.625285625457764,
+      "policy/approxkl_avg": 0.1163678765296936,
+      "policy/clipfrac_avg": 0.306640625,
+      "policy/entropy_avg": 0.4532189965248108,
+      "step": 13,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3123,
+      "val/ratio": 0.9998478889465332,
+      "val/ratio_var": 1.6296273770421976e-06
+    },
+    {
+      "episode": 7680,
+      "epoch": 0.03289924605894448,
+      "eps": 6,
+      "loss/policy_avg": 0.12691491842269897,
+      "lr": 2.8359375e-06,
+      "objective/entropy": -62.55009841918945,
+      "objective/kl": 9.323993682861328,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 6.616338729858398,
+      "objective/scores": 6.616338729858398,
+      "policy/approxkl_avg": 0.15570726990699768,
+      "policy/clipfrac_avg": 0.328125,
+      "policy/entropy_avg": 0.4586066007614136,
+      "step": 14,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2864,
+      "val/ratio": 0.9998098611831665,
+      "val/ratio_var": 1.96517794393003e-06
+    },
+    {
+      "episode": 8192,
+      "epoch": 0.03509252912954078,
+      "eps": 6,
+      "loss/policy_avg": 0.23178045451641083,
+      "lr": 2.82421875e-06,
+      "objective/entropy": -57.576053619384766,
+      "objective/kl": 11.68783950805664,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 6.858205795288086,
+      "objective/scores": 6.858205795288086,
+      "policy/approxkl_avg": 0.17012163996696472,
+      "policy/clipfrac_avg": 0.322265625,
+      "policy/entropy_avg": 0.49115192890167236,
+      "step": 15,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2729,
+      "val/ratio": 1.0000524520874023,
+      "val/ratio_var": 2.174719611502951e-06
+    },
+    {
+      "episode": 8704,
+      "epoch": 0.03728581220013708,
+      "eps": 6,
+      "loss/policy_avg": 0.22957196831703186,
+      "lr": 2.8125e-06,
+      "objective/entropy": -50.27229690551758,
+      "objective/kl": 15.081552505493164,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 7.474477767944336,
+      "objective/scores": 7.474477767944336,
+      "policy/approxkl_avg": 0.18791517615318298,
+      "policy/clipfrac_avg": 0.3671875,
+      "policy/entropy_avg": 0.4883398115634918,
+      "step": 16,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2471,
+      "val/ratio": 1.00034499168396,
+      "val/ratio_var": 3.5397249575908063e-06
+    },
+    {
+      "episode": 9216,
+      "epoch": 0.03947909527073338,
+      "eps": 6,
+      "loss/policy_avg": 0.49715545773506165,
+      "lr": 2.80078125e-06,
+      "objective/entropy": -45.675880432128906,
+      "objective/kl": 18.226818084716797,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 7.699345588684082,
+      "objective/scores": 7.699345588684082,
+      "policy/approxkl_avg": 0.25131434202194214,
+      "policy/clipfrac_avg": 0.341796875,
+      "policy/entropy_avg": 0.5121623277664185,
+      "step": 17,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2467,
+      "val/ratio": 1.0001544952392578,
+      "val/ratio_var": 4.669571353588253e-06
+    },
+    {
+      "episode": 9728,
+      "epoch": 0.041672378341329676,
+      "eps": 6,
+      "loss/policy_avg": 0.346825510263443,
+      "lr": 2.7890625e-06,
+      "objective/entropy": -37.562652587890625,
+      "objective/kl": 23.889034271240234,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 7.912500381469727,
+      "objective/scores": 7.912500381469727,
+      "policy/approxkl_avg": 0.2981986999511719,
+      "policy/clipfrac_avg": 0.38671875,
+      "policy/entropy_avg": 0.5075176954269409,
+      "step": 18,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2109,
+      "val/ratio": 0.9998643398284912,
+      "val/ratio_var": 3.7719155443483032e-06
+    },
+    {
+      "episode": 10240,
+      "epoch": 0.04386566141192598,
+      "eps": 6,
+      "loss/policy_avg": 0.35793837904930115,
+      "lr": 2.77734375e-06,
+      "objective/entropy": -38.90000534057617,
+      "objective/kl": 24.49026870727539,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 7.865503311157227,
+      "objective/scores": 7.865503311157227,
+      "policy/approxkl_avg": 0.28773218393325806,
+      "policy/clipfrac_avg": 0.373046875,
+      "policy/entropy_avg": 0.4393501877784729,
+      "step": 19,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1740,
+      "val/ratio": 1.0000510215759277,
+      "val/ratio_var": 4.729288775706664e-06
+    },
+    {
+      "episode": 10752,
+      "epoch": 0.04605894448252228,
+      "eps": 6,
+      "loss/policy_avg": 0.39905059337615967,
+      "lr": 2.765625e-06,
+      "objective/entropy": -37.25298309326172,
+      "objective/kl": 26.252962112426758,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 7.933384895324707,
+      "objective/scores": 7.933384895324707,
+      "policy/approxkl_avg": 0.33784598112106323,
+      "policy/clipfrac_avg": 0.365234375,
+      "policy/entropy_avg": 0.45024600625038147,
+      "step": 20,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1676,
+      "val/ratio": 0.9996903538703918,
+      "val/ratio_var": 4.557784450298641e-06
+    },
+    {
+      "episode": 11264,
+      "epoch": 0.048252227553118573,
+      "eps": 6,
+      "loss/policy_avg": 0.47556352615356445,
+      "lr": 2.75390625e-06,
+      "objective/entropy": -35.37464141845703,
+      "objective/kl": 26.959781646728516,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.056526184082031,
+      "objective/scores": 8.056526184082031,
+      "policy/approxkl_avg": 0.29800647497177124,
+      "policy/clipfrac_avg": 0.37109375,
+      "policy/entropy_avg": 0.4291992783546448,
+      "step": 21,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1512,
+      "val/ratio": 0.9997580051422119,
+      "val/ratio_var": 4.297321993362857e-06
+    },
+    {
+      "episode": 11776,
+      "epoch": 0.05044551062371487,
+      "eps": 6,
+      "loss/policy_avg": 0.4392542243003845,
+      "lr": 2.7421875e-06,
+      "objective/entropy": -41.550724029541016,
+      "objective/kl": 24.043529510498047,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.027580261230469,
+      "objective/scores": 8.027580261230469,
+      "policy/approxkl_avg": 0.26326847076416016,
+      "policy/clipfrac_avg": 0.36328125,
+      "policy/entropy_avg": 0.4284892678260803,
+      "step": 22,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1592,
+      "val/ratio": 1.0000464916229248,
+      "val/ratio_var": 4.475009518500883e-06
+    },
+    {
+      "episode": 12288,
+      "epoch": 0.052638793694311174,
+      "eps": 6,
+      "loss/policy_avg": 0.3007674515247345,
+      "lr": 2.73046875e-06,
+      "objective/entropy": -52.200408935546875,
+      "objective/kl": 21.862667083740234,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.28553581237793,
+      "objective/scores": 8.28553581237793,
+      "policy/approxkl_avg": 0.2857671082019806,
+      "policy/clipfrac_avg": 0.37109375,
+      "policy/entropy_avg": 0.420071542263031,
+      "step": 23,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1880,
+      "val/ratio": 0.999809980392456,
+      "val/ratio_var": 4.868047199124703e-06
+    },
+    {
+      "episode": 12800,
+      "epoch": 0.05483207676490747,
+      "eps": 6,
+      "loss/policy_avg": 0.2275318056344986,
+      "lr": 2.71875e-06,
+      "objective/entropy": -52.59136962890625,
+      "objective/kl": 21.929889678955078,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.46428108215332,
+      "objective/scores": 8.46428108215332,
+      "policy/approxkl_avg": 0.24716982245445251,
+      "policy/clipfrac_avg": 0.34765625,
+      "policy/entropy_avg": 0.4286070466041565,
+      "step": 24,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1946,
+      "val/ratio": 1.0000059604644775,
+      "val/ratio_var": 3.2738601021264913e-06
+    },
+    {
+      "episode": 13312,
+      "epoch": 0.05702535983550377,
+      "eps": 6,
+      "loss/policy_avg": 0.21806836128234863,
+      "lr": 2.70703125e-06,
+      "objective/entropy": -52.577674865722656,
+      "objective/kl": 20.580398559570312,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.337871551513672,
+      "objective/scores": 8.337871551513672,
+      "policy/approxkl_avg": 0.26169508695602417,
+      "policy/clipfrac_avg": 0.361328125,
+      "policy/entropy_avg": 0.47688111662864685,
+      "step": 25,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2202,
+      "val/ratio": 0.9996628165245056,
+      "val/ratio_var": 3.0032485938136233e-06
+    },
+    {
+      "episode": 13824,
+      "epoch": 0.05921864290610007,
+      "eps": 6,
+      "loss/policy_avg": 0.2163274586200714,
+      "lr": 2.6953125e-06,
+      "objective/entropy": -56.968421936035156,
+      "objective/kl": 19.904338836669922,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.275657653808594,
+      "objective/scores": 8.275657653808594,
+      "policy/approxkl_avg": 0.23852765560150146,
+      "policy/clipfrac_avg": 0.35546875,
+      "policy/entropy_avg": 0.4648514986038208,
+      "step": 26,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2346,
+      "val/ratio": 0.9999961853027344,
+      "val/ratio_var": 3.942259809264215e-06
+    },
+    {
+      "episode": 14336,
+      "epoch": 0.06141192597669637,
+      "eps": 6,
+      "loss/policy_avg": 0.1764034628868103,
+      "lr": 2.68359375e-06,
+      "objective/entropy": -56.630027770996094,
+      "objective/kl": 20.158096313476562,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.34037971496582,
+      "objective/scores": 8.34037971496582,
+      "policy/approxkl_avg": 0.30687031149864197,
+      "policy/clipfrac_avg": 0.3828125,
+      "policy/entropy_avg": 0.4918164312839508,
+      "step": 27,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2430,
+      "val/ratio": 0.999588131904602,
+      "val/ratio_var": 3.445997435846948e-06
+    },
+    {
+      "episode": 14848,
+      "epoch": 0.06360520904729267,
+      "eps": 6,
+      "loss/policy_avg": 0.248679518699646,
+      "lr": 2.671875e-06,
+      "objective/entropy": -56.24310302734375,
+      "objective/kl": 19.94865608215332,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.386137008666992,
+      "objective/scores": 8.386137008666992,
+      "policy/approxkl_avg": 0.2245277464389801,
+      "policy/clipfrac_avg": 0.361328125,
+      "policy/entropy_avg": 0.49217647314071655,
+      "step": 28,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2555,
+      "val/ratio": 0.9999157786369324,
+      "val/ratio_var": 3.2236650895356433e-06
+    },
+    {
+      "episode": 15360,
+      "epoch": 0.06579849211788896,
+      "eps": 6,
+      "loss/policy_avg": 0.26906225085258484,
+      "lr": 2.66015625e-06,
+      "objective/entropy": -56.197872161865234,
+      "objective/kl": 19.09078025817871,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.399325370788574,
+      "objective/scores": 8.399325370788574,
+      "policy/approxkl_avg": 0.2715945839881897,
+      "policy/clipfrac_avg": 0.375,
+      "policy/entropy_avg": 0.52540123462677,
+      "step": 29,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2600,
+      "val/ratio": 1.0000545978546143,
+      "val/ratio_var": 4.418570824782364e-06
+    },
+    {
+      "episode": 15872,
+      "epoch": 0.06799177518848526,
+      "eps": 6,
+      "loss/policy_avg": 0.2973303198814392,
+      "lr": 2.6484375e-06,
+      "objective/entropy": -62.014461517333984,
+      "objective/kl": 19.058860778808594,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.694437026977539,
+      "objective/scores": 8.694437026977539,
+      "policy/approxkl_avg": 0.2335938811302185,
+      "policy/clipfrac_avg": 0.400390625,
+      "policy/entropy_avg": 0.5033716559410095,
+      "step": 30,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2786,
+      "val/ratio": 0.9996792674064636,
+      "val/ratio_var": 2.6598288513923762e-06
+    },
+    {
+      "episode": 16384,
+      "epoch": 0.07018505825908156,
+      "eps": 6,
+      "loss/policy_avg": 0.19679370522499084,
+      "lr": 2.63671875e-06,
+      "objective/entropy": -58.170814514160156,
+      "objective/kl": 18.921110153198242,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.450736045837402,
+      "objective/scores": 8.450736045837402,
+      "policy/approxkl_avg": 0.21560396254062653,
+      "policy/clipfrac_avg": 0.359375,
+      "policy/entropy_avg": 0.5557649731636047,
+      "step": 31,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2790,
+      "val/ratio": 1.0001705884933472,
+      "val/ratio_var": 2.6364186851424165e-06
+    },
+    {
+      "episode": 16896,
+      "epoch": 0.07237834132967787,
+      "eps": 6,
+      "loss/policy_avg": 0.1725379377603531,
+      "lr": 2.6250000000000003e-06,
+      "objective/entropy": -59.47856140136719,
+      "objective/kl": 19.420867919921875,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.567611694335938,
+      "objective/scores": 8.567611694335938,
+      "policy/approxkl_avg": 0.28828394412994385,
+      "policy/clipfrac_avg": 0.345703125,
+      "policy/entropy_avg": 0.5723543167114258,
+      "step": 32,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3227,
+      "val/ratio": 0.9996543526649475,
+      "val/ratio_var": 2.863696181520936e-06
+    },
+    {
+      "episode": 17408,
+      "epoch": 0.07457162440027416,
+      "eps": 6,
+      "loss/policy_avg": 0.3564334511756897,
+      "lr": 2.61328125e-06,
+      "objective/entropy": -59.503028869628906,
+      "objective/kl": 19.913440704345703,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.610246658325195,
+      "objective/scores": 8.610246658325195,
+      "policy/approxkl_avg": 0.22897233068943024,
+      "policy/clipfrac_avg": 0.359375,
+      "policy/entropy_avg": 0.5609124898910522,
+      "step": 33,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2937,
+      "val/ratio": 0.9998010396957397,
+      "val/ratio_var": 3.5082136946584797e-06
+    },
+    {
+      "episode": 17920,
+      "epoch": 0.07676490747087046,
+      "eps": 6,
+      "loss/policy_avg": 0.2554902136325836,
+      "lr": 2.6015625e-06,
+      "objective/entropy": -59.88847732543945,
+      "objective/kl": 18.962574005126953,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.546546936035156,
+      "objective/scores": 8.546546936035156,
+      "policy/approxkl_avg": 0.26599401235580444,
+      "policy/clipfrac_avg": 0.328125,
+      "policy/entropy_avg": 0.5997524857521057,
+      "step": 34,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3095,
+      "val/ratio": 1.00026535987854,
+      "val/ratio_var": 3.7096822325111134e-06
+    },
+    {
+      "episode": 18432,
+      "epoch": 0.07895819054146676,
+      "eps": 6,
+      "loss/policy_avg": 0.3109360337257385,
+      "lr": 2.5898437500000003e-06,
+      "objective/entropy": -62.252044677734375,
+      "objective/kl": 19.137222290039062,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.649435043334961,
+      "objective/scores": 8.649435043334961,
+      "policy/approxkl_avg": 0.2142583727836609,
+      "policy/clipfrac_avg": 0.345703125,
+      "policy/entropy_avg": 0.607221245765686,
+      "step": 35,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3229,
+      "val/ratio": 1.0000017881393433,
+      "val/ratio_var": 2.4627622678963235e-06
+    },
+    {
+      "episode": 18944,
+      "epoch": 0.08115147361206305,
+      "eps": 6,
+      "loss/policy_avg": 0.254541277885437,
+      "lr": 2.578125e-06,
+      "objective/entropy": -63.467262268066406,
+      "objective/kl": 18.373220443725586,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.687257766723633,
+      "objective/scores": 8.687257766723633,
+      "policy/approxkl_avg": 0.19740049540996552,
+      "policy/clipfrac_avg": 0.326171875,
+      "policy/entropy_avg": 0.6166640520095825,
+      "step": 36,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3321,
+      "val/ratio": 0.9999884366989136,
+      "val/ratio_var": 3.2546743113925913e-06
+    },
+    {
+      "episode": 19456,
+      "epoch": 0.08334475668265935,
+      "eps": 6,
+      "loss/policy_avg": 0.17846369743347168,
+      "lr": 2.56640625e-06,
+      "objective/entropy": -63.46952819824219,
+      "objective/kl": 19.115474700927734,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.613758087158203,
+      "objective/scores": 8.613758087158203,
+      "policy/approxkl_avg": 0.23868654668331146,
+      "policy/clipfrac_avg": 0.375,
+      "policy/entropy_avg": 0.5862524509429932,
+      "step": 37,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3248,
+      "val/ratio": 0.9998351335525513,
+      "val/ratio_var": 2.714186166485888e-06
+    },
+    {
+      "episode": 19968,
+      "epoch": 0.08553803975325565,
+      "eps": 6,
+      "loss/policy_avg": 0.19094648957252502,
+      "lr": 2.5546875000000003e-06,
+      "objective/entropy": -63.260986328125,
+      "objective/kl": 19.352394104003906,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.733724594116211,
+      "objective/scores": 8.733724594116211,
+      "policy/approxkl_avg": 0.2169877141714096,
+      "policy/clipfrac_avg": 0.3203125,
+      "policy/entropy_avg": 0.5959694981575012,
+      "step": 38,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3289,
+      "val/ratio": 1.0003280639648438,
+      "val/ratio_var": 3.3813628306234023e-06
+    },
+    {
+      "episode": 20480,
+      "epoch": 0.08773132282385196,
+      "eps": 6,
+      "loss/policy_avg": 0.2243296056985855,
+      "lr": 2.54296875e-06,
+      "objective/entropy": -59.47791290283203,
+      "objective/kl": 19.82537841796875,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.559447288513184,
+      "objective/scores": 8.559447288513184,
+      "policy/approxkl_avg": 0.30144354701042175,
+      "policy/clipfrac_avg": 0.3515625,
+      "policy/entropy_avg": 0.6129642724990845,
+      "step": 39,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3091,
+      "val/ratio": 0.9997188448905945,
+      "val/ratio_var": 4.063951564603485e-06
+    },
+    {
+      "episode": 20992,
+      "epoch": 0.08992460589444826,
+      "eps": 6,
+      "loss/policy_avg": 0.14945922791957855,
+      "lr": 2.53125e-06,
+      "objective/entropy": -63.837181091308594,
+      "objective/kl": 19.955276489257812,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.650920867919922,
+      "objective/scores": 8.650920867919922,
+      "policy/approxkl_avg": 0.26656606793403625,
+      "policy/clipfrac_avg": 0.326171875,
+      "policy/entropy_avg": 0.6257542371749878,
+      "step": 40,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2992,
+      "val/ratio": 1.0000975131988525,
+      "val/ratio_var": 3.132868187094573e-06
+    },
+    {
+      "episode": 21504,
+      "epoch": 0.09211788896504455,
+      "eps": 6,
+      "loss/policy_avg": 0.15452629327774048,
+      "lr": 2.5195312500000003e-06,
+      "objective/entropy": -63.7586784362793,
+      "objective/kl": 20.086402893066406,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.547872543334961,
+      "objective/scores": 8.547872543334961,
+      "policy/approxkl_avg": 0.20186936855316162,
+      "policy/clipfrac_avg": 0.3515625,
+      "policy/entropy_avg": 0.5954059362411499,
+      "step": 41,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3124,
+      "val/ratio": 1.0001468658447266,
+      "val/ratio_var": 3.0967376005719416e-06
+    },
+    {
+      "episode": 22016,
+      "epoch": 0.09431117203564085,
+      "eps": 6,
+      "loss/policy_avg": 0.23816156387329102,
+      "lr": 2.5078125e-06,
+      "objective/entropy": -61.8187141418457,
+      "objective/kl": 20.341209411621094,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.72125244140625,
+      "objective/scores": 8.72125244140625,
+      "policy/approxkl_avg": 0.22096118330955505,
+      "policy/clipfrac_avg": 0.357421875,
+      "policy/entropy_avg": 0.6372725963592529,
+      "step": 42,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2821,
+      "val/ratio": 1.0003211498260498,
+      "val/ratio_var": 6.170133019622881e-06
+    },
+    {
+      "episode": 22528,
+      "epoch": 0.09650445510623715,
+      "eps": 6,
+      "loss/policy_avg": 0.31142109632492065,
+      "lr": 2.49609375e-06,
+      "objective/entropy": -60.373191833496094,
+      "objective/kl": 21.558738708496094,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.520058631896973,
+      "objective/scores": 8.520058631896973,
+      "policy/approxkl_avg": 0.3703784942626953,
+      "policy/clipfrac_avg": 0.333984375,
+      "policy/entropy_avg": 0.6228446960449219,
+      "step": 43,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2950,
+      "val/ratio": 0.9998255372047424,
+      "val/ratio_var": 3.503203515720088e-06
+    },
+    {
+      "episode": 23040,
+      "epoch": 0.09869773817683344,
+      "eps": 6,
+      "loss/policy_avg": 0.20921407639980316,
+      "lr": 2.4843750000000002e-06,
+      "objective/entropy": -62.82386779785156,
+      "objective/kl": 21.283199310302734,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.563481330871582,
+      "objective/scores": 8.563481330871582,
+      "policy/approxkl_avg": 0.245059072971344,
+      "policy/clipfrac_avg": 0.37890625,
+      "policy/entropy_avg": 0.6021170616149902,
+      "step": 44,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3006,
+      "val/ratio": 1.000809907913208,
+      "val/ratio_var": 3.419398490223102e-05
+    },
+    {
+      "episode": 23552,
+      "epoch": 0.10089102124742974,
+      "eps": 6,
+      "loss/policy_avg": 0.22898651659488678,
+      "lr": 2.47265625e-06,
+      "objective/entropy": -61.859039306640625,
+      "objective/kl": 21.692474365234375,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.69731330871582,
+      "objective/scores": 8.69731330871582,
+      "policy/approxkl_avg": 0.26175111532211304,
+      "policy/clipfrac_avg": 0.328125,
+      "policy/entropy_avg": 0.6003079414367676,
+      "step": 45,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2893,
+      "val/ratio": 0.9998016357421875,
+      "val/ratio_var": 5.31077648702194e-06
+    },
+    {
+      "episode": 24064,
+      "epoch": 0.10308430431802605,
+      "eps": 6,
+      "loss/policy_avg": 0.280842125415802,
+      "lr": 2.4609375e-06,
+      "objective/entropy": -61.460044860839844,
+      "objective/kl": 21.635478973388672,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.86025333404541,
+      "objective/scores": 8.86025333404541,
+      "policy/approxkl_avg": 0.26439762115478516,
+      "policy/clipfrac_avg": 0.3359375,
+      "policy/entropy_avg": 0.6330952644348145,
+      "step": 46,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2932,
+      "val/ratio": 0.9998016357421875,
+      "val/ratio_var": 2.7805131139757577e-06
+    },
+    {
+      "episode": 24576,
+      "epoch": 0.10527758738862235,
+      "eps": 6,
+      "loss/policy_avg": 0.33989423513412476,
+      "lr": 2.4492187500000002e-06,
+      "objective/entropy": -62.55036163330078,
+      "objective/kl": 21.94875717163086,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.946436882019043,
+      "objective/scores": 8.946436882019043,
+      "policy/approxkl_avg": 0.21244999766349792,
+      "policy/clipfrac_avg": 0.359375,
+      "policy/entropy_avg": 0.6167851686477661,
+      "step": 47,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2966,
+      "val/ratio": 1.000269889831543,
+      "val/ratio_var": 3.7640895698132226e-06
+    },
+    {
+      "episode": 25088,
+      "epoch": 0.10747087045921865,
+      "eps": 6,
+      "loss/policy_avg": 0.18155589699745178,
+      "lr": 2.4375e-06,
+      "objective/entropy": -62.10002136230469,
+      "objective/kl": 22.01198959350586,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.775335311889648,
+      "objective/scores": 8.775335311889648,
+      "policy/approxkl_avg": 0.2533162236213684,
+      "policy/clipfrac_avg": 0.36328125,
+      "policy/entropy_avg": 0.6513728499412537,
+      "step": 48,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2958,
+      "val/ratio": 0.9997414350509644,
+      "val/ratio_var": 2.8600227324204752e-06
+    },
+    {
+      "episode": 25600,
+      "epoch": 0.10966415352981494,
+      "eps": 6,
+      "loss/policy_avg": 0.22356639802455902,
+      "lr": 2.42578125e-06,
+      "objective/entropy": -60.96599578857422,
+      "objective/kl": 22.15947723388672,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.907562255859375,
+      "objective/scores": 8.907562255859375,
+      "policy/approxkl_avg": 0.21106398105621338,
+      "policy/clipfrac_avg": 0.369140625,
+      "policy/entropy_avg": 0.6532235145568848,
+      "step": 49,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3107,
+      "val/ratio": 0.9998568296432495,
+      "val/ratio_var": 2.8039498829457443e-06
+    },
+    {
+      "episode": 26112,
+      "epoch": 0.11185743660041124,
+      "eps": 6,
+      "loss/policy_avg": 0.2772133946418762,
+      "lr": 2.4140625000000002e-06,
+      "objective/entropy": -60.50039291381836,
+      "objective/kl": 22.602027893066406,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.589601516723633,
+      "objective/scores": 8.589601516723633,
+      "policy/approxkl_avg": 0.2172616720199585,
+      "policy/clipfrac_avg": 0.349609375,
+      "policy/entropy_avg": 0.6324688792228699,
+      "step": 50,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3201,
+      "val/ratio": 1.0002442598342896,
+      "val/ratio_var": 2.36768028116785e-06
+    },
+    {
+      "episode": 26624,
+      "epoch": 0.11405071967100754,
+      "eps": 6,
+      "loss/policy_avg": 0.17067407071590424,
+      "lr": 2.40234375e-06,
+      "objective/entropy": -58.01458740234375,
+      "objective/kl": 23.73537826538086,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.827688217163086,
+      "objective/scores": 8.827688217163086,
+      "policy/approxkl_avg": 0.23201116919517517,
+      "policy/clipfrac_avg": 0.37109375,
+      "policy/entropy_avg": 0.6225080490112305,
+      "step": 51,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3656,
+      "val/ratio": 1.000308871269226,
+      "val/ratio_var": 2.6556886041362304e-06
+    },
+    {
+      "episode": 27136,
+      "epoch": 0.11624400274160383,
+      "eps": 6,
+      "loss/policy_avg": 0.18175722658634186,
+      "lr": 2.390625e-06,
+      "objective/entropy": -58.348419189453125,
+      "objective/kl": 23.386844635009766,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.72817611694336,
+      "objective/scores": 8.72817611694336,
+      "policy/approxkl_avg": 0.2951381504535675,
+      "policy/clipfrac_avg": 0.37890625,
+      "policy/entropy_avg": 0.6238501667976379,
+      "step": 52,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3423,
+      "val/ratio": 1.000365972518921,
+      "val/ratio_var": 2.8732440569001483e-06
+    },
+    {
+      "episode": 27648,
+      "epoch": 0.11843728581220014,
+      "eps": 6,
+      "loss/policy_avg": 0.3416135609149933,
+      "lr": 2.3789062500000002e-06,
+      "objective/entropy": -57.76121520996094,
+      "objective/kl": 23.697275161743164,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 9.052407264709473,
+      "objective/scores": 9.052407264709473,
+      "policy/approxkl_avg": 0.2662898600101471,
+      "policy/clipfrac_avg": 0.380859375,
+      "policy/entropy_avg": 0.6075088977813721,
+      "step": 53,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3651,
+      "val/ratio": 1.0000418424606323,
+      "val/ratio_var": 4.378874109534081e-06
+    },
+    {
+      "episode": 28160,
+      "epoch": 0.12063056888279644,
+      "eps": 6,
+      "loss/policy_avg": 0.265788733959198,
+      "lr": 2.3671875e-06,
+      "objective/entropy": -59.14204788208008,
+      "objective/kl": 23.922271728515625,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.96934700012207,
+      "objective/scores": 8.96934700012207,
+      "policy/approxkl_avg": 0.29015815258026123,
+      "policy/clipfrac_avg": 0.365234375,
+      "policy/entropy_avg": 0.6201518177986145,
+      "step": 54,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3441,
+      "val/ratio": 0.9999364614486694,
+      "val/ratio_var": 2.862538394765579e-06
+    },
+    {
+      "episode": 28672,
+      "epoch": 0.12282385195339274,
+      "eps": 6,
+      "loss/policy_avg": 0.2591712474822998,
+      "lr": 2.35546875e-06,
+      "objective/entropy": -58.195640563964844,
+      "objective/kl": 24.767526626586914,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.780952453613281,
+      "objective/scores": 8.780952453613281,
+      "policy/approxkl_avg": 0.3050951361656189,
+      "policy/clipfrac_avg": 0.3203125,
+      "policy/entropy_avg": 0.6067063808441162,
+      "step": 55,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3560,
+      "val/ratio": 0.9996646642684937,
+      "val/ratio_var": 3.838819793600123e-06
+    },
+    {
+      "episode": 29184,
+      "epoch": 0.12501713502398903,
+      "eps": 6,
+      "loss/policy_avg": 0.1893739104270935,
+      "lr": 2.3437500000000002e-06,
+      "objective/entropy": -61.51493835449219,
+      "objective/kl": 24.346567153930664,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.864131927490234,
+      "objective/scores": 8.864131927490234,
+      "policy/approxkl_avg": 0.2597726583480835,
+      "policy/clipfrac_avg": 0.357421875,
+      "policy/entropy_avg": 0.5995157957077026,
+      "step": 56,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3598,
+      "val/ratio": 0.9998759031295776,
+      "val/ratio_var": 3.2004745662561618e-06
+    },
+    {
+      "episode": 29696,
+      "epoch": 0.12721041809458533,
+      "eps": 6,
+      "loss/policy_avg": 0.17370668053627014,
+      "lr": 2.33203125e-06,
+      "objective/entropy": -60.52989196777344,
+      "objective/kl": 25.029788970947266,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.666007041931152,
+      "objective/scores": 8.666007041931152,
+      "policy/approxkl_avg": 0.2449224293231964,
+      "policy/clipfrac_avg": 0.361328125,
+      "policy/entropy_avg": 0.587178111076355,
+      "step": 57,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3571,
+      "val/ratio": 1.0002458095550537,
+      "val/ratio_var": 3.114056880804128e-06
+    },
+    {
+      "episode": 30208,
+      "epoch": 0.12940370116518163,
+      "eps": 6,
+      "loss/policy_avg": 0.22951845824718475,
+      "lr": 2.3203125e-06,
+      "objective/entropy": -61.93848419189453,
+      "objective/kl": 25.332937240600586,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.495318412780762,
+      "objective/scores": 8.495318412780762,
+      "policy/approxkl_avg": 0.27128294110298157,
+      "policy/clipfrac_avg": 0.349609375,
+      "policy/entropy_avg": 0.5786547064781189,
+      "step": 58,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3478,
+      "val/ratio": 1.0003738403320312,
+      "val/ratio_var": 1.3942043551651295e-05
+    },
+    {
+      "episode": 30720,
+      "epoch": 0.13159698423577793,
+      "eps": 6,
+      "loss/policy_avg": 0.22642306983470917,
+      "lr": 2.30859375e-06,
+      "objective/entropy": -65.1754150390625,
+      "objective/kl": 24.58494758605957,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.604547500610352,
+      "objective/scores": 8.604547500610352,
+      "policy/approxkl_avg": 0.28292161226272583,
+      "policy/clipfrac_avg": 0.310546875,
+      "policy/entropy_avg": 0.5747346878051758,
+      "step": 59,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3528,
+      "val/ratio": 0.9998360276222229,
+      "val/ratio_var": 3.136041641482734e-06
+    },
+    {
+      "episode": 31232,
+      "epoch": 0.13379026730637422,
+      "eps": 6,
+      "loss/policy_avg": 0.1339200735092163,
+      "lr": 2.296875e-06,
+      "objective/entropy": -65.92912292480469,
+      "objective/kl": 24.22686004638672,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.52584457397461,
+      "objective/scores": 8.52584457397461,
+      "policy/approxkl_avg": 0.21744689345359802,
+      "policy/clipfrac_avg": 0.32421875,
+      "policy/entropy_avg": 0.5810856223106384,
+      "step": 60,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3584,
+      "val/ratio": 0.9998077154159546,
+      "val/ratio_var": 2.1918256152275717e-06
+    },
+    {
+      "episode": 31744,
+      "epoch": 0.13598355037697052,
+      "eps": 6,
+      "loss/policy_avg": 0.19992566108703613,
+      "lr": 2.28515625e-06,
+      "objective/entropy": -65.07493591308594,
+      "objective/kl": 25.419109344482422,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.409069061279297,
+      "objective/scores": 8.409069061279297,
+      "policy/approxkl_avg": 0.1962146908044815,
+      "policy/clipfrac_avg": 0.310546875,
+      "policy/entropy_avg": 0.5588007569313049,
+      "step": 61,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3595,
+      "val/ratio": 1.0000882148742676,
+      "val/ratio_var": 2.9485702270903857e-06
+    },
+    {
+      "episode": 32256,
+      "epoch": 0.13817683344756682,
+      "eps": 6,
+      "loss/policy_avg": 0.19996881484985352,
+      "lr": 2.2734375e-06,
+      "objective/entropy": -67.38516235351562,
+      "objective/kl": 23.758987426757812,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.108330726623535,
+      "objective/scores": 8.108330726623535,
+      "policy/approxkl_avg": 0.1896720677614212,
+      "policy/clipfrac_avg": 0.26953125,
+      "policy/entropy_avg": 0.5402642488479614,
+      "step": 62,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3944,
+      "val/ratio": 1.0001147985458374,
+      "val/ratio_var": 3.3895339583978057e-06
+    },
+    {
+      "episode": 32768,
+      "epoch": 0.1403701165181631,
+      "eps": 6,
+      "loss/policy_avg": 0.13677772879600525,
+      "lr": 2.26171875e-06,
+      "objective/entropy": -71.50131225585938,
+      "objective/kl": 23.17635726928711,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.140105247497559,
+      "objective/scores": 8.140105247497559,
+      "policy/approxkl_avg": 0.20787930488586426,
+      "policy/clipfrac_avg": 0.32421875,
+      "policy/entropy_avg": 0.5493172407150269,
+      "step": 63,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3830,
+      "val/ratio": 0.9997697472572327,
+      "val/ratio_var": 2.91177570943546e-06
+    },
+    {
+      "episode": 33280,
+      "epoch": 0.14256339958875944,
+      "eps": 6,
+      "loss/policy_avg": 0.20616966485977173,
+      "lr": 2.25e-06,
+      "objective/entropy": -70.77236938476562,
+      "objective/kl": 24.19706153869629,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.366374015808105,
+      "objective/scores": 8.366374015808105,
+      "policy/approxkl_avg": 0.18721507489681244,
+      "policy/clipfrac_avg": 0.26171875,
+      "policy/entropy_avg": 0.555661678314209,
+      "step": 64,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3883,
+      "val/ratio": 0.9998935461044312,
+      "val/ratio_var": 2.8920271688548382e-06
+    },
+    {
+      "episode": 33792,
+      "epoch": 0.14475668265935573,
+      "eps": 6,
+      "loss/policy_avg": 0.14455527067184448,
+      "lr": 2.23828125e-06,
+      "objective/entropy": -71.4616928100586,
+      "objective/kl": 24.86420440673828,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.183595657348633,
+      "objective/scores": 8.183595657348633,
+      "policy/approxkl_avg": 0.16570612788200378,
+      "policy/clipfrac_avg": 0.267578125,
+      "policy/entropy_avg": 0.527056872844696,
+      "step": 65,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4113,
+      "val/ratio": 1.0001296997070312,
+      "val/ratio_var": 2.1795549400849268e-06
+    },
+    {
+      "episode": 34304,
+      "epoch": 0.14694996572995203,
+      "eps": 6,
+      "loss/policy_avg": 0.1541716307401657,
+      "lr": 2.2265625e-06,
+      "objective/entropy": -73.83854675292969,
+      "objective/kl": 23.82467269897461,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.430439949035645,
+      "objective/scores": 8.430439949035645,
+      "policy/approxkl_avg": 0.17509257793426514,
+      "policy/clipfrac_avg": 0.30078125,
+      "policy/entropy_avg": 0.5003349781036377,
+      "step": 66,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4152,
+      "val/ratio": 0.999921441078186,
+      "val/ratio_var": 2.0571305867633782e-06
+    },
+    {
+      "episode": 34816,
+      "epoch": 0.14914324880054833,
+      "eps": 6,
+      "loss/policy_avg": 0.2405724823474884,
+      "lr": 2.21484375e-06,
+      "objective/entropy": -73.2585220336914,
+      "objective/kl": 24.3407039642334,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.195615768432617,
+      "objective/scores": 8.195615768432617,
+      "policy/approxkl_avg": 0.17023462057113647,
+      "policy/clipfrac_avg": 0.2890625,
+      "policy/entropy_avg": 0.46991419792175293,
+      "step": 67,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4058,
+      "val/ratio": 0.9999005198478699,
+      "val/ratio_var": 1.952504135260824e-06
+    },
+    {
+      "episode": 35328,
+      "epoch": 0.15133653187114463,
+      "eps": 6,
+      "loss/policy_avg": 0.17481882870197296,
+      "lr": 2.203125e-06,
+      "objective/entropy": -74.03469848632812,
+      "objective/kl": 24.17955207824707,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.267098426818848,
+      "objective/scores": 8.267098426818848,
+      "policy/approxkl_avg": 0.1741240918636322,
+      "policy/clipfrac_avg": 0.26171875,
+      "policy/entropy_avg": 0.4679723083972931,
+      "step": 68,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3994,
+      "val/ratio": 1.000124216079712,
+      "val/ratio_var": 2.194252829212928e-06
+    },
+    {
+      "episode": 35840,
+      "epoch": 0.15352981494174092,
+      "eps": 6,
+      "loss/policy_avg": 0.2236326038837433,
+      "lr": 2.19140625e-06,
+      "objective/entropy": -74.11293029785156,
+      "objective/kl": 25.54970932006836,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.242114067077637,
+      "objective/scores": 8.242114067077637,
+      "policy/approxkl_avg": 0.16678233444690704,
+      "policy/clipfrac_avg": 0.275390625,
+      "policy/entropy_avg": 0.459768146276474,
+      "step": 69,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3875,
+      "val/ratio": 0.9999948740005493,
+      "val/ratio_var": 2.9980392355355434e-06
+    },
+    {
+      "episode": 36352,
+      "epoch": 0.15572309801233722,
+      "eps": 6,
+      "loss/policy_avg": 0.2104223519563675,
+      "lr": 2.1796875e-06,
+      "objective/entropy": -70.23101806640625,
+      "objective/kl": 26.61813735961914,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.240058898925781,
+      "objective/scores": 8.240058898925781,
+      "policy/approxkl_avg": 0.17932359874248505,
+      "policy/clipfrac_avg": 0.26171875,
+      "policy/entropy_avg": 0.44719618558883667,
+      "step": 70,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3672,
+      "val/ratio": 1.0002201795578003,
+      "val/ratio_var": 2.288593123012106e-06
+    },
+    {
+      "episode": 36864,
+      "epoch": 0.15791638108293352,
+      "eps": 6,
+      "loss/policy_avg": 0.18882869184017181,
+      "lr": 2.16796875e-06,
+      "objective/entropy": -72.48193359375,
+      "objective/kl": 26.345766067504883,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.317778587341309,
+      "objective/scores": 8.317778587341309,
+      "policy/approxkl_avg": 0.2453937828540802,
+      "policy/clipfrac_avg": 0.2578125,
+      "policy/entropy_avg": 0.43665581941604614,
+      "step": 71,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3908,
+      "val/ratio": 1.0000020265579224,
+      "val/ratio_var": 4.2595497689035255e-06
+    },
+    {
+      "episode": 37376,
+      "epoch": 0.1601096641535298,
+      "eps": 6,
+      "loss/policy_avg": 0.19868019223213196,
+      "lr": 2.15625e-06,
+      "objective/entropy": -70.25636291503906,
+      "objective/kl": 27.215253829956055,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.491509437561035,
+      "objective/scores": 8.491509437561035,
+      "policy/approxkl_avg": 0.18058954179286957,
+      "policy/clipfrac_avg": 0.265625,
+      "policy/entropy_avg": 0.4683910608291626,
+      "step": 72,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3603,
+      "val/ratio": 0.999990701675415,
+      "val/ratio_var": 2.4616608698124764e-06
+    },
+    {
+      "episode": 37888,
+      "epoch": 0.1623029472241261,
+      "eps": 6,
+      "loss/policy_avg": 0.1968342512845993,
+      "lr": 2.14453125e-06,
+      "objective/entropy": -68.2578353881836,
+      "objective/kl": 28.091264724731445,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.216062545776367,
+      "objective/scores": 8.216062545776367,
+      "policy/approxkl_avg": 0.19425219297409058,
+      "policy/clipfrac_avg": 0.298828125,
+      "policy/entropy_avg": 0.4262210726737976,
+      "step": 73,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3467,
+      "val/ratio": 1.000177264213562,
+      "val/ratio_var": 2.9407167403405765e-06
+    },
+    {
+      "episode": 38400,
+      "epoch": 0.1644962302947224,
+      "eps": 6,
+      "loss/policy_avg": 0.21408286690711975,
+      "lr": 2.1328125e-06,
+      "objective/entropy": -70.67361450195312,
+      "objective/kl": 27.946510314941406,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.111261367797852,
+      "objective/scores": 8.111261367797852,
+      "policy/approxkl_avg": 0.21669934689998627,
+      "policy/clipfrac_avg": 0.232421875,
+      "policy/entropy_avg": 0.4281734228134155,
+      "step": 74,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3512,
+      "val/ratio": 1.0006786584854126,
+      "val/ratio_var": 2.5324472517240793e-05
+    },
+    {
+      "episode": 38912,
+      "epoch": 0.1666895133653187,
+      "eps": 6,
+      "loss/policy_avg": 0.1531296968460083,
+      "lr": 2.12109375e-06,
+      "objective/entropy": -69.2296142578125,
+      "objective/kl": 28.850563049316406,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.479706764221191,
+      "objective/scores": 8.479706764221191,
+      "policy/approxkl_avg": 0.20146965980529785,
+      "policy/clipfrac_avg": 0.306640625,
+      "policy/entropy_avg": 0.4345015287399292,
+      "step": 75,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3428,
+      "val/ratio": 1.0000364780426025,
+      "val/ratio_var": 2.9701880066568265e-06
+    },
+    {
+      "episode": 39424,
+      "epoch": 0.168882796435915,
+      "eps": 6,
+      "loss/policy_avg": 0.17767590284347534,
+      "lr": 2.109375e-06,
+      "objective/entropy": -70.01402282714844,
+      "objective/kl": 28.191417694091797,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.29566764831543,
+      "objective/scores": 8.29566764831543,
+      "policy/approxkl_avg": 0.1757415235042572,
+      "policy/clipfrac_avg": 0.26953125,
+      "policy/entropy_avg": 0.4495161175727844,
+      "step": 76,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3349,
+      "val/ratio": 0.9998478889465332,
+      "val/ratio_var": 2.1308803752617678e-06
+    },
+    {
+      "episode": 39936,
+      "epoch": 0.1710760795065113,
+      "eps": 6,
+      "loss/policy_avg": 0.33594539761543274,
+      "lr": 2.09765625e-06,
+      "objective/entropy": -69.42768859863281,
+      "objective/kl": 28.3437442779541,
+      "objective/non_score_reward": 0.0,
+      "objective/rlhf_reward": 8.23046875,
+      "objective/scores": 8.23046875,
+      "policy/approxkl_avg": 0.20253780484199524,
+      "policy/clipfrac_avg": 0.2734375,
+      "policy/entropy_avg": 0.433658242225647,
+      "step": 77,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3187,
+      "val/ratio": 0.9998530149459839,
+      "val/ratio_var": 1.98079601432255e-06
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 128,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1.122960932145305,
+  "save_steps": 26,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": true,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0,
+  "train_batch_size": null,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-78/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5e69611bee011a56c9eda92fc9354b22f6efc4357ae21e957602e49335de635
+size 6584