| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "episode": 7168, | |
| "epoch": 0.030705962988348184, | |
| "eval_steps": 500, | |
| "global_step": 14, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "episode": 512, | |
| "epoch": 0.0021932830705962986, | |
| "eps": 6, | |
| "loss/policy_avg": 0.18060211837291718, | |
| "lr": 3e-06, | |
| "objective/entropy": -44.39540100097656, | |
| "objective/kl": 0.23749515414237976, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 4.621584892272949, | |
| "objective/scores": 4.621584892272949, | |
| "policy/approxkl_avg": 0.09178640693426132, | |
| "policy/clipfrac_avg": 0.271484375, | |
| "policy/entropy_avg": 0.7151485681533813, | |
| "step": 0, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 5024, | |
| "val/ratio": 1.0001094341278076, | |
| "val/ratio_var": 1.1996758075838443e-06 | |
| }, | |
| { | |
| "episode": 1024, | |
| "epoch": 0.004386566141192597, | |
| "eps": 6, | |
| "loss/policy_avg": 0.10823587328195572, | |
| "lr": 2.9882812500000002e-06, | |
| "objective/entropy": -40.33261489868164, | |
| "objective/kl": 0.8562099933624268, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 4.949611186981201, | |
| "objective/scores": 4.949611186981201, | |
| "policy/approxkl_avg": 0.09872549772262573, | |
| "policy/clipfrac_avg": 0.29296875, | |
| "policy/entropy_avg": 0.6909540891647339, | |
| "step": 1, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3988, | |
| "val/ratio": 1.0000126361846924, | |
| "val/ratio_var": 1.5850138197492925e-06 | |
| }, | |
| { | |
| "episode": 1536, | |
| "epoch": 0.006579849211788897, | |
| "eps": 6, | |
| "loss/policy_avg": 0.24130496382713318, | |
| "lr": 2.9765625e-06, | |
| "objective/entropy": -37.140586853027344, | |
| "objective/kl": 2.1461260318756104, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 5.433126449584961, | |
| "objective/scores": 5.433126449584961, | |
| "policy/approxkl_avg": 0.1151532381772995, | |
| "policy/clipfrac_avg": 0.30078125, | |
| "policy/entropy_avg": 0.6479897499084473, | |
| "step": 2, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2924, | |
| "val/ratio": 0.999896764755249, | |
| "val/ratio_var": 1.4944761232982273e-06 | |
| }, | |
| { | |
| "episode": 2048, | |
| "epoch": 0.008773132282385195, | |
| "eps": 6, | |
| "loss/policy_avg": 0.11201904714107513, | |
| "lr": 2.96484375e-06, | |
| "objective/entropy": -23.162431716918945, | |
| "objective/kl": 4.706851005554199, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 5.616954803466797, | |
| "objective/scores": 5.616954803466797, | |
| "policy/approxkl_avg": 0.13956432044506073, | |
| "policy/clipfrac_avg": 0.365234375, | |
| "policy/entropy_avg": 0.6664378643035889, | |
| "step": 3, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2146, | |
| "val/ratio": 0.9996535778045654, | |
| "val/ratio_var": 2.1856692455912707e-06 | |
| }, | |
| { | |
| "episode": 2560, | |
| "epoch": 0.010966415352981495, | |
| "eps": 6, | |
| "loss/policy_avg": 0.21055178344249725, | |
| "lr": 2.953125e-06, | |
| "objective/entropy": -17.498517990112305, | |
| "objective/kl": 6.893625259399414, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 5.975703716278076, | |
| "objective/scores": 5.975703716278076, | |
| "policy/approxkl_avg": 0.14695778489112854, | |
| "policy/clipfrac_avg": 0.33203125, | |
| "policy/entropy_avg": 0.6391204595565796, | |
| "step": 4, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1715, | |
| "val/ratio": 0.9999969005584717, | |
| "val/ratio_var": 2.4044477413553977e-06 | |
| }, | |
| { | |
| "episode": 3072, | |
| "epoch": 0.013159698423577794, | |
| "eps": 6, | |
| "loss/policy_avg": 0.28335219621658325, | |
| "lr": 2.94140625e-06, | |
| "objective/entropy": -16.186603546142578, | |
| "objective/kl": 8.215164184570312, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 6.314523696899414, | |
| "objective/scores": 6.314523696899414, | |
| "policy/approxkl_avg": 0.15406616032123566, | |
| "policy/clipfrac_avg": 0.34375, | |
| "policy/entropy_avg": 0.6234536170959473, | |
| "step": 5, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1743, | |
| "val/ratio": 1.0000813007354736, | |
| "val/ratio_var": 2.2491783511213725e-06 | |
| }, | |
| { | |
| "episode": 3584, | |
| "epoch": 0.015352981494174092, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2581750154495239, | |
| "lr": 2.9296875e-06, | |
| "objective/entropy": -16.82525634765625, | |
| "objective/kl": 10.012128829956055, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 6.68141508102417, | |
| "objective/scores": 6.68141508102417, | |
| "policy/approxkl_avg": 0.15727216005325317, | |
| "policy/clipfrac_avg": 0.306640625, | |
| "policy/entropy_avg": 0.6279575824737549, | |
| "step": 6, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2024, | |
| "val/ratio": 1.0000149011611938, | |
| "val/ratio_var": 3.910876330337487e-06 | |
| }, | |
| { | |
| "episode": 4096, | |
| "epoch": 0.01754626456477039, | |
| "eps": 6, | |
| "loss/policy_avg": 0.4508611857891083, | |
| "lr": 2.91796875e-06, | |
| "objective/entropy": -13.531968116760254, | |
| "objective/kl": 11.674339294433594, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 6.797825813293457, | |
| "objective/scores": 6.797825813293457, | |
| "policy/approxkl_avg": 0.17787906527519226, | |
| "policy/clipfrac_avg": 0.333984375, | |
| "policy/entropy_avg": 0.6140714287757874, | |
| "step": 7, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2248, | |
| "val/ratio": 0.9998712539672852, | |
| "val/ratio_var": 2.3673501345911063e-06 | |
| }, | |
| { | |
| "episode": 4608, | |
| "epoch": 0.01973954763536669, | |
| "eps": 6, | |
| "loss/policy_avg": 0.25109419226646423, | |
| "lr": 2.90625e-06, | |
| "objective/entropy": -15.348489761352539, | |
| "objective/kl": 12.88166332244873, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.097723484039307, | |
| "objective/scores": 7.097723484039307, | |
| "policy/approxkl_avg": 0.18050235509872437, | |
| "policy/clipfrac_avg": 0.302734375, | |
| "policy/entropy_avg": 0.6190727949142456, | |
| "step": 8, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2698, | |
| "val/ratio": 1.000298023223877, | |
| "val/ratio_var": 2.175341933252639e-06 | |
| }, | |
| { | |
| "episode": 5120, | |
| "epoch": 0.02193283070596299, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2478555142879486, | |
| "lr": 2.89453125e-06, | |
| "objective/entropy": -20.807220458984375, | |
| "objective/kl": 12.126293182373047, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.294116020202637, | |
| "objective/scores": 7.294116020202637, | |
| "policy/approxkl_avg": 0.16978076100349426, | |
| "policy/clipfrac_avg": 0.31640625, | |
| "policy/entropy_avg": 0.5916793346405029, | |
| "step": 9, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3166, | |
| "val/ratio": 0.999906063079834, | |
| "val/ratio_var": 2.3764121124258963e-06 | |
| }, | |
| { | |
| "episode": 5632, | |
| "epoch": 0.024126113776559287, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2194754183292389, | |
| "lr": 2.8828125e-06, | |
| "objective/entropy": -23.919815063476562, | |
| "objective/kl": 13.048431396484375, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.783618927001953, | |
| "objective/scores": 7.783618927001953, | |
| "policy/approxkl_avg": 0.1719370186328888, | |
| "policy/clipfrac_avg": 0.3359375, | |
| "policy/entropy_avg": 0.6023513078689575, | |
| "step": 10, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3395, | |
| "val/ratio": 0.9998449683189392, | |
| "val/ratio_var": 1.784112214409106e-06 | |
| }, | |
| { | |
| "episode": 6144, | |
| "epoch": 0.026319396847155587, | |
| "eps": 6, | |
| "loss/policy_avg": 0.13436169922351837, | |
| "lr": 2.87109375e-06, | |
| "objective/entropy": -21.395191192626953, | |
| "objective/kl": 15.173453330993652, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.682435512542725, | |
| "objective/scores": 7.682435512542725, | |
| "policy/approxkl_avg": 0.19314789772033691, | |
| "policy/clipfrac_avg": 0.369140625, | |
| "policy/entropy_avg": 0.6016405820846558, | |
| "step": 11, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3629, | |
| "val/ratio": 0.9996920824050903, | |
| "val/ratio_var": 1.8341333998250775e-06 | |
| }, | |
| { | |
| "episode": 6656, | |
| "epoch": 0.028512679917751884, | |
| "eps": 6, | |
| "loss/policy_avg": 0.27384719252586365, | |
| "lr": 2.859375e-06, | |
| "objective/entropy": -23.660961151123047, | |
| "objective/kl": 15.459722518920898, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.303886413574219, | |
| "objective/scores": 8.303886413574219, | |
| "policy/approxkl_avg": 0.1933055967092514, | |
| "policy/clipfrac_avg": 0.359375, | |
| "policy/entropy_avg": 0.5889922976493835, | |
| "step": 12, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3434, | |
| "val/ratio": 1.0000567436218262, | |
| "val/ratio_var": 2.589923496998381e-06 | |
| }, | |
| { | |
| "episode": 7168, | |
| "epoch": 0.030705962988348184, | |
| "eps": 6, | |
| "loss/policy_avg": 0.20046143233776093, | |
| "lr": 2.84765625e-06, | |
| "objective/entropy": -24.642715454101562, | |
| "objective/kl": 17.541698455810547, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.798042297363281, | |
| "objective/scores": 8.798042297363281, | |
| "policy/approxkl_avg": 0.1510610282421112, | |
| "policy/clipfrac_avg": 0.337890625, | |
| "policy/entropy_avg": 0.552618145942688, | |
| "step": 13, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3045, | |
| "val/ratio": 1.0000343322753906, | |
| "val/ratio_var": 2.4570517780375667e-06 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 128, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1.122960932145305, | |
| "save_steps": 2, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": true, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0, | |
| "train_batch_size": null, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |