| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "episode": 17408, | |
| "epoch": 0.07457162440027416, | |
| "eval_steps": 500, | |
| "global_step": 34, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "episode": 512, | |
| "epoch": 0.0021932830705962986, | |
| "eps": 6, | |
| "loss/policy_avg": 0.18060211837291718, | |
| "lr": 3e-06, | |
| "objective/entropy": -44.39540100097656, | |
| "objective/kl": 0.23749515414237976, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 4.621584892272949, | |
| "objective/scores": 4.621584892272949, | |
| "policy/approxkl_avg": 0.09178640693426132, | |
| "policy/clipfrac_avg": 0.271484375, | |
| "policy/entropy_avg": 0.7151485681533813, | |
| "step": 0, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 5024, | |
| "val/ratio": 1.0001094341278076, | |
| "val/ratio_var": 1.1996758075838443e-06 | |
| }, | |
| { | |
| "episode": 1024, | |
| "epoch": 0.004386566141192597, | |
| "eps": 6, | |
| "loss/policy_avg": 0.10823587328195572, | |
| "lr": 2.9882812500000002e-06, | |
| "objective/entropy": -40.33261489868164, | |
| "objective/kl": 0.8562099933624268, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 4.949611186981201, | |
| "objective/scores": 4.949611186981201, | |
| "policy/approxkl_avg": 0.09872549772262573, | |
| "policy/clipfrac_avg": 0.29296875, | |
| "policy/entropy_avg": 0.6909540891647339, | |
| "step": 1, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3988, | |
| "val/ratio": 1.0000126361846924, | |
| "val/ratio_var": 1.5850138197492925e-06 | |
| }, | |
| { | |
| "episode": 1536, | |
| "epoch": 0.006579849211788897, | |
| "eps": 6, | |
| "loss/policy_avg": 0.24130496382713318, | |
| "lr": 2.9765625e-06, | |
| "objective/entropy": -37.140586853027344, | |
| "objective/kl": 2.1461260318756104, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 5.433126449584961, | |
| "objective/scores": 5.433126449584961, | |
| "policy/approxkl_avg": 0.1151532381772995, | |
| "policy/clipfrac_avg": 0.30078125, | |
| "policy/entropy_avg": 0.6479897499084473, | |
| "step": 2, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2924, | |
| "val/ratio": 0.999896764755249, | |
| "val/ratio_var": 1.4944761232982273e-06 | |
| }, | |
| { | |
| "episode": 2048, | |
| "epoch": 0.008773132282385195, | |
| "eps": 6, | |
| "loss/policy_avg": 0.11201904714107513, | |
| "lr": 2.96484375e-06, | |
| "objective/entropy": -23.162431716918945, | |
| "objective/kl": 4.706851005554199, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 5.616954803466797, | |
| "objective/scores": 5.616954803466797, | |
| "policy/approxkl_avg": 0.13956432044506073, | |
| "policy/clipfrac_avg": 0.365234375, | |
| "policy/entropy_avg": 0.6664378643035889, | |
| "step": 3, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2146, | |
| "val/ratio": 0.9996535778045654, | |
| "val/ratio_var": 2.1856692455912707e-06 | |
| }, | |
| { | |
| "episode": 2560, | |
| "epoch": 0.010966415352981495, | |
| "eps": 6, | |
| "loss/policy_avg": 0.21055178344249725, | |
| "lr": 2.953125e-06, | |
| "objective/entropy": -17.498517990112305, | |
| "objective/kl": 6.893625259399414, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 5.975703716278076, | |
| "objective/scores": 5.975703716278076, | |
| "policy/approxkl_avg": 0.14695778489112854, | |
| "policy/clipfrac_avg": 0.33203125, | |
| "policy/entropy_avg": 0.6391204595565796, | |
| "step": 4, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1715, | |
| "val/ratio": 0.9999969005584717, | |
| "val/ratio_var": 2.4044477413553977e-06 | |
| }, | |
| { | |
| "episode": 3072, | |
| "epoch": 0.013159698423577794, | |
| "eps": 6, | |
| "loss/policy_avg": 0.28335219621658325, | |
| "lr": 2.94140625e-06, | |
| "objective/entropy": -16.186603546142578, | |
| "objective/kl": 8.215164184570312, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 6.314523696899414, | |
| "objective/scores": 6.314523696899414, | |
| "policy/approxkl_avg": 0.15406616032123566, | |
| "policy/clipfrac_avg": 0.34375, | |
| "policy/entropy_avg": 0.6234536170959473, | |
| "step": 5, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1743, | |
| "val/ratio": 1.0000813007354736, | |
| "val/ratio_var": 2.2491783511213725e-06 | |
| }, | |
| { | |
| "episode": 3584, | |
| "epoch": 0.015352981494174092, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2581750154495239, | |
| "lr": 2.9296875e-06, | |
| "objective/entropy": -16.82525634765625, | |
| "objective/kl": 10.012128829956055, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 6.68141508102417, | |
| "objective/scores": 6.68141508102417, | |
| "policy/approxkl_avg": 0.15727216005325317, | |
| "policy/clipfrac_avg": 0.306640625, | |
| "policy/entropy_avg": 0.6279575824737549, | |
| "step": 6, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2024, | |
| "val/ratio": 1.0000149011611938, | |
| "val/ratio_var": 3.910876330337487e-06 | |
| }, | |
| { | |
| "episode": 4096, | |
| "epoch": 0.01754626456477039, | |
| "eps": 6, | |
| "loss/policy_avg": 0.4508611857891083, | |
| "lr": 2.91796875e-06, | |
| "objective/entropy": -13.531968116760254, | |
| "objective/kl": 11.674339294433594, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 6.797825813293457, | |
| "objective/scores": 6.797825813293457, | |
| "policy/approxkl_avg": 0.17787906527519226, | |
| "policy/clipfrac_avg": 0.333984375, | |
| "policy/entropy_avg": 0.6140714287757874, | |
| "step": 7, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2248, | |
| "val/ratio": 0.9998712539672852, | |
| "val/ratio_var": 2.3673501345911063e-06 | |
| }, | |
| { | |
| "episode": 4608, | |
| "epoch": 0.01973954763536669, | |
| "eps": 6, | |
| "loss/policy_avg": 0.25109419226646423, | |
| "lr": 2.90625e-06, | |
| "objective/entropy": -15.348489761352539, | |
| "objective/kl": 12.88166332244873, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.097723484039307, | |
| "objective/scores": 7.097723484039307, | |
| "policy/approxkl_avg": 0.18050235509872437, | |
| "policy/clipfrac_avg": 0.302734375, | |
| "policy/entropy_avg": 0.6190727949142456, | |
| "step": 8, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2698, | |
| "val/ratio": 1.000298023223877, | |
| "val/ratio_var": 2.175341933252639e-06 | |
| }, | |
| { | |
| "episode": 5120, | |
| "epoch": 0.02193283070596299, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2478555142879486, | |
| "lr": 2.89453125e-06, | |
| "objective/entropy": -20.807220458984375, | |
| "objective/kl": 12.126293182373047, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.294116020202637, | |
| "objective/scores": 7.294116020202637, | |
| "policy/approxkl_avg": 0.16978076100349426, | |
| "policy/clipfrac_avg": 0.31640625, | |
| "policy/entropy_avg": 0.5916793346405029, | |
| "step": 9, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3166, | |
| "val/ratio": 0.999906063079834, | |
| "val/ratio_var": 2.3764121124258963e-06 | |
| }, | |
| { | |
| "episode": 5632, | |
| "epoch": 0.024126113776559287, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2194754183292389, | |
| "lr": 2.8828125e-06, | |
| "objective/entropy": -23.919815063476562, | |
| "objective/kl": 13.048431396484375, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.783618927001953, | |
| "objective/scores": 7.783618927001953, | |
| "policy/approxkl_avg": 0.1719370186328888, | |
| "policy/clipfrac_avg": 0.3359375, | |
| "policy/entropy_avg": 0.6023513078689575, | |
| "step": 10, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3395, | |
| "val/ratio": 0.9998449683189392, | |
| "val/ratio_var": 1.784112214409106e-06 | |
| }, | |
| { | |
| "episode": 6144, | |
| "epoch": 0.026319396847155587, | |
| "eps": 6, | |
| "loss/policy_avg": 0.13436169922351837, | |
| "lr": 2.87109375e-06, | |
| "objective/entropy": -21.395191192626953, | |
| "objective/kl": 15.173453330993652, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 7.682435512542725, | |
| "objective/scores": 7.682435512542725, | |
| "policy/approxkl_avg": 0.19314789772033691, | |
| "policy/clipfrac_avg": 0.369140625, | |
| "policy/entropy_avg": 0.6016405820846558, | |
| "step": 11, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3629, | |
| "val/ratio": 0.9996920824050903, | |
| "val/ratio_var": 1.8341333998250775e-06 | |
| }, | |
| { | |
| "episode": 6656, | |
| "epoch": 0.028512679917751884, | |
| "eps": 6, | |
| "loss/policy_avg": 0.27384719252586365, | |
| "lr": 2.859375e-06, | |
| "objective/entropy": -23.660961151123047, | |
| "objective/kl": 15.459722518920898, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.303886413574219, | |
| "objective/scores": 8.303886413574219, | |
| "policy/approxkl_avg": 0.1933055967092514, | |
| "policy/clipfrac_avg": 0.359375, | |
| "policy/entropy_avg": 0.5889922976493835, | |
| "step": 12, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3434, | |
| "val/ratio": 1.0000567436218262, | |
| "val/ratio_var": 2.589923496998381e-06 | |
| }, | |
| { | |
| "episode": 7168, | |
| "epoch": 0.030705962988348184, | |
| "eps": 6, | |
| "loss/policy_avg": 0.20046143233776093, | |
| "lr": 2.84765625e-06, | |
| "objective/entropy": -24.642715454101562, | |
| "objective/kl": 17.541698455810547, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.798042297363281, | |
| "objective/scores": 8.798042297363281, | |
| "policy/approxkl_avg": 0.1510610282421112, | |
| "policy/clipfrac_avg": 0.337890625, | |
| "policy/entropy_avg": 0.552618145942688, | |
| "step": 13, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 3045, | |
| "val/ratio": 1.0000343322753906, | |
| "val/ratio_var": 2.4570517780375667e-06 | |
| }, | |
| { | |
| "episode": 7680, | |
| "epoch": 0.03289924605894448, | |
| "eps": 6, | |
| "loss/policy_avg": 0.22966830432415009, | |
| "lr": 2.8359375e-06, | |
| "objective/entropy": -19.635862350463867, | |
| "objective/kl": 18.844629287719727, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.394830703735352, | |
| "objective/scores": 8.394830703735352, | |
| "policy/approxkl_avg": 0.1538720726966858, | |
| "policy/clipfrac_avg": 0.3359375, | |
| "policy/entropy_avg": 0.5296885371208191, | |
| "step": 14, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2402, | |
| "val/ratio": 1.0000078678131104, | |
| "val/ratio_var": 2.192526608268963e-06 | |
| }, | |
| { | |
| "episode": 8192, | |
| "epoch": 0.03509252912954078, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2523574233055115, | |
| "lr": 2.82421875e-06, | |
| "objective/entropy": -19.664382934570312, | |
| "objective/kl": 19.9348201751709, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.683082580566406, | |
| "objective/scores": 8.683082580566406, | |
| "policy/approxkl_avg": 0.216833233833313, | |
| "policy/clipfrac_avg": 0.34375, | |
| "policy/entropy_avg": 0.5138433575630188, | |
| "step": 15, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 2232, | |
| "val/ratio": 0.9998021721839905, | |
| "val/ratio_var": 1.7273125649808208e-06 | |
| }, | |
| { | |
| "episode": 8704, | |
| "epoch": 0.03728581220013708, | |
| "eps": 6, | |
| "loss/policy_avg": 0.1616867184638977, | |
| "lr": 2.8125e-06, | |
| "objective/entropy": -19.297401428222656, | |
| "objective/kl": 20.52892303466797, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.915764808654785, | |
| "objective/scores": 8.915764808654785, | |
| "policy/approxkl_avg": 0.1600520759820938, | |
| "policy/clipfrac_avg": 0.318359375, | |
| "policy/entropy_avg": 0.47873735427856445, | |
| "step": 16, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1931, | |
| "val/ratio": 0.9997690916061401, | |
| "val/ratio_var": 2.2947442630538717e-06 | |
| }, | |
| { | |
| "episode": 9216, | |
| "epoch": 0.03947909527073338, | |
| "eps": 6, | |
| "loss/policy_avg": 0.163500115275383, | |
| "lr": 2.80078125e-06, | |
| "objective/entropy": -16.193389892578125, | |
| "objective/kl": 22.477802276611328, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.87047290802002, | |
| "objective/scores": 8.87047290802002, | |
| "policy/approxkl_avg": 0.17684966325759888, | |
| "policy/clipfrac_avg": 0.33984375, | |
| "policy/entropy_avg": 0.4807279706001282, | |
| "step": 17, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1798, | |
| "val/ratio": 0.9998293519020081, | |
| "val/ratio_var": 2.4208854938478908e-06 | |
| }, | |
| { | |
| "episode": 9728, | |
| "epoch": 0.041672378341329676, | |
| "eps": 6, | |
| "loss/policy_avg": 0.19272606074810028, | |
| "lr": 2.7890625e-06, | |
| "objective/entropy": -12.483476638793945, | |
| "objective/kl": 24.110105514526367, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.77183723449707, | |
| "objective/scores": 8.77183723449707, | |
| "policy/approxkl_avg": 0.1895884871482849, | |
| "policy/clipfrac_avg": 0.36328125, | |
| "policy/entropy_avg": 0.4732131361961365, | |
| "step": 18, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1580, | |
| "val/ratio": 1.0000896453857422, | |
| "val/ratio_var": 2.6988111585524166e-06 | |
| }, | |
| { | |
| "episode": 10240, | |
| "epoch": 0.04386566141192598, | |
| "eps": 6, | |
| "loss/policy_avg": 0.31339162588119507, | |
| "lr": 2.77734375e-06, | |
| "objective/entropy": -14.506279945373535, | |
| "objective/kl": 23.571977615356445, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.617586135864258, | |
| "objective/scores": 8.617586135864258, | |
| "policy/approxkl_avg": 0.18291257321834564, | |
| "policy/clipfrac_avg": 0.349609375, | |
| "policy/entropy_avg": 0.4734336733818054, | |
| "step": 19, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1565, | |
| "val/ratio": 0.9999023675918579, | |
| "val/ratio_var": 2.692267571546836e-06 | |
| }, | |
| { | |
| "episode": 10752, | |
| "epoch": 0.04605894448252228, | |
| "eps": 6, | |
| "loss/policy_avg": 0.4666588306427002, | |
| "lr": 2.765625e-06, | |
| "objective/entropy": -12.953289985656738, | |
| "objective/kl": 24.32935333251953, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 8.778571128845215, | |
| "objective/scores": 8.778571128845215, | |
| "policy/approxkl_avg": 0.20027068257331848, | |
| "policy/clipfrac_avg": 0.30859375, | |
| "policy/entropy_avg": 0.4659819006919861, | |
| "step": 20, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1506, | |
| "val/ratio": 1.0001583099365234, | |
| "val/ratio_var": 3.233054258089396e-06 | |
| }, | |
| { | |
| "episode": 11264, | |
| "epoch": 0.048252227553118573, | |
| "eps": 6, | |
| "loss/policy_avg": 0.27946847677230835, | |
| "lr": 2.75390625e-06, | |
| "objective/entropy": -11.48613166809082, | |
| "objective/kl": 24.395483016967773, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.02904224395752, | |
| "objective/scores": 9.02904224395752, | |
| "policy/approxkl_avg": 0.18909631669521332, | |
| "policy/clipfrac_avg": 0.333984375, | |
| "policy/entropy_avg": 0.48570871353149414, | |
| "step": 21, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1560, | |
| "val/ratio": 1.0002957582473755, | |
| "val/ratio_var": 2.959979838124127e-06 | |
| }, | |
| { | |
| "episode": 11776, | |
| "epoch": 0.05044551062371487, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2717439830303192, | |
| "lr": 2.7421875e-06, | |
| "objective/entropy": -9.29305362701416, | |
| "objective/kl": 26.03940200805664, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.028656005859375, | |
| "objective/scores": 9.028656005859375, | |
| "policy/approxkl_avg": 0.1726618856191635, | |
| "policy/clipfrac_avg": 0.34765625, | |
| "policy/entropy_avg": 0.5084494352340698, | |
| "step": 22, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1547, | |
| "val/ratio": 1.0000401735305786, | |
| "val/ratio_var": 3.0132259780657478e-06 | |
| }, | |
| { | |
| "episode": 12288, | |
| "epoch": 0.052638793694311174, | |
| "eps": 6, | |
| "loss/policy_avg": 0.19815856218338013, | |
| "lr": 2.73046875e-06, | |
| "objective/entropy": -13.52733039855957, | |
| "objective/kl": 25.79766273498535, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.250341415405273, | |
| "objective/scores": 9.250341415405273, | |
| "policy/approxkl_avg": 0.1738530993461609, | |
| "policy/clipfrac_avg": 0.3125, | |
| "policy/entropy_avg": 0.4977012574672699, | |
| "step": 23, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1648, | |
| "val/ratio": 0.9999476671218872, | |
| "val/ratio_var": 2.497435616533039e-06 | |
| }, | |
| { | |
| "episode": 12800, | |
| "epoch": 0.05483207676490747, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2345161885023117, | |
| "lr": 2.71875e-06, | |
| "objective/entropy": -9.971322059631348, | |
| "objective/kl": 26.595800399780273, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.097506523132324, | |
| "objective/scores": 9.097506523132324, | |
| "policy/approxkl_avg": 0.18735162913799286, | |
| "policy/clipfrac_avg": 0.33203125, | |
| "policy/entropy_avg": 0.4894530177116394, | |
| "step": 24, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1580, | |
| "val/ratio": 1.000110149383545, | |
| "val/ratio_var": 2.1261726033117156e-06 | |
| }, | |
| { | |
| "episode": 13312, | |
| "epoch": 0.05702535983550377, | |
| "eps": 6, | |
| "loss/policy_avg": 0.2765505909919739, | |
| "lr": 2.70703125e-06, | |
| "objective/entropy": -8.664510726928711, | |
| "objective/kl": 27.207202911376953, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.248998641967773, | |
| "objective/scores": 9.248998641967773, | |
| "policy/approxkl_avg": 0.21922221779823303, | |
| "policy/clipfrac_avg": 0.3671875, | |
| "policy/entropy_avg": 0.4936632513999939, | |
| "step": 25, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1689, | |
| "val/ratio": 0.9997528195381165, | |
| "val/ratio_var": 2.2092231120041106e-06 | |
| }, | |
| { | |
| "episode": 13824, | |
| "epoch": 0.05921864290610007, | |
| "eps": 6, | |
| "loss/policy_avg": 0.20894497632980347, | |
| "lr": 2.6953125e-06, | |
| "objective/entropy": -13.794546127319336, | |
| "objective/kl": 27.585330963134766, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.134411811828613, | |
| "objective/scores": 9.134411811828613, | |
| "policy/approxkl_avg": 0.19758901000022888, | |
| "policy/clipfrac_avg": 0.337890625, | |
| "policy/entropy_avg": 0.4570607841014862, | |
| "step": 26, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1664, | |
| "val/ratio": 0.999937117099762, | |
| "val/ratio_var": 3.2459099656989565e-06 | |
| }, | |
| { | |
| "episode": 14336, | |
| "epoch": 0.06141192597669637, | |
| "eps": 6, | |
| "loss/policy_avg": 0.187041237950325, | |
| "lr": 2.68359375e-06, | |
| "objective/entropy": -18.817930221557617, | |
| "objective/kl": 26.306095123291016, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.336585998535156, | |
| "objective/scores": 9.336585998535156, | |
| "policy/approxkl_avg": 0.20577886700630188, | |
| "policy/clipfrac_avg": 0.279296875, | |
| "policy/entropy_avg": 0.4518057107925415, | |
| "step": 27, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1731, | |
| "val/ratio": 1.0001513957977295, | |
| "val/ratio_var": 2.708766032810672e-06 | |
| }, | |
| { | |
| "episode": 14848, | |
| "epoch": 0.06360520904729267, | |
| "eps": 6, | |
| "loss/policy_avg": 0.15964296460151672, | |
| "lr": 2.671875e-06, | |
| "objective/entropy": -19.800785064697266, | |
| "objective/kl": 27.51376724243164, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.47825813293457, | |
| "objective/scores": 9.47825813293457, | |
| "policy/approxkl_avg": 0.19460758566856384, | |
| "policy/clipfrac_avg": 0.34765625, | |
| "policy/entropy_avg": 0.4188844859600067, | |
| "step": 28, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1687, | |
| "val/ratio": 0.9997619390487671, | |
| "val/ratio_var": 2.6442203306942247e-06 | |
| }, | |
| { | |
| "episode": 15360, | |
| "epoch": 0.06579849211788896, | |
| "eps": 6, | |
| "loss/policy_avg": 0.15885861217975616, | |
| "lr": 2.66015625e-06, | |
| "objective/entropy": -17.79833221435547, | |
| "objective/kl": 27.949443817138672, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.496625900268555, | |
| "objective/scores": 9.496625900268555, | |
| "policy/approxkl_avg": 0.16567930579185486, | |
| "policy/clipfrac_avg": 0.34765625, | |
| "policy/entropy_avg": 0.4135560691356659, | |
| "step": 29, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1689, | |
| "val/ratio": 1.0000743865966797, | |
| "val/ratio_var": 3.2084180929814465e-06 | |
| }, | |
| { | |
| "episode": 15872, | |
| "epoch": 0.06799177518848526, | |
| "eps": 6, | |
| "loss/policy_avg": 0.16605864465236664, | |
| "lr": 2.6484375e-06, | |
| "objective/entropy": -18.510845184326172, | |
| "objective/kl": 29.722782135009766, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.869793891906738, | |
| "objective/scores": 9.869793891906738, | |
| "policy/approxkl_avg": 0.1928042322397232, | |
| "policy/clipfrac_avg": 0.421875, | |
| "policy/entropy_avg": 0.3933394253253937, | |
| "step": 30, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1600, | |
| "val/ratio": 1.0000554323196411, | |
| "val/ratio_var": 2.9572597668447997e-06 | |
| }, | |
| { | |
| "episode": 16384, | |
| "epoch": 0.07018505825908156, | |
| "eps": 6, | |
| "loss/policy_avg": 0.19288870692253113, | |
| "lr": 2.63671875e-06, | |
| "objective/entropy": -17.214672088623047, | |
| "objective/kl": 30.913219451904297, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.56223201751709, | |
| "objective/scores": 9.56223201751709, | |
| "policy/approxkl_avg": 0.1885090172290802, | |
| "policy/clipfrac_avg": 0.345703125, | |
| "policy/entropy_avg": 0.3990550935268402, | |
| "step": 31, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1621, | |
| "val/ratio": 0.9998235106468201, | |
| "val/ratio_var": 2.50669563683914e-06 | |
| }, | |
| { | |
| "episode": 16896, | |
| "epoch": 0.07237834132967787, | |
| "eps": 6, | |
| "loss/policy_avg": 0.23322975635528564, | |
| "lr": 2.6250000000000003e-06, | |
| "objective/entropy": -18.789386749267578, | |
| "objective/kl": 30.845539093017578, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.760330200195312, | |
| "objective/scores": 9.760330200195312, | |
| "policy/approxkl_avg": 0.17573946714401245, | |
| "policy/clipfrac_avg": 0.30859375, | |
| "policy/entropy_avg": 0.38172662258148193, | |
| "step": 32, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1705, | |
| "val/ratio": 1.0002737045288086, | |
| "val/ratio_var": 2.0741492789966287e-06 | |
| }, | |
| { | |
| "episode": 17408, | |
| "epoch": 0.07457162440027416, | |
| "eps": 6, | |
| "loss/policy_avg": 0.21965008974075317, | |
| "lr": 2.61328125e-06, | |
| "objective/entropy": -14.802291870117188, | |
| "objective/kl": 32.42792510986328, | |
| "objective/non_score_reward": 0.0, | |
| "objective/rlhf_reward": 9.586631774902344, | |
| "objective/scores": 9.586631774902344, | |
| "policy/approxkl_avg": 0.22661904990673065, | |
| "policy/clipfrac_avg": 0.36328125, | |
| "policy/entropy_avg": 0.40180861949920654, | |
| "step": 33, | |
| "val/clipfrac_avg": 0.0, | |
| "val/num_eos_tokens": 1797, | |
| "val/ratio": 1.0000510215759277, | |
| "val/ratio_var": 2.326285539311357e-06 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 128, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1.122960932145305, | |
| "save_steps": 2, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": true, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0, | |
| "train_batch_size": null, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |