| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.13381506757660913, | |
| "eval_steps": 500, | |
| "global_step": 250, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 281.1666851043701, | |
| "epoch": 0.0005352602703064365, | |
| "grad_norm": 0.5917035937309265, | |
| "kl": 0.0, | |
| "learning_rate": 0.0, | |
| "loss": -0.0, | |
| "reward": -0.10487502068281174, | |
| "reward_std": 0.644918380305171, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.1041666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.37570834904909134, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 590.8750171661377, | |
| "epoch": 0.001070520540612873, | |
| "grad_norm": 0.7477704286575317, | |
| "kl": 0.0, | |
| "learning_rate": 2.0000000000000002e-07, | |
| "loss": 0.0, | |
| "reward": 0.22162500163540244, | |
| "reward_std": 0.09485530573874712, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.09662500163540244, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 539.6666870117188, | |
| "epoch": 0.0016057808109193096, | |
| "grad_norm": 0.4429571032524109, | |
| "kl": 0.002077269156870898, | |
| "learning_rate": 4.0000000000000003e-07, | |
| "loss": 0.0001, | |
| "reward": 0.016208335757255554, | |
| "reward_std": 0.6246479228138924, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.2083333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.1921250014565885, | |
| "step": 3 | |
| }, | |
| { | |
| "completion_length": 185.7916717529297, | |
| "epoch": 0.002141041081225746, | |
| "grad_norm": 0.6855182647705078, | |
| "kl": 0.0009879921708488837, | |
| "learning_rate": 6.000000000000001e-07, | |
| "loss": 0.0, | |
| "reward": 1.0669583305716515, | |
| "reward_std": 0.5203845072537661, | |
| "rewards/correctness_reward_func": 0.7500000298023224, | |
| "rewards/int_reward_func": 0.3541666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.03720833268016577, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 587.3750076293945, | |
| "epoch": 0.0026763013515321826, | |
| "grad_norm": 0.7880218029022217, | |
| "kl": 0.0007925744503154419, | |
| "learning_rate": 8.000000000000001e-07, | |
| "loss": 0.0, | |
| "reward": 0.09533333079889417, | |
| "reward_std": 0.5927679911255836, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.1041666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.09216667525470257, | |
| "step": 5 | |
| }, | |
| { | |
| "completion_length": 245.70834159851074, | |
| "epoch": 0.003211561621838619, | |
| "grad_norm": 0.9220851063728333, | |
| "kl": 0.001330614773905836, | |
| "learning_rate": 1.0000000000000002e-06, | |
| "loss": 0.0001, | |
| "reward": 0.08295834437012672, | |
| "reward_std": 0.5829638005234301, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.0416666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.04204164445400238, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 448.7083549499512, | |
| "epoch": 0.0037468218921450553, | |
| "grad_norm": 0.8303655385971069, | |
| "kl": 0.0012542481999844313, | |
| "learning_rate": 1.2000000000000002e-06, | |
| "loss": 0.0001, | |
| "reward": 0.03099999949336052, | |
| "reward_std": 0.7598095312714577, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.16666666977107525, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.13566668145358562, | |
| "step": 7 | |
| }, | |
| { | |
| "completion_length": 197.95833587646484, | |
| "epoch": 0.004282082162451492, | |
| "grad_norm": 1.0582274198532104, | |
| "kl": 0.0008733256690902635, | |
| "learning_rate": 1.4000000000000001e-06, | |
| "loss": 0.0, | |
| "reward": 0.12341666966676712, | |
| "reward_std": 0.09142086654901505, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.0, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.12341666780412197, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 176.41666984558105, | |
| "epoch": 0.004817342432757929, | |
| "grad_norm": 1.0369133949279785, | |
| "kl": 0.005098502180771902, | |
| "learning_rate": 1.6000000000000001e-06, | |
| "loss": 0.0002, | |
| "reward": 0.32220835238695145, | |
| "reward_std": 0.3896215371787548, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.11387500539422035, | |
| "step": 9 | |
| }, | |
| { | |
| "completion_length": 267.66667556762695, | |
| "epoch": 0.005352602703064365, | |
| "grad_norm": 0.4552709460258484, | |
| "kl": 0.001112774269131478, | |
| "learning_rate": 1.8000000000000001e-06, | |
| "loss": 0.0, | |
| "reward": 0.31854166090488434, | |
| "reward_std": 0.3717608004808426, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.1250000037252903, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.02687499998137355, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 319.58333587646484, | |
| "epoch": 0.005887862973370801, | |
| "grad_norm": 0.7992357015609741, | |
| "kl": 0.0012863876472692937, | |
| "learning_rate": 2.0000000000000003e-06, | |
| "loss": 0.0001, | |
| "reward": 0.2948333490639925, | |
| "reward_std": 0.38963131979107857, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.10416666977107525, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.10733332857489586, | |
| "step": 11 | |
| }, | |
| { | |
| "completion_length": 375.29166984558105, | |
| "epoch": 0.006423123243677238, | |
| "grad_norm": 0.965032160282135, | |
| "kl": 0.0008803782802715432, | |
| "learning_rate": 2.2e-06, | |
| "loss": 0.0, | |
| "reward": 1.012750007212162, | |
| "reward_std": 0.5616761147975922, | |
| "rewards/correctness_reward_func": 0.8333333432674408, | |
| "rewards/int_reward_func": 0.22916666977107525, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.04975000023841858, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 482.4583435058594, | |
| "epoch": 0.0069583835139836745, | |
| "grad_norm": 1.502334713935852, | |
| "kl": 0.0028791724907932803, | |
| "learning_rate": 2.4000000000000003e-06, | |
| "loss": 0.0001, | |
| "reward": -0.32758333161473274, | |
| "reward_std": 0.7293146029114723, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.02083333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.34841667115688324, | |
| "step": 13 | |
| }, | |
| { | |
| "completion_length": 236.41667366027832, | |
| "epoch": 0.007493643784290111, | |
| "grad_norm": 0.8761454820632935, | |
| "kl": 0.0018893379892688245, | |
| "learning_rate": 2.6e-06, | |
| "loss": 0.0001, | |
| "reward": -0.017458327114582062, | |
| "reward_std": 0.31026666425168514, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.06250000186264515, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.07995832804590464, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 306.6250104904175, | |
| "epoch": 0.008028904054596548, | |
| "grad_norm": 1.070983648300171, | |
| "kl": 0.0017425262776669115, | |
| "learning_rate": 2.8000000000000003e-06, | |
| "loss": 0.0001, | |
| "reward": -0.08808333426713943, | |
| "reward_std": 0.37953382171690464, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.0, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.08808333426713943, | |
| "step": 15 | |
| }, | |
| { | |
| "completion_length": 366.6666793823242, | |
| "epoch": 0.008564164324902984, | |
| "grad_norm": 0.6342064738273621, | |
| "kl": 0.0009713478648336604, | |
| "learning_rate": 3e-06, | |
| "loss": 0.0, | |
| "reward": 0.03991668112576008, | |
| "reward_std": 0.8613052181899548, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.0416666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.16841666959226131, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 297.75000381469727, | |
| "epoch": 0.00909942459520942, | |
| "grad_norm": 0.629483163356781, | |
| "kl": 0.0008868449804140255, | |
| "learning_rate": 3.2000000000000003e-06, | |
| "loss": 0.0, | |
| "reward": 1.0445833504199982, | |
| "reward_std": 0.3152644243091345, | |
| "rewards/correctness_reward_func": 0.6666666716337204, | |
| "rewards/int_reward_func": 0.3333333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.044583337381482124, | |
| "step": 17 | |
| }, | |
| { | |
| "completion_length": 185.79166793823242, | |
| "epoch": 0.009634684865515858, | |
| "grad_norm": 1.0232924222946167, | |
| "kl": 0.0012610588310053572, | |
| "learning_rate": 3.4000000000000005e-06, | |
| "loss": 0.0001, | |
| "reward": 0.21658334136009216, | |
| "reward_std": 0.18211832642555237, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.0625, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.15408333763480186, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 361.5416679382324, | |
| "epoch": 0.010169945135822294, | |
| "grad_norm": 0.8113482594490051, | |
| "kl": 0.0015902465383987874, | |
| "learning_rate": 3.6000000000000003e-06, | |
| "loss": 0.0001, | |
| "reward": 0.44666668586432934, | |
| "reward_std": 0.5656739473342896, | |
| "rewards/correctness_reward_func": 0.2500000074505806, | |
| "rewards/int_reward_func": 0.18750000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.009166665724478662, | |
| "step": 19 | |
| }, | |
| { | |
| "completion_length": 254.66666984558105, | |
| "epoch": 0.01070520540612873, | |
| "grad_norm": 1.1768397092819214, | |
| "kl": 0.004655150449252687, | |
| "learning_rate": 3.8000000000000005e-06, | |
| "loss": 0.0002, | |
| "reward": 0.48504166305065155, | |
| "reward_std": 0.4368314128369093, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.0833333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.06837500259280205, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 137.91667366027832, | |
| "epoch": 0.011240465676435166, | |
| "grad_norm": 1.0621542930603027, | |
| "kl": 0.0022582018573302776, | |
| "learning_rate": 4.000000000000001e-06, | |
| "loss": 0.0001, | |
| "reward": 0.3660416714847088, | |
| "reward_std": 0.5700555201619864, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.1875, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.011874992400407791, | |
| "step": 21 | |
| }, | |
| { | |
| "completion_length": 88.91667175292969, | |
| "epoch": 0.011775725946741603, | |
| "grad_norm": 1.2175222635269165, | |
| "kl": 0.0020840048528043553, | |
| "learning_rate": 4.2000000000000004e-06, | |
| "loss": 0.0001, | |
| "reward": 0.8160417033359408, | |
| "reward_std": 0.26041645370423794, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.14937500189989805, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 150.2083339691162, | |
| "epoch": 0.01231098621704804, | |
| "grad_norm": 0.8459586501121521, | |
| "kl": 0.0017513818893348798, | |
| "learning_rate": 4.4e-06, | |
| "loss": 0.0001, | |
| "reward": 0.5479583460837603, | |
| "reward_std": 0.6487118303775787, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.2916666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.0062916697934269905, | |
| "step": 23 | |
| }, | |
| { | |
| "completion_length": 154.87500381469727, | |
| "epoch": 0.012846246487354477, | |
| "grad_norm": 0.8675287961959839, | |
| "kl": 0.001371489226585254, | |
| "learning_rate": 4.600000000000001e-06, | |
| "loss": 0.0001, | |
| "reward": 0.17075001262128353, | |
| "reward_std": 0.19458706118166447, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.14583333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.024916673079133034, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 150.7500057220459, | |
| "epoch": 0.013381506757660913, | |
| "grad_norm": 0.8717443943023682, | |
| "kl": 0.0020001856610178947, | |
| "learning_rate": 4.800000000000001e-06, | |
| "loss": 0.0001, | |
| "reward": 0.20204169023782015, | |
| "reward_std": 0.4490640014410019, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.06250000186264515, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.027124996297061443, | |
| "step": 25 | |
| }, | |
| { | |
| "completion_length": 467.9583568572998, | |
| "epoch": 0.013916767027967349, | |
| "grad_norm": 1.1116266250610352, | |
| "kl": 0.0012415697274263948, | |
| "learning_rate": 5e-06, | |
| "loss": 0.0, | |
| "reward": 0.304541677236557, | |
| "reward_std": 0.28451096825301647, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.14583333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.07537499908357859, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 433.5833339691162, | |
| "epoch": 0.014452027298273785, | |
| "grad_norm": 0.8203855156898499, | |
| "kl": 0.001017560571199283, | |
| "learning_rate": 4.999756310023261e-06, | |
| "loss": 0.0, | |
| "reward": 0.2795000094920397, | |
| "reward_std": 0.11828233953565359, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.14583333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.13366666994988918, | |
| "step": 27 | |
| }, | |
| { | |
| "completion_length": 126.00000381469727, | |
| "epoch": 0.014987287568580221, | |
| "grad_norm": 1.2663164138793945, | |
| "kl": 0.0014343319344334304, | |
| "learning_rate": 4.999025287600886e-06, | |
| "loss": 0.0001, | |
| "reward": 0.6117500364780426, | |
| "reward_std": 0.4831337593495846, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.2083333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.2367500104010105, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 229.66667556762695, | |
| "epoch": 0.01552254783888666, | |
| "grad_norm": 0.8950901627540588, | |
| "kl": 0.0015667550032958388, | |
| "learning_rate": 4.997807075247147e-06, | |
| "loss": 0.0001, | |
| "reward": 0.1262916720006615, | |
| "reward_std": 0.148749228566885, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.02083333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.1054583361838013, | |
| "step": 29 | |
| }, | |
| { | |
| "completion_length": 210.8750114440918, | |
| "epoch": 0.016057808109193095, | |
| "grad_norm": 1.1234443187713623, | |
| "kl": 0.0019746975012822077, | |
| "learning_rate": 4.996101910454953e-06, | |
| "loss": 0.0001, | |
| "reward": 0.3709999993443489, | |
| "reward_std": 0.3687104620039463, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.20833333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.0793333351612091, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 177.62500190734863, | |
| "epoch": 0.01659306837949953, | |
| "grad_norm": 0.9238327145576477, | |
| "kl": 0.0028813415410695598, | |
| "learning_rate": 4.993910125649561e-06, | |
| "loss": 0.0001, | |
| "reward": 0.39008335024118423, | |
| "reward_std": 0.3615882135927677, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.27083333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.11925000231713057, | |
| "step": 31 | |
| }, | |
| { | |
| "completion_length": 173.79167366027832, | |
| "epoch": 0.017128328649805968, | |
| "grad_norm": 1.0448777675628662, | |
| "kl": 0.006160023040138185, | |
| "learning_rate": 4.9912321481237616e-06, | |
| "loss": 0.0002, | |
| "reward": 0.14554167166352272, | |
| "reward_std": 0.16822291910648346, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.0416666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.10387500375509262, | |
| "step": 32 | |
| }, | |
| { | |
| "completion_length": 486.75000762939453, | |
| "epoch": 0.017663588920112404, | |
| "grad_norm": 0.6601158380508423, | |
| "kl": 0.001103398812119849, | |
| "learning_rate": 4.988068499954578e-06, | |
| "loss": 0.0, | |
| "reward": -0.5304166711866856, | |
| "reward_std": 1.0785409808158875, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.06250000186264515, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.6762500237673521, | |
| "step": 33 | |
| }, | |
| { | |
| "completion_length": 268.12501525878906, | |
| "epoch": 0.01819884919041884, | |
| "grad_norm": 0.910798966884613, | |
| "kl": 0.0013230827316874638, | |
| "learning_rate": 4.984419797901491e-06, | |
| "loss": 0.0001, | |
| "reward": 0.6703750789165497, | |
| "reward_std": 0.36514274775981903, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.12500000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.1287083402276039, | |
| "step": 34 | |
| }, | |
| { | |
| "completion_length": 267.708345413208, | |
| "epoch": 0.018734109460725276, | |
| "grad_norm": 1.027286171913147, | |
| "kl": 0.004269710392691195, | |
| "learning_rate": 4.980286753286196e-06, | |
| "loss": 0.0002, | |
| "reward": 0.4232500046491623, | |
| "reward_std": 0.2645573355257511, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.27083334140479565, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.1524166688323021, | |
| "step": 35 | |
| }, | |
| { | |
| "completion_length": 245.8750114440918, | |
| "epoch": 0.019269369731031716, | |
| "grad_norm": 1.0046695470809937, | |
| "kl": 0.005482323234900832, | |
| "learning_rate": 4.975670171853926e-06, | |
| "loss": 0.0002, | |
| "reward": 0.2149583324790001, | |
| "reward_std": 0.7910580635070801, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.2083333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.07670832984149456, | |
| "step": 36 | |
| }, | |
| { | |
| "completion_length": 560.7916870117188, | |
| "epoch": 0.019804630001338152, | |
| "grad_norm": 0.8885159492492676, | |
| "kl": 0.002885772308218293, | |
| "learning_rate": 4.970570953616383e-06, | |
| "loss": 0.0001, | |
| "reward": 0.1944583347067237, | |
| "reward_std": 0.05626108031719923, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.0, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.1944583347067237, | |
| "step": 37 | |
| }, | |
| { | |
| "completion_length": 522.8750133514404, | |
| "epoch": 0.020339890271644588, | |
| "grad_norm": 0.5502146482467651, | |
| "kl": 0.009517412836430594, | |
| "learning_rate": 4.964990092676263e-06, | |
| "loss": 0.0004, | |
| "reward": 0.2919999957084656, | |
| "reward_std": 0.674026682972908, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.08299997518770397, | |
| "step": 38 | |
| }, | |
| { | |
| "completion_length": 640.2083511352539, | |
| "epoch": 0.020875150541951024, | |
| "grad_norm": 0.7062350511550903, | |
| "kl": 0.0034277847153134644, | |
| "learning_rate": 4.958928677033465e-06, | |
| "loss": 0.0001, | |
| "reward": -0.004291646182537079, | |
| "reward_std": 0.7502853199839592, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.1250000037252903, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.21262501180171967, | |
| "step": 39 | |
| }, | |
| { | |
| "completion_length": 491.08335876464844, | |
| "epoch": 0.02141041081225746, | |
| "grad_norm": 0.7743443250656128, | |
| "kl": 0.004902548622339964, | |
| "learning_rate": 4.9523878883729794e-06, | |
| "loss": 0.0002, | |
| "reward": 0.06212499737739563, | |
| "reward_std": 0.8643132671713829, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.1458333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.1670416765846312, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 301.45834159851074, | |
| "epoch": 0.021945671082563897, | |
| "grad_norm": 0.7774906754493713, | |
| "kl": 0.007360402669291943, | |
| "learning_rate": 4.9453690018345144e-06, | |
| "loss": 0.0003, | |
| "reward": -0.09349998086690903, | |
| "reward_std": 0.7777874618768692, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.1041666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": -0.2810000069439411, | |
| "step": 41 | |
| }, | |
| { | |
| "completion_length": 455.5833435058594, | |
| "epoch": 0.022480931352870333, | |
| "grad_norm": 0.6825084090232849, | |
| "kl": 0.0033266296959482133, | |
| "learning_rate": 4.937873385763909e-06, | |
| "loss": 0.0001, | |
| "reward": 0.2603750079870224, | |
| "reward_std": 1.1693618446588516, | |
| "rewards/correctness_reward_func": 0.2500000074505806, | |
| "rewards/int_reward_func": 0.1458333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.02083333395421505, | |
| "rewards/xmlcount_reward_func": -0.15629167575389147, | |
| "step": 42 | |
| }, | |
| { | |
| "completion_length": 254.7500057220459, | |
| "epoch": 0.02301619162317677, | |
| "grad_norm": 1.0424985885620117, | |
| "kl": 0.013458715460728854, | |
| "learning_rate": 4.9299025014463665e-06, | |
| "loss": 0.0005, | |
| "reward": 0.6490416824817657, | |
| "reward_std": 0.6297374591231346, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.3125000111758709, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.16987500933464617, | |
| "step": 43 | |
| }, | |
| { | |
| "completion_length": 243.8333339691162, | |
| "epoch": 0.023551451893483205, | |
| "grad_norm": 0.6618691682815552, | |
| "kl": 0.013514326536096632, | |
| "learning_rate": 4.921457902821578e-06, | |
| "loss": 0.0005, | |
| "reward": 0.235000004991889, | |
| "reward_std": 0.4793561212718487, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.12500000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.02666667103767395, | |
| "step": 44 | |
| }, | |
| { | |
| "completion_length": 247.1250057220459, | |
| "epoch": 0.02408671216378964, | |
| "grad_norm": 0.8150098919868469, | |
| "kl": 0.006718623684719205, | |
| "learning_rate": 4.912541236180779e-06, | |
| "loss": 0.0003, | |
| "reward": 0.3942916840314865, | |
| "reward_std": 0.4090446010231972, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.1458333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.08179166866466403, | |
| "step": 45 | |
| }, | |
| { | |
| "completion_length": 311.95833587646484, | |
| "epoch": 0.02462197243409608, | |
| "grad_norm": 0.9185469746589661, | |
| "kl": 0.011615818890277296, | |
| "learning_rate": 4.903154239845798e-06, | |
| "loss": 0.0005, | |
| "reward": 0.6513333357870579, | |
| "reward_std": 0.26453326642513275, | |
| "rewards/correctness_reward_func": 0.5, | |
| "rewards/int_reward_func": 0.14583333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.005500006955116987, | |
| "step": 46 | |
| }, | |
| { | |
| "completion_length": 347.25000762939453, | |
| "epoch": 0.025157232704402517, | |
| "grad_norm": 0.9208869338035583, | |
| "kl": 0.019979659002274275, | |
| "learning_rate": 4.893298743830168e-06, | |
| "loss": 0.0008, | |
| "reward": 0.4205416589975357, | |
| "reward_std": 0.4555768258869648, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.0625, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.108041662722826, | |
| "step": 47 | |
| }, | |
| { | |
| "completion_length": 245.6666717529297, | |
| "epoch": 0.025692492974708953, | |
| "grad_norm": 0.8743010759353638, | |
| "kl": 0.01431413902901113, | |
| "learning_rate": 4.882976669482368e-06, | |
| "loss": 0.0006, | |
| "reward": 0.269333329051733, | |
| "reward_std": 0.6685181586071849, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.06099999323487282, | |
| "step": 48 | |
| }, | |
| { | |
| "completion_length": 431.12501525878906, | |
| "epoch": 0.02622775324501539, | |
| "grad_norm": 0.6829207539558411, | |
| "kl": 0.010582708870060742, | |
| "learning_rate": 4.8721900291112415e-06, | |
| "loss": 0.0004, | |
| "reward": 1.1573750227689743, | |
| "reward_std": 0.5191336497664452, | |
| "rewards/correctness_reward_func": 0.75, | |
| "rewards/int_reward_func": 0.27083333767950535, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.1365416720509529, | |
| "step": 49 | |
| }, | |
| { | |
| "completion_length": 362.2916793823242, | |
| "epoch": 0.026763013515321826, | |
| "grad_norm": 0.9591223001480103, | |
| "kl": 0.021306635811924934, | |
| "learning_rate": 4.860940925593703e-06, | |
| "loss": 0.0009, | |
| "reward": 0.5036666616797447, | |
| "reward_std": 0.6060219556093216, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.16666666977107525, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.06250000186264515, | |
| "rewards/xmlcount_reward_func": 0.19116667530033737, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 105.25, | |
| "epoch": 0.02729827378562826, | |
| "grad_norm": 1.2040213346481323, | |
| "kl": 0.03592631733044982, | |
| "learning_rate": 4.849231551964771e-06, | |
| "loss": 0.0014, | |
| "reward": 0.9875416681170464, | |
| "reward_std": 0.5127010717988014, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.20833334140479565, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0416666679084301, | |
| "rewards/xmlcount_reward_func": 0.32087502628564835, | |
| "step": 51 | |
| }, | |
| { | |
| "completion_length": 105.50000190734863, | |
| "epoch": 0.027833534055934698, | |
| "grad_norm": 0.9755409359931946, | |
| "kl": 0.03177254740148783, | |
| "learning_rate": 4.837064190990036e-06, | |
| "loss": 0.0013, | |
| "reward": 0.36112499982118607, | |
| "reward_std": 0.23680441081523895, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.14583333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.02083333395421505, | |
| "rewards/xmlcount_reward_func": 0.19445833936333656, | |
| "step": 52 | |
| }, | |
| { | |
| "completion_length": 115.08333778381348, | |
| "epoch": 0.028368794326241134, | |
| "grad_norm": 1.0774939060211182, | |
| "kl": 0.04096163995563984, | |
| "learning_rate": 4.824441214720629e-06, | |
| "loss": 0.0016, | |
| "reward": 0.39137500151991844, | |
| "reward_std": 0.21243570372462273, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0416666679084301, | |
| "rewards/xmlcount_reward_func": 0.22470833733677864, | |
| "step": 53 | |
| }, | |
| { | |
| "completion_length": 71.87500095367432, | |
| "epoch": 0.02890405459654757, | |
| "grad_norm": 1.5860177278518677, | |
| "kl": 0.07264666631817818, | |
| "learning_rate": 4.811365084030784e-06, | |
| "loss": 0.0029, | |
| "reward": 0.7297500222921371, | |
| "reward_std": 0.40594012290239334, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.25000000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0416666679084301, | |
| "rewards/xmlcount_reward_func": 0.35475001111626625, | |
| "step": 54 | |
| }, | |
| { | |
| "completion_length": 335.0416831970215, | |
| "epoch": 0.029439314866854006, | |
| "grad_norm": 0.8996623158454895, | |
| "kl": 0.03017168352380395, | |
| "learning_rate": 4.7978383481380865e-06, | |
| "loss": 0.0012, | |
| "reward": 0.21566667163278908, | |
| "reward_std": 0.38637750223279, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.0625, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0416666679084301, | |
| "rewards/xmlcount_reward_func": 0.11149999999906868, | |
| "step": 55 | |
| }, | |
| { | |
| "completion_length": 116.00000476837158, | |
| "epoch": 0.029974575137160443, | |
| "grad_norm": 1.1853581666946411, | |
| "kl": 0.03844497771933675, | |
| "learning_rate": 4.783863644106502e-06, | |
| "loss": 0.0015, | |
| "reward": 0.35095833986997604, | |
| "reward_std": 0.21118691470474005, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0416666679084301, | |
| "rewards/xmlcount_reward_func": 0.18429166823625565, | |
| "step": 56 | |
| }, | |
| { | |
| "completion_length": 203.58334350585938, | |
| "epoch": 0.030509835407466882, | |
| "grad_norm": 0.9677571654319763, | |
| "kl": 0.03654019068926573, | |
| "learning_rate": 4.769443696332272e-06, | |
| "loss": 0.0015, | |
| "reward": 0.867875000461936, | |
| "reward_std": 0.6674522012472153, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.2708333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.08333333395421505, | |
| "rewards/xmlcount_reward_func": 0.1803750041872263, | |
| "step": 57 | |
| }, | |
| { | |
| "completion_length": 226.58334159851074, | |
| "epoch": 0.03104509567777332, | |
| "grad_norm": 0.7546242475509644, | |
| "kl": 0.10618894919753075, | |
| "learning_rate": 4.754581316012785e-06, | |
| "loss": 0.0042, | |
| "reward": 0.7405833136290312, | |
| "reward_std": 1.0614993423223495, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.2708333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.125, | |
| "rewards/xmlcount_reward_func": 0.09474998340010643, | |
| "step": 58 | |
| }, | |
| { | |
| "completion_length": 96.75000190734863, | |
| "epoch": 0.03158035594807975, | |
| "grad_norm": 0.9959896206855774, | |
| "kl": 0.040049958042800426, | |
| "learning_rate": 4.7392794005985324e-06, | |
| "loss": 0.0016, | |
| "reward": 1.0202916860580444, | |
| "reward_std": 0.5125212594866753, | |
| "rewards/correctness_reward_func": 0.2500000074505806, | |
| "rewards/int_reward_func": 0.4166666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.35362500697374344, | |
| "step": 59 | |
| }, | |
| { | |
| "completion_length": 77.83333587646484, | |
| "epoch": 0.03211561621838619, | |
| "grad_norm": 1.0476152896881104, | |
| "kl": 0.08419935218989849, | |
| "learning_rate": 4.723540933228245e-06, | |
| "loss": 0.0034, | |
| "reward": 0.6385000422596931, | |
| "reward_std": 0.23676574788987637, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.27083333767950535, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "rewards/xmlcount_reward_func": 0.36766667664051056, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 76.0416669845581, | |
| "epoch": 0.03265087648869262, | |
| "grad_norm": 1.6621463298797607, | |
| "kl": 0.06288609141483903, | |
| "learning_rate": 4.707368982147318e-06, | |
| "loss": 0.0025, | |
| "reward": 0.7950416952371597, | |
| "reward_std": 0.2815567087382078, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.2916666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.1458333395421505, | |
| "rewards/xmlcount_reward_func": 0.35754168033599854, | |
| "step": 61 | |
| }, | |
| { | |
| "completion_length": 97.87500190734863, | |
| "epoch": 0.03318613675899906, | |
| "grad_norm": 1.3513109683990479, | |
| "kl": 0.07356535829603672, | |
| "learning_rate": 4.690766700109659e-06, | |
| "loss": 0.0029, | |
| "reward": 1.054708331823349, | |
| "reward_std": 0.3879717066884041, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.4791666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.06250000186264515, | |
| "rewards/xmlcount_reward_func": 0.3463750060182065, | |
| "step": 62 | |
| }, | |
| { | |
| "completion_length": 356.79167556762695, | |
| "epoch": 0.0337213970293055, | |
| "grad_norm": 0.9732988476753235, | |
| "kl": 0.0708354264497757, | |
| "learning_rate": 4.673737323763048e-06, | |
| "loss": 0.0028, | |
| "reward": 0.3889166936278343, | |
| "reward_std": 1.0406904257833958, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.0833333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0833333358168602, | |
| "rewards/xmlcount_reward_func": 0.05558334290981293, | |
| "step": 63 | |
| }, | |
| { | |
| "completion_length": 124.16666984558105, | |
| "epoch": 0.034256657299611935, | |
| "grad_norm": 1.4622650146484375, | |
| "kl": 0.08344197925180197, | |
| "learning_rate": 4.656284173018144e-06, | |
| "loss": 0.0033, | |
| "reward": 1.0468750149011612, | |
| "reward_std": 0.7830385342240334, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.2708333432674408, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.16666666977107525, | |
| "rewards/xmlcount_reward_func": 0.3593750074505806, | |
| "step": 64 | |
| }, | |
| { | |
| "completion_length": 233.333345413208, | |
| "epoch": 0.034791917569918375, | |
| "grad_norm": 2.0504956245422363, | |
| "kl": 0.07171727810055017, | |
| "learning_rate": 4.638410650401267e-06, | |
| "loss": 0.0029, | |
| "reward": 1.2617916613817215, | |
| "reward_std": 0.8660007119178772, | |
| "rewards/correctness_reward_func": 0.5000000149011612, | |
| "rewards/int_reward_func": 0.3750000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0625, | |
| "rewards/xmlcount_reward_func": 0.3242916837334633, | |
| "step": 65 | |
| }, | |
| { | |
| "completion_length": 147.5833339691162, | |
| "epoch": 0.03532717784022481, | |
| "grad_norm": 0.6744219660758972, | |
| "kl": 0.11512961238622665, | |
| "learning_rate": 4.620120240391065e-06, | |
| "loss": 0.0046, | |
| "reward": 0.8016250282526016, | |
| "reward_std": 0.29210690781474113, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.2291666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.0416666679084301, | |
| "rewards/xmlcount_reward_func": 0.44745834171772003, | |
| "step": 66 | |
| }, | |
| { | |
| "completion_length": 76.41666793823242, | |
| "epoch": 0.03586243811053125, | |
| "grad_norm": 0.8137429356575012, | |
| "kl": 0.11052755452692509, | |
| "learning_rate": 4.601416508739211e-06, | |
| "loss": 0.0044, | |
| "reward": 1.2232083678245544, | |
| "reward_std": 0.6085939556360245, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.18750000558793545, | |
| "rewards/xmlcount_reward_func": 0.45237500965595245, | |
| "step": 67 | |
| }, | |
| { | |
| "completion_length": 103.91666984558105, | |
| "epoch": 0.03639769838083768, | |
| "grad_norm": 1.3938682079315186, | |
| "kl": 0.10067875497043133, | |
| "learning_rate": 4.582303101775249e-06, | |
| "loss": 0.004, | |
| "reward": 0.7972083538770676, | |
| "reward_std": 0.43218278884887695, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.14583333767950535, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.22916667349636555, | |
| "rewards/xmlcount_reward_func": 0.42220834642648697, | |
| "step": 68 | |
| }, | |
| { | |
| "completion_length": 607.5000114440918, | |
| "epoch": 0.03693295865114412, | |
| "grad_norm": 0.7114788293838501, | |
| "kl": 0.059856235020561144, | |
| "learning_rate": 4.562783745695738e-06, | |
| "loss": 0.0024, | |
| "reward": 0.6020833402872086, | |
| "reward_std": 0.26822593063116074, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.1875000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.1041666716337204, | |
| "rewards/xmlcount_reward_func": 0.31041666865348816, | |
| "step": 69 | |
| }, | |
| { | |
| "completion_length": 366.1666736602783, | |
| "epoch": 0.03746821892145055, | |
| "grad_norm": 1.5701572895050049, | |
| "kl": 0.12799374386668205, | |
| "learning_rate": 4.542862245837821e-06, | |
| "loss": 0.0051, | |
| "reward": 0.6387916915118694, | |
| "reward_std": 0.3272341303527355, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.16666667349636555, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.14583333395421505, | |
| "rewards/xmlcount_reward_func": 0.3262916784733534, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 150.9166717529297, | |
| "epoch": 0.03800347919175699, | |
| "grad_norm": 1.3559887409210205, | |
| "kl": 0.17991142719984055, | |
| "learning_rate": 4.522542485937369e-06, | |
| "loss": 0.0072, | |
| "reward": 0.8541667014360428, | |
| "reward_std": 0.28774577379226685, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.2500000074505806, | |
| "rewards/xmlcount_reward_func": 0.4791666716337204, | |
| "step": 71 | |
| }, | |
| { | |
| "completion_length": 92.08333778381348, | |
| "epoch": 0.03853873946206343, | |
| "grad_norm": 0.9042914509773254, | |
| "kl": 0.21130416169762611, | |
| "learning_rate": 4.501828427371834e-06, | |
| "loss": 0.0085, | |
| "reward": 0.7811249941587448, | |
| "reward_std": 0.20265305042266846, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.1666666679084301, | |
| "rewards/xmlcount_reward_func": 0.4894583374261856, | |
| "step": 72 | |
| }, | |
| { | |
| "completion_length": 125.45833778381348, | |
| "epoch": 0.039073999732369864, | |
| "grad_norm": 0.8635703921318054, | |
| "kl": 0.13566209375858307, | |
| "learning_rate": 4.4807241083879774e-06, | |
| "loss": 0.0054, | |
| "reward": 1.1882500350475311, | |
| "reward_std": 0.8479792177677155, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.1875000037252903, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.2708333358168602, | |
| "rewards/xmlcount_reward_func": 0.39658334106206894, | |
| "step": 73 | |
| }, | |
| { | |
| "completion_length": 68.79166984558105, | |
| "epoch": 0.039609260002676304, | |
| "grad_norm": 1.192706823348999, | |
| "kl": 0.27350207418203354, | |
| "learning_rate": 4.4592336433146e-06, | |
| "loss": 0.0109, | |
| "reward": 1.7708333730697632, | |
| "reward_std": 0.11558076366782188, | |
| "rewards/correctness_reward_func": 0.5, | |
| "rewards/int_reward_func": 0.3125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 74 | |
| }, | |
| { | |
| "completion_length": 90.79166984558105, | |
| "epoch": 0.04014452027298274, | |
| "grad_norm": 1.3415632247924805, | |
| "kl": 0.15339597314596176, | |
| "learning_rate": 4.437361221760449e-06, | |
| "loss": 0.0061, | |
| "reward": 1.0463333874940872, | |
| "reward_std": 0.5073548853397369, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.2083333358168602, | |
| "rewards/xmlcount_reward_func": 0.46300000697374344, | |
| "step": 75 | |
| }, | |
| { | |
| "completion_length": 163.20834159851074, | |
| "epoch": 0.040679780543289176, | |
| "grad_norm": 1.5307588577270508, | |
| "kl": 0.1565667698159814, | |
| "learning_rate": 4.415111107797445e-06, | |
| "loss": 0.0063, | |
| "reward": 1.239583358168602, | |
| "reward_std": 0.48138320073485374, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.2083333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3125000074505806, | |
| "rewards/xmlcount_reward_func": 0.4687500074505806, | |
| "step": 76 | |
| }, | |
| { | |
| "completion_length": 162.20834159851074, | |
| "epoch": 0.04121504081359561, | |
| "grad_norm": 1.7075368165969849, | |
| "kl": 0.2482675537467003, | |
| "learning_rate": 4.3924876391293915e-06, | |
| "loss": 0.0099, | |
| "reward": 1.5780000388622284, | |
| "reward_std": 0.7741712592542171, | |
| "rewards/correctness_reward_func": 0.416666679084301, | |
| "rewards/int_reward_func": 0.3333333469927311, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.37500000558793545, | |
| "rewards/xmlcount_reward_func": 0.453000009059906, | |
| "step": 77 | |
| }, | |
| { | |
| "completion_length": 90.83333587646484, | |
| "epoch": 0.04175030108390205, | |
| "grad_norm": 1.605960726737976, | |
| "kl": 0.1952410712838173, | |
| "learning_rate": 4.36949522624633e-06, | |
| "loss": 0.0078, | |
| "reward": 1.7031250596046448, | |
| "reward_std": 0.5450708866119385, | |
| "rewards/correctness_reward_func": 0.6666666865348816, | |
| "rewards/int_reward_func": 0.2083333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3333333395421505, | |
| "rewards/xmlcount_reward_func": 0.4947916716337204, | |
| "step": 78 | |
| }, | |
| { | |
| "completion_length": 136.16666984558105, | |
| "epoch": 0.04228556135420848, | |
| "grad_norm": 0.673092246055603, | |
| "kl": 0.1753321774303913, | |
| "learning_rate": 4.346138351564711e-06, | |
| "loss": 0.007, | |
| "reward": 1.2291666716337204, | |
| "reward_std": 0.4924144148826599, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.291666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.29166667349636555, | |
| "rewards/xmlcount_reward_func": 0.4791666716337204, | |
| "step": 79 | |
| }, | |
| { | |
| "completion_length": 71.20833683013916, | |
| "epoch": 0.04282082162451492, | |
| "grad_norm": 1.5928924083709717, | |
| "kl": 0.31612617522478104, | |
| "learning_rate": 4.322421568553529e-06, | |
| "loss": 0.0126, | |
| "reward": 2.0416666865348816, | |
| "reward_std": 0.20412416756153107, | |
| "rewards/correctness_reward_func": 0.5833333358168602, | |
| "rewards/int_reward_func": 0.4791666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 112.33333492279053, | |
| "epoch": 0.043356081894821354, | |
| "grad_norm": 0.4769633412361145, | |
| "kl": 0.21202785894274712, | |
| "learning_rate": 4.2983495008466285e-06, | |
| "loss": 0.0085, | |
| "reward": 1.7968750447034836, | |
| "reward_std": 0.15385404229164124, | |
| "rewards/correctness_reward_func": 0.5, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375000074505806, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 81 | |
| }, | |
| { | |
| "completion_length": 175.41667366027832, | |
| "epoch": 0.04389134216512779, | |
| "grad_norm": 0.9674685597419739, | |
| "kl": 0.13749209698289633, | |
| "learning_rate": 4.273926841341303e-06, | |
| "loss": 0.0055, | |
| "reward": 1.6510417461395264, | |
| "reward_std": 0.5754482969641685, | |
| "rewards/correctness_reward_func": 0.416666679084301, | |
| "rewards/int_reward_func": 0.4166666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.33333334140479565, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 82 | |
| }, | |
| { | |
| "completion_length": 67.87500190734863, | |
| "epoch": 0.04442660243543423, | |
| "grad_norm": 1.0911532640457153, | |
| "kl": 0.28480928763747215, | |
| "learning_rate": 4.249158351283414e-06, | |
| "loss": 0.0114, | |
| "reward": 1.4166666865348816, | |
| "reward_std": 0.12909945845603943, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.4791666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375000074505806, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 83 | |
| }, | |
| { | |
| "completion_length": 71.91666889190674, | |
| "epoch": 0.044961862705740666, | |
| "grad_norm": 1.9787883758544922, | |
| "kl": 0.2760180849581957, | |
| "learning_rate": 4.224048859339175e-06, | |
| "loss": 0.011, | |
| "reward": 1.1250000596046448, | |
| "reward_std": 0.20412414520978928, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.25000000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.37500000558793545, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 84 | |
| }, | |
| { | |
| "completion_length": 124.37500190734863, | |
| "epoch": 0.045497122976047105, | |
| "grad_norm": 0.5233182907104492, | |
| "kl": 0.20504293218255043, | |
| "learning_rate": 4.198603260653792e-06, | |
| "loss": 0.0082, | |
| "reward": 1.2343750298023224, | |
| "reward_std": 0.3668263405561447, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.3541666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.25, | |
| "rewards/xmlcount_reward_func": 0.4635416716337204, | |
| "step": 85 | |
| }, | |
| { | |
| "completion_length": 101.62500381469727, | |
| "epoch": 0.04603238324635354, | |
| "grad_norm": 1.2110995054244995, | |
| "kl": 0.15592870488762856, | |
| "learning_rate": 4.172826515897146e-06, | |
| "loss": 0.0062, | |
| "reward": 1.7083334028720856, | |
| "reward_std": 0.5623037368059158, | |
| "rewards/correctness_reward_func": 0.5000000223517418, | |
| "rewards/int_reward_func": 0.2291666753590107, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 86 | |
| }, | |
| { | |
| "completion_length": 126.25000762939453, | |
| "epoch": 0.04656764351665998, | |
| "grad_norm": 0.7313621640205383, | |
| "kl": 0.18805699050426483, | |
| "learning_rate": 4.146723650296701e-06, | |
| "loss": 0.0075, | |
| "reward": 1.3646250218153, | |
| "reward_std": 0.3509002774953842, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.14583333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.416666679084301, | |
| "rewards/xmlcount_reward_func": 0.4687916710972786, | |
| "step": 87 | |
| }, | |
| { | |
| "completion_length": 209.45833683013916, | |
| "epoch": 0.04710290378696641, | |
| "grad_norm": 0.725437343120575, | |
| "kl": 0.16482173651456833, | |
| "learning_rate": 4.120299752657828e-06, | |
| "loss": 0.0066, | |
| "reward": 1.046916663646698, | |
| "reward_std": 0.4301242418587208, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.1875000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.2291666716337204, | |
| "rewards/xmlcount_reward_func": 0.4635833352804184, | |
| "step": 88 | |
| }, | |
| { | |
| "completion_length": 165.0833396911621, | |
| "epoch": 0.04763816405727285, | |
| "grad_norm": 0.7724325656890869, | |
| "kl": 0.20376655086874962, | |
| "learning_rate": 4.093559974371725e-06, | |
| "loss": 0.0082, | |
| "reward": 1.0260416865348816, | |
| "reward_std": 0.1994822435081005, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.1250000037252903, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 89 | |
| }, | |
| { | |
| "completion_length": 98.66666984558105, | |
| "epoch": 0.04817342432757928, | |
| "grad_norm": 1.395373821258545, | |
| "kl": 0.2529403530061245, | |
| "learning_rate": 4.066509528411151e-06, | |
| "loss": 0.0101, | |
| "reward": 1.0780000239610672, | |
| "reward_std": 0.2602536380290985, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.1875000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333432674408, | |
| "rewards/xmlcount_reward_func": 0.4946666657924652, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 89.25, | |
| "epoch": 0.04870868459788572, | |
| "grad_norm": 0.8370155692100525, | |
| "kl": 0.19209491834044456, | |
| "learning_rate": 4.039153688314146e-06, | |
| "loss": 0.0077, | |
| "reward": 1.2864583879709244, | |
| "reward_std": 0.43756843730807304, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.0833333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.4947916716337204, | |
| "step": 91 | |
| }, | |
| { | |
| "completion_length": 63.333335876464844, | |
| "epoch": 0.04924394486819216, | |
| "grad_norm": 2.1854286193847656, | |
| "kl": 0.21549397706985474, | |
| "learning_rate": 4.011497787155938e-06, | |
| "loss": 0.0086, | |
| "reward": 1.2964583337306976, | |
| "reward_std": 0.1316254585981369, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3541666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333358168602, | |
| "rewards/xmlcount_reward_func": 0.48395833373069763, | |
| "step": 92 | |
| }, | |
| { | |
| "completion_length": 88.62500190734863, | |
| "epoch": 0.049779205138498595, | |
| "grad_norm": 1.6499943733215332, | |
| "kl": 0.2516642101109028, | |
| "learning_rate": 3.983547216509254e-06, | |
| "loss": 0.0101, | |
| "reward": 1.6613333523273468, | |
| "reward_std": 0.6053861007094383, | |
| "rewards/correctness_reward_func": 0.4166666716337204, | |
| "rewards/int_reward_func": 0.3541666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333358168602, | |
| "rewards/xmlcount_reward_func": 0.4946666657924652, | |
| "step": 93 | |
| }, | |
| { | |
| "completion_length": 134.12500381469727, | |
| "epoch": 0.050314465408805034, | |
| "grad_norm": 0.9040616154670715, | |
| "kl": 0.18027858808636665, | |
| "learning_rate": 3.955307425393224e-06, | |
| "loss": 0.0072, | |
| "reward": 1.1510417014360428, | |
| "reward_std": 0.4075661599636078, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3750000037252903, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 94 | |
| }, | |
| { | |
| "completion_length": 143.12500381469727, | |
| "epoch": 0.05084972567911147, | |
| "grad_norm": 0.8769562244415283, | |
| "kl": 0.2623859569430351, | |
| "learning_rate": 3.92678391921108e-06, | |
| "loss": 0.0105, | |
| "reward": 1.3593750298023224, | |
| "reward_std": 0.3879491835832596, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.3958333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333358168602, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 95 | |
| }, | |
| { | |
| "completion_length": 80.58333587646484, | |
| "epoch": 0.051384985949417906, | |
| "grad_norm": 1.6260957717895508, | |
| "kl": 0.2531866990029812, | |
| "learning_rate": 3.897982258676867e-06, | |
| "loss": 0.0101, | |
| "reward": 1.3281250298023224, | |
| "reward_std": 0.37103308364748955, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.33333334140479565, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375000074505806, | |
| "rewards/xmlcount_reward_func": 0.4739583432674408, | |
| "step": 96 | |
| }, | |
| { | |
| "completion_length": 104.91666889190674, | |
| "epoch": 0.05192024621972434, | |
| "grad_norm": 0.8191968202590942, | |
| "kl": 0.27327974885702133, | |
| "learning_rate": 3.868908058731376e-06, | |
| "loss": 0.0109, | |
| "reward": 1.1041666865348816, | |
| "reward_std": 0.16337091475725174, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.2500000037252903, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.375, | |
| "rewards/xmlcount_reward_func": 0.4791666716337204, | |
| "step": 97 | |
| }, | |
| { | |
| "completion_length": 62.00000190734863, | |
| "epoch": 0.05245550649003078, | |
| "grad_norm": 0.3426864743232727, | |
| "kl": 0.2767893858253956, | |
| "learning_rate": 3.839566987447492e-06, | |
| "loss": 0.0111, | |
| "reward": 1.3541666865348816, | |
| "reward_std": 0.05103103816509247, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 98 | |
| }, | |
| { | |
| "completion_length": 389.2500114440918, | |
| "epoch": 0.05299076676033721, | |
| "grad_norm": 1.0328341722488403, | |
| "kl": 0.1228889636695385, | |
| "learning_rate": 3.8099647649251984e-06, | |
| "loss": 0.0049, | |
| "reward": 1.067708358168602, | |
| "reward_std": 0.422527939081192, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.27083333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.2916666716337204, | |
| "rewards/xmlcount_reward_func": 0.421875, | |
| "step": 99 | |
| }, | |
| { | |
| "completion_length": 73.62500286102295, | |
| "epoch": 0.05352602703064365, | |
| "grad_norm": 0.7198861241340637, | |
| "kl": 0.2295570969581604, | |
| "learning_rate": 3.780107162176429e-06, | |
| "loss": 0.0092, | |
| "reward": 1.375, | |
| "reward_std": 0.2885505259037018, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.2916666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 247.083345413208, | |
| "epoch": 0.054061287300950084, | |
| "grad_norm": 0.8113954663276672, | |
| "kl": 0.2137942397966981, | |
| "learning_rate": 3.7500000000000005e-06, | |
| "loss": 0.0086, | |
| "reward": 1.0833750367164612, | |
| "reward_std": 0.4043814614415169, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.29166667349636555, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.2916666716337204, | |
| "rewards/xmlcount_reward_func": 0.4167083315551281, | |
| "step": 101 | |
| }, | |
| { | |
| "completion_length": 161.08334159851074, | |
| "epoch": 0.05459654757125652, | |
| "grad_norm": 0.9631187915802002, | |
| "kl": 0.320892296731472, | |
| "learning_rate": 3.7196491478468322e-06, | |
| "loss": 0.0128, | |
| "reward": 1.7552083432674408, | |
| "reward_std": 0.20395417511463165, | |
| "rewards/correctness_reward_func": 0.5, | |
| "rewards/int_reward_func": 0.3541666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 102 | |
| }, | |
| { | |
| "completion_length": 86.87500286102295, | |
| "epoch": 0.05513180784156296, | |
| "grad_norm": 0.5522407293319702, | |
| "kl": 0.19866621680557728, | |
| "learning_rate": 3.689060522675689e-06, | |
| "loss": 0.0079, | |
| "reward": 1.6974583566188812, | |
| "reward_std": 0.43488648533821106, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.3333333432674408, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.48912499845027924, | |
| "step": 103 | |
| }, | |
| { | |
| "completion_length": 116.37500381469727, | |
| "epoch": 0.055667068111869396, | |
| "grad_norm": 0.39440569281578064, | |
| "kl": 0.2107317578047514, | |
| "learning_rate": 3.658240087799655e-06, | |
| "loss": 0.0084, | |
| "reward": 1.4635416865348816, | |
| "reward_std": 0.3243303596973419, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.4375000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 104 | |
| }, | |
| { | |
| "completion_length": 115.41666984558105, | |
| "epoch": 0.056202328382175835, | |
| "grad_norm": 1.1680481433868408, | |
| "kl": 0.4403987228870392, | |
| "learning_rate": 3.627193851723577e-06, | |
| "loss": 0.0176, | |
| "reward": 1.2552083730697632, | |
| "reward_std": 0.2559161148965359, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3125000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 105 | |
| }, | |
| { | |
| "completion_length": 69.83333396911621, | |
| "epoch": 0.05673758865248227, | |
| "grad_norm": 0.4940861463546753, | |
| "kl": 0.27071962505578995, | |
| "learning_rate": 3.595927866972694e-06, | |
| "loss": 0.0108, | |
| "reward": 1.1666666865348816, | |
| "reward_std": 0.06454972922801971, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.1666666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 106 | |
| }, | |
| { | |
| "completion_length": 94.45833587646484, | |
| "epoch": 0.05727284892278871, | |
| "grad_norm": 1.4608356952667236, | |
| "kl": 0.16904586926102638, | |
| "learning_rate": 3.564448228912682e-06, | |
| "loss": 0.0068, | |
| "reward": 1.4895833432674408, | |
| "reward_std": 0.4765794351696968, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.4166666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.416666679084301, | |
| "rewards/xmlcount_reward_func": 0.4895833432674408, | |
| "step": 107 | |
| }, | |
| { | |
| "completion_length": 130.75000190734863, | |
| "epoch": 0.05780810919309514, | |
| "grad_norm": 1.187828779220581, | |
| "kl": 0.2347713652998209, | |
| "learning_rate": 3.532761074561355e-06, | |
| "loss": 0.0094, | |
| "reward": 1.0625000298023224, | |
| "reward_std": 0.11558076366782188, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.2500000037252903, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3125, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 108 | |
| }, | |
| { | |
| "completion_length": 81.08333587646484, | |
| "epoch": 0.05834336946340158, | |
| "grad_norm": 0.4293256103992462, | |
| "kl": 0.3195993173867464, | |
| "learning_rate": 3.5008725813922383e-06, | |
| "loss": 0.0128, | |
| "reward": 1.1666666865348816, | |
| "reward_std": 0.06454972922801971, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.1666666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 109 | |
| }, | |
| { | |
| "completion_length": 153.29166984558105, | |
| "epoch": 0.05887862973370801, | |
| "grad_norm": 1.0056191682815552, | |
| "kl": 0.20048995688557625, | |
| "learning_rate": 3.4687889661302577e-06, | |
| "loss": 0.008, | |
| "reward": 1.4843750298023224, | |
| "reward_std": 0.41315262764692307, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.37500000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.37500000558793545, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 88.54167175292969, | |
| "epoch": 0.05941389000401445, | |
| "grad_norm": 1.9036917686462402, | |
| "kl": 0.2402002513408661, | |
| "learning_rate": 3.436516483539781e-06, | |
| "loss": 0.0096, | |
| "reward": 1.833333358168602, | |
| "reward_std": 0.5183059275150299, | |
| "rewards/correctness_reward_func": 0.5000000223517418, | |
| "rewards/int_reward_func": 0.39583333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375000074505806, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 111 | |
| }, | |
| { | |
| "completion_length": 73.37500190734863, | |
| "epoch": 0.059949150274320885, | |
| "grad_norm": 1.473577857017517, | |
| "kl": 0.31748900189995766, | |
| "learning_rate": 3.4040614252052305e-06, | |
| "loss": 0.0127, | |
| "reward": 1.3645833730697632, | |
| "reward_std": 0.3358423411846161, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.3541666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375, | |
| "rewards/xmlcount_reward_func": 0.4895833358168602, | |
| "step": 112 | |
| }, | |
| { | |
| "completion_length": 75.04166793823242, | |
| "epoch": 0.060484410544627325, | |
| "grad_norm": 0.9336056709289551, | |
| "kl": 0.29590417072176933, | |
| "learning_rate": 3.3714301183045382e-06, | |
| "loss": 0.0118, | |
| "reward": 1.270833358168602, | |
| "reward_std": 0.1530931033194065, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 113 | |
| }, | |
| { | |
| "completion_length": 71.25000190734863, | |
| "epoch": 0.061019670814933764, | |
| "grad_norm": 1.1127337217330933, | |
| "kl": 0.2616447024047375, | |
| "learning_rate": 3.338628924375638e-06, | |
| "loss": 0.0105, | |
| "reward": 1.291666716337204, | |
| "reward_std": 0.16661179810762405, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3125000149011612, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 114 | |
| }, | |
| { | |
| "completion_length": 75.50000286102295, | |
| "epoch": 0.0615549310852402, | |
| "grad_norm": 0.7082913517951965, | |
| "kl": 0.2341964803636074, | |
| "learning_rate": 3.3056642380762783e-06, | |
| "loss": 0.0094, | |
| "reward": 1.4791666865348816, | |
| "reward_std": 0.3881191611289978, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.3333333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 115 | |
| }, | |
| { | |
| "completion_length": 206.91667556762695, | |
| "epoch": 0.06209019135554664, | |
| "grad_norm": 0.41269803047180176, | |
| "kl": 0.18453531339764595, | |
| "learning_rate": 3.272542485937369e-06, | |
| "loss": 0.0074, | |
| "reward": 1.5364583730697632, | |
| "reward_std": 0.7113124281167984, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.4375000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3125000037252903, | |
| "rewards/xmlcount_reward_func": 0.453125, | |
| "step": 116 | |
| }, | |
| { | |
| "completion_length": 88.50000381469727, | |
| "epoch": 0.06262545162585308, | |
| "grad_norm": 0.8284731507301331, | |
| "kl": 0.18175287544727325, | |
| "learning_rate": 3.2392701251101172e-06, | |
| "loss": 0.0073, | |
| "reward": 1.3958333730697632, | |
| "reward_std": 0.3092299550771713, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3541666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 117 | |
| }, | |
| { | |
| "completion_length": 151.66666984558105, | |
| "epoch": 0.0631607118961595, | |
| "grad_norm": 1.4958237409591675, | |
| "kl": 0.2391066513955593, | |
| "learning_rate": 3.205853642107192e-06, | |
| "loss": 0.0096, | |
| "reward": 1.1510416865348816, | |
| "reward_std": 0.5006890743970871, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.22916666977107525, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.2708333432674408, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 118 | |
| }, | |
| { | |
| "completion_length": 123.37500381469727, | |
| "epoch": 0.06369597216646594, | |
| "grad_norm": 0.7593151926994324, | |
| "kl": 0.23146136105060577, | |
| "learning_rate": 3.1722995515381644e-06, | |
| "loss": 0.0093, | |
| "reward": 1.4270833432674408, | |
| "reward_std": 0.48038505017757416, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.3750000037252903, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.46875, | |
| "step": 119 | |
| }, | |
| { | |
| "completion_length": 123.75000762939453, | |
| "epoch": 0.06423123243677238, | |
| "grad_norm": 0.3002820909023285, | |
| "kl": 0.18683998472988605, | |
| "learning_rate": 3.1386143948394764e-06, | |
| "loss": 0.0075, | |
| "reward": 1.3802083730697632, | |
| "reward_std": 0.2934284619987011, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.1041666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.375, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 71.62500190734863, | |
| "epoch": 0.06476649270707882, | |
| "grad_norm": 0.894112229347229, | |
| "kl": 0.2145114541053772, | |
| "learning_rate": 3.1048047389991693e-06, | |
| "loss": 0.0086, | |
| "reward": 1.2291666865348816, | |
| "reward_std": 0.05103103443980217, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 121 | |
| }, | |
| { | |
| "completion_length": 77.50000095367432, | |
| "epoch": 0.06530175297738525, | |
| "grad_norm": 0.9729853272438049, | |
| "kl": 0.25685518980026245, | |
| "learning_rate": 3.0708771752766397e-06, | |
| "loss": 0.0103, | |
| "reward": 1.645833358168602, | |
| "reward_std": 0.25515517219901085, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 122 | |
| }, | |
| { | |
| "completion_length": 115.25000381469727, | |
| "epoch": 0.06583701324769169, | |
| "grad_norm": 0.60560142993927, | |
| "kl": 0.26780444383621216, | |
| "learning_rate": 3.0368383179176584e-06, | |
| "loss": 0.0107, | |
| "reward": 1.3333333432674408, | |
| "reward_std": 0.06454972922801971, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333358168602, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 123 | |
| }, | |
| { | |
| "completion_length": 65.25000190734863, | |
| "epoch": 0.06637227351799813, | |
| "grad_norm": 0.47688770294189453, | |
| "kl": 0.2243332415819168, | |
| "learning_rate": 3.002694802864912e-06, | |
| "loss": 0.009, | |
| "reward": 1.2291666865348816, | |
| "reward_std": 0.05103103816509247, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 124 | |
| }, | |
| { | |
| "completion_length": 349.833345413208, | |
| "epoch": 0.06690753378830457, | |
| "grad_norm": 0.6750498414039612, | |
| "kl": 0.27263053273782134, | |
| "learning_rate": 2.9684532864643123e-06, | |
| "loss": 0.0109, | |
| "reward": 1.5260833650827408, | |
| "reward_std": 0.4318152070045471, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.3333333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3541666716337204, | |
| "rewards/xmlcount_reward_func": 0.421916663646698, | |
| "step": 125 | |
| }, | |
| { | |
| "completion_length": 186.62500762939453, | |
| "epoch": 0.067442794058611, | |
| "grad_norm": 0.8187605142593384, | |
| "kl": 0.2676307410001755, | |
| "learning_rate": 2.9341204441673267e-06, | |
| "loss": 0.0107, | |
| "reward": 1.55266672372818, | |
| "reward_std": 0.4699760675430298, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.4166666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375000074505806, | |
| "rewards/xmlcount_reward_func": 0.44849999994039536, | |
| "step": 126 | |
| }, | |
| { | |
| "completion_length": 72.50000190734863, | |
| "epoch": 0.06797805432891743, | |
| "grad_norm": 0.06586437672376633, | |
| "kl": 0.25656602531671524, | |
| "learning_rate": 2.8997029692295875e-06, | |
| "loss": 0.0103, | |
| "reward": 1.375, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 127 | |
| }, | |
| { | |
| "completion_length": 67.66666889190674, | |
| "epoch": 0.06851331459922387, | |
| "grad_norm": 0.13435645401477814, | |
| "kl": 0.23636912554502487, | |
| "learning_rate": 2.8652075714060296e-06, | |
| "loss": 0.0095, | |
| "reward": 1.25, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 128 | |
| }, | |
| { | |
| "completion_length": 71.12500381469727, | |
| "epoch": 0.06904857486953031, | |
| "grad_norm": 0.6717380285263062, | |
| "kl": 0.3070458807051182, | |
| "learning_rate": 2.8306409756428067e-06, | |
| "loss": 0.0123, | |
| "reward": 1.4375000298023224, | |
| "reward_std": 0.11558076366782188, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.4583333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 129 | |
| }, | |
| { | |
| "completion_length": 73.16666889190674, | |
| "epoch": 0.06958383513983675, | |
| "grad_norm": 0.9400418996810913, | |
| "kl": 0.22653049230575562, | |
| "learning_rate": 2.7960099207662535e-06, | |
| "loss": 0.0091, | |
| "reward": 2.0000000596046448, | |
| "reward_std": 0.556085180491209, | |
| "rewards/correctness_reward_func": 0.6666666865348816, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 130 | |
| }, | |
| { | |
| "completion_length": 71.66666889190674, | |
| "epoch": 0.07011909541014318, | |
| "grad_norm": 0.6696528792381287, | |
| "kl": 0.26699281856417656, | |
| "learning_rate": 2.761321158169134e-06, | |
| "loss": 0.0107, | |
| "reward": 1.3958333730697632, | |
| "reward_std": 0.25515518337488174, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.4583333432674408, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375000149011612, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 131 | |
| }, | |
| { | |
| "completion_length": 98.91666984558105, | |
| "epoch": 0.07065435568044962, | |
| "grad_norm": 0.8539110422134399, | |
| "kl": 0.23305394127964973, | |
| "learning_rate": 2.726581450494451e-06, | |
| "loss": 0.0093, | |
| "reward": 1.1562500298023224, | |
| "reward_std": 0.22562336921691895, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.2916666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3750000037252903, | |
| "rewards/xmlcount_reward_func": 0.4895833358168602, | |
| "step": 132 | |
| }, | |
| { | |
| "completion_length": 76.66666984558105, | |
| "epoch": 0.07118961595075605, | |
| "grad_norm": 0.2167089283466339, | |
| "kl": 0.20909808576107025, | |
| "learning_rate": 2.6917975703170466e-06, | |
| "loss": 0.0084, | |
| "reward": 1.25, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 133 | |
| }, | |
| { | |
| "completion_length": 69.50000190734863, | |
| "epoch": 0.0717248762210625, | |
| "grad_norm": 1.07498037815094, | |
| "kl": 0.2273651361465454, | |
| "learning_rate": 2.6569762988232838e-06, | |
| "loss": 0.0091, | |
| "reward": 1.2291666865348816, | |
| "reward_std": 0.11558076366782188, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.22916666977107525, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 134 | |
| }, | |
| { | |
| "completion_length": 78.87500190734863, | |
| "epoch": 0.07226013649136893, | |
| "grad_norm": 1.082901954650879, | |
| "kl": 0.21143031865358353, | |
| "learning_rate": 2.6221244244890336e-06, | |
| "loss": 0.0085, | |
| "reward": 1.2691666781902313, | |
| "reward_std": 0.16395629942417145, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.4775000065565109, | |
| "step": 135 | |
| }, | |
| { | |
| "completion_length": 84.79166793823242, | |
| "epoch": 0.07279539676167536, | |
| "grad_norm": 0.5148155093193054, | |
| "kl": 0.19925828650593758, | |
| "learning_rate": 2.587248741756253e-06, | |
| "loss": 0.008, | |
| "reward": 1.375, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 136 | |
| }, | |
| { | |
| "completion_length": 102.62500190734863, | |
| "epoch": 0.0733306570319818, | |
| "grad_norm": 0.6643544435501099, | |
| "kl": 0.1562279723584652, | |
| "learning_rate": 2.5523560497083927e-06, | |
| "loss": 0.0062, | |
| "reward": 1.248750001192093, | |
| "reward_std": 0.2717357352375984, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3333333432674408, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375000149011612, | |
| "rewards/xmlcount_reward_func": 0.4779166728258133, | |
| "step": 137 | |
| }, | |
| { | |
| "completion_length": 67.0416669845581, | |
| "epoch": 0.07386591730228824, | |
| "grad_norm": 0.4721544086933136, | |
| "kl": 0.19591450318694115, | |
| "learning_rate": 2.517453150744904e-06, | |
| "loss": 0.0078, | |
| "reward": 1.3541666865348816, | |
| "reward_std": 0.05103103816509247, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 138 | |
| }, | |
| { | |
| "completion_length": 79.37500190734863, | |
| "epoch": 0.07440117757259468, | |
| "grad_norm": 0.4948488771915436, | |
| "kl": 0.20949136465787888, | |
| "learning_rate": 2.482546849255096e-06, | |
| "loss": 0.0084, | |
| "reward": 1.5416666865348816, | |
| "reward_std": 0.25819891691207886, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 139 | |
| }, | |
| { | |
| "completion_length": 64.29166889190674, | |
| "epoch": 0.0749364378429011, | |
| "grad_norm": 0.8793404698371887, | |
| "kl": 0.22966529056429863, | |
| "learning_rate": 2.447643950291608e-06, | |
| "loss": 0.0092, | |
| "reward": 1.3541666865348816, | |
| "reward_std": 0.25515519082546234, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.27083333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 140 | |
| }, | |
| { | |
| "completion_length": 139.62500190734863, | |
| "epoch": 0.07547169811320754, | |
| "grad_norm": 0.5459631681442261, | |
| "kl": 0.19374394416809082, | |
| "learning_rate": 2.4127512582437486e-06, | |
| "loss": 0.0077, | |
| "reward": 1.177083358168602, | |
| "reward_std": 0.18890930339694023, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.2916666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666679084301, | |
| "rewards/xmlcount_reward_func": 0.46875, | |
| "step": 141 | |
| }, | |
| { | |
| "completion_length": 153.16666984558105, | |
| "epoch": 0.07600695838351398, | |
| "grad_norm": 0.9452311396598816, | |
| "kl": 0.1499454267323017, | |
| "learning_rate": 2.377875575510967e-06, | |
| "loss": 0.006, | |
| "reward": 1.7239583730697632, | |
| "reward_std": 0.8880488127470016, | |
| "rewards/correctness_reward_func": 0.666666679084301, | |
| "rewards/int_reward_func": 0.3958333432674408, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.25, | |
| "rewards/xmlcount_reward_func": 0.4114583358168602, | |
| "step": 142 | |
| }, | |
| { | |
| "completion_length": 61.29166793823242, | |
| "epoch": 0.07654221865382042, | |
| "grad_norm": 0.09438279271125793, | |
| "kl": 0.3462410867214203, | |
| "learning_rate": 2.3430237011767166e-06, | |
| "loss": 0.0138, | |
| "reward": 1.5, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.5, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 143 | |
| }, | |
| { | |
| "completion_length": 88.37500286102295, | |
| "epoch": 0.07707747892412686, | |
| "grad_norm": 1.7993136644363403, | |
| "kl": 0.32219041138887405, | |
| "learning_rate": 2.3082024296829538e-06, | |
| "loss": 0.0129, | |
| "reward": 1.4218750596046448, | |
| "reward_std": 0.4565740302205086, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.37500000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333358168602, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 144 | |
| }, | |
| { | |
| "completion_length": 72.62500381469727, | |
| "epoch": 0.07761273919443329, | |
| "grad_norm": 0.09042877703905106, | |
| "kl": 0.20896168053150177, | |
| "learning_rate": 2.2734185495055503e-06, | |
| "loss": 0.0084, | |
| "reward": 1.125, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 145 | |
| }, | |
| { | |
| "completion_length": 163.75000190734863, | |
| "epoch": 0.07814799946473973, | |
| "grad_norm": 1.1228388547897339, | |
| "kl": 0.21238887682557106, | |
| "learning_rate": 2.238678841830867e-06, | |
| "loss": 0.0085, | |
| "reward": 1.1927083730697632, | |
| "reward_std": 0.2637527585029602, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3333333432674408, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3750000074505806, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 146 | |
| }, | |
| { | |
| "completion_length": 71.95833683013916, | |
| "epoch": 0.07868325973504617, | |
| "grad_norm": 0.661372721195221, | |
| "kl": 0.3126152493059635, | |
| "learning_rate": 2.2039900792337477e-06, | |
| "loss": 0.0125, | |
| "reward": 1.4375, | |
| "reward_std": 0.22008520364761353, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.39583333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333358168602, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 147 | |
| }, | |
| { | |
| "completion_length": 72.33333492279053, | |
| "epoch": 0.07921852000535261, | |
| "grad_norm": 0.6126328706741333, | |
| "kl": 0.18934645876288414, | |
| "learning_rate": 2.1693590243571937e-06, | |
| "loss": 0.0076, | |
| "reward": 1.8333333432674408, | |
| "reward_std": 0.20412413775920868, | |
| "rewards/correctness_reward_func": 0.5833333358168602, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 148 | |
| }, | |
| { | |
| "completion_length": 103.37500381469727, | |
| "epoch": 0.07975378027565903, | |
| "grad_norm": 0.8722951412200928, | |
| "kl": 0.14480971172451973, | |
| "learning_rate": 2.134792428593971e-06, | |
| "loss": 0.0058, | |
| "reward": 1.895833432674408, | |
| "reward_std": 0.5449064522981644, | |
| "rewards/correctness_reward_func": 0.5000000149011612, | |
| "rewards/int_reward_func": 0.4791666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 149 | |
| }, | |
| { | |
| "completion_length": 69.54166984558105, | |
| "epoch": 0.08028904054596547, | |
| "grad_norm": 0.7660404443740845, | |
| "kl": 0.20250581949949265, | |
| "learning_rate": 2.1002970307704134e-06, | |
| "loss": 0.0081, | |
| "reward": 1.3958333432674408, | |
| "reward_std": 0.35721728205680847, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 150 | |
| }, | |
| { | |
| "completion_length": 180.7083396911621, | |
| "epoch": 0.08082430081627191, | |
| "grad_norm": 0.5615190267562866, | |
| "kl": 0.23339027352631092, | |
| "learning_rate": 2.0658795558326745e-06, | |
| "loss": 0.0093, | |
| "reward": 1.5729166865348816, | |
| "reward_std": 0.31237732619047165, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375, | |
| "rewards/xmlcount_reward_func": 0.46875, | |
| "step": 151 | |
| }, | |
| { | |
| "completion_length": 63.83333396911621, | |
| "epoch": 0.08135956108657835, | |
| "grad_norm": 0.1392926722764969, | |
| "kl": 0.26523152738809586, | |
| "learning_rate": 2.031546713535688e-06, | |
| "loss": 0.0106, | |
| "reward": 1.375, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 152 | |
| }, | |
| { | |
| "completion_length": 81.08333778381348, | |
| "epoch": 0.08189482135688479, | |
| "grad_norm": 1.1122794151306152, | |
| "kl": 0.25071796402335167, | |
| "learning_rate": 1.997305197135089e-06, | |
| "loss": 0.01, | |
| "reward": 1.5200416892766953, | |
| "reward_std": 0.3365423232316971, | |
| "rewards/correctness_reward_func": 0.5, | |
| "rewards/int_reward_func": 0.1875, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.354166679084301, | |
| "rewards/xmlcount_reward_func": 0.47837500274181366, | |
| "step": 153 | |
| }, | |
| { | |
| "completion_length": 93.37500381469727, | |
| "epoch": 0.08243008162719122, | |
| "grad_norm": 1.3372734785079956, | |
| "kl": 0.19058941677212715, | |
| "learning_rate": 1.963161682082342e-06, | |
| "loss": 0.0076, | |
| "reward": 1.6250000298023224, | |
| "reward_std": 0.47279806435108185, | |
| "rewards/correctness_reward_func": 0.5000000223517418, | |
| "rewards/int_reward_func": 0.22916666977107525, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.39583333395421505, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 154 | |
| }, | |
| { | |
| "completion_length": 104.83333396911621, | |
| "epoch": 0.08296534189749766, | |
| "grad_norm": 2.2292211055755615, | |
| "kl": 0.16792716644704342, | |
| "learning_rate": 1.9291228247233607e-06, | |
| "loss": 0.0067, | |
| "reward": 2.0000000596046448, | |
| "reward_std": 0.4289814233779907, | |
| "rewards/correctness_reward_func": 0.583333358168602, | |
| "rewards/int_reward_func": 0.5, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666679084301, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 155 | |
| }, | |
| { | |
| "completion_length": 90.87500476837158, | |
| "epoch": 0.0835006021678041, | |
| "grad_norm": 0.4122345745563507, | |
| "kl": 0.18274019937962294, | |
| "learning_rate": 1.895195261000831e-06, | |
| "loss": 0.0073, | |
| "reward": 1.2708333432674408, | |
| "reward_std": 0.05103103443980217, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.27083333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 156 | |
| }, | |
| { | |
| "completion_length": 172.45833587646484, | |
| "epoch": 0.08403586243811054, | |
| "grad_norm": 0.8472205400466919, | |
| "kl": 0.2094765491783619, | |
| "learning_rate": 1.8613856051605242e-06, | |
| "loss": 0.0084, | |
| "reward": 1.2343750298023224, | |
| "reward_std": 0.2048850804567337, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3541666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333395421505, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 157 | |
| }, | |
| { | |
| "completion_length": 127.00000762939453, | |
| "epoch": 0.08457112270841696, | |
| "grad_norm": 0.6804733276367188, | |
| "kl": 0.20304612442851067, | |
| "learning_rate": 1.827700448461836e-06, | |
| "loss": 0.0081, | |
| "reward": 1.192708358168602, | |
| "reward_std": 0.17030073329806328, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.25000000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333358168602, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 158 | |
| }, | |
| { | |
| "completion_length": 85.29166793823242, | |
| "epoch": 0.0851063829787234, | |
| "grad_norm": 0.7838725447654724, | |
| "kl": 0.29594049230217934, | |
| "learning_rate": 1.7941463578928088e-06, | |
| "loss": 0.0118, | |
| "reward": 1.2708333432674408, | |
| "reward_std": 0.16913224011659622, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3541666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375000074505806, | |
| "rewards/xmlcount_reward_func": 0.4791666716337204, | |
| "step": 159 | |
| }, | |
| { | |
| "completion_length": 126.00000381469727, | |
| "epoch": 0.08564164324902984, | |
| "grad_norm": 1.384969711303711, | |
| "kl": 0.16587615385651588, | |
| "learning_rate": 1.7607298748898844e-06, | |
| "loss": 0.0066, | |
| "reward": 1.4843750149011612, | |
| "reward_std": 0.5336930006742477, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.2916666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 160 | |
| }, | |
| { | |
| "completion_length": 72.12500286102295, | |
| "epoch": 0.08617690351933628, | |
| "grad_norm": 0.0433061420917511, | |
| "kl": 0.1850288063287735, | |
| "learning_rate": 1.7274575140626318e-06, | |
| "loss": 0.0074, | |
| "reward": 1.125, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 161 | |
| }, | |
| { | |
| "completion_length": 174.58334159851074, | |
| "epoch": 0.08671216378964271, | |
| "grad_norm": 0.8271235823631287, | |
| "kl": 0.18862449377775192, | |
| "learning_rate": 1.6943357619237227e-06, | |
| "loss": 0.0075, | |
| "reward": 1.2083333432674408, | |
| "reward_std": 0.30994437262415886, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.3333333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3333333432674408, | |
| "rewards/xmlcount_reward_func": 0.4583333358168602, | |
| "step": 162 | |
| }, | |
| { | |
| "completion_length": 78.20833587646484, | |
| "epoch": 0.08724742405994915, | |
| "grad_norm": 0.908470869064331, | |
| "kl": 0.1706334725022316, | |
| "learning_rate": 1.661371075624363e-06, | |
| "loss": 0.0068, | |
| "reward": 1.2708333432674408, | |
| "reward_std": 0.33958156406879425, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.27083333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 163 | |
| }, | |
| { | |
| "completion_length": 137.91666984558105, | |
| "epoch": 0.08778268433025559, | |
| "grad_norm": 0.7333254814147949, | |
| "kl": 0.1247784998267889, | |
| "learning_rate": 1.6285698816954626e-06, | |
| "loss": 0.005, | |
| "reward": 1.5208333432674408, | |
| "reward_std": 0.767971470952034, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.29166667349636555, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.39583333395421505, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 164 | |
| }, | |
| { | |
| "completion_length": 78.08333683013916, | |
| "epoch": 0.08831794460056203, | |
| "grad_norm": 0.8487056493759155, | |
| "kl": 0.22962494008243084, | |
| "learning_rate": 1.5959385747947697e-06, | |
| "loss": 0.0092, | |
| "reward": 1.2916666865348816, | |
| "reward_std": 0.14360667020082474, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.31250000186264515, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 165 | |
| }, | |
| { | |
| "completion_length": 200.79166793823242, | |
| "epoch": 0.08885320487086847, | |
| "grad_norm": 1.4497365951538086, | |
| "kl": 0.1650528460741043, | |
| "learning_rate": 1.56348351646022e-06, | |
| "loss": 0.0066, | |
| "reward": 1.4895834028720856, | |
| "reward_std": 0.5747665874660015, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.4375000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.46875, | |
| "step": 166 | |
| }, | |
| { | |
| "completion_length": 176.12500762939453, | |
| "epoch": 0.08938846514117489, | |
| "grad_norm": 0.4245186746120453, | |
| "kl": 0.15275901928544044, | |
| "learning_rate": 1.5312110338697427e-06, | |
| "loss": 0.0061, | |
| "reward": 1.1822916865348816, | |
| "reward_std": 0.3088150769472122, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3125000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.453125, | |
| "step": 167 | |
| }, | |
| { | |
| "completion_length": 312.9583396911621, | |
| "epoch": 0.08992372541148133, | |
| "grad_norm": 0.7641220092773438, | |
| "kl": 0.14276206120848656, | |
| "learning_rate": 1.4991274186077632e-06, | |
| "loss": 0.0057, | |
| "reward": 1.166666716337204, | |
| "reward_std": 0.49721667170524597, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.22916666977107525, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666679084301, | |
| "rewards/xmlcount_reward_func": 0.4375, | |
| "step": 168 | |
| }, | |
| { | |
| "completion_length": 101.45833778381348, | |
| "epoch": 0.09045898568178777, | |
| "grad_norm": 0.7765936255455017, | |
| "kl": 0.1949683390557766, | |
| "learning_rate": 1.467238925438646e-06, | |
| "loss": 0.0078, | |
| "reward": 1.2916666865348816, | |
| "reward_std": 0.28610818088054657, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.3125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333395421505, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 169 | |
| }, | |
| { | |
| "completion_length": 172.45833778381348, | |
| "epoch": 0.09099424595209421, | |
| "grad_norm": 0.41969409584999084, | |
| "kl": 0.13137296214699745, | |
| "learning_rate": 1.4355517710873184e-06, | |
| "loss": 0.0053, | |
| "reward": 2.114583343267441, | |
| "reward_std": 0.5827288627624512, | |
| "rewards/correctness_reward_func": 0.8333333358168602, | |
| "rewards/int_reward_func": 0.4583333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3541666716337204, | |
| "rewards/xmlcount_reward_func": 0.46875, | |
| "step": 170 | |
| }, | |
| { | |
| "completion_length": 79.95833587646484, | |
| "epoch": 0.09152950622240064, | |
| "grad_norm": 0.16617827117443085, | |
| "kl": 0.16961714625358582, | |
| "learning_rate": 1.4040721330273063e-06, | |
| "loss": 0.0068, | |
| "reward": 1.25, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 171 | |
| }, | |
| { | |
| "completion_length": 80.87500190734863, | |
| "epoch": 0.09206476649270708, | |
| "grad_norm": 0.36631259322166443, | |
| "kl": 0.1460169106721878, | |
| "learning_rate": 1.3728061482764238e-06, | |
| "loss": 0.0058, | |
| "reward": 1.3333333432674408, | |
| "reward_std": 0.06454972922801971, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3333333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 172 | |
| }, | |
| { | |
| "completion_length": 97.12500190734863, | |
| "epoch": 0.09260002676301352, | |
| "grad_norm": 1.1213421821594238, | |
| "kl": 0.14688214287161827, | |
| "learning_rate": 1.3417599122003464e-06, | |
| "loss": 0.0059, | |
| "reward": 1.2291666865348816, | |
| "reward_std": 0.05103103816509247, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.2291666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 173 | |
| }, | |
| { | |
| "completion_length": 85.58333587646484, | |
| "epoch": 0.09313528703331996, | |
| "grad_norm": 0.7063998579978943, | |
| "kl": 0.17591003328561783, | |
| "learning_rate": 1.3109394773243117e-06, | |
| "loss": 0.007, | |
| "reward": 1.3958333730697632, | |
| "reward_std": 0.11558076739311218, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3958333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 174 | |
| }, | |
| { | |
| "completion_length": 89.04166793823242, | |
| "epoch": 0.0936705473036264, | |
| "grad_norm": 0.7872856855392456, | |
| "kl": 0.17240377515554428, | |
| "learning_rate": 1.280350852153168e-06, | |
| "loss": 0.0069, | |
| "reward": 1.2500000447034836, | |
| "reward_std": 0.2728445753455162, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.2083333432674408, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 175 | |
| }, | |
| { | |
| "completion_length": 252.00000762939453, | |
| "epoch": 0.09420580757393282, | |
| "grad_norm": 1.0295542478561401, | |
| "kl": 0.20453453436493874, | |
| "learning_rate": 1.2500000000000007e-06, | |
| "loss": 0.0082, | |
| "reward": 1.3854167014360428, | |
| "reward_std": 0.5866826139390469, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.3958333432674408, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.35416666977107525, | |
| "rewards/xmlcount_reward_func": 0.46875, | |
| "step": 176 | |
| }, | |
| { | |
| "completion_length": 147.75000381469727, | |
| "epoch": 0.09474106784423926, | |
| "grad_norm": 0.834882915019989, | |
| "kl": 0.134497981518507, | |
| "learning_rate": 1.2198928378235717e-06, | |
| "loss": 0.0054, | |
| "reward": 1.4427083730697632, | |
| "reward_std": 0.5198761932551861, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.37500000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 177 | |
| }, | |
| { | |
| "completion_length": 100.08333778381348, | |
| "epoch": 0.0952763281145457, | |
| "grad_norm": 1.221563458442688, | |
| "kl": 0.16201673820614815, | |
| "learning_rate": 1.1900352350748026e-06, | |
| "loss": 0.0065, | |
| "reward": 1.338541716337204, | |
| "reward_std": 0.2559161148965359, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3958333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 178 | |
| }, | |
| { | |
| "completion_length": 127.83333969116211, | |
| "epoch": 0.09581158838485214, | |
| "grad_norm": 0.886989176273346, | |
| "kl": 0.15441275481134653, | |
| "learning_rate": 1.160433012552508e-06, | |
| "loss": 0.0062, | |
| "reward": 0.9375000149011612, | |
| "reward_std": 0.18744874745607376, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.0625, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3750000037252903, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 179 | |
| }, | |
| { | |
| "completion_length": 86.87500190734863, | |
| "epoch": 0.09634684865515857, | |
| "grad_norm": 1.1908172369003296, | |
| "kl": 0.15353485196828842, | |
| "learning_rate": 1.1310919412686248e-06, | |
| "loss": 0.0061, | |
| "reward": 1.0833333432674408, | |
| "reward_std": 0.10206207260489464, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 180 | |
| }, | |
| { | |
| "completion_length": 89.58333587646484, | |
| "epoch": 0.096882108925465, | |
| "grad_norm": 1.44786536693573, | |
| "kl": 0.1816324070096016, | |
| "learning_rate": 1.1020177413231334e-06, | |
| "loss": 0.0073, | |
| "reward": 1.510416716337204, | |
| "reward_std": 0.472514558583498, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.3750000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333358168602, | |
| "rewards/xmlcount_reward_func": 0.4895833358168602, | |
| "step": 181 | |
| }, | |
| { | |
| "completion_length": 82.58333778381348, | |
| "epoch": 0.09741736919577144, | |
| "grad_norm": 0.13121654093265533, | |
| "kl": 0.18894518539309502, | |
| "learning_rate": 1.073216080788921e-06, | |
| "loss": 0.0076, | |
| "reward": 1.25, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 182 | |
| }, | |
| { | |
| "completion_length": 221.0000123977661, | |
| "epoch": 0.09795262946607788, | |
| "grad_norm": 1.4435704946517944, | |
| "kl": 0.18115888815373182, | |
| "learning_rate": 1.0446925746067768e-06, | |
| "loss": 0.0072, | |
| "reward": 1.1875, | |
| "reward_std": 0.28912585973739624, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.2916666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.375, | |
| "rewards/xmlcount_reward_func": 0.4375, | |
| "step": 183 | |
| }, | |
| { | |
| "completion_length": 125.9583387374878, | |
| "epoch": 0.09848788973638432, | |
| "grad_norm": 1.2450430393218994, | |
| "kl": 0.14993033185601234, | |
| "learning_rate": 1.0164527834907468e-06, | |
| "loss": 0.006, | |
| "reward": 1.6093750596046448, | |
| "reward_std": 0.7003048211336136, | |
| "rewards/correctness_reward_func": 0.583333358168602, | |
| "rewards/int_reward_func": 0.1875000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3541666716337204, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 184 | |
| }, | |
| { | |
| "completion_length": 250.50000381469727, | |
| "epoch": 0.09902315000669075, | |
| "grad_norm": 0.6676502823829651, | |
| "kl": 0.14374011009931564, | |
| "learning_rate": 9.88502212844063e-07, | |
| "loss": 0.0058, | |
| "reward": 1.6502083837985992, | |
| "reward_std": 0.4829741967841983, | |
| "rewards/correctness_reward_func": 0.6666666716337204, | |
| "rewards/int_reward_func": 0.2916666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.2708333395421505, | |
| "rewards/xmlcount_reward_func": 0.42104167491197586, | |
| "step": 185 | |
| }, | |
| { | |
| "completion_length": 95.41666984558105, | |
| "epoch": 0.09955841027699719, | |
| "grad_norm": 0.14050611853599548, | |
| "kl": 0.19362322241067886, | |
| "learning_rate": 9.608463116858544e-07, | |
| "loss": 0.0077, | |
| "reward": 1.5, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.5, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.375, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 186 | |
| }, | |
| { | |
| "completion_length": 69.25000190734863, | |
| "epoch": 0.10009367054730363, | |
| "grad_norm": 0.7105045318603516, | |
| "kl": 0.22476506605744362, | |
| "learning_rate": 9.334904715888496e-07, | |
| "loss": 0.009, | |
| "reward": 1.2291666865348816, | |
| "reward_std": 0.05103103443980217, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.2291666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 187 | |
| }, | |
| { | |
| "completion_length": 81.08333778381348, | |
| "epoch": 0.10062893081761007, | |
| "grad_norm": 1.2536767721176147, | |
| "kl": 0.20191873610019684, | |
| "learning_rate": 9.064400256282757e-07, | |
| "loss": 0.0081, | |
| "reward": 1.0000000298023224, | |
| "reward_std": 0.20479072630405426, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.10416666977107525, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.4791666716337204, | |
| "step": 188 | |
| }, | |
| { | |
| "completion_length": 79.95833587646484, | |
| "epoch": 0.1011641910879165, | |
| "grad_norm": 0.1358516663312912, | |
| "kl": 0.2150093950331211, | |
| "learning_rate": 8.797002473421729e-07, | |
| "loss": 0.0086, | |
| "reward": 1.375, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 189 | |
| }, | |
| { | |
| "completion_length": 255.83334350585938, | |
| "epoch": 0.10169945135822293, | |
| "grad_norm": 0.8230968713760376, | |
| "kl": 0.1948665827512741, | |
| "learning_rate": 8.532763497032987e-07, | |
| "loss": 0.0078, | |
| "reward": 1.1041666865348816, | |
| "reward_std": 0.3931647092103958, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.2500000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.4375, | |
| "step": 190 | |
| }, | |
| { | |
| "completion_length": 72.45833492279053, | |
| "epoch": 0.10223471162852937, | |
| "grad_norm": 0.6617278456687927, | |
| "kl": 0.17371252551674843, | |
| "learning_rate": 8.271734841028553e-07, | |
| "loss": 0.0069, | |
| "reward": 1.4166666865348816, | |
| "reward_std": 0.4518480896949768, | |
| "rewards/correctness_reward_func": 0.2500000074505806, | |
| "rewards/int_reward_func": 0.1875, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 191 | |
| }, | |
| { | |
| "completion_length": 92.29166889190674, | |
| "epoch": 0.10276997189883581, | |
| "grad_norm": 0.7337960600852966, | |
| "kl": 0.18250929936766624, | |
| "learning_rate": 8.013967393462094e-07, | |
| "loss": 0.0073, | |
| "reward": 1.4375000596046448, | |
| "reward_std": 0.4927079305052757, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.2708333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 192 | |
| }, | |
| { | |
| "completion_length": 93.79166984558105, | |
| "epoch": 0.10330523216914224, | |
| "grad_norm": 1.3416187763214111, | |
| "kl": 0.19713782332837582, | |
| "learning_rate": 7.759511406608255e-07, | |
| "loss": 0.0079, | |
| "reward": 1.5156250596046448, | |
| "reward_std": 0.5118480771780014, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.37500000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333432674408, | |
| "rewards/xmlcount_reward_func": 0.4947916716337204, | |
| "step": 193 | |
| }, | |
| { | |
| "completion_length": 96.08333969116211, | |
| "epoch": 0.10384049243944868, | |
| "grad_norm": 0.9250803589820862, | |
| "kl": 0.21085454896092415, | |
| "learning_rate": 7.508416487165862e-07, | |
| "loss": 0.0084, | |
| "reward": 1.8125000596046448, | |
| "reward_std": 0.31970491632819176, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.4166666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 194 | |
| }, | |
| { | |
| "completion_length": 74.00000190734863, | |
| "epoch": 0.10437575270975512, | |
| "grad_norm": 1.441701889038086, | |
| "kl": 0.18313675373792648, | |
| "learning_rate": 7.260731586586983e-07, | |
| "loss": 0.0073, | |
| "reward": 1.7500000298023224, | |
| "reward_std": 0.3624359965324402, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.4583333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333432674408, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 195 | |
| }, | |
| { | |
| "completion_length": 116.54167175292969, | |
| "epoch": 0.10491101298006156, | |
| "grad_norm": 1.0711389780044556, | |
| "kl": 0.13384228572249413, | |
| "learning_rate": 7.016504991533727e-07, | |
| "loss": 0.0054, | |
| "reward": 1.1666667014360428, | |
| "reward_std": 0.16661180183291435, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.27083333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333395421505, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 196 | |
| }, | |
| { | |
| "completion_length": 78.83333587646484, | |
| "epoch": 0.105446273250368, | |
| "grad_norm": 1.2510522603988647, | |
| "kl": 0.18244327045977116, | |
| "learning_rate": 6.775784314464717e-07, | |
| "loss": 0.0073, | |
| "reward": 1.0781250298023224, | |
| "reward_std": 0.17936956882476807, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.10416666977107525, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.4947916716337204, | |
| "step": 197 | |
| }, | |
| { | |
| "completion_length": 201.83333492279053, | |
| "epoch": 0.10598153352067442, | |
| "grad_norm": 1.4353007078170776, | |
| "kl": 0.17845631763339043, | |
| "learning_rate": 6.538616484352902e-07, | |
| "loss": 0.0071, | |
| "reward": 1.4947916865348816, | |
| "reward_std": 0.45137757435441017, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.33333334140479565, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.375, | |
| "rewards/xmlcount_reward_func": 0.453125, | |
| "step": 198 | |
| }, | |
| { | |
| "completion_length": 315.6666736602783, | |
| "epoch": 0.10651679379098086, | |
| "grad_norm": 0.7709922790527344, | |
| "kl": 0.15007262770086527, | |
| "learning_rate": 6.305047737536707e-07, | |
| "loss": 0.006, | |
| "reward": 1.3333333730697632, | |
| "reward_std": 0.43266693875193596, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.4166666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333395421505, | |
| "rewards/xmlcount_reward_func": 0.4375, | |
| "step": 199 | |
| }, | |
| { | |
| "completion_length": 226.25001335144043, | |
| "epoch": 0.1070520540612873, | |
| "grad_norm": 0.4058845043182373, | |
| "kl": 0.24160834029316902, | |
| "learning_rate": 6.075123608706093e-07, | |
| "loss": 0.0097, | |
| "reward": 1.6354166865348816, | |
| "reward_std": 0.6109069883823395, | |
| "rewards/correctness_reward_func": 0.3333333358168602, | |
| "rewards/int_reward_func": 0.3958333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375, | |
| "rewards/xmlcount_reward_func": 0.46875, | |
| "step": 200 | |
| }, | |
| { | |
| "completion_length": 78.66666793823242, | |
| "epoch": 0.10758731433159374, | |
| "grad_norm": 0.5436657071113586, | |
| "kl": 0.16075460240244865, | |
| "learning_rate": 5.848888922025553e-07, | |
| "loss": 0.0064, | |
| "reward": 1.4375, | |
| "reward_std": 0.22008520364761353, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.3541666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 201 | |
| }, | |
| { | |
| "completion_length": 82.58333396911621, | |
| "epoch": 0.10812257460190017, | |
| "grad_norm": 0.7532062530517578, | |
| "kl": 0.1826519127935171, | |
| "learning_rate": 5.626387782395512e-07, | |
| "loss": 0.0073, | |
| "reward": 1.7916666865348816, | |
| "reward_std": 0.20412413775920868, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 202 | |
| }, | |
| { | |
| "completion_length": 79.16666984558105, | |
| "epoch": 0.10865783487220661, | |
| "grad_norm": 1.078783631324768, | |
| "kl": 0.1677638739347458, | |
| "learning_rate": 5.407663566854008e-07, | |
| "loss": 0.0067, | |
| "reward": 1.4375000298023224, | |
| "reward_std": 0.31970491632819176, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.35416666977107525, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 203 | |
| }, | |
| { | |
| "completion_length": 92.33333396911621, | |
| "epoch": 0.10919309514251305, | |
| "grad_norm": 0.8154336214065552, | |
| "kl": 0.18606754019856453, | |
| "learning_rate": 5.192758916120236e-07, | |
| "loss": 0.0074, | |
| "reward": 1.9375000298023224, | |
| "reward_std": 0.44672293961048126, | |
| "rewards/correctness_reward_func": 0.5000000223517418, | |
| "rewards/int_reward_func": 0.4375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 204 | |
| }, | |
| { | |
| "completion_length": 215.95834159851074, | |
| "epoch": 0.10972835541281949, | |
| "grad_norm": 0.8592817783355713, | |
| "kl": 0.15069226268678904, | |
| "learning_rate": 4.981715726281666e-07, | |
| "loss": 0.006, | |
| "reward": 1.2552083730697632, | |
| "reward_std": 0.21166006475687027, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3958333432674408, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.39583333395421505, | |
| "rewards/xmlcount_reward_func": 0.4635416716337204, | |
| "step": 205 | |
| }, | |
| { | |
| "completion_length": 160.62500381469727, | |
| "epoch": 0.11026361568312593, | |
| "grad_norm": 1.0854992866516113, | |
| "kl": 0.1672863345593214, | |
| "learning_rate": 4.774575140626317e-07, | |
| "loss": 0.0067, | |
| "reward": 1.3802083730697632, | |
| "reward_std": 0.4717924892902374, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.4166666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333358168602, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 206 | |
| }, | |
| { | |
| "completion_length": 85.62500190734863, | |
| "epoch": 0.11079887595343235, | |
| "grad_norm": 2.937671184539795, | |
| "kl": 0.22150231339037418, | |
| "learning_rate": 4.5713775416217884e-07, | |
| "loss": 0.0089, | |
| "reward": 1.1458333730697632, | |
| "reward_std": 0.3955717794597149, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.18750000186264515, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3750000074505806, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 207 | |
| }, | |
| { | |
| "completion_length": 86.54166889190674, | |
| "epoch": 0.11133413622373879, | |
| "grad_norm": 0.791983962059021, | |
| "kl": 0.2634577229619026, | |
| "learning_rate": 4.372162543042624e-07, | |
| "loss": 0.0105, | |
| "reward": 1.3541667014360428, | |
| "reward_std": 0.31970490142703056, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.3541666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666679084301, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 208 | |
| }, | |
| { | |
| "completion_length": 103.04166984558105, | |
| "epoch": 0.11186939649404523, | |
| "grad_norm": 0.7941485643386841, | |
| "kl": 0.15498985722661018, | |
| "learning_rate": 4.1769689822475147e-07, | |
| "loss": 0.0062, | |
| "reward": 1.2343750298023224, | |
| "reward_std": 0.3260645717382431, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.4375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.25000000186264515, | |
| "rewards/xmlcount_reward_func": 0.4635416716337204, | |
| "step": 209 | |
| }, | |
| { | |
| "completion_length": 126.79166984558105, | |
| "epoch": 0.11240465676435167, | |
| "grad_norm": 1.0149474143981934, | |
| "kl": 0.12467027455568314, | |
| "learning_rate": 3.9858349126078945e-07, | |
| "loss": 0.005, | |
| "reward": 1.7271667420864105, | |
| "reward_std": 0.5239234380424023, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.4166666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.416666679084301, | |
| "rewards/xmlcount_reward_func": 0.47716666758060455, | |
| "step": 210 | |
| }, | |
| { | |
| "completion_length": 85.79166984558105, | |
| "epoch": 0.1129399170346581, | |
| "grad_norm": 0.8396437764167786, | |
| "kl": 0.18657264113426208, | |
| "learning_rate": 3.798797596089351e-07, | |
| "loss": 0.0075, | |
| "reward": 1.5000000298023224, | |
| "reward_std": 0.32274864614009857, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.3333333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 211 | |
| }, | |
| { | |
| "completion_length": 108.00000381469727, | |
| "epoch": 0.11347517730496454, | |
| "grad_norm": 0.934079110622406, | |
| "kl": 0.1696683205664158, | |
| "learning_rate": 3.615893495987335e-07, | |
| "loss": 0.0068, | |
| "reward": 1.8125000298023224, | |
| "reward_std": 0.6319277845323086, | |
| "rewards/correctness_reward_func": 0.5000000149011612, | |
| "rewards/int_reward_func": 0.3958333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.416666679084301, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 212 | |
| }, | |
| { | |
| "completion_length": 193.0416669845581, | |
| "epoch": 0.11401043757527098, | |
| "grad_norm": 0.6677061319351196, | |
| "kl": 0.1520245149731636, | |
| "learning_rate": 3.4371582698185636e-07, | |
| "loss": 0.0061, | |
| "reward": 1.713541716337204, | |
| "reward_std": 0.5496542304754257, | |
| "rewards/correctness_reward_func": 0.583333358168602, | |
| "rewards/int_reward_func": 0.3333333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3333333395421505, | |
| "rewards/xmlcount_reward_func": 0.4635416716337204, | |
| "step": 213 | |
| }, | |
| { | |
| "completion_length": 86.66666984558105, | |
| "epoch": 0.11454569784557742, | |
| "grad_norm": 0.4593207538127899, | |
| "kl": 0.1518435962498188, | |
| "learning_rate": 3.262626762369525e-07, | |
| "loss": 0.0061, | |
| "reward": 1.0833333432674408, | |
| "reward_std": 0.23273734748363495, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.0625, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 214 | |
| }, | |
| { | |
| "completion_length": 104.45833778381348, | |
| "epoch": 0.11508095811588386, | |
| "grad_norm": 0.42647111415863037, | |
| "kl": 0.21028569713234901, | |
| "learning_rate": 3.092332998903416e-07, | |
| "loss": 0.0084, | |
| "reward": 1.6458333730697632, | |
| "reward_std": 0.2753772810101509, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.4166666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3333333358168602, | |
| "rewards/xmlcount_reward_func": 0.4791666716337204, | |
| "step": 215 | |
| }, | |
| { | |
| "completion_length": 325.2500114440918, | |
| "epoch": 0.11561621838619028, | |
| "grad_norm": 0.5365056991577148, | |
| "kl": 0.134639460593462, | |
| "learning_rate": 2.9263101785268253e-07, | |
| "loss": 0.0054, | |
| "reward": 1.604166679084301, | |
| "reward_std": 0.14088044688105583, | |
| "rewards/correctness_reward_func": 0.5, | |
| "rewards/int_reward_func": 0.2916666679084301, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.375, | |
| "rewards/xmlcount_reward_func": 0.4375, | |
| "step": 216 | |
| }, | |
| { | |
| "completion_length": 89.54166793823242, | |
| "epoch": 0.11615147865649672, | |
| "grad_norm": 0.6397629976272583, | |
| "kl": 0.21499066427350044, | |
| "learning_rate": 2.764590667717562e-07, | |
| "loss": 0.0086, | |
| "reward": 1.5, | |
| "reward_std": 0.19364917278289795, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.4583333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333358168602, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 217 | |
| }, | |
| { | |
| "completion_length": 85.50000095367432, | |
| "epoch": 0.11668673892680316, | |
| "grad_norm": 1.3944755792617798, | |
| "kl": 0.13307987339794636, | |
| "learning_rate": 2.6072059940146775e-07, | |
| "loss": 0.0053, | |
| "reward": 1.541666716337204, | |
| "reward_std": 0.5103103779256344, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.39583333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 218 | |
| }, | |
| { | |
| "completion_length": 80.66666793823242, | |
| "epoch": 0.1172219991971096, | |
| "grad_norm": 0.08477512001991272, | |
| "kl": 0.16065896674990654, | |
| "learning_rate": 2.454186839872158e-07, | |
| "loss": 0.0064, | |
| "reward": 1.125, | |
| "reward_std": 0.0, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 219 | |
| }, | |
| { | |
| "completion_length": 72.79166793823242, | |
| "epoch": 0.11775725946741603, | |
| "grad_norm": 0.6473060250282288, | |
| "kl": 0.2047443389892578, | |
| "learning_rate": 2.3055630366772857e-07, | |
| "loss": 0.0082, | |
| "reward": 1.6666666865348816, | |
| "reward_std": 0.20412413775920868, | |
| "rewards/correctness_reward_func": 0.4166666865348816, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 220 | |
| }, | |
| { | |
| "completion_length": 255.87500381469727, | |
| "epoch": 0.11829251973772247, | |
| "grad_norm": 1.165757656097412, | |
| "kl": 0.34885890036821365, | |
| "learning_rate": 2.1613635589349756e-07, | |
| "loss": 0.014, | |
| "reward": 1.2083333432674408, | |
| "reward_std": 0.5354874432086945, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.3125, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3750000074505806, | |
| "rewards/xmlcount_reward_func": 0.4375, | |
| "step": 221 | |
| }, | |
| { | |
| "completion_length": 68.04166889190674, | |
| "epoch": 0.1188277800080289, | |
| "grad_norm": 1.5734628438949585, | |
| "kl": 0.35767246037721634, | |
| "learning_rate": 2.0216165186191406e-07, | |
| "loss": 0.0143, | |
| "reward": 1.729166716337204, | |
| "reward_std": 0.4259376786649227, | |
| "rewards/correctness_reward_func": 0.5000000223517418, | |
| "rewards/int_reward_func": 0.25000000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 222 | |
| }, | |
| { | |
| "completion_length": 93.95833587646484, | |
| "epoch": 0.11936304027833534, | |
| "grad_norm": 0.3049551248550415, | |
| "kl": 0.14186285808682442, | |
| "learning_rate": 1.8863491596921745e-07, | |
| "loss": 0.0057, | |
| "reward": 1.375, | |
| "reward_std": 0.25, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666679084301, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 223 | |
| }, | |
| { | |
| "completion_length": 170.7500057220459, | |
| "epoch": 0.11989830054864177, | |
| "grad_norm": 0.8757752180099487, | |
| "kl": 0.17503754422068596, | |
| "learning_rate": 1.7555878527937164e-07, | |
| "loss": 0.007, | |
| "reward": 1.1865417063236237, | |
| "reward_std": 0.413798563182354, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.2291666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333358168602, | |
| "rewards/xmlcount_reward_func": 0.4782083332538605, | |
| "step": 224 | |
| }, | |
| { | |
| "completion_length": 191.8333396911621, | |
| "epoch": 0.12043356081894821, | |
| "grad_norm": 1.3906254768371582, | |
| "kl": 0.11161109246313572, | |
| "learning_rate": 1.629358090099639e-07, | |
| "loss": 0.0045, | |
| "reward": 0.96875, | |
| "reward_std": 0.29255440831184387, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.1875000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3125000037252903, | |
| "rewards/xmlcount_reward_func": 0.46875, | |
| "step": 225 | |
| }, | |
| { | |
| "completion_length": 150.75000762939453, | |
| "epoch": 0.12096882108925465, | |
| "grad_norm": 0.7032321691513062, | |
| "kl": 0.23093389347195625, | |
| "learning_rate": 1.507684480352292e-07, | |
| "loss": 0.0092, | |
| "reward": 1.651041716337204, | |
| "reward_std": 0.36715345084667206, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.4375000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.39583333395421505, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 226 | |
| }, | |
| { | |
| "completion_length": 173.04166793823242, | |
| "epoch": 0.12150408135956109, | |
| "grad_norm": 0.777036726474762, | |
| "kl": 0.22725828364491463, | |
| "learning_rate": 1.3905907440629752e-07, | |
| "loss": 0.0091, | |
| "reward": 1.333333358168602, | |
| "reward_std": 0.3425312591716647, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.3750000037252903, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333358168602, | |
| "rewards/xmlcount_reward_func": 0.4791666716337204, | |
| "step": 227 | |
| }, | |
| { | |
| "completion_length": 114.66667079925537, | |
| "epoch": 0.12203934162986753, | |
| "grad_norm": 2.417041778564453, | |
| "kl": 0.15211265347898006, | |
| "learning_rate": 1.278099708887587e-07, | |
| "loss": 0.0061, | |
| "reward": 2.067708373069763, | |
| "reward_std": 0.8522266149520874, | |
| "rewards/correctness_reward_func": 0.6666666716337204, | |
| "rewards/int_reward_func": 0.4791666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 228 | |
| }, | |
| { | |
| "completion_length": 71.45833587646484, | |
| "epoch": 0.12257460190017395, | |
| "grad_norm": 1.4324641227722168, | |
| "kl": 0.2629435919225216, | |
| "learning_rate": 1.1702333051763271e-07, | |
| "loss": 0.0105, | |
| "reward": 1.4375000149011612, | |
| "reward_std": 0.4443886801600456, | |
| "rewards/correctness_reward_func": 0.25, | |
| "rewards/int_reward_func": 0.20833333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 229 | |
| }, | |
| { | |
| "completion_length": 265.95834159851074, | |
| "epoch": 0.1231098621704804, | |
| "grad_norm": 0.49865278601646423, | |
| "kl": 0.08768011070787907, | |
| "learning_rate": 1.067012561698319e-07, | |
| "loss": 0.0035, | |
| "reward": 0.8750000149011612, | |
| "reward_std": 0.42222120985388756, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.02083333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3333333432674408, | |
| "rewards/xmlcount_reward_func": 0.4375, | |
| "step": 230 | |
| }, | |
| { | |
| "completion_length": 365.291672706604, | |
| "epoch": 0.12364512244078683, | |
| "grad_norm": 0.5076226592063904, | |
| "kl": 0.13591468706727028, | |
| "learning_rate": 9.684576015420277e-08, | |
| "loss": 0.0054, | |
| "reward": 1.083333358168602, | |
| "reward_std": 0.49768130481243134, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.25, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.27083333395421505, | |
| "rewards/xmlcount_reward_func": 0.3958333358168602, | |
| "step": 231 | |
| }, | |
| { | |
| "completion_length": 95.41666793823242, | |
| "epoch": 0.12418038271109327, | |
| "grad_norm": 1.2022953033447266, | |
| "kl": 0.14907664991915226, | |
| "learning_rate": 8.745876381922147e-08, | |
| "loss": 0.006, | |
| "reward": 1.3697917461395264, | |
| "reward_std": 0.4918174706399441, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.31250000186264515, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.4739583358168602, | |
| "step": 232 | |
| }, | |
| { | |
| "completion_length": 113.79166793823242, | |
| "epoch": 0.1247156429813997, | |
| "grad_norm": 0.5673840641975403, | |
| "kl": 0.26192033290863037, | |
| "learning_rate": 7.854209717842231e-08, | |
| "loss": 0.0105, | |
| "reward": 1.3125000149011612, | |
| "reward_std": 0.15864631533622742, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.37500000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375000074505806, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 233 | |
| }, | |
| { | |
| "completion_length": 112.12500190734863, | |
| "epoch": 0.12525090325170615, | |
| "grad_norm": 1.3854151964187622, | |
| "kl": 0.17152241989970207, | |
| "learning_rate": 7.009749855363457e-08, | |
| "loss": 0.0069, | |
| "reward": 1.4375000596046448, | |
| "reward_std": 0.4543575756251812, | |
| "rewards/correctness_reward_func": 0.1666666716337204, | |
| "rewards/int_reward_func": 0.33333333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375000074505806, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 234 | |
| }, | |
| { | |
| "completion_length": 98.75000381469727, | |
| "epoch": 0.12578616352201258, | |
| "grad_norm": 0.790249764919281, | |
| "kl": 0.14792344719171524, | |
| "learning_rate": 6.212661423609184e-08, | |
| "loss": 0.0059, | |
| "reward": 1.7291666865348816, | |
| "reward_std": 0.5564306005835533, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.4791666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 235 | |
| }, | |
| { | |
| "completion_length": 144.8333396911621, | |
| "epoch": 0.126321423792319, | |
| "grad_norm": 0.2076501101255417, | |
| "kl": 0.18184524960815907, | |
| "learning_rate": 5.463099816548578e-08, | |
| "loss": 0.0073, | |
| "reward": 1.4479166865348816, | |
| "reward_std": 0.2685803771018982, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.4583333358168602, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375, | |
| "rewards/xmlcount_reward_func": 0.46875, | |
| "step": 236 | |
| }, | |
| { | |
| "completion_length": 142.12500190734863, | |
| "epoch": 0.12685668406262546, | |
| "grad_norm": 1.9539458751678467, | |
| "kl": 0.24447984993457794, | |
| "learning_rate": 4.761211162702117e-08, | |
| "loss": 0.0098, | |
| "reward": 1.2968750596046448, | |
| "reward_std": 0.29226116091012955, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.4791666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3333333469927311, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 237 | |
| }, | |
| { | |
| "completion_length": 72.37500381469727, | |
| "epoch": 0.12739194433293188, | |
| "grad_norm": 1.3912091255187988, | |
| "kl": 0.29350727051496506, | |
| "learning_rate": 4.1071322966535487e-08, | |
| "loss": 0.0117, | |
| "reward": 1.291666716337204, | |
| "reward_std": 0.2096773497760296, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.3125000074505806, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 238 | |
| }, | |
| { | |
| "completion_length": 82.41667175292969, | |
| "epoch": 0.12792720460323834, | |
| "grad_norm": 0.43811848759651184, | |
| "kl": 0.202481709420681, | |
| "learning_rate": 3.5009907323737826e-08, | |
| "loss": 0.0081, | |
| "reward": 1.2083333432674408, | |
| "reward_std": 0.06454972922801971, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.2291666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 239 | |
| }, | |
| { | |
| "completion_length": 85.45833587646484, | |
| "epoch": 0.12846246487354476, | |
| "grad_norm": 0.8119534850120544, | |
| "kl": 0.23333512246608734, | |
| "learning_rate": 2.9429046383618042e-08, | |
| "loss": 0.0093, | |
| "reward": 1.1666666865348816, | |
| "reward_std": 0.11949635669589043, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.1666666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 240 | |
| }, | |
| { | |
| "completion_length": 160.91666984558105, | |
| "epoch": 0.1289977251438512, | |
| "grad_norm": 1.1854231357574463, | |
| "kl": 0.15807193890213966, | |
| "learning_rate": 2.4329828146074096e-08, | |
| "loss": 0.0063, | |
| "reward": 0.8219999894499779, | |
| "reward_std": 0.6791375987231731, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.1875, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4375000074505806, | |
| "rewards/xmlcount_reward_func": 0.19699998199939728, | |
| "step": 241 | |
| }, | |
| { | |
| "completion_length": 72.75000190734863, | |
| "epoch": 0.12953298541415764, | |
| "grad_norm": 0.45305031538009644, | |
| "kl": 0.20153097435832024, | |
| "learning_rate": 1.9713246713805588e-08, | |
| "loss": 0.0081, | |
| "reward": 1.3125, | |
| "reward_std": 0.22008520364761353, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.2291666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 242 | |
| }, | |
| { | |
| "completion_length": 152.1666717529297, | |
| "epoch": 0.13006824568446407, | |
| "grad_norm": 0.4157455861568451, | |
| "kl": 0.1709270216524601, | |
| "learning_rate": 1.5580202098509078e-08, | |
| "loss": 0.0068, | |
| "reward": 1.4427083432674408, | |
| "reward_std": 0.345734566450119, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.4583333432674408, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666679084301, | |
| "rewards/xmlcount_reward_func": 0.484375, | |
| "step": 243 | |
| }, | |
| { | |
| "completion_length": 72.79166889190674, | |
| "epoch": 0.1306035059547705, | |
| "grad_norm": 1.6172090768814087, | |
| "kl": 0.1881002075970173, | |
| "learning_rate": 1.193150004542204e-08, | |
| "loss": 0.0075, | |
| "reward": 1.4322916865348816, | |
| "reward_std": 0.26791293919086456, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.375, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.4947916716337204, | |
| "step": 244 | |
| }, | |
| { | |
| "completion_length": 63.541666984558105, | |
| "epoch": 0.13113876622507695, | |
| "grad_norm": 0.6279511451721191, | |
| "kl": 0.26533466950058937, | |
| "learning_rate": 8.767851876239075e-09, | |
| "loss": 0.0106, | |
| "reward": 1.2708333432674408, | |
| "reward_std": 0.05103103816509247, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.27083333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 245 | |
| }, | |
| { | |
| "completion_length": 188.20833587646484, | |
| "epoch": 0.13167402649538337, | |
| "grad_norm": 0.917432963848114, | |
| "kl": 0.16238786652684212, | |
| "learning_rate": 6.089874350439507e-09, | |
| "loss": 0.0065, | |
| "reward": 1.6145834028720856, | |
| "reward_std": 0.7706367075443268, | |
| "rewards/correctness_reward_func": 0.3333333432674408, | |
| "rewards/int_reward_func": 0.3958333432674408, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4166666716337204, | |
| "rewards/xmlcount_reward_func": 0.46875, | |
| "step": 246 | |
| }, | |
| { | |
| "completion_length": 102.25000381469727, | |
| "epoch": 0.13220928676568983, | |
| "grad_norm": 0.6094866394996643, | |
| "kl": 0.1517469845712185, | |
| "learning_rate": 3.8980895450474455e-09, | |
| "loss": 0.0061, | |
| "reward": 1.2708333432674408, | |
| "reward_std": 0.27258947491645813, | |
| "rewards/correctness_reward_func": 0.0833333358168602, | |
| "rewards/int_reward_func": 0.2291666716337204, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4583333358168602, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 247 | |
| }, | |
| { | |
| "completion_length": 265.0000114440918, | |
| "epoch": 0.13274454703599625, | |
| "grad_norm": 1.4663077592849731, | |
| "kl": 0.1445157825946808, | |
| "learning_rate": 2.192924752854042e-09, | |
| "loss": 0.0058, | |
| "reward": 0.9218750074505806, | |
| "reward_std": 0.21220333129167557, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.18750000186264515, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3125, | |
| "rewards/xmlcount_reward_func": 0.421875, | |
| "step": 248 | |
| }, | |
| { | |
| "completion_length": 78.83333587646484, | |
| "epoch": 0.13327980730630268, | |
| "grad_norm": 1.0312261581420898, | |
| "kl": 0.16930826753377914, | |
| "learning_rate": 9.747123991141193e-10, | |
| "loss": 0.0068, | |
| "reward": 1.229166716337204, | |
| "reward_std": 0.1530931033194065, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/int_reward_func": 0.25000000558793545, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.4791666716337204, | |
| "rewards/xmlcount_reward_func": 0.5, | |
| "step": 249 | |
| }, | |
| { | |
| "completion_length": 271.9166793823242, | |
| "epoch": 0.13381506757660913, | |
| "grad_norm": 0.3518020808696747, | |
| "kl": 0.12262176536023617, | |
| "learning_rate": 2.43689976739403e-10, | |
| "loss": 0.0049, | |
| "reward": 1.5572916716337204, | |
| "reward_std": 0.25069504231214523, | |
| "rewards/correctness_reward_func": 0.5, | |
| "rewards/int_reward_func": 0.20833333395421505, | |
| "rewards/soft_format_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.3958333395421505, | |
| "rewards/xmlcount_reward_func": 0.453125, | |
| "step": 250 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 250, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 250, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 6, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |