{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 4.995499549954996, "eval_steps": 15, "global_step": 395, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0279017857142857, "completions/max_length": 2048.0, "completions/max_terminated_length": 1978.0, "completions/mean_length": 538.3058471679688, "completions/mean_terminated_length": 494.97357177734375, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "epoch": 0.012601260126012601, "grad_norm": 0.602237343788147, "learning_rate": 0.0, "loss": 0.0039, "num_tokens": 609906.0, "reward": 0.2795497477054596, "reward_std": 0.41905471682548523, "rewards/accuracy_reward/mean": 0.1685267835855484, "rewards/accuracy_reward/std": 0.3745425045490265, "rewards/cosine_scaled_reward/mean": 0.1684744656085968, "rewards/cosine_scaled_reward/std": 0.37442949414253235, "rewards/format_reward/mean": 0.0535714291036129, "rewards/format_reward/std": 0.2252957671880722, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.021484375, "completions/max_length": 2048.0, "completions/max_terminated_length": 1946.5, "completions/mean_length": 523.0030899047852, "completions/mean_terminated_length": 489.49188232421875, "completions/min_length": 2.5, "completions/min_terminated_length": 2.5, "epoch": 0.063006300630063, "grad_norm": 0.32099631428718567, "learning_rate": 1e-07, "loss": 0.0035, "num_tokens": 3031821.0, "reward": 0.2492716759443283, "reward_std": 0.3989013135433197, "rewards/accuracy_reward/mean": 0.1512276753783226, "rewards/accuracy_reward/std": 0.3584018871188164, "rewards/cosine_scaled_reward/mean": 0.15116609632968903, "rewards/cosine_scaled_reward/std": 0.35826219618320465, "rewards/format_reward/mean": 0.044921875931322575, "rewards/format_reward/std": 0.20668350905179977, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.024107142857142883, "completions/max_length": 2048.0, "completions/max_terminated_length": 1979.6, "completions/mean_length": 547.3105102539063, "completions/mean_terminated_length": 510.2965393066406, "completions/min_length": 4.8, "completions/min_terminated_length": 4.8, "epoch": 0.126012601260126, "grad_norm": 0.4403175711631775, "learning_rate": 2.25e-07, "loss": 0.0027, "num_tokens": 6168628.0, "reward": 0.2597510188817978, "reward_std": 0.39840335249900816, "rewards/accuracy_reward/mean": 0.15892857015132905, "rewards/accuracy_reward/std": 0.365299665927887, "rewards/cosine_scaled_reward/mean": 0.15878771841526032, "rewards/cosine_scaled_reward/std": 0.3649946331977844, "rewards/format_reward/mean": 0.04285714328289032, "rewards/format_reward/std": 0.2020670711994171, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02589285714285714, "completions/max_length": 2048.0, "completions/max_terminated_length": 1994.4, "completions/mean_length": 555.5024780273437, "completions/mean_terminated_length": 515.7921813964844, "completions/min_length": 6.2, "completions/min_terminated_length": 6.2, "epoch": 0.18901890189018902, "grad_norm": 1.8242275714874268, "learning_rate": 3.5e-07, "loss": 0.0031, "num_tokens": 9301535.0, "reward": 0.27981547117233274, "reward_std": 0.40177412033081056, "rewards/accuracy_reward/mean": 0.16138392984867095, "rewards/accuracy_reward/std": 0.36725109815597534, "rewards/cosine_scaled_reward/mean": 0.16119343042373657, "rewards/cosine_scaled_reward/std": 0.36687323451042175, "rewards/format_reward/mean": 0.07566964402794837, "rewards/format_reward/std": 0.2617058604955673, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02209821428571428, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 548.893994140625, "completions/mean_terminated_length": 515.1586364746094, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "epoch": 0.252025202520252, "grad_norm": 0.6227929592132568, "learning_rate": 4.7499999999999995e-07, "loss": -0.0047, "num_tokens": 12415812.0, "reward": 0.32004910707473755, "reward_std": 0.4481643557548523, "rewards/accuracy_reward/mean": 0.17611607313156127, "rewards/accuracy_reward/std": 0.3801506757736206, "rewards/cosine_scaled_reward/mean": 0.1758124738931656, "rewards/cosine_scaled_reward/std": 0.37958953976631166, "rewards/format_reward/mean": 0.11205357164144517, "rewards/format_reward/std": 0.3129072666168213, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.02075892857142858, "completions/max_length": 2048.0, "completions/max_terminated_length": 1966.2, "completions/mean_length": 565.0823852539063, "completions/mean_terminated_length": 533.6526733398438, "completions/min_length": 2.8, "completions/min_terminated_length": 2.8, "epoch": 0.31503150315031503, "grad_norm": 0.38101691007614136, "learning_rate": 6e-07, "loss": -0.0097, "num_tokens": 15816741.0, "reward": 0.5020053923130036, "reward_std": 0.6075942158699036, "rewards/accuracy_reward/mean": 0.2622948855161667, "rewards/accuracy_reward/std": 0.4395259916782379, "rewards/cosine_scaled_reward/mean": 0.26026553511619566, "rewards/cosine_scaled_reward/std": 0.43781105875968934, "rewards/format_reward/mean": 0.22053571343421935, "rewards/format_reward/std": 0.40703149437904357, "step": 25 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.017187499999999977, "completions/max_length": 2048.0, "completions/max_terminated_length": 1967.0, "completions/mean_length": 578.9469116210937, "completions/mean_terminated_length": 553.24716796875, "completions/min_length": 6.8, "completions/min_terminated_length": 6.8, "epoch": 0.37803780378037805, "grad_norm": 0.33144351840019226, "learning_rate": 7.249999999999999e-07, "loss": -0.0055, "num_tokens": 19248647.0, "reward": 0.7754053592681884, "reward_std": 0.677242922782898, "rewards/accuracy_reward/mean": 0.35334821343421935, "rewards/accuracy_reward/std": 0.4751245081424713, "rewards/cosine_scaled_reward/mean": 0.3530427932739258, "rewards/cosine_scaled_reward/std": 0.4747708261013031, "rewards/format_reward/mean": 0.4910714328289032, "rewards/format_reward/std": 0.48938958048820497, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.014285714285714258, "completions/max_length": 2048.0, "completions/max_terminated_length": 1959.6, "completions/mean_length": 608.9741455078125, "completions/mean_terminated_length": 588.129736328125, "completions/min_length": 12.0, "completions/min_terminated_length": 12.0, "epoch": 0.44104410441044106, "grad_norm": 0.2479850798845291, "learning_rate": 8.499999999999999e-07, "loss": 0.0144, "num_tokens": 22822291.0, "reward": 1.1359302043914794, "reward_std": 0.633991277217865, "rewards/accuracy_reward/mean": 0.5194196462631225, "rewards/accuracy_reward/std": 0.49612280130386355, "rewards/cosine_scaled_reward/mean": 0.518735259771347, "rewards/cosine_scaled_reward/std": 0.495624178647995, "rewards/format_reward/mean": 0.7142857193946839, "rewards/format_reward/std": 0.4489860713481903, "step": 35 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 2048.0, "completions/max_terminated_length": 1967.4, "completions/mean_length": 672.230615234375, "completions/mean_terminated_length": 654.8196044921875, "completions/min_length": 70.6, "completions/min_terminated_length": 70.6, "epoch": 0.504050405040504, "grad_norm": 0.19197942316532135, "learning_rate": 9.75e-07, "loss": 0.0261, "num_tokens": 26671876.0, "reward": 1.3178162574768066, "reward_std": 0.5191608130931854, "rewards/accuracy_reward/mean": 0.5977678537368775, "rewards/accuracy_reward/std": 0.48948062062263487, "rewards/cosine_scaled_reward/mean": 0.5967931866645813, "rewards/cosine_scaled_reward/std": 0.4888714671134949, "rewards/format_reward/mean": 0.8433035612106323, "rewards/format_reward/std": 0.35585846900939944, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01383928571428572, "completions/max_length": 2048.0, "completions/max_terminated_length": 2024.0, "completions/mean_length": 691.8661010742187, "completions/mean_terminated_length": 672.869873046875, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "epoch": 0.5670567056705671, "grad_norm": 0.12823984026908875, "learning_rate": 9.99686774088805e-07, "loss": 0.0293, "num_tokens": 30644708.0, "reward": 1.4662254571914672, "reward_std": 0.4329299807548523, "rewards/accuracy_reward/mean": 0.6647321343421936, "rewards/accuracy_reward/std": 0.47048051953315734, "rewards/cosine_scaled_reward/mean": 0.6637008666992188, "rewards/cosine_scaled_reward/std": 0.46995849609375, "rewards/format_reward/mean": 0.9392857074737548, "rewards/format_reward/std": 0.23625308275222778, "step": 45 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01517857142857142, "completions/max_length": 2048.0, "completions/max_terminated_length": 2006.2, "completions/mean_length": 724.4636352539062, "completions/mean_terminated_length": 704.2110107421875, "completions/min_length": 183.8, "completions/min_terminated_length": 183.8, "epoch": 0.6300630063006301, "grad_norm": 0.11830825358629227, "learning_rate": 9.984149663879992e-07, "loss": 0.0277, "num_tokens": 34704233.0, "reward": 1.4932293176651001, "reward_std": 0.39716021418571473, "rewards/accuracy_reward/mean": 0.6720982074737549, "rewards/accuracy_reward/std": 0.46862396597862244, "rewards/cosine_scaled_reward/mean": 0.6715031266212463, "rewards/cosine_scaled_reward/std": 0.4683354377746582, "rewards/format_reward/mean": 0.9707589268684387, "rewards/format_reward/std": 0.1669136554002762, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.009821428571428559, "completions/max_length": 2048.0, "completions/max_terminated_length": 1976.2, "completions/mean_length": 713.6201171875, "completions/mean_terminated_length": 700.3927490234375, "completions/min_length": 167.4, "completions/min_terminated_length": 167.4, "epoch": 0.693069306930693, "grad_norm": 0.10666404664516449, "learning_rate": 9.961674879848956e-07, "loss": 0.0233, "num_tokens": 38681571.0, "reward": 1.5286698818206788, "reward_std": 0.36640719771385194, "rewards/accuracy_reward/mean": 0.6915178418159484, "rewards/accuracy_reward/std": 0.4611486494541168, "rewards/cosine_scaled_reward/mean": 0.6908218264579773, "rewards/cosine_scaled_reward/std": 0.4608599662780762, "rewards/format_reward/mean": 0.9834821462631226, "rewards/format_reward/std": 0.12500424087047576, "step": 55 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.012053571428571419, "completions/max_length": 2048.0, "completions/max_terminated_length": 1924.6, "completions/mean_length": 711.6618530273438, "completions/mean_terminated_length": 695.3810913085938, "completions/min_length": 164.0, "completions/min_terminated_length": 164.0, "epoch": 0.7560756075607561, "grad_norm": 0.1039816215634346, "learning_rate": 9.929487384240101e-07, "loss": 0.0233, "num_tokens": 42628176.0, "reward": 1.5549091339111327, "reward_std": 0.3488038718700409, "rewards/accuracy_reward/mean": 0.7089285612106323, "rewards/accuracy_reward/std": 0.45444890260696413, "rewards/cosine_scaled_reward/mean": 0.7075859427452087, "rewards/cosine_scaled_reward/std": 0.4539263606071472, "rewards/format_reward/mean": 0.984375, "rewards/format_reward/std": 0.1233037456870079, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01339285714285714, "completions/max_length": 2048.0, "completions/max_terminated_length": 1963.2, "completions/mean_length": 713.0527099609375, "completions/mean_terminated_length": 694.9294799804687, "completions/min_length": 205.6, "completions/min_terminated_length": 205.6, "epoch": 0.819081908190819, "grad_norm": 0.11131048947572708, "learning_rate": 9.88765018558761e-07, "loss": 0.0214, "num_tokens": 46613412.0, "reward": 1.553797197341919, "reward_std": 0.34091995358467103, "rewards/accuracy_reward/mean": 0.7078125, "rewards/accuracy_reward/std": 0.45394955277442933, "rewards/cosine_scaled_reward/mean": 0.7067014455795289, "rewards/cosine_scaled_reward/std": 0.4534592032432556, "rewards/format_reward/mean": 0.9852678537368774, "rewards/format_reward/std": 0.11978632807731629, "step": 65 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010267857142857162, "completions/max_length": 2048.0, "completions/max_terminated_length": 1950.2, "completions/mean_length": 687.0343994140625, "completions/mean_terminated_length": 672.891162109375, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "epoch": 0.8820882088208821, "grad_norm": 0.11210864782333374, "learning_rate": 9.836245182172519e-07, "loss": 0.0192, "num_tokens": 50425302.0, "reward": 1.568692970275879, "reward_std": 0.3520396828651428, "rewards/accuracy_reward/mean": 0.7180039405822753, "rewards/accuracy_reward/std": 0.44985153079032897, "rewards/cosine_scaled_reward/mean": 0.7167718410491943, "rewards/cosine_scaled_reward/std": 0.44936810731887816, "rewards/format_reward/mean": 0.9883928537368775, "rewards/format_reward/std": 0.10585158914327622, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.007366071428571397, "completions/max_length": 2048.0, "completions/max_terminated_length": 1996.0, "completions/mean_length": 702.3578369140625, "completions/mean_terminated_length": 692.3953125, "completions/min_length": 167.2, "completions/min_terminated_length": 167.2, "epoch": 0.9450945094509451, "grad_norm": 0.11102982610464096, "learning_rate": 9.775373001702997e-07, "loss": 0.0159, "num_tokens": 54367609.0, "reward": 1.5509341955184937, "reward_std": 0.3260269582271576, "rewards/accuracy_reward/mean": 0.7037946343421936, "rewards/accuracy_reward/std": 0.45627689361572266, "rewards/cosine_scaled_reward/mean": 0.7023146390914917, "rewards/cosine_scaled_reward/std": 0.45560853481292723, "rewards/format_reward/mean": 0.9919642925262451, "rewards/format_reward/std": 0.08732087463140488, "step": 75 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.008453970180305114, "completions/max_length": 2048.0, "completions/max_terminated_length": 1951.4, "completions/mean_length": 707.547412109375, "completions/mean_terminated_length": 696.137548828125, "completions/min_length": 175.0, "completions/min_terminated_length": 175.0, "epoch": 1.0126012601260126, "grad_norm": 0.10652182251214981, "learning_rate": 9.705152804330871e-07, "loss": 0.0192, "num_tokens": 58375847.0, "reward": 1.5461820125579835, "reward_std": 0.3652195274829865, "rewards/accuracy_reward/mean": 0.7006696343421936, "rewards/accuracy_reward/std": 0.4574929714202881, "rewards/cosine_scaled_reward/mean": 0.6995066642761231, "rewards/cosine_scaled_reward/std": 0.45710301399230957, "rewards/format_reward/mean": 0.9915178656578064, "rewards/format_reward/std": 0.0895774394273758, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.005580357142857162, "completions/max_length": 2048.0, "completions/max_terminated_length": 1953.8, "completions/mean_length": 682.2417724609375, "completions/mean_terminated_length": 674.6181396484375, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "epoch": 1.0756075607560756, "grad_norm": 0.11581874638795853, "learning_rate": 9.625722049390045e-07, "loss": 0.0092, "num_tokens": 62230690.0, "reward": 1.5838431358337401, "reward_std": 0.3170156180858612, "rewards/accuracy_reward/mean": 0.7247767925262452, "rewards/accuracy_reward/std": 0.4461586236953735, "rewards/cosine_scaled_reward/mean": 0.723936116695404, "rewards/cosine_scaled_reward/std": 0.4458396315574646, "rewards/format_reward/mean": 0.9941964268684387, "rewards/format_reward/std": 0.07385859042406082, "step": 85 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.010714285714285721, "completions/max_length": 2048.0, "completions/max_terminated_length": 2015.2, "completions/mean_length": 713.357861328125, "completions/mean_terminated_length": 698.86904296875, "completions/min_length": 212.2, "completions/min_terminated_length": 212.2, "epoch": 1.1386138613861387, "grad_norm": 0.10194957256317139, "learning_rate": 9.537236226313388e-07, "loss": 0.0201, "num_tokens": 66250349.0, "reward": 1.5542301416397095, "reward_std": 0.33158175349235536, "rewards/accuracy_reward/mean": 0.7066964268684387, "rewards/accuracy_reward/std": 0.453377491235733, "rewards/cosine_scaled_reward/mean": 0.7060048222541809, "rewards/cosine_scaled_reward/std": 0.45305579900741577, "rewards/format_reward/mean": 0.989062488079071, "rewards/format_reward/std": 0.1036216139793396, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00959821428571428, "completions/max_length": 2048.0, "completions/max_terminated_length": 1982.4, "completions/mean_length": 696.29912109375, "completions/mean_terminated_length": 683.2262084960937, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "epoch": 1.2016201620162017, "grad_norm": 0.11849887669086456, "learning_rate": 9.439868550254868e-07, "loss": 0.0179, "num_tokens": 70144897.0, "reward": 1.5924715518951416, "reward_std": 0.3465422332286835, "rewards/accuracy_reward/mean": 0.7321428537368775, "rewards/accuracy_reward/std": 0.4421205699443817, "rewards/cosine_scaled_reward/mean": 0.7307018756866455, "rewards/cosine_scaled_reward/std": 0.44154341220855714, "rewards/format_reward/mean": 0.9899553656578064, "rewards/format_reward/std": 0.09479753226041794, "step": 95 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0046875, "completions/max_length": 2023.8, "completions/max_terminated_length": 1962.0, "completions/mean_length": 685.8980224609375, "completions/mean_terminated_length": 679.4762451171875, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "epoch": 1.2646264626462647, "grad_norm": 0.11047360301017761, "learning_rate": 9.333809623012763e-07, "loss": 0.0137, "num_tokens": 73963520.0, "reward": 1.6014429330825806, "reward_std": 0.3224481403827667, "rewards/accuracy_reward/mean": 0.7363839387893677, "rewards/accuracy_reward/std": 0.4394433259963989, "rewards/cosine_scaled_reward/mean": 0.7348053455352783, "rewards/cosine_scaled_reward/std": 0.4390797853469849, "rewards/format_reward/mean": 0.995312511920929, "rewards/format_reward/std": 0.060620172321796416, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.006026785714285698, "completions/max_length": 2048.0, "completions/max_terminated_length": 1955.6, "completions/mean_length": 685.2373168945312, "completions/mean_terminated_length": 677.0635864257813, "completions/min_length": 165.2, "completions/min_terminated_length": 165.2, "epoch": 1.3276327632763276, "grad_norm": 0.11911053210496902, "learning_rate": 9.219267059917681e-07, "loss": 0.0179, "num_tokens": 77867639.0, "reward": 1.5839551210403442, "reward_std": 0.3121181547641754, "rewards/accuracy_reward/mean": 0.725, "rewards/accuracy_reward/std": 0.44613986015319823, "rewards/cosine_scaled_reward/mean": 0.7243833780288697, "rewards/cosine_scaled_reward/std": 0.4459415376186371, "rewards/format_reward/mean": 0.9935267806053162, "rewards/format_reward/std": 0.07562160044908524, "step": 105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0049107142857142795, "completions/max_length": 1989.2, "completions/max_terminated_length": 1903.0, "completions/mean_length": 677.8884155273438, "completions/mean_terminated_length": 671.1434326171875, "completions/min_length": 192.4, "completions/min_terminated_length": 192.4, "epoch": 1.3906390639063906, "grad_norm": 0.12140108644962311, "learning_rate": 9.096465083415807e-07, "loss": 0.0172, "num_tokens": 81711467.0, "reward": 1.6193986415863038, "reward_std": 0.3027470111846924, "rewards/accuracy_reward/mean": 0.7484375, "rewards/accuracy_reward/std": 0.4324600279331207, "rewards/cosine_scaled_reward/mean": 0.7475025296211243, "rewards/cosine_scaled_reward/std": 0.43219619393348696, "rewards/format_reward/mean": 0.9944196462631225, "rewards/format_reward/std": 0.07256761714816093, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00825892857142858, "completions/max_length": 2032.2, "completions/max_terminated_length": 1932.0, "completions/mean_length": 692.1948974609375, "completions/mean_terminated_length": 680.9073486328125, "completions/min_length": 187.8, "completions/min_terminated_length": 187.8, "epoch": 1.4536453645364538, "grad_norm": 0.13913048803806305, "learning_rate": 8.96564408414293e-07, "loss": 0.014, "num_tokens": 85604932.0, "reward": 1.5589273691177368, "reward_std": 0.3327997088432312, "rewards/accuracy_reward/mean": 0.7130610585212708, "rewards/accuracy_reward/std": 0.4512468934059143, "rewards/cosine_scaled_reward/mean": 0.70667005777359, "rewards/cosine_scaled_reward/std": 0.45351189374923706, "rewards/format_reward/mean": 0.9915178656578064, "rewards/format_reward/std": 0.07997382432222366, "step": 115 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.005357142857142861, "completions/max_length": 2048.0, "completions/max_terminated_length": 1993.8, "completions/mean_length": 657.3136474609375, "completions/mean_terminated_length": 649.76474609375, "completions/min_length": 174.0, "completions/min_terminated_length": 174.0, "epoch": 1.5166516651665165, "grad_norm": 0.11739910393953323, "learning_rate": 8.827060150348489e-07, "loss": 0.0121, "num_tokens": 89320697.0, "reward": 1.6037935495376587, "reward_std": 0.2636448621749878, "rewards/accuracy_reward/mean": 0.7381696462631225, "rewards/accuracy_reward/std": 0.4392316222190857, "rewards/cosine_scaled_reward/mean": 0.7377209424972534, "rewards/cosine_scaled_reward/std": 0.4390567898750305, "rewards/format_reward/mean": 0.9935267925262451, "rewards/format_reward/std": 0.07422313913702965, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.009821428571428559, "completions/max_length": 2048.0, "completions/max_terminated_length": 1931.4, "completions/mean_length": 687.1069580078125, "completions/mean_terminated_length": 673.6233276367187, "completions/min_length": 177.8, "completions/min_terminated_length": 177.8, "epoch": 1.5796579657965797, "grad_norm": 0.11269731819629669, "learning_rate": 8.680984566590792e-07, "loss": 0.0104, "num_tokens": 93191536.0, "reward": 1.6061813592910767, "reward_std": 0.3020260453224182, "rewards/accuracy_reward/mean": 0.7410714387893677, "rewards/accuracy_reward/std": 0.4377245783805847, "rewards/cosine_scaled_reward/mean": 0.7404875159263611, "rewards/cosine_scaled_reward/std": 0.43749932646751405, "rewards/format_reward/mean": 0.9897321462631226, "rewards/format_reward/std": 0.10028065890073776, "step": 125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0078124999999999774, "completions/max_length": 2048.0, "completions/max_terminated_length": 1875.6, "completions/mean_length": 655.5540405273438, "completions/mean_terminated_length": 644.6221923828125, "completions/min_length": 189.6, "completions/min_terminated_length": 189.6, "epoch": 1.6426642664266426, "grad_norm": 0.13505014777183533, "learning_rate": 8.527703282684765e-07, "loss": 0.014, "num_tokens": 96899722.0, "reward": 1.6154436826705934, "reward_std": 0.30025253891944886, "rewards/accuracy_reward/mean": 0.7470982193946838, "rewards/accuracy_reward/std": 0.43418333530426023, "rewards/cosine_scaled_reward/mean": 0.7465121984481812, "rewards/cosine_scaled_reward/std": 0.43400861620903014, "rewards/format_reward/mean": 0.9901785850524902, "rewards/format_reward/std": 0.09706691503524781, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00401785714285714, "completions/max_length": 2048.0, "completions/max_terminated_length": 1906.4, "completions/mean_length": 651.9241333007812, "completions/mean_terminated_length": 646.2785034179688, "completions/min_length": 181.4, "completions/min_terminated_length": 181.4, "epoch": 1.7056705670567056, "grad_norm": 0.1649669110774994, "learning_rate": 8.36751635394175e-07, "loss": 0.0071, "num_tokens": 100597958.0, "reward": 1.621180534362793, "reward_std": 0.27085027396678923, "rewards/accuracy_reward/mean": 0.7497767925262451, "rewards/accuracy_reward/std": 0.4303715705871582, "rewards/cosine_scaled_reward/mean": 0.7490574598312378, "rewards/cosine_scaled_reward/std": 0.43027289509773253, "rewards/format_reward/mean": 0.993749988079071, "rewards/format_reward/std": 0.07439046055078506, "step": 135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00625, "completions/max_length": 1999.2, "completions/max_terminated_length": 1840.8, "completions/mean_length": 629.5085205078125, "completions/mean_terminated_length": 620.7006591796875, "completions/min_length": 158.0, "completions/min_terminated_length": 158.0, "epoch": 1.7686768676867688, "grad_norm": 0.15633437037467957, "learning_rate": 8.200737353797148e-07, "loss": 0.0149, "num_tokens": 104226908.0, "reward": 1.6223199844360352, "reward_std": 0.28274320662021635, "rewards/accuracy_reward/mean": 0.7506696343421936, "rewards/accuracy_reward/std": 0.43105919361114503, "rewards/cosine_scaled_reward/mean": 0.7499970197677612, "rewards/cosine_scaled_reward/std": 0.43102300763130186, "rewards/format_reward/mean": 0.9933035612106323, "rewards/format_reward/std": 0.06884920671582222, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.004241071428571441, "completions/max_length": 2048.0, "completions/max_terminated_length": 1935.0, "completions/mean_length": 616.7654296875, "completions/mean_terminated_length": 610.688623046875, "completions/min_length": 171.4, "completions/min_terminated_length": 171.4, "epoch": 1.8316831683168315, "grad_norm": 0.16371622681617737, "learning_rate": 8.027692759975693e-07, "loss": 0.0091, "num_tokens": 107827689.0, "reward": 1.5669083833694457, "reward_std": 0.308967524766922, "rewards/accuracy_reward/mean": 0.7129464268684387, "rewards/accuracy_reward/std": 0.44984045028686526, "rewards/cosine_scaled_reward/mean": 0.7126113176345825, "rewards/cosine_scaled_reward/std": 0.44967737793922424, "rewards/format_reward/mean": 0.995312488079071, "rewards/format_reward/std": 0.06629456728696823, "step": 145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00379464285714286, "completions/max_length": 2048.0, "completions/max_terminated_length": 1910.4, "completions/mean_length": 605.9803833007812, "completions/mean_terminated_length": 600.4914794921875, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "epoch": 1.8946894689468947, "grad_norm": 0.16951541602611542, "learning_rate": 7.848721315395974e-07, "loss": 0.0083, "num_tokens": 111359681.0, "reward": 1.6116993427276611, "reward_std": 0.3007046103477478, "rewards/accuracy_reward/mean": 0.7428571343421936, "rewards/accuracy_reward/std": 0.43679211139678953, "rewards/cosine_scaled_reward/mean": 0.7423717975616455, "rewards/cosine_scaled_reward/std": 0.4366111934185028, "rewards/format_reward/mean": 0.995312488079071, "rewards/format_reward/std": 0.06531310081481934, "step": 150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00401785714285714, "completions/max_length": 2048.0, "completions/max_terminated_length": 1928.0, "completions/mean_length": 610.7569458007813, "completions/mean_terminated_length": 604.954296875, "completions/min_length": 176.0, "completions/min_terminated_length": 176.0, "epoch": 1.9576957695769577, "grad_norm": 0.17909295856952667, "learning_rate": 7.664173365065265e-07, "loss": 0.0103, "num_tokens": 114894120.0, "reward": 1.6185973644256593, "reward_std": 0.29384649991989137, "rewards/accuracy_reward/mean": 0.7475446581840515, "rewards/accuracy_reward/std": 0.4323570549488068, "rewards/cosine_scaled_reward/mean": 0.7470160365104676, "rewards/cosine_scaled_reward/std": 0.4321545004844666, "rewards/format_reward/mean": 0.9950892806053162, "rewards/format_reward/std": 0.06607652679085732, "step": 155 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0036299410540915344, "completions/max_length": 2048.0, "completions/max_terminated_length": 1968.8, "completions/mean_length": 619.8013061523437, "completions/mean_terminated_length": 614.59990234375, "completions/min_length": 153.6, "completions/min_terminated_length": 153.6, "epoch": 2.025202520252025, "grad_norm": 0.14987502992153168, "learning_rate": 7.474410170262718e-07, "loss": 0.0109, "num_tokens": 118441067.0, "reward": 1.608110237121582, "reward_std": 0.2891895234584808, "rewards/accuracy_reward/mean": 0.7404017925262452, "rewards/accuracy_reward/std": 0.437438040971756, "rewards/cosine_scaled_reward/mean": 0.7396578192710876, "rewards/cosine_scaled_reward/std": 0.4371993124485016, "rewards/format_reward/mean": 0.9957589268684387, "rewards/format_reward/std": 0.0631766937673092, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.006026785714285721, "completions/max_length": 2048.0, "completions/max_terminated_length": 1911.4, "completions/mean_length": 626.1058227539063, "completions/mean_terminated_length": 617.5382690429688, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "epoch": 2.0882088208820884, "grad_norm": 0.17082159221172333, "learning_rate": 7.279803201353449e-07, "loss": 0.0123, "num_tokens": 122031925.0, "reward": 1.6282689809799193, "reward_std": 0.2621865808963776, "rewards/accuracy_reward/mean": 0.754687511920929, "rewards/accuracy_reward/std": 0.42926387786865233, "rewards/cosine_scaled_reward/mean": 0.7534128069877625, "rewards/cosine_scaled_reward/std": 0.4291880249977112, "rewards/format_reward/mean": 0.99375, "rewards/format_reward/std": 0.07523396834731103, "step": 165 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.002678571428571419, "completions/max_length": 2017.2, "completions/max_terminated_length": 1854.8, "completions/mean_length": 626.8839477539062, "completions/mean_terminated_length": 623.02763671875, "completions/min_length": 191.6, "completions/min_terminated_length": 191.6, "epoch": 2.151215121512151, "grad_norm": 0.14197254180908203, "learning_rate": 7.080733410617851e-07, "loss": 0.0094, "num_tokens": 125637397.0, "reward": 1.6435120105743408, "reward_std": 0.2684548318386078, "rewards/accuracy_reward/mean": 0.7638392806053161, "rewards/accuracy_reward/std": 0.42319408655166624, "rewards/cosine_scaled_reward/mean": 0.7626935601234436, "rewards/cosine_scaled_reward/std": 0.4231561243534088, "rewards/format_reward/mean": 0.9966517925262451, "rewards/format_reward/std": 0.04984662607312203, "step": 170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0033482142857143016, "completions/max_length": 2048.0, "completions/max_terminated_length": 1970.8, "completions/mean_length": 644.8076293945312, "completions/mean_terminated_length": 640.0797729492188, "completions/min_length": 151.6, "completions/min_terminated_length": 151.6, "epoch": 2.2142214221422143, "grad_norm": 0.16446265578269958, "learning_rate": 6.87759048651962e-07, "loss": 0.0102, "num_tokens": 129311063.0, "reward": 1.6173259973526002, "reward_std": 0.29779149889945983, "rewards/accuracy_reward/mean": 0.7466517806053161, "rewards/accuracy_reward/std": 0.43190850019454957, "rewards/cosine_scaled_reward/mean": 0.7460358500480652, "rewards/cosine_scaled_reward/std": 0.43167824745178224, "rewards/format_reward/mean": 0.9953125, "rewards/format_reward/std": 0.06686954125761986, "step": 175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.005357142857142861, "completions/max_length": 2048.0, "completions/max_terminated_length": 1889.6, "completions/mean_length": 629.3937744140625, "completions/mean_terminated_length": 621.8263305664062, "completions/min_length": 170.8, "completions/min_terminated_length": 170.8, "epoch": 2.2772277227722775, "grad_norm": 0.13965024054050446, "learning_rate": 6.670772090872278e-07, "loss": 0.0114, "num_tokens": 132892795.0, "reward": 1.6336923837661743, "reward_std": 0.26696730852127076, "rewards/accuracy_reward/mean": 0.7592241644859314, "rewards/accuracy_reward/std": 0.424642938375473, "rewards/cosine_scaled_reward/mean": 0.7585270166397095, "rewards/cosine_scaled_reward/std": 0.4244441330432892, "rewards/format_reward/mean": 0.9946428418159485, "rewards/format_reward/std": 0.07060872837901115, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00401785714285714, "completions/max_length": 1994.0, "completions/max_terminated_length": 1871.0, "completions/mean_length": 623.195556640625, "completions/mean_terminated_length": 617.4927368164062, "completions/min_length": 170.2, "completions/min_terminated_length": 170.2, "epoch": 2.34023402340234, "grad_norm": 0.13541771471500397, "learning_rate": 6.460683080397508e-07, "loss": 0.0108, "num_tokens": 136497919.0, "reward": 1.638669991493225, "reward_std": 0.2737217009067535, "rewards/accuracy_reward/mean": 0.7620797157287598, "rewards/accuracy_reward/std": 0.4253371238708496, "rewards/cosine_scaled_reward/mean": 0.7601523399353027, "rewards/cosine_scaled_reward/std": 0.42597259283065797, "rewards/format_reward/mean": 0.9957589387893677, "rewards/format_reward/std": 0.05754158571362496, "step": 185 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.004687500000000022, "completions/max_length": 2048.0, "completions/max_terminated_length": 1951.2, "completions/mean_length": 617.7080688476562, "completions/mean_terminated_length": 610.93955078125, "completions/min_length": 158.4, "completions/min_terminated_length": 158.4, "epoch": 2.4032403240324034, "grad_norm": 0.1562889963388443, "learning_rate": 6.247734714199105e-07, "loss": 0.018, "num_tokens": 140050915.0, "reward": 1.6390478134155273, "reward_std": 0.2830404281616211, "rewards/accuracy_reward/mean": 0.7611607193946839, "rewards/accuracy_reward/std": 0.4261560559272766, "rewards/cosine_scaled_reward/mean": 0.7606848120689392, "rewards/cosine_scaled_reward/std": 0.42603325843811035, "rewards/format_reward/mean": 0.995089304447174, "rewards/format_reward/std": 0.06784935072064399, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0049107142857142795, "completions/max_length": 2048.0, "completions/max_terminated_length": 1952.4, "completions/mean_length": 629.9161010742188, "completions/mean_terminated_length": 622.9476318359375, "completions/min_length": 170.0, "completions/min_terminated_length": 170.0, "epoch": 2.466246624662466, "grad_norm": 0.16037462651729584, "learning_rate": 6.032343848703969e-07, "loss": 0.0111, "num_tokens": 143659179.0, "reward": 1.6348266124725341, "reward_std": 0.2890359699726105, "rewards/accuracy_reward/mean": 0.7587053537368774, "rewards/accuracy_reward/std": 0.4270632088184357, "rewards/cosine_scaled_reward/mean": 0.7578226804733277, "rewards/cosine_scaled_reward/std": 0.4268454909324646, "rewards/format_reward/mean": 0.9944196343421936, "rewards/format_reward/std": 0.0683063194155693, "step": 195 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0046875, "completions/max_length": 2011.6, "completions/max_terminated_length": 1926.0, "completions/mean_length": 632.8710083007812, "completions/mean_terminated_length": 626.33193359375, "completions/min_length": 161.6, "completions/min_terminated_length": 161.6, "epoch": 2.5292529252925293, "grad_norm": 0.14154884219169617, "learning_rate": 5.814932121646073e-07, "loss": 0.0079, "num_tokens": 147280177.0, "reward": 1.6299890995025634, "reward_std": 0.27802737057209015, "rewards/accuracy_reward/mean": 0.7558035850524902, "rewards/accuracy_reward/std": 0.42795242071151735, "rewards/cosine_scaled_reward/mean": 0.7550673365592957, "rewards/cosine_scaled_reward/std": 0.4278149545192719, "rewards/format_reward/mean": 0.9933035731315613, "rewards/format_reward/std": 0.06978957653045655, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0051339285714286035, "completions/max_length": 2048.0, "completions/max_terminated_length": 1918.8, "completions/mean_length": 615.8734741210938, "completions/mean_terminated_length": 608.469384765625, "completions/min_length": 168.8, "completions/min_terminated_length": 168.8, "epoch": 2.592259225922592, "grad_norm": 0.12673905491828918, "learning_rate": 5.595925126690801e-07, "loss": 0.0128, "num_tokens": 150787250.0, "reward": 1.6726323127746583, "reward_std": 0.2540514647960663, "rewards/accuracy_reward/mean": 0.7837053537368774, "rewards/accuracy_reward/std": 0.40963547229766845, "rewards/cosine_scaled_reward/mean": 0.782987654209137, "rewards/cosine_scaled_reward/std": 0.40952636003494264, "rewards/format_reward/mean": 0.9948660850524902, "rewards/format_reward/std": 0.06850367933511733, "step": 205 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.003794642857142838, "completions/max_length": 2048.0, "completions/max_terminated_length": 1862.4, "completions/mean_length": 612.8245849609375, "completions/mean_terminated_length": 607.3509399414063, "completions/min_length": 166.8, "completions/min_terminated_length": 166.8, "epoch": 2.6552655265526552, "grad_norm": 0.12310470640659332, "learning_rate": 5.375751580315354e-07, "loss": 0.0103, "num_tokens": 154286136.0, "reward": 1.634846067428589, "reward_std": 0.25745274126529694, "rewards/accuracy_reward/mean": 0.7580357193946838, "rewards/accuracy_reward/std": 0.42683074474334715, "rewards/cosine_scaled_reward/mean": 0.7574151754379272, "rewards/cosine_scaled_reward/std": 0.42679900527000425, "rewards/format_reward/mean": 0.9962053537368775, "rewards/format_reward/std": 0.0598295733332634, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0035714285714285587, "completions/max_length": 2048.0, "completions/max_terminated_length": 1913.4, "completions/mean_length": 600.7576049804687, "completions/mean_terminated_length": 595.5764038085938, "completions/min_length": 178.4, "completions/min_terminated_length": 178.4, "epoch": 2.7182718271827184, "grad_norm": 0.12419384717941284, "learning_rate": 5.154842482576126e-07, "loss": 0.0126, "num_tokens": 157779418.0, "reward": 1.658141851425171, "reward_std": 0.259933602809906, "rewards/accuracy_reward/mean": 0.7734375, "rewards/accuracy_reward/std": 0.41526424288749697, "rewards/cosine_scaled_reward/mean": 0.7729800581932068, "rewards/cosine_scaled_reward/std": 0.4152462720870972, "rewards/format_reward/mean": 0.9964285612106323, "rewards/format_reward/std": 0.05749904364347458, "step": 215 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0040178571428571615, "completions/max_length": 2048.0, "completions/max_terminated_length": 1845.8, "completions/mean_length": 606.7765747070313, "completions/mean_terminated_length": 600.97294921875, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "epoch": 2.781278127812781, "grad_norm": 0.16448700428009033, "learning_rate": 4.933630273405846e-07, "loss": 0.0162, "num_tokens": 161255897.0, "reward": 1.6536934852600098, "reward_std": 0.2780122488737106, "rewards/accuracy_reward/mean": 0.7709821581840515, "rewards/accuracy_reward/std": 0.41886167526245116, "rewards/cosine_scaled_reward/mean": 0.7701100707054138, "rewards/cosine_scaled_reward/std": 0.4186771154403687, "rewards/format_reward/mean": 0.9953125, "rewards/format_reward/std": 0.06595941931009293, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.002901785714285743, "completions/max_length": 2048.0, "completions/max_terminated_length": 2013.6, "completions/mean_length": 597.0652099609375, "completions/mean_terminated_length": 592.8518188476562, "completions/min_length": 162.6, "completions/min_terminated_length": 162.6, "epoch": 2.8442844284428443, "grad_norm": 0.14639931917190552, "learning_rate": 4.712547986092099e-07, "loss": 0.0067, "num_tokens": 164732445.0, "reward": 1.6427595376968385, "reward_std": 0.27772180140018465, "rewards/accuracy_reward/mean": 0.7633928656578064, "rewards/accuracy_reward/std": 0.42306692004203794, "rewards/cosine_scaled_reward/mean": 0.7627511620521545, "rewards/cosine_scaled_reward/std": 0.4228369653224945, "rewards/format_reward/mean": 0.9959821462631225, "rewards/format_reward/std": 0.062224668264389035, "step": 225 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0008928571428571397, "completions/max_length": 1944.6, "completions/max_terminated_length": 1889.6, "completions/mean_length": 599.8812866210938, "completions/mean_terminated_length": 598.5843017578125, "completions/min_length": 169.6, "completions/min_terminated_length": 169.6, "epoch": 2.9072907290729075, "grad_norm": 0.1395055204629898, "learning_rate": 4.492028399594338e-07, "loss": 0.0043, "num_tokens": 168207073.0, "reward": 1.6468029737472534, "reward_std": 0.26623707115650175, "rewards/accuracy_reward/mean": 0.7649553537368774, "rewards/accuracy_reward/std": 0.42270631790161134, "rewards/cosine_scaled_reward/mean": 0.764587914943695, "rewards/cosine_scaled_reward/std": 0.42256171107292173, "rewards/format_reward/mean": 0.9991071343421936, "rewards/format_reward/std": 0.022806894034147263, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.006026785714285721, "completions/max_length": 2048.0, "completions/max_terminated_length": 1864.6, "completions/mean_length": 641.97099609375, "completions/mean_terminated_length": 633.473095703125, "completions/min_length": 186.6, "completions/min_terminated_length": 186.6, "epoch": 2.9702970297029703, "grad_norm": 0.12765838205814362, "learning_rate": 4.2725031913587426e-07, "loss": 0.0096, "num_tokens": 171879167.0, "reward": 1.6136202335357666, "reward_std": 0.26737120449543, "rewards/accuracy_reward/mean": 0.7448660731315613, "rewards/accuracy_reward/std": 0.43231003880500796, "rewards/cosine_scaled_reward/mean": 0.7442046284675599, "rewards/cosine_scaled_reward/std": 0.43226224184036255, "rewards/format_reward/mean": 0.9933035612106323, "rewards/format_reward/std": 0.07773758172988891, "step": 235 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.005047243411927904, "completions/max_length": 2048.0, "completions/max_terminated_length": 1930.4, "completions/mean_length": 646.4539672851563, "completions/mean_terminated_length": 639.4364501953125, "completions/min_length": 183.2, "completions/min_terminated_length": 183.2, "epoch": 3.037803780378038, "grad_norm": 0.1862199455499649, "learning_rate": 4.054402092289354e-07, "loss": 0.0103, "num_tokens": 175632429.0, "reward": 1.5804502010345458, "reward_std": 0.29361810684204104, "rewards/accuracy_reward/mean": 0.7220982074737549, "rewards/accuracy_reward/std": 0.44716750979423525, "rewards/cosine_scaled_reward/mean": 0.7216144323348999, "rewards/cosine_scaled_reward/std": 0.44697086215019227, "rewards/format_reward/mean": 0.9950892925262451, "rewards/format_reward/std": 0.06746494099497795, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0033482142857142795, "completions/max_length": 2020.6, "completions/max_terminated_length": 1812.2, "completions/mean_length": 615.8056030273438, "completions/mean_terminated_length": 611.0432861328125, "completions/min_length": 194.2, "completions/min_terminated_length": 194.2, "epoch": 3.1008100810081007, "grad_norm": 0.16215628385543823, "learning_rate": 3.838152045529636e-07, "loss": 0.0143, "num_tokens": 179124806.0, "reward": 1.6801328897476195, "reward_std": 0.2461833655834198, "rewards/accuracy_reward/mean": 0.7881696343421936, "rewards/accuracy_reward/std": 0.4080040454864502, "rewards/cosine_scaled_reward/mean": 0.7877209305763244, "rewards/cosine_scaled_reward/std": 0.4078561782836914, "rewards/format_reward/mean": 0.9962053537368775, "rewards/format_reward/std": 0.05407858937978745, "step": 245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.002678571428571419, "completions/max_length": 2029.4, "completions/max_terminated_length": 1823.4, "completions/mean_length": 606.2763671875, "completions/mean_terminated_length": 602.3788940429688, "completions/min_length": 189.6, "completions/min_terminated_length": 189.6, "epoch": 3.163816381638164, "grad_norm": 0.14493921399116516, "learning_rate": 3.624176370701235e-07, "loss": 0.0047, "num_tokens": 182588044.0, "reward": 1.6811640739440918, "reward_std": 0.24738809168338777, "rewards/accuracy_reward/mean": 0.7888392806053162, "rewards/accuracy_reward/std": 0.4081935465335846, "rewards/cosine_scaled_reward/mean": 0.7882209300994873, "rewards/cosine_scaled_reward/std": 0.40814857482910155, "rewards/format_reward/mean": 0.9964285731315613, "rewards/format_reward/std": 0.053148042410612106, "step": 250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0022321428571428605, "completions/max_length": 2048.0, "completions/max_terminated_length": 1797.2, "completions/mean_length": 610.3625122070313, "completions/mean_terminated_length": 607.1727905273438, "completions/min_length": 166.6, "completions/min_terminated_length": 166.6, "epoch": 3.2268226822682267, "grad_norm": 0.12905967235565186, "learning_rate": 3.412893935235936e-07, "loss": 0.0075, "num_tokens": 186106108.0, "reward": 1.664694619178772, "reward_std": 0.2503175616264343, "rewards/accuracy_reward/mean": 0.7774553537368775, "rewards/accuracy_reward/std": 0.41434150338172915, "rewards/cosine_scaled_reward/mean": 0.7767105340957642, "rewards/cosine_scaled_reward/std": 0.4142517387866974, "rewards/format_reward/mean": 0.9977678418159485, "rewards/format_reward/std": 0.04592652022838593, "step": 255 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.003571428571428581, "completions/max_length": 2048.0, "completions/max_terminated_length": 1915.6, "completions/mean_length": 619.6020385742188, "completions/mean_terminated_length": 614.4874267578125, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "epoch": 3.28982898289829, "grad_norm": 0.1830361932516098, "learning_rate": 3.204718334422998e-07, "loss": 0.0095, "num_tokens": 189646853.0, "reward": 1.6342453956604004, "reward_std": 0.30342464447021483, "rewards/accuracy_reward/mean": 0.7575892806053162, "rewards/accuracy_reward/std": 0.42738271951675416, "rewards/cosine_scaled_reward/mean": 0.7568834662437439, "rewards/cosine_scaled_reward/std": 0.4271435678005219, "rewards/format_reward/mean": 0.9964285731315613, "rewards/format_reward/std": 0.059244754165410994, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0020089285714285808, "completions/max_length": 1969.0, "completions/max_terminated_length": 1903.2, "completions/mean_length": 618.0531494140625, "completions/mean_terminated_length": 615.2063110351562, "completions/min_length": 160.0, "completions/min_terminated_length": 160.0, "epoch": 3.352835283528353, "grad_norm": 0.13739930093288422, "learning_rate": 3.0000570817769464e-07, "loss": 0.0078, "num_tokens": 193209859.0, "reward": 1.6563403844833373, "reward_std": 0.2727595895528793, "rewards/accuracy_reward/mean": 0.7716517925262452, "rewards/accuracy_reward/std": 0.4175321161746979, "rewards/cosine_scaled_reward/mean": 0.7711627960205079, "rewards/cosine_scaled_reward/std": 0.4173714339733124, "rewards/format_reward/mean": 0.9982142806053161, "rewards/format_reward/std": 0.031032296270132063, "step": 265 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.002678571428571441, "completions/max_length": 1993.2, "completions/max_terminated_length": 1915.8, "completions/mean_length": 627.6781494140625, "completions/mean_terminated_length": 623.89052734375, "completions/min_length": 156.4, "completions/min_terminated_length": 156.4, "epoch": 3.4158415841584158, "grad_norm": 0.14828138053417206, "learning_rate": 2.7993108113107443e-07, "loss": 0.0066, "num_tokens": 196821017.0, "reward": 1.6634482383728026, "reward_std": 0.23966350257396699, "rewards/accuracy_reward/mean": 0.7767857193946839, "rewards/accuracy_reward/std": 0.41428372263908386, "rewards/cosine_scaled_reward/mean": 0.7762267231941223, "rewards/cosine_scaled_reward/std": 0.41410806179046633, "rewards/format_reward/mean": 0.9970982074737549, "rewards/format_reward/std": 0.0474703922867775, "step": 270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00401785714285714, "completions/max_length": 2048.0, "completions/max_terminated_length": 1884.6, "completions/mean_length": 609.0343994140625, "completions/mean_terminated_length": 603.2148071289063, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "epoch": 3.478847884788479, "grad_norm": 0.1737632006406784, "learning_rate": 2.602872493275881e-07, "loss": 0.0051, "num_tokens": 200339011.0, "reward": 1.6778597354888916, "reward_std": 0.25451319515705106, "rewards/accuracy_reward/mean": 0.7868303537368775, "rewards/accuracy_reward/std": 0.4092303454875946, "rewards/cosine_scaled_reward/mean": 0.7860764384269714, "rewards/cosine_scaled_reward/std": 0.4091233015060425, "rewards/format_reward/mean": 0.9959821343421936, "rewards/format_reward/std": 0.056319963186979294, "step": 275 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0024553571428571398, "completions/max_length": 2048.0, "completions/max_terminated_length": 1807.8, "completions/mean_length": 627.6569458007813, "completions/mean_terminated_length": 624.14736328125, "completions/min_length": 171.6, "completions/min_terminated_length": 171.6, "epoch": 3.5418541854185417, "grad_norm": 0.14920973777770996, "learning_rate": 2.411126664904665e-07, "loss": 0.0122, "num_tokens": 203921794.0, "reward": 1.640373420715332, "reward_std": 0.27687210142612456, "rewards/accuracy_reward/mean": 0.7618303537368775, "rewards/accuracy_reward/std": 0.424312424659729, "rewards/cosine_scaled_reward/mean": 0.7613270163536072, "rewards/cosine_scaled_reward/std": 0.42429112195968627, "rewards/format_reward/mean": 0.9957589149475098, "rewards/format_reward/std": 0.06131577789783478, "step": 280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0033482142857142795, "completions/max_length": 2048.0, "completions/max_terminated_length": 1812.4, "completions/mean_length": 599.8631958007812, "completions/mean_terminated_length": 595.0171997070313, "completions/min_length": 178.0, "completions/min_terminated_length": 178.0, "epoch": 3.604860486048605, "grad_norm": 0.16235579550266266, "learning_rate": 2.2244486776605241e-07, "loss": 0.0082, "num_tokens": 207393885.0, "reward": 1.6748189449310302, "reward_std": 0.25130158960819243, "rewards/accuracy_reward/mean": 0.784374988079071, "rewards/accuracy_reward/std": 0.4097891986370087, "rewards/cosine_scaled_reward/mean": 0.7840126991271973, "rewards/cosine_scaled_reward/std": 0.40970866084098817, "rewards/format_reward/mean": 0.996875, "rewards/format_reward/std": 0.05381677523255348, "step": 285 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0020089285714285808, "completions/max_length": 1973.4, "completions/max_terminated_length": 1706.2, "completions/mean_length": 593.27236328125, "completions/mean_terminated_length": 590.3545043945312, "completions/min_length": 170.4, "completions/min_terminated_length": 170.4, "epoch": 3.667866786678668, "grad_norm": 0.14174528419971466, "learning_rate": 2.0432039624699015e-07, "loss": 0.0072, "num_tokens": 210886881.0, "reward": 1.6687217950820923, "reward_std": 0.2413029581308365, "rewards/accuracy_reward/mean": 0.7799107074737549, "rewards/accuracy_reward/std": 0.41347917914390564, "rewards/cosine_scaled_reward/mean": 0.779630982875824, "rewards/cosine_scaled_reward/std": 0.4133819997310638, "rewards/format_reward/mean": 0.9979910731315613, "rewards/format_reward/std": 0.038909842073917386, "step": 290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.004241071428571419, "completions/max_length": 2048.0, "completions/max_terminated_length": 1969.2, "completions/mean_length": 635.64912109375, "completions/mean_terminated_length": 629.61572265625, "completions/min_length": 174.6, "completions/min_terminated_length": 174.6, "epoch": 3.730873087308731, "grad_norm": 0.13457916676998138, "learning_rate": 1.867747314374041e-07, "loss": 0.014, "num_tokens": 214550253.0, "reward": 1.6441724061965943, "reward_std": 0.25683614015579226, "rewards/accuracy_reward/mean": 0.7647321343421936, "rewards/accuracy_reward/std": 0.42399376034736636, "rewards/cosine_scaled_reward/mean": 0.7633446335792542, "rewards/cosine_scaled_reward/std": 0.4237023591995239, "rewards/format_reward/mean": 0.9955357193946839, "rewards/format_reward/std": 0.06432178840041161, "step": 295 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.006026785714285698, "completions/max_length": 2048.0, "completions/max_terminated_length": 1899.4, "completions/mean_length": 625.6288208007812, "completions/mean_terminated_length": 617.0573120117188, "completions/min_length": 183.8, "completions/min_terminated_length": 183.8, "epoch": 3.793879387938794, "grad_norm": 0.16838262975215912, "learning_rate": 1.6984221980010454e-07, "loss": 0.0107, "num_tokens": 218094638.0, "reward": 1.6642211198806762, "reward_std": 0.2543728470802307, "rewards/accuracy_reward/mean": 0.7783482193946838, "rewards/accuracy_reward/std": 0.41493446230888364, "rewards/cosine_scaled_reward/mean": 0.7779957056045532, "rewards/cosine_scaled_reward/std": 0.41481839418411254, "rewards/format_reward/mean": 0.993749988079071, "rewards/format_reward/std": 0.07161349579691886, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.004464285714285721, "completions/max_length": 2048.0, "completions/max_terminated_length": 1934.0, "completions/mean_length": 627.9725708007812, "completions/mean_terminated_length": 621.595361328125, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "epoch": 3.856885688568857, "grad_norm": 0.14313475787639618, "learning_rate": 1.5355600752177312e-07, "loss": 0.0125, "num_tokens": 221700083.0, "reward": 1.6535644292831422, "reward_std": 0.2506650030612946, "rewards/accuracy_reward/mean": 0.7707589268684387, "rewards/accuracy_reward/std": 0.4196486592292786, "rewards/cosine_scaled_reward/mean": 0.7700751662254334, "rewards/cosine_scaled_reward/std": 0.41945862770080566, "rewards/format_reward/mean": 0.9955357074737549, "rewards/format_reward/std": 0.06590693667531014, "step": 305 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.003125, "completions/max_length": 2014.6, "completions/max_terminated_length": 1926.0, "completions/mean_length": 613.800927734375, "completions/mean_terminated_length": 609.321435546875, "completions/min_length": 179.8, "completions/min_terminated_length": 179.8, "epoch": 3.91989198919892, "grad_norm": 0.13859793543815613, "learning_rate": 1.3794797562774596e-07, "loss": 0.0068, "num_tokens": 225211495.0, "reward": 1.6338534832000733, "reward_std": 0.2609659105539322, "rewards/accuracy_reward/mean": 0.7595358729362488, "rewards/accuracy_reward/std": 0.4260308504104614, "rewards/cosine_scaled_reward/mean": 0.7553341269493103, "rewards/cosine_scaled_reward/std": 0.4275141417980194, "rewards/format_reward/mean": 0.9970982074737549, "rewards/format_reward/std": 0.0464889258146286, "step": 310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00625, "completions/max_length": 2048.0, "completions/max_terminated_length": 1900.4, "completions/mean_length": 611.440869140625, "completions/mean_terminated_length": 602.3813110351563, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "epoch": 3.9828982898289826, "grad_norm": 0.1344553679227829, "learning_rate": 1.230486775734097e-07, "loss": 0.013, "num_tokens": 228791646.0, "reward": 1.6422872066497802, "reward_std": 0.23485693633556365, "rewards/accuracy_reward/mean": 0.7667692184448243, "rewards/accuracy_reward/std": 0.42179493308067323, "rewards/cosine_scaled_reward/mean": 0.761570417881012, "rewards/cosine_scaled_reward/std": 0.4240230083465576, "rewards/format_reward/mean": 0.9935267806053162, "rewards/format_reward/std": 0.07590627297759056, "step": 315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0031640083217753024, "completions/max_length": 2048.0, "completions/max_terminated_length": 1952.0, "completions/mean_length": 617.7764038085937, "completions/mean_terminated_length": 613.2669799804687, "completions/min_length": 164.4, "completions/min_terminated_length": 164.4, "epoch": 4.05040504050405, "grad_norm": 0.15917585790157318, "learning_rate": 1.0888727943437748e-07, "loss": 0.0084, "num_tokens": 232312551.0, "reward": 1.6375758409500123, "reward_std": 0.2651157915592194, "rewards/accuracy_reward/mean": 0.7629625201225281, "rewards/accuracy_reward/std": 0.42385138273239137, "rewards/cosine_scaled_reward/mean": 0.7563670516014099, "rewards/cosine_scaled_reward/std": 0.4267165422439575, "rewards/format_reward/mean": 0.996875, "rewards/format_reward/std": 0.052468103170394895, "step": 320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0022321428571428826, "completions/max_length": 2048.0, "completions/max_terminated_length": 1919.4, "completions/mean_length": 608.4109619140625, "completions/mean_terminated_length": 605.1753173828125, "completions/min_length": 191.8, "completions/min_terminated_length": 191.8, "epoch": 4.113411341134113, "grad_norm": 0.1608886420726776, "learning_rate": 9.549150281252632e-08, "loss": 0.0043, "num_tokens": 235787696.0, "reward": 1.66022846698761, "reward_std": 0.2489331156015396, "rewards/accuracy_reward/mean": 0.7745535850524903, "rewards/accuracy_reward/std": 0.41753928661346434, "rewards/cosine_scaled_reward/mean": 0.7740282416343689, "rewards/cosine_scaled_reward/std": 0.4174140512943268, "rewards/format_reward/mean": 0.9973214268684387, "rewards/format_reward/std": 0.05046965554356575, "step": 325 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.003794642857142838, "completions/max_length": 2048.0, "completions/max_terminated_length": 1945.6, "completions/mean_length": 622.467431640625, "completions/mean_terminated_length": 617.0385375976563, "completions/min_length": 152.6, "completions/min_terminated_length": 152.6, "epoch": 4.176417641764177, "grad_norm": 0.15062691271305084, "learning_rate": 8.288757056966017e-08, "loss": 0.0112, "num_tokens": 239378142.0, "reward": 1.6499414920806885, "reward_std": 0.2437061160802841, "rewards/accuracy_reward/mean": 0.7680803537368774, "rewards/accuracy_reward/std": 0.4208991706371307, "rewards/cosine_scaled_reward/mean": 0.7675167083740234, "rewards/cosine_scaled_reward/std": 0.4206974029541016, "rewards/format_reward/mean": 0.9962053418159484, "rewards/format_reward/std": 0.05708769783377647, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.005133928571428558, "completions/max_length": 2048.0, "completions/max_terminated_length": 1929.2, "completions/mean_length": 633.4506958007812, "completions/mean_terminated_length": 626.1871459960937, "completions/min_length": 175.4, "completions/min_terminated_length": 175.4, "epoch": 4.2394239423942395, "grad_norm": 0.14156123995780945, "learning_rate": 7.110015549502546e-08, "loss": 0.0082, "num_tokens": 243021681.0, "reward": 1.6500544548034668, "reward_std": 0.26026159822940825, "rewards/accuracy_reward/mean": 0.768749988079071, "rewards/accuracy_reward/std": 0.42072436213493347, "rewards/cosine_scaled_reward/mean": 0.7679658651351928, "rewards/cosine_scaled_reward/std": 0.4205094575881958, "rewards/format_reward/mean": 0.9946428418159485, "rewards/format_reward/std": 0.06923034116625786, "step": 335 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0024553571428571177, "completions/max_length": 1975.6, "completions/max_terminated_length": 1895.2, "completions/mean_length": 627.3937744140625, "completions/mean_terminated_length": 623.917529296875, "completions/min_length": 161.8, "completions/min_terminated_length": 161.8, "epoch": 4.302430243024302, "grad_norm": 0.13976019620895386, "learning_rate": 6.015233200716874e-08, "loss": 0.0064, "num_tokens": 246632541.0, "reward": 1.6928518533706665, "reward_std": 0.2341456115245819, "rewards/accuracy_reward/mean": 0.7964285731315612, "rewards/accuracy_reward/std": 0.40205827355384827, "rewards/cosine_scaled_reward/mean": 0.7953016996383667, "rewards/cosine_scaled_reward/std": 0.40191030502319336, "rewards/format_reward/mean": 0.9975446343421936, "rewards/format_reward/std": 0.04161064326763153, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.005133928571428581, "completions/max_length": 2048.0, "completions/max_terminated_length": 1957.8, "completions/mean_length": 618.9397705078125, "completions/mean_terminated_length": 611.7075561523437, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "epoch": 4.365436543654366, "grad_norm": 0.15124908089637756, "learning_rate": 5.006553098467764e-08, "loss": 0.0103, "num_tokens": 250178143.0, "reward": 1.6755474090576172, "reward_std": 0.2548744171857834, "rewards/accuracy_reward/mean": 0.7857142925262451, "rewards/accuracy_reward/std": 0.4092557728290558, "rewards/cosine_scaled_reward/mean": 0.7848000407218934, "rewards/cosine_scaled_reward/std": 0.4092385172843933, "rewards/format_reward/mean": 0.9948660850524902, "rewards/format_reward/std": 0.06760494336485863, "step": 345 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.004464285714285721, "completions/max_length": 2048.0, "completions/max_terminated_length": 1865.8, "completions/mean_length": 626.4183349609375, "completions/mean_terminated_length": 620.0564453125, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "epoch": 4.428442844284429, "grad_norm": 0.14921754598617554, "learning_rate": 4.085949781423048e-08, "loss": 0.007, "num_tokens": 253769705.0, "reward": 1.6789589166641234, "reward_std": 0.23221377432346343, "rewards/accuracy_reward/mean": 0.7877232074737549, "rewards/accuracy_reward/std": 0.40675231218338015, "rewards/cosine_scaled_reward/mean": 0.7873820304870606, "rewards/cosine_scaled_reward/std": 0.4066896378993988, "rewards/format_reward/mean": 0.9950892806053162, "rewards/format_reward/std": 0.06925405710935592, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.002678571428571419, "completions/max_length": 2048.0, "completions/max_terminated_length": 1934.2, "completions/mean_length": 618.8587280273438, "completions/mean_terminated_length": 615.0157958984375, "completions/min_length": 160.0, "completions/min_terminated_length": 160.0, "epoch": 4.491449144914491, "grad_norm": 0.16535188257694244, "learning_rate": 3.2552253738074074e-08, "loss": 0.008, "num_tokens": 257334160.0, "reward": 1.6443732500076294, "reward_std": 0.2480471521615982, "rewards/accuracy_reward/mean": 0.7647321343421936, "rewards/accuracy_reward/std": 0.423707377910614, "rewards/cosine_scaled_reward/mean": 0.7641927599906921, "rewards/cosine_scaled_reward/std": 0.4235422372817993, "rewards/format_reward/mean": 0.9950892925262451, "rewards/format_reward/std": 0.06574137881398201, "step": 355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0024553571428571398, "completions/max_length": 1972.6, "completions/max_terminated_length": 1843.2, "completions/mean_length": 613.258740234375, "completions/mean_terminated_length": 609.7564453125, "completions/min_length": 191.4, "completions/min_terminated_length": 191.4, "epoch": 4.554455445544555, "grad_norm": 0.14730912446975708, "learning_rate": 2.5160060576596365e-08, "loss": 0.0072, "num_tokens": 260865495.0, "reward": 1.6269155740737915, "reward_std": 0.2688676297664642, "rewards/accuracy_reward/mean": 0.7524553418159485, "rewards/accuracy_reward/std": 0.42926363945007323, "rewards/cosine_scaled_reward/mean": 0.7520452857017517, "rewards/cosine_scaled_reward/std": 0.4292009472846985, "rewards/format_reward/mean": 0.996874988079071, "rewards/format_reward/std": 0.053170456737279895, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0029017857142857205, "completions/max_length": 2048.0, "completions/max_terminated_length": 1827.2, "completions/mean_length": 618.671240234375, "completions/mean_terminated_length": 614.5585571289063, "completions/min_length": 169.0, "completions/min_terminated_length": 169.0, "epoch": 4.617461746174618, "grad_norm": 0.1736821085214615, "learning_rate": 1.8697388895048515e-08, "loss": 0.01, "num_tokens": 264474382.0, "reward": 1.6391252517700194, "reward_std": 0.25250489115715025, "rewards/accuracy_reward/mean": 0.760937488079071, "rewards/accuracy_reward/std": 0.4259650409221649, "rewards/cosine_scaled_reward/mean": 0.7601699948310852, "rewards/cosine_scaled_reward/std": 0.42582063674926757, "rewards/format_reward/mean": 0.9962053656578064, "rewards/format_reward/std": 0.05821031853556633, "step": 365 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0013392857142857206, "completions/max_length": 1962.0, "completions/max_terminated_length": 1836.8, "completions/mean_length": 595.1326293945312, "completions/mean_terminated_length": 593.2115600585937, "completions/min_length": 174.4, "completions/min_terminated_length": 174.4, "epoch": 4.68046804680468, "grad_norm": 0.16435028612613678, "learning_rate": 1.3176889676734692e-08, "loss": 0.0088, "num_tokens": 267903232.0, "reward": 1.6919838905334472, "reward_std": 0.2522643059492111, "rewards/accuracy_reward/mean": 0.7953125, "rewards/accuracy_reward/std": 0.40310133099555967, "rewards/cosine_scaled_reward/mean": 0.7946819305419922, "rewards/cosine_scaled_reward/std": 0.40297635197639464, "rewards/format_reward/mean": 0.9986607074737549, "rewards/format_reward/std": 0.027685175836086272, "step": 370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00379464285714286, "completions/max_length": 2048.0, "completions/max_terminated_length": 1986.6, "completions/mean_length": 633.7323852539063, "completions/mean_terminated_length": 628.3538696289063, "completions/min_length": 181.8, "completions/min_terminated_length": 181.8, "epoch": 4.743474347434743, "grad_norm": 0.16870158910751343, "learning_rate": 8.60936955811753e-09, "loss": 0.0053, "num_tokens": 271506625.0, "reward": 1.677209448814392, "reward_std": 0.24304699897766113, "rewards/accuracy_reward/mean": 0.7866071343421936, "rewards/accuracy_reward/std": 0.4092015206813812, "rewards/cosine_scaled_reward/mean": 0.7849991202354432, "rewards/cosine_scaled_reward/std": 0.409130734205246, "rewards/format_reward/mean": 0.9962053537368775, "rewards/format_reward/std": 0.058644138276576996, "step": 375 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.003125, "completions/max_length": 2033.8, "completions/max_terminated_length": 1926.0, "completions/mean_length": 596.5832885742187, "completions/mean_terminated_length": 592.036572265625, "completions/min_length": 161.2, "completions/min_terminated_length": 161.2, "epoch": 4.806480648064807, "grad_norm": 0.18181964755058289, "learning_rate": 5.003769674320368e-09, "loss": 0.011, "num_tokens": 274925870.0, "reward": 1.6703053951263427, "reward_std": 0.24658992886543274, "rewards/accuracy_reward/mean": 0.7816964268684388, "rewards/accuracy_reward/std": 0.41211633682250975, "rewards/cosine_scaled_reward/mean": 0.7807891845703125, "rewards/cosine_scaled_reward/std": 0.4120659828186035, "rewards/format_reward/mean": 0.9964285731315613, "rewards/format_reward/std": 0.05088246762752533, "step": 380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00401785714285714, "completions/max_length": 2025.6, "completions/max_terminated_length": 1959.0, "completions/mean_length": 629.3553833007812, "completions/mean_terminated_length": 623.6705200195313, "completions/min_length": 182.2, "completions/min_terminated_length": 182.2, "epoch": 4.8694869486948695, "grad_norm": 0.15454868972301483, "learning_rate": 2.3671481564347396e-09, "loss": 0.0112, "num_tokens": 278501238.0, "reward": 1.6728654146194457, "reward_std": 0.24723841547966002, "rewards/accuracy_reward/mean": 0.7837053537368774, "rewards/accuracy_reward/std": 0.41015875339508057, "rewards/cosine_scaled_reward/mean": 0.7823377847671509, "rewards/cosine_scaled_reward/std": 0.40985342860221863, "rewards/format_reward/mean": 0.9959821343421936, "rewards/format_reward/std": 0.04546051248908043, "step": 385 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0033482142857142795, "completions/max_length": 2048.0, "completions/max_terminated_length": 1895.4, "completions/mean_length": 639.8073974609375, "completions/mean_terminated_length": 635.0597290039062, "completions/min_length": 183.0, "completions/min_terminated_length": 183.0, "epoch": 4.932493249324932, "grad_norm": 0.15312208235263824, "learning_rate": 7.046663148979615e-10, "loss": 0.0143, "num_tokens": 282142359.0, "reward": 1.6685362339019776, "reward_std": 0.25656993985176085, "rewards/accuracy_reward/mean": 0.7805803537368774, "rewards/accuracy_reward/std": 0.4132432878017426, "rewards/cosine_scaled_reward/mean": 0.7792598366737366, "rewards/cosine_scaled_reward/std": 0.4132158994674683, "rewards/format_reward/mean": 0.9966517806053161, "rewards/format_reward/std": 0.05593275725841522, "step": 390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.003833651178918163, "completions/max_length": 2048.0, "completions/max_terminated_length": 1935.2, "completions/mean_length": 615.3019409179688, "completions/mean_terminated_length": 609.7786987304687, "completions/min_length": 197.0, "completions/min_terminated_length": 197.0, "epoch": 4.995499549954996, "grad_norm": 0.15891171991825104, "learning_rate": 1.95785359849987e-11, "loss": 0.01, "num_tokens": 285681294.0, "reward": 1.6595622539520263, "reward_std": 0.2522480070590973, "rewards/accuracy_reward/mean": 0.7747767686843872, "rewards/accuracy_reward/std": 0.41717290282249453, "rewards/cosine_scaled_reward/mean": 0.7740350365638733, "rewards/cosine_scaled_reward/std": 0.4170451521873474, "rewards/format_reward/mean": 0.9955357193946839, "rewards/format_reward/std": 0.06459032222628594, "step": 395 }, { "epoch": 4.995499549954996, "step": 395, "total_flos": 0.0, "train_loss": 0.011041237461038782, "train_runtime": 79107.9261, "train_samples_per_second": 0.562, "train_steps_per_second": 0.005 } ], "logging_steps": 5, "max_steps": 395, "num_input_tokens_seen": 285681294, "num_train_epochs": 5, "save_steps": 15, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }