| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 3.0, | |
| "eval_steps": 500, | |
| "global_step": 1200, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 1.0416666666666667e-07, | |
| "logps/chosen": -50.84611511230469, | |
| "logps/rejected": -68.16911315917969, | |
| "loss": 0.7128, | |
| "losses/dpo": 0.7152940630912781, | |
| "losses/sft": 1.772851586341858, | |
| "losses/total": 0.7152940630912781, | |
| "ref_logps/chosen": -50.76835632324219, | |
| "ref_logps/rejected": -68.15713500976562, | |
| "rewards/accuracies": 0.4920000433921814, | |
| "rewards/chosen": -0.007775493897497654, | |
| "rewards/margins": -0.006577868480235338, | |
| "rewards/rejected": -0.0011976259993389249, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 2.0833333333333333e-07, | |
| "logps/chosen": -51.992218017578125, | |
| "logps/rejected": -67.8126220703125, | |
| "loss": 0.7059, | |
| "losses/dpo": 0.7131974697113037, | |
| "losses/sft": 1.781252384185791, | |
| "losses/total": 0.7131974697113037, | |
| "ref_logps/chosen": -52.02339172363281, | |
| "ref_logps/rejected": -67.7878646850586, | |
| "rewards/accuracies": 0.5110000371932983, | |
| "rewards/chosen": 0.003117597894743085, | |
| "rewards/margins": 0.005593638867139816, | |
| "rewards/rejected": -0.0024760416708886623, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logps/chosen": -52.34601593017578, | |
| "logps/rejected": -67.67086029052734, | |
| "loss": 0.7097, | |
| "losses/dpo": 0.7076306939125061, | |
| "losses/sft": 1.7903097867965698, | |
| "losses/total": 0.7076306939125061, | |
| "ref_logps/chosen": -52.40769577026367, | |
| "ref_logps/rejected": -67.70091247558594, | |
| "rewards/accuracies": 0.49650001525878906, | |
| "rewards/chosen": 0.006168678868561983, | |
| "rewards/margins": 0.003163211978971958, | |
| "rewards/rejected": 0.003005466191098094, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 4.1666666666666667e-07, | |
| "logps/chosen": -51.974483489990234, | |
| "logps/rejected": -68.44440460205078, | |
| "loss": 0.7115, | |
| "losses/dpo": 0.7194903492927551, | |
| "losses/sft": 1.777998924255371, | |
| "losses/total": 0.7194903492927551, | |
| "ref_logps/chosen": -51.95515441894531, | |
| "ref_logps/rejected": -68.4430923461914, | |
| "rewards/accuracies": 0.4794999957084656, | |
| "rewards/chosen": -0.0019327354384586215, | |
| "rewards/margins": -0.0018010810017585754, | |
| "rewards/rejected": -0.00013165354903321713, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 4.976851851851852e-07, | |
| "logps/chosen": -51.21366882324219, | |
| "logps/rejected": -68.10318756103516, | |
| "loss": 0.7096, | |
| "losses/dpo": 0.7016645073890686, | |
| "losses/sft": 1.7784287929534912, | |
| "losses/total": 0.7016645073890686, | |
| "ref_logps/chosen": -51.175315856933594, | |
| "ref_logps/rejected": -68.03705596923828, | |
| "rewards/accuracies": 0.5074999928474426, | |
| "rewards/chosen": -0.003834722563624382, | |
| "rewards/margins": 0.002778256544843316, | |
| "rewards/rejected": -0.006612981669604778, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 4.861111111111111e-07, | |
| "logps/chosen": -49.84953689575195, | |
| "logps/rejected": -66.59363555908203, | |
| "loss": 0.6989, | |
| "losses/dpo": 0.6881049275398254, | |
| "losses/sft": 1.7335822582244873, | |
| "losses/total": 0.6881049275398254, | |
| "ref_logps/chosen": -49.83342361450195, | |
| "ref_logps/rejected": -66.36032104492188, | |
| "rewards/accuracies": 0.5224999785423279, | |
| "rewards/chosen": -0.0016115426551550627, | |
| "rewards/margins": 0.021719880402088165, | |
| "rewards/rejected": -0.023331422358751297, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 4.74537037037037e-07, | |
| "logps/chosen": -51.20610427856445, | |
| "logps/rejected": -68.37096405029297, | |
| "loss": 0.6965, | |
| "losses/dpo": 0.6989789605140686, | |
| "losses/sft": 1.8248052597045898, | |
| "losses/total": 0.6989789605140686, | |
| "ref_logps/chosen": -51.07723617553711, | |
| "ref_logps/rejected": -67.93907165527344, | |
| "rewards/accuracies": 0.5259999632835388, | |
| "rewards/chosen": -0.012886175885796547, | |
| "rewards/margins": 0.0303032249212265, | |
| "rewards/rejected": -0.0431894026696682, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 4.6296296296296297e-07, | |
| "logps/chosen": -53.1021842956543, | |
| "logps/rejected": -69.64921569824219, | |
| "loss": 0.6821, | |
| "losses/dpo": 0.6905575394630432, | |
| "losses/sft": 1.7901018857955933, | |
| "losses/total": 0.6905575394630432, | |
| "ref_logps/chosen": -52.89280319213867, | |
| "ref_logps/rejected": -68.83776092529297, | |
| "rewards/accuracies": 0.5610000491142273, | |
| "rewards/chosen": -0.02093815989792347, | |
| "rewards/margins": 0.0602080263197422, | |
| "rewards/rejected": -0.08114618062973022, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 4.513888888888889e-07, | |
| "logps/chosen": -51.60838317871094, | |
| "logps/rejected": -68.19721984863281, | |
| "loss": 0.6783, | |
| "losses/dpo": 0.6758668422698975, | |
| "losses/sft": 1.7487859725952148, | |
| "losses/total": 0.6758668422698975, | |
| "ref_logps/chosen": -51.219181060791016, | |
| "ref_logps/rejected": -67.0694580078125, | |
| "rewards/accuracies": 0.5529999732971191, | |
| "rewards/chosen": -0.03891964256763458, | |
| "rewards/margins": 0.0738559141755104, | |
| "rewards/rejected": -0.11277556419372559, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 4.398148148148148e-07, | |
| "logps/chosen": -51.5633544921875, | |
| "logps/rejected": -70.04096221923828, | |
| "loss": 0.6606, | |
| "losses/dpo": 0.6762534976005554, | |
| "losses/sft": 1.7945468425750732, | |
| "losses/total": 0.6762534976005554, | |
| "ref_logps/chosen": -51.164485931396484, | |
| "ref_logps/rejected": -68.47702026367188, | |
| "rewards/accuracies": 0.5925000309944153, | |
| "rewards/chosen": -0.03988657891750336, | |
| "rewards/margins": 0.11650750786066055, | |
| "rewards/rejected": -0.1563940793275833, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 4.2824074074074075e-07, | |
| "logps/chosen": -52.760066986083984, | |
| "logps/rejected": -71.27080535888672, | |
| "loss": 0.6541, | |
| "losses/dpo": 0.6559135317802429, | |
| "losses/sft": 1.8313995599746704, | |
| "losses/total": 0.6559135317802429, | |
| "ref_logps/chosen": -52.14683151245117, | |
| "ref_logps/rejected": -69.2181625366211, | |
| "rewards/accuracies": 0.6005000472068787, | |
| "rewards/chosen": -0.0613233856856823, | |
| "rewards/margins": 0.14394104480743408, | |
| "rewards/rejected": -0.20526443421840668, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 4.1666666666666667e-07, | |
| "logps/chosen": -53.022369384765625, | |
| "logps/rejected": -70.22279357910156, | |
| "loss": 0.6444, | |
| "losses/dpo": 0.6332589387893677, | |
| "losses/sft": 1.8123092651367188, | |
| "losses/total": 0.6332589387893677, | |
| "ref_logps/chosen": -52.272945404052734, | |
| "ref_logps/rejected": -67.84944152832031, | |
| "rewards/accuracies": 0.6039999723434448, | |
| "rewards/chosen": -0.07494210451841354, | |
| "rewards/margins": 0.16239266097545624, | |
| "rewards/rejected": -0.2373347282409668, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 4.050925925925926e-07, | |
| "logps/chosen": -52.53852462768555, | |
| "logps/rejected": -69.21473693847656, | |
| "loss": 0.6472, | |
| "losses/dpo": 0.6568192839622498, | |
| "losses/sft": 1.808381199836731, | |
| "losses/total": 0.6568192839622498, | |
| "ref_logps/chosen": -51.50153350830078, | |
| "ref_logps/rejected": -66.50575256347656, | |
| "rewards/accuracies": 0.6184999942779541, | |
| "rewards/chosen": -0.10369860380887985, | |
| "rewards/margins": 0.1671994924545288, | |
| "rewards/rejected": -0.27089810371398926, | |
| "step": 325 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 3.9351851851851854e-07, | |
| "logps/chosen": -52.68033218383789, | |
| "logps/rejected": -69.9216079711914, | |
| "loss": 0.6322, | |
| "losses/dpo": 0.6414695978164673, | |
| "losses/sft": 1.7517987489700317, | |
| "losses/total": 0.6414695978164673, | |
| "ref_logps/chosen": -51.4389533996582, | |
| "ref_logps/rejected": -66.67315673828125, | |
| "rewards/accuracies": 0.6355000138282776, | |
| "rewards/chosen": -0.12413790076971054, | |
| "rewards/margins": 0.20070795714855194, | |
| "rewards/rejected": -0.3248458802700043, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.819444444444444e-07, | |
| "logps/chosen": -53.01972579956055, | |
| "logps/rejected": -69.83106231689453, | |
| "loss": 0.6254, | |
| "losses/dpo": 0.6416024565696716, | |
| "losses/sft": 1.806755781173706, | |
| "losses/total": 0.6416024565696716, | |
| "ref_logps/chosen": -51.760948181152344, | |
| "ref_logps/rejected": -66.29093170166016, | |
| "rewards/accuracies": 0.6260000467300415, | |
| "rewards/chosen": -0.12587812542915344, | |
| "rewards/margins": 0.22813528776168823, | |
| "rewards/rejected": -0.3540134131908417, | |
| "step": 375 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 3.703703703703703e-07, | |
| "logps/chosen": -53.866825103759766, | |
| "logps/rejected": -73.0006332397461, | |
| "loss": 0.6223, | |
| "losses/dpo": 0.6281777024269104, | |
| "losses/sft": 1.8359609842300415, | |
| "losses/total": 0.6281777024269104, | |
| "ref_logps/chosen": -52.29502868652344, | |
| "ref_logps/rejected": -68.76205444335938, | |
| "rewards/accuracies": 0.624500036239624, | |
| "rewards/chosen": -0.15717901289463043, | |
| "rewards/margins": 0.26667752861976624, | |
| "rewards/rejected": -0.42385655641555786, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 1.06, | |
| "learning_rate": 3.587962962962963e-07, | |
| "logps/chosen": -52.83345031738281, | |
| "logps/rejected": -73.19969940185547, | |
| "loss": 0.6044, | |
| "losses/dpo": 0.6066713929176331, | |
| "losses/sft": 1.8482356071472168, | |
| "losses/total": 0.6066713929176331, | |
| "ref_logps/chosen": -51.23924255371094, | |
| "ref_logps/rejected": -68.46554565429688, | |
| "rewards/accuracies": 0.658500075340271, | |
| "rewards/chosen": -0.15942102670669556, | |
| "rewards/margins": 0.31399470567703247, | |
| "rewards/rejected": -0.4734157621860504, | |
| "step": 425 | |
| }, | |
| { | |
| "epoch": 1.12, | |
| "learning_rate": 3.472222222222222e-07, | |
| "logps/chosen": -53.00102996826172, | |
| "logps/rejected": -71.68830871582031, | |
| "loss": 0.6107, | |
| "losses/dpo": 0.5940313935279846, | |
| "losses/sft": 1.8907349109649658, | |
| "losses/total": 0.5940313935279846, | |
| "ref_logps/chosen": -51.17875671386719, | |
| "ref_logps/rejected": -66.69224548339844, | |
| "rewards/accuracies": 0.6355000138282776, | |
| "rewards/chosen": -0.18222743272781372, | |
| "rewards/margins": 0.3173789978027344, | |
| "rewards/rejected": -0.4996064305305481, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 1.19, | |
| "learning_rate": 3.3564814814814815e-07, | |
| "logps/chosen": -54.13747024536133, | |
| "logps/rejected": -75.39285278320312, | |
| "loss": 0.5906, | |
| "losses/dpo": 0.6001964807510376, | |
| "losses/sft": 1.837428331375122, | |
| "losses/total": 0.6001964807510376, | |
| "ref_logps/chosen": -52.013328552246094, | |
| "ref_logps/rejected": -69.39274597167969, | |
| "rewards/accuracies": 0.6559999585151672, | |
| "rewards/chosen": -0.21241408586502075, | |
| "rewards/margins": 0.387596994638443, | |
| "rewards/rejected": -0.6000111103057861, | |
| "step": 475 | |
| }, | |
| { | |
| "epoch": 1.25, | |
| "learning_rate": 3.2407407407407406e-07, | |
| "logps/chosen": -52.84211730957031, | |
| "logps/rejected": -73.44519805908203, | |
| "loss": 0.588, | |
| "losses/dpo": 0.5720361471176147, | |
| "losses/sft": 1.82402765750885, | |
| "losses/total": 0.5720361471176147, | |
| "ref_logps/chosen": -50.860145568847656, | |
| "ref_logps/rejected": -67.45731353759766, | |
| "rewards/accuracies": 0.6580000519752502, | |
| "rewards/chosen": -0.1981971114873886, | |
| "rewards/margins": 0.40059128403663635, | |
| "rewards/rejected": -0.5987883806228638, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 1.31, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logps/chosen": -54.73566436767578, | |
| "logps/rejected": -73.35672760009766, | |
| "loss": 0.5864, | |
| "losses/dpo": 0.5692346096038818, | |
| "losses/sft": 1.8467037677764893, | |
| "losses/total": 0.5692346096038818, | |
| "ref_logps/chosen": -52.32421875, | |
| "ref_logps/rejected": -66.84322357177734, | |
| "rewards/accuracies": 0.6690000295639038, | |
| "rewards/chosen": -0.24114486575126648, | |
| "rewards/margins": 0.4102060794830322, | |
| "rewards/rejected": -0.6513509154319763, | |
| "step": 525 | |
| }, | |
| { | |
| "epoch": 1.38, | |
| "learning_rate": 3.0092592592592594e-07, | |
| "logps/chosen": -54.00200653076172, | |
| "logps/rejected": -77.5279312133789, | |
| "loss": 0.5712, | |
| "losses/dpo": 0.5749830007553101, | |
| "losses/sft": 1.8555569648742676, | |
| "losses/total": 0.5749830007553101, | |
| "ref_logps/chosen": -51.46855163574219, | |
| "ref_logps/rejected": -70.09744262695312, | |
| "rewards/accuracies": 0.6720000505447388, | |
| "rewards/chosen": -0.25334587693214417, | |
| "rewards/margins": 0.48970410227775574, | |
| "rewards/rejected": -0.7430499196052551, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 1.44, | |
| "learning_rate": 2.8935185185185185e-07, | |
| "logps/chosen": -53.216957092285156, | |
| "logps/rejected": -74.7680892944336, | |
| "loss": 0.5712, | |
| "losses/dpo": 0.5629280209541321, | |
| "losses/sft": 1.8214166164398193, | |
| "losses/total": 0.5629280209541321, | |
| "ref_logps/chosen": -50.563720703125, | |
| "ref_logps/rejected": -67.18008422851562, | |
| "rewards/accuracies": 0.6755000352859497, | |
| "rewards/chosen": -0.2653237581253052, | |
| "rewards/margins": 0.4934755265712738, | |
| "rewards/rejected": -0.7587993144989014, | |
| "step": 575 | |
| }, | |
| { | |
| "epoch": 1.5, | |
| "learning_rate": 2.7777777777777776e-07, | |
| "logps/chosen": -55.059600830078125, | |
| "logps/rejected": -76.23825073242188, | |
| "loss": 0.5577, | |
| "losses/dpo": 0.5477968454360962, | |
| "losses/sft": 1.9264967441558838, | |
| "losses/total": 0.5477968454360962, | |
| "ref_logps/chosen": -52.09434509277344, | |
| "ref_logps/rejected": -67.781005859375, | |
| "rewards/accuracies": 0.6880000233650208, | |
| "rewards/chosen": -0.2965255081653595, | |
| "rewards/margins": 0.5491979122161865, | |
| "rewards/rejected": -0.8457233905792236, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 1.56, | |
| "learning_rate": 2.662037037037037e-07, | |
| "logps/chosen": -56.8978271484375, | |
| "logps/rejected": -78.7895736694336, | |
| "loss": 0.5667, | |
| "losses/dpo": 0.5594893097877502, | |
| "losses/sft": 1.9116603136062622, | |
| "losses/total": 0.5594893097877502, | |
| "ref_logps/chosen": -53.51485824584961, | |
| "ref_logps/rejected": -69.69821166992188, | |
| "rewards/accuracies": 0.6595000624656677, | |
| "rewards/chosen": -0.338296502828598, | |
| "rewards/margins": 0.5708397030830383, | |
| "rewards/rejected": -0.9091362953186035, | |
| "step": 625 | |
| }, | |
| { | |
| "epoch": 1.62, | |
| "learning_rate": 2.5462962962962963e-07, | |
| "logps/chosen": -55.704219818115234, | |
| "logps/rejected": -77.06362915039062, | |
| "loss": 0.5734, | |
| "losses/dpo": 0.583986222743988, | |
| "losses/sft": 1.8731328248977661, | |
| "losses/total": 0.583986222743988, | |
| "ref_logps/chosen": -52.17438888549805, | |
| "ref_logps/rejected": -67.96855926513672, | |
| "rewards/accuracies": 0.6725000143051147, | |
| "rewards/chosen": -0.3529825508594513, | |
| "rewards/margins": 0.5565236806869507, | |
| "rewards/rejected": -0.9095062017440796, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 1.69, | |
| "learning_rate": 2.4305555555555555e-07, | |
| "logps/chosen": -54.6390266418457, | |
| "logps/rejected": -76.69291687011719, | |
| "loss": 0.5568, | |
| "losses/dpo": 0.5715891718864441, | |
| "losses/sft": 1.8541003465652466, | |
| "losses/total": 0.5715891718864441, | |
| "ref_logps/chosen": -51.134674072265625, | |
| "ref_logps/rejected": -67.13263702392578, | |
| "rewards/accuracies": 0.687000036239624, | |
| "rewards/chosen": -0.3504358232021332, | |
| "rewards/margins": 0.605591893196106, | |
| "rewards/rejected": -0.9560276865959167, | |
| "step": 675 | |
| }, | |
| { | |
| "epoch": 1.75, | |
| "learning_rate": 2.3148148148148148e-07, | |
| "logps/chosen": -54.50948715209961, | |
| "logps/rejected": -77.46994018554688, | |
| "loss": 0.5657, | |
| "losses/dpo": 0.5799014568328857, | |
| "losses/sft": 1.8863662481307983, | |
| "losses/total": 0.5799014568328857, | |
| "ref_logps/chosen": -50.88352966308594, | |
| "ref_logps/rejected": -67.8268051147461, | |
| "rewards/accuracies": 0.6755000352859497, | |
| "rewards/chosen": -0.36259526014328003, | |
| "rewards/margins": 0.6017180681228638, | |
| "rewards/rejected": -0.9643133282661438, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.81, | |
| "learning_rate": 2.199074074074074e-07, | |
| "logps/chosen": -55.129295349121094, | |
| "logps/rejected": -77.04840087890625, | |
| "loss": 0.5636, | |
| "losses/dpo": 0.5747159123420715, | |
| "losses/sft": 1.926343560218811, | |
| "losses/total": 0.5747159123420715, | |
| "ref_logps/chosen": -51.24678039550781, | |
| "ref_logps/rejected": -67.05614471435547, | |
| "rewards/accuracies": 0.6730000376701355, | |
| "rewards/chosen": -0.3882514238357544, | |
| "rewards/margins": 0.6109741926193237, | |
| "rewards/rejected": -0.9992256760597229, | |
| "step": 725 | |
| }, | |
| { | |
| "epoch": 1.88, | |
| "learning_rate": 2.0833333333333333e-07, | |
| "logps/chosen": -55.22993087768555, | |
| "logps/rejected": -78.04380798339844, | |
| "loss": 0.5575, | |
| "losses/dpo": 0.5746473073959351, | |
| "losses/sft": 1.8473073244094849, | |
| "losses/total": 0.5746473073959351, | |
| "ref_logps/chosen": -51.49522399902344, | |
| "ref_logps/rejected": -67.86705017089844, | |
| "rewards/accuracies": 0.6645000576972961, | |
| "rewards/chosen": -0.3734706938266754, | |
| "rewards/margins": 0.6442059278488159, | |
| "rewards/rejected": -1.0176767110824585, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 1.94, | |
| "learning_rate": 1.9675925925925927e-07, | |
| "logps/chosen": -54.95744323730469, | |
| "logps/rejected": -77.55332946777344, | |
| "loss": 0.5428, | |
| "losses/dpo": 0.5475051999092102, | |
| "losses/sft": 1.8516862392425537, | |
| "losses/total": 0.5475051999092102, | |
| "ref_logps/chosen": -51.13042449951172, | |
| "ref_logps/rejected": -66.90882110595703, | |
| "rewards/accuracies": 0.6959999799728394, | |
| "rewards/chosen": -0.382702112197876, | |
| "rewards/margins": 0.6817485690116882, | |
| "rewards/rejected": -1.064450740814209, | |
| "step": 775 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "learning_rate": 1.8518518518518516e-07, | |
| "logps/chosen": -56.311126708984375, | |
| "logps/rejected": -79.54733276367188, | |
| "loss": 0.5453, | |
| "losses/dpo": 0.529507577419281, | |
| "losses/sft": 1.9264029264450073, | |
| "losses/total": 0.529507577419281, | |
| "ref_logps/chosen": -51.99349594116211, | |
| "ref_logps/rejected": -68.2928237915039, | |
| "rewards/accuracies": 0.7039999961853027, | |
| "rewards/chosen": -0.4317638874053955, | |
| "rewards/margins": 0.6936854720115662, | |
| "rewards/rejected": -1.1254491806030273, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 2.06, | |
| "learning_rate": 1.736111111111111e-07, | |
| "logps/chosen": -55.88136672973633, | |
| "logps/rejected": -78.89193725585938, | |
| "loss": 0.5536, | |
| "losses/dpo": 0.5027932524681091, | |
| "losses/sft": 1.888191819190979, | |
| "losses/total": 0.5027932524681091, | |
| "ref_logps/chosen": -51.50459289550781, | |
| "ref_logps/rejected": -67.58809661865234, | |
| "rewards/accuracies": 0.7039999961853027, | |
| "rewards/chosen": -0.43767789006233215, | |
| "rewards/margins": 0.6927057504653931, | |
| "rewards/rejected": -1.1303836107254028, | |
| "step": 825 | |
| }, | |
| { | |
| "epoch": 2.12, | |
| "learning_rate": 1.6203703703703703e-07, | |
| "logps/chosen": -56.851905822753906, | |
| "logps/rejected": -79.84717559814453, | |
| "loss": 0.5519, | |
| "losses/dpo": 0.5627533197402954, | |
| "losses/sft": 1.950637936592102, | |
| "losses/total": 0.5627533197402954, | |
| "ref_logps/chosen": -52.47712326049805, | |
| "ref_logps/rejected": -68.6231918334961, | |
| "rewards/accuracies": 0.6880000233650208, | |
| "rewards/chosen": -0.4374789893627167, | |
| "rewards/margins": 0.6849193572998047, | |
| "rewards/rejected": -1.1223982572555542, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 2.19, | |
| "learning_rate": 1.5046296296296297e-07, | |
| "logps/chosen": -56.792240142822266, | |
| "logps/rejected": -80.210205078125, | |
| "loss": 0.5433, | |
| "losses/dpo": 0.562707245349884, | |
| "losses/sft": 1.9036242961883545, | |
| "losses/total": 0.562707245349884, | |
| "ref_logps/chosen": -52.128883361816406, | |
| "ref_logps/rejected": -68.35295104980469, | |
| "rewards/accuracies": 0.6974999904632568, | |
| "rewards/chosen": -0.46633559465408325, | |
| "rewards/margins": 0.719389796257019, | |
| "rewards/rejected": -1.185725450515747, | |
| "step": 875 | |
| }, | |
| { | |
| "epoch": 2.25, | |
| "learning_rate": 1.3888888888888888e-07, | |
| "logps/chosen": -56.366416931152344, | |
| "logps/rejected": -81.07738494873047, | |
| "loss": 0.5341, | |
| "losses/dpo": 0.5471935868263245, | |
| "losses/sft": 1.9060190916061401, | |
| "losses/total": 0.5471935868263245, | |
| "ref_logps/chosen": -51.77671432495117, | |
| "ref_logps/rejected": -68.86495971679688, | |
| "rewards/accuracies": 0.6914999485015869, | |
| "rewards/chosen": -0.45896950364112854, | |
| "rewards/margins": 0.7622722387313843, | |
| "rewards/rejected": -1.2212417125701904, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 2.31, | |
| "learning_rate": 1.2731481481481482e-07, | |
| "logps/chosen": -56.640018463134766, | |
| "logps/rejected": -81.28205871582031, | |
| "loss": 0.5195, | |
| "losses/dpo": 0.5150865316390991, | |
| "losses/sft": 1.918342113494873, | |
| "losses/total": 0.5150865316390991, | |
| "ref_logps/chosen": -52.07315444946289, | |
| "ref_logps/rejected": -68.57891082763672, | |
| "rewards/accuracies": 0.7275000214576721, | |
| "rewards/chosen": -0.4566872715950012, | |
| "rewards/margins": 0.8136279582977295, | |
| "rewards/rejected": -1.2703152894973755, | |
| "step": 925 | |
| }, | |
| { | |
| "epoch": 2.38, | |
| "learning_rate": 1.1574074074074074e-07, | |
| "logps/chosen": -56.610965728759766, | |
| "logps/rejected": -82.6984634399414, | |
| "loss": 0.5267, | |
| "losses/dpo": 0.5297408103942871, | |
| "losses/sft": 1.9613755941390991, | |
| "losses/total": 0.5297408103942871, | |
| "ref_logps/chosen": -51.79991149902344, | |
| "ref_logps/rejected": -69.79439544677734, | |
| "rewards/accuracies": 0.7074999809265137, | |
| "rewards/chosen": -0.4811055660247803, | |
| "rewards/margins": 0.8093020915985107, | |
| "rewards/rejected": -1.2904075384140015, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 2.44, | |
| "learning_rate": 1.0416666666666667e-07, | |
| "logps/chosen": -55.42496109008789, | |
| "logps/rejected": -80.11758422851562, | |
| "loss": 0.5346, | |
| "losses/dpo": 0.5188722610473633, | |
| "losses/sft": 1.9635555744171143, | |
| "losses/total": 0.5188722610473633, | |
| "ref_logps/chosen": -50.690673828125, | |
| "ref_logps/rejected": -67.66686248779297, | |
| "rewards/accuracies": 0.7055000066757202, | |
| "rewards/chosen": -0.47342926263809204, | |
| "rewards/margins": 0.7716432213783264, | |
| "rewards/rejected": -1.2450724840164185, | |
| "step": 975 | |
| }, | |
| { | |
| "epoch": 2.5, | |
| "learning_rate": 9.259259259259258e-08, | |
| "logps/chosen": -55.2717399597168, | |
| "logps/rejected": -78.23765563964844, | |
| "loss": 0.5358, | |
| "losses/dpo": 0.522473156452179, | |
| "losses/sft": 1.9380919933319092, | |
| "losses/total": 0.522473156452179, | |
| "ref_logps/chosen": -50.576473236083984, | |
| "ref_logps/rejected": -65.8857192993164, | |
| "rewards/accuracies": 0.6880000233650208, | |
| "rewards/chosen": -0.46952569484710693, | |
| "rewards/margins": 0.7656675577163696, | |
| "rewards/rejected": -1.2351933717727661, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 2.56, | |
| "learning_rate": 8.101851851851852e-08, | |
| "logps/chosen": -57.238101959228516, | |
| "logps/rejected": -80.15925598144531, | |
| "loss": 0.5477, | |
| "losses/dpo": 0.5454632043838501, | |
| "losses/sft": 1.9586001634597778, | |
| "losses/total": 0.5454632043838501, | |
| "ref_logps/chosen": -51.92245101928711, | |
| "ref_logps/rejected": -67.11357879638672, | |
| "rewards/accuracies": 0.6919999718666077, | |
| "rewards/chosen": -0.5315641164779663, | |
| "rewards/margins": 0.7730042934417725, | |
| "rewards/rejected": -1.3045685291290283, | |
| "step": 1025 | |
| }, | |
| { | |
| "epoch": 2.62, | |
| "learning_rate": 6.944444444444444e-08, | |
| "logps/chosen": -56.340274810791016, | |
| "logps/rejected": -78.74750518798828, | |
| "loss": 0.5353, | |
| "losses/dpo": 0.564401388168335, | |
| "losses/sft": 1.9238648414611816, | |
| "losses/total": 0.564401388168335, | |
| "ref_logps/chosen": -51.298805236816406, | |
| "ref_logps/rejected": -66.08538818359375, | |
| "rewards/accuracies": 0.7080000042915344, | |
| "rewards/chosen": -0.5041469931602478, | |
| "rewards/margins": 0.7620646953582764, | |
| "rewards/rejected": -1.2662116289138794, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 2.69, | |
| "learning_rate": 5.787037037037037e-08, | |
| "logps/chosen": -58.54380416870117, | |
| "logps/rejected": -82.85121154785156, | |
| "loss": 0.5379, | |
| "losses/dpo": 0.4951947033405304, | |
| "losses/sft": 1.927669882774353, | |
| "losses/total": 0.4951947033405304, | |
| "ref_logps/chosen": -53.125526428222656, | |
| "ref_logps/rejected": -69.30874633789062, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.5418279767036438, | |
| "rewards/margins": 0.8124186992645264, | |
| "rewards/rejected": -1.3542466163635254, | |
| "step": 1075 | |
| }, | |
| { | |
| "epoch": 2.75, | |
| "learning_rate": 4.629629629629629e-08, | |
| "logps/chosen": -56.923858642578125, | |
| "logps/rejected": -81.64384460449219, | |
| "loss": 0.5352, | |
| "losses/dpo": 0.5503857135772705, | |
| "losses/sft": 1.9750890731811523, | |
| "losses/total": 0.5503857135772705, | |
| "ref_logps/chosen": -51.61204528808594, | |
| "ref_logps/rejected": -68.07569122314453, | |
| "rewards/accuracies": 0.7145000696182251, | |
| "rewards/chosen": -0.5311816930770874, | |
| "rewards/margins": 0.8256343603134155, | |
| "rewards/rejected": -1.356816053390503, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 2.81, | |
| "learning_rate": 3.472222222222222e-08, | |
| "logps/chosen": -57.147186279296875, | |
| "logps/rejected": -79.592529296875, | |
| "loss": 0.541, | |
| "losses/dpo": 0.5187271237373352, | |
| "losses/sft": 1.9453308582305908, | |
| "losses/total": 0.5187271237373352, | |
| "ref_logps/chosen": -52.0300178527832, | |
| "ref_logps/rejected": -66.7751693725586, | |
| "rewards/accuracies": 0.7010000348091125, | |
| "rewards/chosen": -0.5117166042327881, | |
| "rewards/margins": 0.7700192332267761, | |
| "rewards/rejected": -1.281735897064209, | |
| "step": 1125 | |
| }, | |
| { | |
| "epoch": 2.88, | |
| "learning_rate": 2.3148148148148144e-08, | |
| "logps/chosen": -56.04924011230469, | |
| "logps/rejected": -82.78221893310547, | |
| "loss": 0.5138, | |
| "losses/dpo": 0.5014563202857971, | |
| "losses/sft": 1.892021656036377, | |
| "losses/total": 0.5014563202857971, | |
| "ref_logps/chosen": -50.947792053222656, | |
| "ref_logps/rejected": -68.86062622070312, | |
| "rewards/accuracies": 0.7239999175071716, | |
| "rewards/chosen": -0.5101450085639954, | |
| "rewards/margins": 0.8820143938064575, | |
| "rewards/rejected": -1.3921594619750977, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 2.94, | |
| "learning_rate": 1.1574074074074072e-08, | |
| "logps/chosen": -57.95354461669922, | |
| "logps/rejected": -81.48481750488281, | |
| "loss": 0.5322, | |
| "losses/dpo": 0.5276296138763428, | |
| "losses/sft": 1.914094090461731, | |
| "losses/total": 0.5276296138763428, | |
| "ref_logps/chosen": -52.49882507324219, | |
| "ref_logps/rejected": -68.10453796386719, | |
| "rewards/accuracies": 0.7064999938011169, | |
| "rewards/chosen": -0.5454717874526978, | |
| "rewards/margins": 0.7925567030906677, | |
| "rewards/rejected": -1.3380285501480103, | |
| "step": 1175 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "learning_rate": 0.0, | |
| "logps/chosen": -58.058658599853516, | |
| "logps/rejected": -80.37024688720703, | |
| "loss": 0.5418, | |
| "losses/dpo": 0.5331112146377563, | |
| "losses/sft": 1.9555615186691284, | |
| "losses/total": 0.5331112146377563, | |
| "ref_logps/chosen": -52.748592376708984, | |
| "ref_logps/rejected": -67.33932495117188, | |
| "rewards/accuracies": 0.6899999976158142, | |
| "rewards/chosen": -0.5310066342353821, | |
| "rewards/margins": 0.7720865607261658, | |
| "rewards/rejected": -1.3030930757522583, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "step": 1200, | |
| "total_flos": 0.0, | |
| "train_loss": 0.5943309187889099, | |
| "train_runtime": 16038.0463, | |
| "train_samples_per_second": 5.984, | |
| "train_steps_per_second": 0.075 | |
| } | |
| ], | |
| "logging_steps": 25, | |
| "max_steps": 1200, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": {}, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |