{ "best_metric": null, "best_model_checkpoint": null, "epoch": 2.986666666666667, "eval_steps": 500, "global_step": 315, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.047407407407407405, "grad_norm": 604.0, "learning_rate": 7.8125e-06, "log_odds_chosen": 4.997040271759033, "log_odds_ratio": -9.76394271850586, "logits/chosen": 138.64193725585938, "logits/rejected": 152.19479370117188, "logps/chosen": -20.550189971923828, "logps/rejected": -25.547183990478516, "loss": 284.5915, "nll_loss": 8.063753128051758, "rewards/accuracies": 0.543749988079071, "rewards/chosen": -1.0275095701217651, "rewards/margins": 0.24984964728355408, "rewards/rejected": -1.2773592472076416, "step": 5 }, { "epoch": 0.09481481481481481, "grad_norm": 350.0, "learning_rate": 1.5625e-05, "log_odds_chosen": 2.356145143508911, "log_odds_ratio": -7.16825008392334, "logits/chosen": 131.21401977539062, "logits/rejected": 160.0535430908203, "logps/chosen": -15.259765625, "logps/rejected": -17.6148738861084, "loss": 251.6382, "nll_loss": 6.878444671630859, "rewards/accuracies": 0.5375000238418579, "rewards/chosen": -0.7629883289337158, "rewards/margins": 0.11775543540716171, "rewards/rejected": -0.880743682384491, "step": 10 }, { "epoch": 0.14222222222222222, "grad_norm": 314.0, "learning_rate": 2.34375e-05, "log_odds_chosen": 4.337428092956543, "log_odds_ratio": -7.543389797210693, "logits/chosen": 123.3839340209961, "logits/rejected": 150.43328857421875, "logps/chosen": -19.47274398803711, "logps/rejected": -23.80842399597168, "loss": 257.0097, "nll_loss": 8.022147178649902, "rewards/accuracies": 0.5687500238418579, "rewards/chosen": -0.9736372828483582, "rewards/margins": 0.2167840451002121, "rewards/rejected": -1.190421223640442, "step": 15 }, { "epoch": 0.18962962962962962, "grad_norm": 1352.0, "learning_rate": 3.125e-05, "log_odds_chosen": -0.34767332673072815, "log_odds_ratio": -5.567352771759033, "logits/chosen": 149.3828125, "logits/rejected": 153.21987915039062, "logps/chosen": -13.090100288391113, "logps/rejected": -12.743819236755371, "loss": 207.4182, "nll_loss": 6.128692150115967, "rewards/accuracies": 0.543749988079071, "rewards/chosen": -0.6545050740242004, "rewards/margins": -0.01731405407190323, "rewards/rejected": -0.6371909976005554, "step": 20 }, { "epoch": 0.23703703703703705, "grad_norm": 294.0, "learning_rate": 3.90625e-05, "log_odds_chosen": 1.0955836772918701, "log_odds_ratio": -1.4455451965332031, "logits/chosen": 165.73220825195312, "logits/rejected": 174.95925903320312, "logps/chosen": -3.5200371742248535, "logps/rejected": -4.602563858032227, "loss": 75.9138, "nll_loss": 2.368574380874634, "rewards/accuracies": 0.53125, "rewards/chosen": -0.17600186169147491, "rewards/margins": 0.05412634462118149, "rewards/rejected": -0.230128213763237, "step": 25 }, { "epoch": 0.28444444444444444, "grad_norm": 153.0, "learning_rate": 4.6875e-05, "log_odds_chosen": 0.15411178767681122, "log_odds_ratio": -0.7986702919006348, "logits/chosen": 197.4137725830078, "logits/rejected": 219.61807250976562, "logps/chosen": -1.626283884048462, "logps/rejected": -1.7547603845596313, "loss": 62.0478, "nll_loss": 1.865581750869751, "rewards/accuracies": 0.5687500238418579, "rewards/chosen": -0.08131420612335205, "rewards/margins": 0.006423823535442352, "rewards/rejected": -0.0877380222082138, "step": 30 }, { "epoch": 0.33185185185185184, "grad_norm": 166.0, "learning_rate": 4.998613757348784e-05, "log_odds_chosen": 0.004070779774338007, "log_odds_ratio": -0.8579562902450562, "logits/chosen": 234.51419067382812, "logits/rejected": 230.0107421875, "logps/chosen": -1.581650972366333, "logps/rejected": -1.591902494430542, "loss": 57.0513, "nll_loss": 1.7507221698760986, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -0.07908254116773605, "rewards/margins": 0.0005125798052176833, "rewards/rejected": -0.07959512621164322, "step": 35 }, { "epoch": 0.37925925925925924, "grad_norm": 161.0, "learning_rate": 4.990147841143462e-05, "log_odds_chosen": 0.2214292585849762, "log_odds_ratio": -0.6974400281906128, "logits/chosen": 228.30813598632812, "logits/rejected": 230.08468627929688, "logps/chosen": -1.3625082969665527, "logps/rejected": -1.5368319749832153, "loss": 54.0731, "nll_loss": 1.6455962657928467, "rewards/accuracies": 0.4937500059604645, "rewards/chosen": -0.0681254118680954, "rewards/margins": 0.008716189302504063, "rewards/rejected": -0.07684160768985748, "step": 40 }, { "epoch": 0.4266666666666667, "grad_norm": 78.0, "learning_rate": 4.97401218720448e-05, "log_odds_chosen": 0.18503662943840027, "log_odds_ratio": -0.7793182730674744, "logits/chosen": 209.5319366455078, "logits/rejected": 208.72811889648438, "logps/chosen": -1.389093041419983, "logps/rejected": -1.543234944343567, "loss": 52.3797, "nll_loss": 1.6294572353363037, "rewards/accuracies": 0.5249999761581421, "rewards/chosen": -0.06945464760065079, "rewards/margins": 0.00770709291100502, "rewards/rejected": -0.0771617442369461, "step": 45 }, { "epoch": 0.4740740740740741, "grad_norm": 68.0, "learning_rate": 4.9502564938797946e-05, "log_odds_chosen": 0.1809634566307068, "log_odds_ratio": -0.7305103540420532, "logits/chosen": 201.18109130859375, "logits/rejected": 201.95797729492188, "logps/chosen": -1.3455302715301514, "logps/rejected": -1.4954122304916382, "loss": 54.161, "nll_loss": 1.6752201318740845, "rewards/accuracies": 0.5625, "rewards/chosen": -0.06727652251720428, "rewards/margins": 0.007494096644222736, "rewards/rejected": -0.07477061450481415, "step": 50 }, { "epoch": 0.5214814814814814, "grad_norm": 70.0, "learning_rate": 4.918953929490768e-05, "log_odds_chosen": 0.14337705075740814, "log_odds_ratio": -0.7200931310653687, "logits/chosen": 194.4894256591797, "logits/rejected": 193.81051635742188, "logps/chosen": -1.2621675729751587, "logps/rejected": -1.3799371719360352, "loss": 49.4217, "nll_loss": 1.5453771352767944, "rewards/accuracies": 0.5625, "rewards/chosen": -0.06310837715864182, "rewards/margins": 0.005888479296118021, "rewards/rejected": -0.068996861577034, "step": 55 }, { "epoch": 0.5688888888888889, "grad_norm": 63.25, "learning_rate": 4.88020090697132e-05, "log_odds_chosen": 0.221818208694458, "log_odds_ratio": -0.6779998540878296, "logits/chosen": 194.46102905273438, "logits/rejected": 190.25668334960938, "logps/chosen": -1.1540250778198242, "logps/rejected": -1.3141117095947266, "loss": 47.4294, "nll_loss": 1.4274753332138062, "rewards/accuracies": 0.581250011920929, "rewards/chosen": -0.05770125240087509, "rewards/margins": 0.008004339411854744, "rewards/rejected": -0.06570558995008469, "step": 60 }, { "epoch": 0.6162962962962963, "grad_norm": 83.0, "learning_rate": 4.834116786912897e-05, "log_odds_chosen": 0.1881437748670578, "log_odds_ratio": -0.7192609906196594, "logits/chosen": 202.4822235107422, "logits/rejected": 204.4671173095703, "logps/chosen": -1.2269448041915894, "logps/rejected": -1.3740522861480713, "loss": 47.3934, "nll_loss": 1.461767315864563, "rewards/accuracies": 0.53125, "rewards/chosen": -0.06134723871946335, "rewards/margins": 0.00735537800937891, "rewards/rejected": -0.06870262324810028, "step": 65 }, { "epoch": 0.6637037037037037, "grad_norm": 114.0, "learning_rate": 4.7808435099299045e-05, "log_odds_chosen": 0.30721157789230347, "log_odds_ratio": -0.663455069065094, "logits/chosen": 202.92465209960938, "logits/rejected": 196.5619354248047, "logps/chosen": -1.1372950077056885, "logps/rejected": -1.3888975381851196, "loss": 46.1289, "nll_loss": 1.4218207597732544, "rewards/accuracies": 0.581250011920929, "rewards/chosen": -0.056864749640226364, "rewards/margins": 0.012580126523971558, "rewards/rejected": -0.06944488734006882, "step": 70 }, { "epoch": 0.7111111111111111, "grad_norm": 294.0, "learning_rate": 4.720545159477922e-05, "log_odds_chosen": 0.3042272627353668, "log_odds_ratio": -0.6515589952468872, "logits/chosen": 203.1158447265625, "logits/rejected": 205.7737274169922, "logps/chosen": -1.100188136100769, "logps/rejected": -1.3240959644317627, "loss": 46.1877, "nll_loss": 1.4243879318237305, "rewards/accuracies": 0.637499988079071, "rewards/chosen": -0.05500940605998039, "rewards/margins": 0.011195378378033638, "rewards/rejected": -0.06620479375123978, "step": 75 }, { "epoch": 0.7585185185185185, "grad_norm": 114.5, "learning_rate": 4.653407456471222e-05, "log_odds_chosen": 0.2768544554710388, "log_odds_ratio": -0.6946061849594116, "logits/chosen": 209.74020385742188, "logits/rejected": 206.70449829101562, "logps/chosen": -1.1427994966506958, "logps/rejected": -1.3516017198562622, "loss": 45.0629, "nll_loss": 1.4209954738616943, "rewards/accuracies": 0.59375, "rewards/chosen": -0.05713997408747673, "rewards/margins": 0.010440108366310596, "rewards/rejected": -0.06758008152246475, "step": 80 }, { "epoch": 0.8059259259259259, "grad_norm": 72.0, "learning_rate": 4.579637187256222e-05, "log_odds_chosen": 0.2393846958875656, "log_odds_ratio": -0.6808741688728333, "logits/chosen": 204.6654815673828, "logits/rejected": 199.04537963867188, "logps/chosen": -1.0734455585479736, "logps/rejected": -1.2461655139923096, "loss": 44.4836, "nll_loss": 1.3528298139572144, "rewards/accuracies": 0.59375, "rewards/chosen": -0.05367228388786316, "rewards/margins": 0.00863600242882967, "rewards/rejected": -0.062308281660079956, "step": 85 }, { "epoch": 0.8533333333333334, "grad_norm": 74.5, "learning_rate": 4.499461566702685e-05, "log_odds_chosen": 0.26840656995773315, "log_odds_ratio": -0.660419762134552, "logits/chosen": 203.23355102539062, "logits/rejected": 202.61805725097656, "logps/chosen": -1.0564864873886108, "logps/rejected": -1.2361832857131958, "loss": 45.7849, "nll_loss": 1.381618857383728, "rewards/accuracies": 0.625, "rewards/chosen": -0.05282432958483696, "rewards/margins": 0.008984836749732494, "rewards/rejected": -0.06180916354060173, "step": 90 }, { "epoch": 0.9007407407407407, "grad_norm": 119.0, "learning_rate": 4.413127538374411e-05, "log_odds_chosen": 0.37480229139328003, "log_odds_ratio": -0.5969882011413574, "logits/chosen": 198.45016479492188, "logits/rejected": 195.573974609375, "logps/chosen": -0.9739300608634949, "logps/rejected": -1.2221287488937378, "loss": 44.2264, "nll_loss": 1.2837311029434204, "rewards/accuracies": 0.6812499761581421, "rewards/chosen": -0.048696503043174744, "rewards/margins": 0.012409940361976624, "rewards/rejected": -0.06110644340515137, "step": 95 }, { "epoch": 0.9481481481481482, "grad_norm": 54.0, "learning_rate": 4.320901013934887e-05, "log_odds_chosen": 0.1513899266719818, "log_odds_ratio": -0.7243643403053284, "logits/chosen": 206.537109375, "logits/rejected": 200.01821899414062, "logps/chosen": -1.0972650051116943, "logps/rejected": -1.22805917263031, "loss": 44.4168, "nll_loss": 1.393330693244934, "rewards/accuracies": 0.5687500238418579, "rewards/chosen": -0.054863251745700836, "rewards/margins": 0.006539707537740469, "rewards/rejected": -0.06140295788645744, "step": 100 }, { "epoch": 0.9955555555555555, "grad_norm": 110.0, "learning_rate": 4.223066054130568e-05, "log_odds_chosen": 0.30877920985221863, "log_odds_ratio": -0.6278973817825317, "logits/chosen": 207.3557891845703, "logits/rejected": 204.31600952148438, "logps/chosen": -1.0028679370880127, "logps/rejected": -1.2167755365371704, "loss": 43.3043, "nll_loss": 1.304164171218872, "rewards/accuracies": 0.6312500238418579, "rewards/chosen": -0.05014339089393616, "rewards/margins": 0.010695383884012699, "rewards/rejected": -0.06083878129720688, "step": 105 }, { "epoch": 1.0429629629629629, "grad_norm": 56.25, "learning_rate": 4.1199239938743797e-05, "log_odds_chosen": 0.5618379712104797, "log_odds_ratio": -0.5704431533813477, "logits/chosen": 198.82461547851562, "logits/rejected": 193.56935119628906, "logps/chosen": -0.8706051707267761, "logps/rejected": -1.2394962310791016, "loss": 37.9309, "nll_loss": 1.133053183555603, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.043530263006687164, "rewards/margins": 0.018444549292325974, "rewards/rejected": -0.06197481229901314, "step": 110 }, { "epoch": 1.0903703703703704, "grad_norm": 101.0, "learning_rate": 4.0117925141242174e-05, "log_odds_chosen": 0.7508487701416016, "log_odds_ratio": -0.48277807235717773, "logits/chosen": 194.4977569580078, "logits/rejected": 192.06838989257812, "logps/chosen": -0.8287516832351685, "logps/rejected": -1.3157087564468384, "loss": 36.5937, "nll_loss": 1.1013622283935547, "rewards/accuracies": 0.75, "rewards/chosen": -0.04143758863210678, "rewards/margins": 0.02434784732758999, "rewards/rejected": -0.06578543782234192, "step": 115 }, { "epoch": 1.1377777777777778, "grad_norm": 59.0, "learning_rate": 3.899004663415084e-05, "log_odds_chosen": 0.8108431696891785, "log_odds_ratio": -0.4502410888671875, "logits/chosen": 194.10855102539062, "logits/rejected": 193.88357543945312, "logps/chosen": -0.8010668754577637, "logps/rejected": -1.2895253896713257, "loss": 37.1465, "nll_loss": 1.1015279293060303, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -0.0400533489882946, "rewards/margins": 0.02442292682826519, "rewards/rejected": -0.06447627395391464, "step": 120 }, { "epoch": 1.1851851851851851, "grad_norm": 76.0, "learning_rate": 3.781907832058587e-05, "log_odds_chosen": 0.6857287883758545, "log_odds_ratio": -0.5154197216033936, "logits/chosen": 192.4232635498047, "logits/rejected": 189.86856079101562, "logps/chosen": -0.8564508557319641, "logps/rejected": -1.2979596853256226, "loss": 34.4135, "nll_loss": 1.067600965499878, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.042822543531656265, "rewards/margins": 0.02207544818520546, "rewards/rejected": -0.06489799171686172, "step": 125 }, { "epoch": 1.2325925925925927, "grad_norm": 49.75, "learning_rate": 3.660862682169282e-05, "log_odds_chosen": 0.5871692895889282, "log_odds_ratio": -0.544813334941864, "logits/chosen": 197.4970703125, "logits/rejected": 195.34658813476562, "logps/chosen": -0.8550790548324585, "logps/rejected": -1.1946574449539185, "loss": 36.1597, "nll_loss": 1.1329249143600464, "rewards/accuracies": 0.731249988079071, "rewards/chosen": -0.04275395721197128, "rewards/margins": 0.0169789157807827, "rewards/rejected": -0.059732865542173386, "step": 130 }, { "epoch": 1.28, "grad_norm": 56.75, "learning_rate": 3.5362420368134356e-05, "log_odds_chosen": 0.696057915687561, "log_odds_ratio": -0.500568151473999, "logits/chosen": 191.46945190429688, "logits/rejected": 193.9897918701172, "logps/chosen": -0.7812957763671875, "logps/rejected": -1.2018015384674072, "loss": 35.7705, "nll_loss": 1.0668926239013672, "rewards/accuracies": 0.75, "rewards/chosen": -0.039064787328243256, "rewards/margins": 0.021025285124778748, "rewards/rejected": -0.0600900761783123, "step": 135 }, { "epoch": 1.3274074074074074, "grad_norm": 50.5, "learning_rate": 3.408429731701635e-05, "log_odds_chosen": 0.6391748189926147, "log_odds_ratio": -0.5416852235794067, "logits/chosen": 186.97299194335938, "logits/rejected": 188.9523162841797, "logps/chosen": -0.8818444013595581, "logps/rejected": -1.2928279638290405, "loss": 37.211, "nll_loss": 1.1587209701538086, "rewards/accuracies": 0.737500011920929, "rewards/chosen": -0.044092223048210144, "rewards/margins": 0.02054918184876442, "rewards/rejected": -0.06464140117168427, "step": 140 }, { "epoch": 1.374814814814815, "grad_norm": 62.75, "learning_rate": 3.2778194329621104e-05, "log_odds_chosen": 0.6546341180801392, "log_odds_ratio": -0.5061019062995911, "logits/chosen": 193.48361206054688, "logits/rejected": 189.97891235351562, "logps/chosen": -0.8745110630989075, "logps/rejected": -1.3096916675567627, "loss": 36.6068, "nll_loss": 1.1216226816177368, "rewards/accuracies": 0.731249988079071, "rewards/chosen": -0.043725550174713135, "rewards/margins": 0.021759027615189552, "rewards/rejected": -0.06548458337783813, "step": 145 }, { "epoch": 1.4222222222222223, "grad_norm": 57.5, "learning_rate": 3.144813424636031e-05, "log_odds_chosen": 0.6039055585861206, "log_odds_ratio": -0.5407282114028931, "logits/chosen": 190.10671997070312, "logits/rejected": 191.7057342529297, "logps/chosen": -0.8144019246101379, "logps/rejected": -1.1773698329925537, "loss": 35.5667, "nll_loss": 1.0835626125335693, "rewards/accuracies": 0.71875, "rewards/chosen": -0.040720097720623016, "rewards/margins": 0.01814839243888855, "rewards/rejected": -0.058868490159511566, "step": 150 }, { "epoch": 1.4696296296296296, "grad_norm": 43.25, "learning_rate": 3.0098213696293542e-05, "log_odds_chosen": 0.5702832937240601, "log_odds_ratio": -0.5412538051605225, "logits/chosen": 195.86476135253906, "logits/rejected": 193.18934631347656, "logps/chosen": -0.8274780511856079, "logps/rejected": -1.1580636501312256, "loss": 36.6038, "nll_loss": 1.1131330728530884, "rewards/accuracies": 0.71875, "rewards/chosen": -0.041373901069164276, "rewards/margins": 0.016529280692338943, "rewards/rejected": -0.05790318176150322, "step": 155 }, { "epoch": 1.5170370370370372, "grad_norm": 72.0, "learning_rate": 2.8732590479375165e-05, "log_odds_chosen": 0.6357023119926453, "log_odds_ratio": -0.5035272836685181, "logits/chosen": 199.20773315429688, "logits/rejected": 201.07427978515625, "logps/chosen": -0.898838996887207, "logps/rejected": -1.3042376041412354, "loss": 37.2702, "nll_loss": 1.169263482093811, "rewards/accuracies": 0.762499988079071, "rewards/chosen": -0.04494195431470871, "rewards/margins": 0.020269926637411118, "rewards/rejected": -0.06521187722682953, "step": 160 }, { "epoch": 1.5644444444444443, "grad_norm": 63.0, "learning_rate": 2.7355470760292956e-05, "log_odds_chosen": 0.7361176013946533, "log_odds_ratio": -0.49207085371017456, "logits/chosen": 196.7345428466797, "logits/rejected": 201.42710876464844, "logps/chosen": -0.8479239344596863, "logps/rejected": -1.30262291431427, "loss": 35.9781, "nll_loss": 1.0971934795379639, "rewards/accuracies": 0.768750011920929, "rewards/chosen": -0.042396195232868195, "rewards/margins": 0.022734951227903366, "rewards/rejected": -0.06513114273548126, "step": 165 }, { "epoch": 1.6118518518518519, "grad_norm": 44.25, "learning_rate": 2.597109611334169e-05, "log_odds_chosen": 0.7843419313430786, "log_odds_ratio": -0.47543781995773315, "logits/chosen": 195.94033813476562, "logits/rejected": 194.622314453125, "logps/chosen": -0.8161409497261047, "logps/rejected": -1.3115676641464233, "loss": 35.8517, "nll_loss": 1.0809214115142822, "rewards/accuracies": 0.7875000238418579, "rewards/chosen": -0.040807049721479416, "rewards/margins": 0.02477133832871914, "rewards/rejected": -0.0655783861875534, "step": 170 }, { "epoch": 1.6592592592592592, "grad_norm": 51.5, "learning_rate": 2.458373045823404e-05, "log_odds_chosen": 0.6568929553031921, "log_odds_ratio": -0.5108085870742798, "logits/chosen": 197.05380249023438, "logits/rejected": 193.593994140625, "logps/chosen": -0.8179122805595398, "logps/rejected": -1.2255433797836304, "loss": 35.4159, "nll_loss": 1.1078870296478271, "rewards/accuracies": 0.7562500238418579, "rewards/chosen": -0.04089561477303505, "rewards/margins": 0.02038155123591423, "rewards/rejected": -0.06127716228365898, "step": 175 }, { "epoch": 1.7066666666666666, "grad_norm": 44.75, "learning_rate": 2.3197646927086697e-05, "log_odds_chosen": 0.5102953314781189, "log_odds_ratio": -0.5745335817337036, "logits/chosen": 189.71182250976562, "logits/rejected": 189.39669799804688, "logps/chosen": -0.8822305798530579, "logps/rejected": -1.1973659992218018, "loss": 36.8984, "nll_loss": 1.1632763147354126, "rewards/accuracies": 0.675000011920929, "rewards/chosen": -0.04411153122782707, "rewards/margins": 0.015756776556372643, "rewards/rejected": -0.059868305921554565, "step": 180 }, { "epoch": 1.7540740740740741, "grad_norm": 34.75, "learning_rate": 2.1817114703032176e-05, "log_odds_chosen": 0.6035521030426025, "log_odds_ratio": -0.5137989521026611, "logits/chosen": 192.17518615722656, "logits/rejected": 191.57754516601562, "logps/chosen": -0.8429533839225769, "logps/rejected": -1.228217601776123, "loss": 35.5081, "nll_loss": 1.0759392976760864, "rewards/accuracies": 0.7437499761581421, "rewards/chosen": -0.042147669941186905, "rewards/margins": 0.019263211637735367, "rewards/rejected": -0.06141088157892227, "step": 185 }, { "epoch": 1.8014814814814815, "grad_norm": 58.25, "learning_rate": 2.0446385870993467e-05, "log_odds_chosen": 0.7391771078109741, "log_odds_ratio": -0.4915809631347656, "logits/chosen": 193.21046447753906, "logits/rejected": 190.1485595703125, "logps/chosen": -0.8688086271286011, "logps/rejected": -1.3316614627838135, "loss": 37.3014, "nll_loss": 1.1229429244995117, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -0.043440431356430054, "rewards/margins": 0.023142646998167038, "rewards/rejected": -0.06658308207988739, "step": 190 }, { "epoch": 1.8488888888888888, "grad_norm": 45.25, "learning_rate": 1.9089682321121834e-05, "log_odds_chosen": 0.7794807553291321, "log_odds_ratio": -0.47752276062965393, "logits/chosen": 191.25637817382812, "logits/rejected": 185.43508911132812, "logps/chosen": -0.8267251253128052, "logps/rejected": -1.3205393552780151, "loss": 35.5594, "nll_loss": 1.077235460281372, "rewards/accuracies": 0.762499988079071, "rewards/chosen": -0.04133625701069832, "rewards/margins": 0.024690713733434677, "rewards/rejected": -0.0660269632935524, "step": 195 }, { "epoch": 1.8962962962962964, "grad_norm": 40.25, "learning_rate": 1.775118274523545e-05, "log_odds_chosen": 0.5731266140937805, "log_odds_ratio": -0.5659371614456177, "logits/chosen": 192.36257934570312, "logits/rejected": 191.9702911376953, "logps/chosen": -0.8584796786308289, "logps/rejected": -1.2092317342758179, "loss": 37.8225, "nll_loss": 1.114450216293335, "rewards/accuracies": 0.6875, "rewards/chosen": -0.04292398318648338, "rewards/margins": 0.01753760501742363, "rewards/rejected": -0.06046159192919731, "step": 200 }, { "epoch": 1.9437037037037037, "grad_norm": 47.0, "learning_rate": 1.643500976631037e-05, "log_odds_chosen": 0.6254440546035767, "log_odds_ratio": -0.5245063304901123, "logits/chosen": 195.681640625, "logits/rejected": 188.51748657226562, "logps/chosen": -0.8348779678344727, "logps/rejected": -1.2449554204940796, "loss": 33.9795, "nll_loss": 1.04995858669281, "rewards/accuracies": 0.71875, "rewards/chosen": -0.04174389690160751, "rewards/margins": 0.020503869280219078, "rewards/rejected": -0.06224777549505234, "step": 205 }, { "epoch": 1.991111111111111, "grad_norm": 57.75, "learning_rate": 1.514521724066537e-05, "log_odds_chosen": 0.5750250816345215, "log_odds_ratio": -0.5381630659103394, "logits/chosen": 189.287353515625, "logits/rejected": 191.6526641845703, "logps/chosen": -0.7991558313369751, "logps/rejected": -1.1348118782043457, "loss": 34.3563, "nll_loss": 1.0541894435882568, "rewards/accuracies": 0.706250011920929, "rewards/chosen": -0.039957791566848755, "rewards/margins": 0.01678280346095562, "rewards/rejected": -0.056740593165159225, "step": 210 }, { "epoch": 2.0385185185185186, "grad_norm": 48.0, "learning_rate": 1.3885777771950348e-05, "log_odds_chosen": 1.051464319229126, "log_odds_ratio": -0.3930102288722992, "logits/chosen": 184.33993530273438, "logits/rejected": 184.17428588867188, "logps/chosen": -0.7056841850280762, "logps/rejected": -1.287292242050171, "loss": 30.5978, "nll_loss": 0.9244067072868347, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -0.03528420999646187, "rewards/margins": 0.029080402106046677, "rewards/rejected": -0.06436461955308914, "step": 215 }, { "epoch": 2.0859259259259257, "grad_norm": 65.0, "learning_rate": 1.2660570475395683e-05, "log_odds_chosen": 1.3456884622573853, "log_odds_ratio": -0.33826351165771484, "logits/chosen": 171.13723754882812, "logits/rejected": 175.174072265625, "logps/chosen": -0.6312381029129028, "logps/rejected": -1.3982974290847778, "loss": 27.8654, "nll_loss": 0.8854498863220215, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -0.03156190365552902, "rewards/margins": 0.03835297003388405, "rewards/rejected": -0.06991487741470337, "step": 220 }, { "epoch": 2.1333333333333333, "grad_norm": 39.0, "learning_rate": 1.1473369030008974e-05, "log_odds_chosen": 1.20364248752594, "log_odds_ratio": -0.3720916509628296, "logits/chosen": 179.11940002441406, "logits/rejected": 177.05686950683594, "logps/chosen": -0.643406093120575, "logps/rejected": -1.30879545211792, "loss": 29.0556, "nll_loss": 0.8873918652534485, "rewards/accuracies": 0.875, "rewards/chosen": -0.03217030316591263, "rewards/margins": 0.033269476145505905, "rewards/rejected": -0.06543977558612823, "step": 225 }, { "epoch": 2.180740740740741, "grad_norm": 56.5, "learning_rate": 1.0327830055518842e-05, "log_odds_chosen": 1.3064587116241455, "log_odds_ratio": -0.3238556981086731, "logits/chosen": 175.1729736328125, "logits/rejected": 176.44863891601562, "logps/chosen": -0.6326884627342224, "logps/rejected": -1.3729749917984009, "loss": 27.8203, "nll_loss": 0.854932427406311, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -0.03163442760705948, "rewards/margins": 0.03701432794332504, "rewards/rejected": -0.06864875555038452, "step": 230 }, { "epoch": 2.228148148148148, "grad_norm": 47.0, "learning_rate": 9.227481849865235e-06, "log_odds_chosen": 1.1249010562896729, "log_odds_ratio": -0.3926791250705719, "logits/chosen": 173.18649291992188, "logits/rejected": 177.1532440185547, "logps/chosen": -0.6803759336471558, "logps/rejected": -1.2795072793960571, "loss": 28.6684, "nll_loss": 0.8934875726699829, "rewards/accuracies": 0.84375, "rewards/chosen": -0.03401879593729973, "rewards/margins": 0.02995656430721283, "rewards/rejected": -0.06397536396980286, "step": 235 }, { "epoch": 2.2755555555555556, "grad_norm": 43.75, "learning_rate": 8.175713521924978e-06, "log_odds_chosen": 1.3639342784881592, "log_odds_ratio": -0.3280356526374817, "logits/chosen": 170.26976013183594, "logits/rejected": 175.56011962890625, "logps/chosen": -0.6179399490356445, "logps/rejected": -1.3654717206954956, "loss": 28.216, "nll_loss": 0.8750711679458618, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -0.030897000804543495, "rewards/margins": 0.037376586347818375, "rewards/rejected": -0.06827358901500702, "step": 240 }, { "epoch": 2.322962962962963, "grad_norm": 36.25, "learning_rate": 7.1757645529443665e-06, "log_odds_chosen": 1.3733258247375488, "log_odds_ratio": -0.33095091581344604, "logits/chosen": 170.61001586914062, "logits/rejected": 173.3466339111328, "logps/chosen": -0.643506646156311, "logps/rejected": -1.4111663103103638, "loss": 27.9636, "nll_loss": 0.8818845748901367, "rewards/accuracies": 0.875, "rewards/chosen": -0.03217533975839615, "rewards/margins": 0.03838297724723816, "rewards/rejected": -0.07055831700563431, "step": 245 }, { "epoch": 2.3703703703703702, "grad_norm": 36.25, "learning_rate": 6.230714818829733e-06, "log_odds_chosen": 1.4122543334960938, "log_odds_ratio": -0.3063311278820038, "logits/chosen": 166.13916015625, "logits/rejected": 172.06869506835938, "logps/chosen": -0.6153512001037598, "logps/rejected": -1.4051437377929688, "loss": 27.1152, "nll_loss": 0.8230810165405273, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -0.030767563730478287, "rewards/margins": 0.03948962315917015, "rewards/rejected": -0.07025718688964844, "step": 250 }, { "epoch": 2.417777777777778, "grad_norm": 39.75, "learning_rate": 5.343475104027743e-06, "log_odds_chosen": 1.5338956117630005, "log_odds_ratio": -0.3078998625278473, "logits/chosen": 163.81158447265625, "logits/rejected": 167.9901580810547, "logps/chosen": -0.5699383020401001, "logps/rejected": -1.4373152256011963, "loss": 27.1707, "nll_loss": 0.8324142694473267, "rewards/accuracies": 0.90625, "rewards/chosen": -0.028496911749243736, "rewards/margins": 0.043368853628635406, "rewards/rejected": -0.0718657597899437, "step": 255 }, { "epoch": 2.4651851851851854, "grad_norm": 47.5, "learning_rate": 4.516778136213037e-06, "log_odds_chosen": 1.4830719232559204, "log_odds_ratio": -0.31063929200172424, "logits/chosen": 163.97885131835938, "logits/rejected": 169.63653564453125, "logps/chosen": -0.606620728969574, "logps/rejected": -1.4316565990447998, "loss": 27.166, "nll_loss": 0.8371620178222656, "rewards/accuracies": 0.875, "rewards/chosen": -0.03033103607594967, "rewards/margins": 0.04125178977847099, "rewards/rejected": -0.07158282399177551, "step": 260 }, { "epoch": 2.5125925925925925, "grad_norm": 45.0, "learning_rate": 3.7531701693965554e-06, "log_odds_chosen": 1.2772926092147827, "log_odds_ratio": -0.3710404634475708, "logits/chosen": 166.98779296875, "logits/rejected": 168.69686889648438, "logps/chosen": -0.6445072889328003, "logps/rejected": -1.3713839054107666, "loss": 27.2089, "nll_loss": 0.8413775563240051, "rewards/accuracies": 0.875, "rewards/chosen": -0.032225366681814194, "rewards/margins": 0.036343835294246674, "rewards/rejected": -0.06856919825077057, "step": 265 }, { "epoch": 2.56, "grad_norm": 38.75, "learning_rate": 3.055003141378948e-06, "log_odds_chosen": 1.4745795726776123, "log_odds_ratio": -0.29757434129714966, "logits/chosen": 167.68179321289062, "logits/rejected": 172.54640197753906, "logps/chosen": -0.6402829885482788, "logps/rejected": -1.4750518798828125, "loss": 27.9266, "nll_loss": 0.8752562403678894, "rewards/accuracies": 0.90625, "rewards/chosen": -0.0320141427218914, "rewards/margins": 0.041738446801900864, "rewards/rejected": -0.07375258952379227, "step": 270 }, { "epoch": 2.6074074074074076, "grad_norm": 47.75, "learning_rate": 2.424427429704365e-06, "log_odds_chosen": 1.4180128574371338, "log_odds_ratio": -0.3266395926475525, "logits/chosen": 165.8551483154297, "logits/rejected": 167.0991668701172, "logps/chosen": -0.6501474976539612, "logps/rejected": -1.4511449337005615, "loss": 28.3881, "nll_loss": 0.8757912516593933, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -0.03250737115740776, "rewards/margins": 0.04004988074302673, "rewards/rejected": -0.07255725562572479, "step": 275 }, { "epoch": 2.6548148148148147, "grad_norm": 42.5, "learning_rate": 1.8633852284264508e-06, "log_odds_chosen": 1.3011739253997803, "log_odds_ratio": -0.3382631838321686, "logits/chosen": 167.21853637695312, "logits/rejected": 169.163330078125, "logps/chosen": -0.6230785250663757, "logps/rejected": -1.3411012887954712, "loss": 26.7019, "nll_loss": 0.8079374432563782, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -0.031153922900557518, "rewards/margins": 0.035901136696338654, "rewards/rejected": -0.06705506145954132, "step": 280 }, { "epoch": 2.7022222222222223, "grad_norm": 40.0, "learning_rate": 1.3736045660864034e-06, "log_odds_chosen": 1.3024629354476929, "log_odds_ratio": -0.3390798270702362, "logits/chosen": 169.4490203857422, "logits/rejected": 169.3362274169922, "logps/chosen": -0.6201122403144836, "logps/rejected": -1.3454174995422363, "loss": 28.0871, "nll_loss": 0.8465447425842285, "rewards/accuracies": 0.875, "rewards/chosen": -0.03100561536848545, "rewards/margins": 0.036265257745981216, "rewards/rejected": -0.06727087497711182, "step": 285 }, { "epoch": 2.74962962962963, "grad_norm": 44.75, "learning_rate": 9.565939833279192e-07, "log_odds_chosen": 1.383882761001587, "log_odds_ratio": -0.31699490547180176, "logits/chosen": 167.99237060546875, "logits/rejected": 173.28675842285156, "logps/chosen": -0.6342524290084839, "logps/rejected": -1.401726484298706, "loss": 28.0037, "nll_loss": 0.8696128726005554, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -0.03171262517571449, "rewards/margins": 0.03837370499968529, "rewards/rejected": -0.07008632272481918, "step": 290 }, { "epoch": 2.797037037037037, "grad_norm": 51.5, "learning_rate": 6.136378865420872e-07, "log_odds_chosen": 1.4249391555786133, "log_odds_ratio": -0.35160475969314575, "logits/chosen": 167.20567321777344, "logits/rejected": 170.51210021972656, "logps/chosen": -0.6513175964355469, "logps/rejected": -1.4825408458709717, "loss": 27.9843, "nll_loss": 0.8786695599555969, "rewards/accuracies": 0.887499988079071, "rewards/chosen": -0.0325658805668354, "rewards/margins": 0.04156116023659706, "rewards/rejected": -0.07412704080343246, "step": 295 }, { "epoch": 2.8444444444444446, "grad_norm": 43.5, "learning_rate": 3.45792591853214e-07, "log_odds_chosen": 1.237529993057251, "log_odds_ratio": -0.36313003301620483, "logits/chosen": 175.26461791992188, "logits/rejected": 173.83885192871094, "logps/chosen": -0.6848838925361633, "logps/rejected": -1.3957815170288086, "loss": 28.3691, "nll_loss": 0.9239709973335266, "rewards/accuracies": 0.875, "rewards/chosen": -0.034244198352098465, "rewards/margins": 0.035544879734516144, "rewards/rejected": -0.06978908181190491, "step": 300 }, { "epoch": 2.891851851851852, "grad_norm": 40.0, "learning_rate": 1.538830716302092e-07, "log_odds_chosen": 1.354568362236023, "log_odds_ratio": -0.33406147360801697, "logits/chosen": 168.0280303955078, "logits/rejected": 169.37564086914062, "logps/chosen": -0.6664038300514221, "logps/rejected": -1.40469491481781, "loss": 26.5858, "nll_loss": 0.817513108253479, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -0.033320192247629166, "rewards/margins": 0.036914560943841934, "rewards/rejected": -0.0702347606420517, "step": 305 }, { "epoch": 2.9392592592592592, "grad_norm": 52.5, "learning_rate": 3.8500413544415025e-08, "log_odds_chosen": 1.4047539234161377, "log_odds_ratio": -0.31650859117507935, "logits/chosen": 168.70623779296875, "logits/rejected": 170.0711669921875, "logps/chosen": -0.6374054551124573, "logps/rejected": -1.395747184753418, "loss": 27.6658, "nll_loss": 0.8348110318183899, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -0.031870268285274506, "rewards/margins": 0.037917084991931915, "rewards/rejected": -0.06978735327720642, "step": 310 }, { "epoch": 2.986666666666667, "grad_norm": 61.5, "learning_rate": 0.0, "log_odds_chosen": 1.5434544086456299, "log_odds_ratio": -0.29583439230918884, "logits/chosen": 165.3496856689453, "logits/rejected": 166.1381378173828, "logps/chosen": -0.5781136155128479, "logps/rejected": -1.430616021156311, "loss": 27.0079, "nll_loss": 0.8401652574539185, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -0.028905680403113365, "rewards/margins": 0.042625121772289276, "rewards/rejected": -0.07153080403804779, "step": 315 }, { "epoch": 2.986666666666667, "step": 315, "total_flos": 0.0, "train_loss": 50.88312741234189, "train_runtime": 6896.118, "train_samples_per_second": 2.936, "train_steps_per_second": 0.046 } ], "logging_steps": 5, "max_steps": 315, "num_input_tokens_seen": 0, "num_train_epochs": 3, "save_steps": 100000, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }