{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.986666666666667,
  "eval_steps": 500,
  "global_step": 315,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 350208.0,
      "learning_rate": 7.8125e-06,
      "log_odds_chosen": 5.121338844299316,
      "log_odds_ratio": -9.85741901397705,
      "logits/chosen": 138.62344360351562,
      "logits/rejected": 152.14088439941406,
      "logps/chosen": -20.805774688720703,
      "logps/rejected": -25.92710304260254,
      "loss": 2767.5898,
      "nll_loss": 8.236187934875488,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -10.402887344360352,
      "rewards/margins": 2.5606625080108643,
      "rewards/rejected": -12.96355152130127,
      "step": 5
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 18560.0,
      "learning_rate": 1.5625e-05,
      "log_odds_chosen": 2.4278221130371094,
      "log_odds_ratio": -7.406900882720947,
      "logits/chosen": 129.0441436767578,
      "logits/rejected": 156.9169158935547,
      "logps/chosen": -15.9468355178833,
      "logps/rejected": -18.37396812438965,
      "loss": 222.5123,
      "nll_loss": 7.290040016174316,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -7.97341775894165,
      "rewards/margins": 1.2135658264160156,
      "rewards/rejected": -9.186984062194824,
      "step": 10
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 22144.0,
      "learning_rate": 2.34375e-05,
      "log_odds_chosen": 4.718414306640625,
      "log_odds_ratio": -8.311258316040039,
      "logits/chosen": 109.10520935058594,
      "logits/rejected": 134.63148498535156,
      "logps/chosen": -21.552303314208984,
      "logps/rejected": -26.269397735595703,
      "loss": 4043.0027,
      "nll_loss": 8.928205490112305,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -10.776151657104492,
      "rewards/margins": 2.358546733856201,
      "rewards/rejected": -13.134698867797852,
      "step": 15
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 7392.0,
      "learning_rate": 3.125e-05,
      "log_odds_chosen": -0.21470332145690918,
      "log_odds_ratio": -7.985512733459473,
      "logits/chosen": 124.5482177734375,
      "logits/rejected": 130.01638793945312,
      "logps/chosen": -18.390491485595703,
      "logps/rejected": -18.17677879333496,
      "loss": 2356.9844,
      "nll_loss": 8.458616256713867,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -9.195245742797852,
      "rewards/margins": -0.10685563087463379,
      "rewards/rejected": -9.08838939666748,
      "step": 20
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 15040.0,
      "learning_rate": 3.90625e-05,
      "log_odds_chosen": 4.443707466125488,
      "log_odds_ratio": -7.725132942199707,
      "logits/chosen": 117.22242736816406,
      "logits/rejected": 125.91157531738281,
      "logps/chosen": -18.476272583007812,
      "logps/rejected": -22.920040130615234,
      "loss": 2797.959,
      "nll_loss": 7.461993217468262,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -9.238136291503906,
      "rewards/margins": 2.2218832969665527,
      "rewards/rejected": -11.460020065307617,
      "step": 25
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 44800.0,
      "learning_rate": 4.6875e-05,
      "log_odds_chosen": 1.0098240375518799,
      "log_odds_ratio": -6.719524383544922,
      "logits/chosen": 74.41526794433594,
      "logits/rejected": 114.5694808959961,
      "logps/chosen": -17.375003814697266,
      "logps/rejected": -18.381793975830078,
      "loss": 806.7089,
      "nll_loss": 7.751124382019043,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -8.687501907348633,
      "rewards/margins": 0.5033950209617615,
      "rewards/rejected": -9.190896987915039,
      "step": 30
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 1488.0,
      "learning_rate": 4.998613757348784e-05,
      "log_odds_chosen": -0.7847718000411987,
      "log_odds_ratio": -3.851313352584839,
      "logits/chosen": 95.89997100830078,
      "logits/rejected": 78.72817993164062,
      "logps/chosen": -9.437823295593262,
      "logps/rejected": -8.653680801391602,
      "loss": 540.7871,
      "nll_loss": 5.164363861083984,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -4.718911647796631,
      "rewards/margins": -0.3920711278915405,
      "rewards/rejected": -4.326840400695801,
      "step": 35
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 572.0,
      "learning_rate": 4.990147841143462e-05,
      "log_odds_chosen": 0.18732526898384094,
      "log_odds_ratio": -0.7723637819290161,
      "logits/chosen": 186.49609375,
      "logits/rejected": 200.811767578125,
      "logps/chosen": -1.9941200017929077,
      "logps/rejected": -2.152578830718994,
      "loss": 89.2208,
      "nll_loss": 2.3764045238494873,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.9970600008964539,
      "rewards/margins": 0.07922936975955963,
      "rewards/rejected": -1.076289415359497,
      "step": 40
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 194.0,
      "learning_rate": 4.97401218720448e-05,
      "log_odds_chosen": 0.16486576199531555,
      "log_odds_ratio": -0.8942667245864868,
      "logits/chosen": 235.48745727539062,
      "logits/rejected": 228.4767608642578,
      "logps/chosen": -1.7937572002410889,
      "logps/rejected": -1.9410396814346313,
      "loss": 73.9402,
      "nll_loss": 2.0359139442443848,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.8968786001205444,
      "rewards/margins": 0.07364129275083542,
      "rewards/rejected": -0.9705198407173157,
      "step": 45
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 108.5,
      "learning_rate": 4.9502564938797946e-05,
      "log_odds_chosen": 0.19230754673480988,
      "log_odds_ratio": -0.7629112601280212,
      "logits/chosen": 250.28610229492188,
      "logits/rejected": 250.9467315673828,
      "logps/chosen": -1.563698172569275,
      "logps/rejected": -1.7324550151824951,
      "loss": 71.6374,
      "nll_loss": 1.9435456991195679,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.7818490862846375,
      "rewards/margins": 0.08437846601009369,
      "rewards/rejected": -0.8662275075912476,
      "step": 50
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 158.0,
      "learning_rate": 4.918953929490768e-05,
      "log_odds_chosen": 0.12492300570011139,
      "log_odds_ratio": -0.7422401309013367,
      "logits/chosen": 259.849853515625,
      "logits/rejected": 262.5646667480469,
      "logps/chosen": -1.3962795734405518,
      "logps/rejected": -1.4994373321533203,
      "loss": 62.4039,
      "nll_loss": 1.724207878112793,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.6981397867202759,
      "rewards/margins": 0.05157895013689995,
      "rewards/rejected": -0.7497186660766602,
      "step": 55
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 96.5,
      "learning_rate": 4.88020090697132e-05,
      "log_odds_chosen": 0.20564976334571838,
      "log_odds_ratio": -0.6903149485588074,
      "logits/chosen": 244.59274291992188,
      "logits/rejected": 249.990478515625,
      "logps/chosen": -1.2423059940338135,
      "logps/rejected": -1.395542025566101,
      "loss": 58.7184,
      "nll_loss": 1.5578796863555908,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.6211529970169067,
      "rewards/margins": 0.0766180157661438,
      "rewards/rejected": -0.6977710127830505,
      "step": 60
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 146.0,
      "learning_rate": 4.834116786912897e-05,
      "log_odds_chosen": 0.16068454086780548,
      "log_odds_ratio": -0.7302707433700562,
      "logits/chosen": 241.5931854248047,
      "logits/rejected": 244.501220703125,
      "logps/chosen": -1.264792799949646,
      "logps/rejected": -1.3899643421173096,
      "loss": 56.932,
      "nll_loss": 1.5472662448883057,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.632396399974823,
      "rewards/margins": 0.06258574873209,
      "rewards/rejected": -0.6949821710586548,
      "step": 65
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 296.0,
      "learning_rate": 4.7808435099299045e-05,
      "log_odds_chosen": 0.2955527603626251,
      "log_odds_ratio": -0.6667484641075134,
      "logits/chosen": 230.53378295898438,
      "logits/rejected": 237.8424530029297,
      "logps/chosen": -1.1751458644866943,
      "logps/rejected": -1.4215205907821655,
      "loss": 56.7455,
      "nll_loss": 1.5035736560821533,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.5875729322433472,
      "rewards/margins": 0.12318732589483261,
      "rewards/rejected": -0.7107602953910828,
      "step": 70
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 96.0,
      "learning_rate": 4.720545159477922e-05,
      "log_odds_chosen": 0.2671827971935272,
      "log_odds_ratio": -0.6653175950050354,
      "logits/chosen": 244.9853057861328,
      "logits/rejected": 243.1300811767578,
      "logps/chosen": -1.1448094844818115,
      "logps/rejected": -1.3471866846084595,
      "loss": 54.6994,
      "nll_loss": 1.492123007774353,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.5724047422409058,
      "rewards/margins": 0.10118861496448517,
      "rewards/rejected": -0.6735933423042297,
      "step": 75
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 107.0,
      "learning_rate": 4.653407456471222e-05,
      "log_odds_chosen": 0.24170565605163574,
      "log_odds_ratio": -0.7027624845504761,
      "logits/chosen": 238.3944091796875,
      "logits/rejected": 243.826171875,
      "logps/chosen": -1.1621917486190796,
      "logps/rejected": -1.3395326137542725,
      "loss": 52.7796,
      "nll_loss": 1.4673112630844116,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.5810958743095398,
      "rewards/margins": 0.08867045491933823,
      "rewards/rejected": -0.6697663068771362,
      "step": 80
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 67.5,
      "learning_rate": 4.579637187256222e-05,
      "log_odds_chosen": 0.2149166613817215,
      "log_odds_ratio": -0.6902152895927429,
      "logits/chosen": 237.4712677001953,
      "logits/rejected": 251.7366485595703,
      "logps/chosen": -1.0897717475891113,
      "logps/rejected": -1.2465016841888428,
      "loss": 51.8551,
      "nll_loss": 1.3890597820281982,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5448858737945557,
      "rewards/margins": 0.07836499065160751,
      "rewards/rejected": -0.6232508420944214,
      "step": 85
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 83.0,
      "learning_rate": 4.499461566702685e-05,
      "log_odds_chosen": 0.2485649585723877,
      "log_odds_ratio": -0.6673235297203064,
      "logits/chosen": 234.73666381835938,
      "logits/rejected": 244.18630981445312,
      "logps/chosen": -1.0668323040008545,
      "logps/rejected": -1.230261206626892,
      "loss": 52.656,
      "nll_loss": 1.4122871160507202,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.5334161520004272,
      "rewards/margins": 0.08171448856592178,
      "rewards/rejected": -0.615130603313446,
      "step": 90
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 198.0,
      "learning_rate": 4.413127538374411e-05,
      "log_odds_chosen": 0.36127719283103943,
      "log_odds_ratio": -0.5996707677841187,
      "logits/chosen": 227.44729614257812,
      "logits/rejected": 236.6618194580078,
      "logps/chosen": -0.9856353998184204,
      "logps/rejected": -1.2236969470977783,
      "loss": 50.7134,
      "nll_loss": 1.3108433485031128,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.4928176999092102,
      "rewards/margins": 0.11903069913387299,
      "rewards/rejected": -0.6118484735488892,
      "step": 95
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 71.0,
      "learning_rate": 4.320901013934887e-05,
      "log_odds_chosen": 0.09772248566150665,
      "log_odds_ratio": -0.7446527481079102,
      "logits/chosen": 239.33163452148438,
      "logits/rejected": 244.9424591064453,
      "logps/chosen": -1.1100224256515503,
      "logps/rejected": -1.1970586776733398,
      "loss": 51.5177,
      "nll_loss": 1.4378069639205933,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5550112128257751,
      "rewards/margins": 0.04351811483502388,
      "rewards/rejected": -0.5985293388366699,
      "step": 100
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 64.0,
      "learning_rate": 4.223066054130568e-05,
      "log_odds_chosen": 0.27767232060432434,
      "log_odds_ratio": -0.6391838788986206,
      "logits/chosen": 239.01522827148438,
      "logits/rejected": 248.607421875,
      "logps/chosen": -1.0211066007614136,
      "logps/rejected": -1.21805739402771,
      "loss": 49.8911,
      "nll_loss": 1.339125633239746,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5105533003807068,
      "rewards/margins": 0.09847544133663177,
      "rewards/rejected": -0.609028697013855,
      "step": 105
    },
    {
      "epoch": 1.0429629629629629,
      "grad_norm": 52.0,
      "learning_rate": 4.1199239938743797e-05,
      "log_odds_chosen": 0.2824048697948456,
      "log_odds_ratio": -0.6530550718307495,
      "logits/chosen": 236.0783233642578,
      "logits/rejected": 240.7754364013672,
      "logps/chosen": -0.9296471476554871,
      "logps/rejected": -1.1144460439682007,
      "loss": 46.0534,
      "nll_loss": 1.2121281623840332,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.46482357382774353,
      "rewards/margins": 0.09239944070577621,
      "rewards/rejected": -0.5572230219841003,
      "step": 110
    },
    {
      "epoch": 1.0903703703703704,
      "grad_norm": 90.0,
      "learning_rate": 4.0117925141242174e-05,
      "log_odds_chosen": 0.4364786744117737,
      "log_odds_ratio": -0.5888470411300659,
      "logits/chosen": 231.71511840820312,
      "logits/rejected": 239.5157928466797,
      "logps/chosen": -0.8990312814712524,
      "logps/rejected": -1.1864837408065796,
      "loss": 45.2371,
      "nll_loss": 1.1880322694778442,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.4495156407356262,
      "rewards/margins": 0.14372624456882477,
      "rewards/rejected": -0.5932418704032898,
      "step": 115
    },
    {
      "epoch": 1.1377777777777778,
      "grad_norm": 52.75,
      "learning_rate": 3.899004663415084e-05,
      "log_odds_chosen": 0.433866024017334,
      "log_odds_ratio": -0.5812664031982422,
      "logits/chosen": 228.9931640625,
      "logits/rejected": 234.70193481445312,
      "logps/chosen": -0.8852219581604004,
      "logps/rejected": -1.1457195281982422,
      "loss": 45.4878,
      "nll_loss": 1.1959518194198608,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.4426109790802002,
      "rewards/margins": 0.13024874031543732,
      "rewards/rejected": -0.5728597640991211,
      "step": 120
    },
    {
      "epoch": 1.1851851851851851,
      "grad_norm": 59.75,
      "learning_rate": 3.781907832058587e-05,
      "log_odds_chosen": 0.4279605448246002,
      "log_odds_ratio": -0.6149727702140808,
      "logits/chosen": 228.2986602783203,
      "logits/rejected": 231.1459503173828,
      "logps/chosen": -0.9138356447219849,
      "logps/rejected": -1.196605920791626,
      "loss": 42.8771,
      "nll_loss": 1.1416022777557373,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.45691782236099243,
      "rewards/margins": 0.14138510823249817,
      "rewards/rejected": -0.598302960395813,
      "step": 125
    },
    {
      "epoch": 1.2325925925925927,
      "grad_norm": 116.0,
      "learning_rate": 3.660862682169282e-05,
      "log_odds_chosen": 0.29699453711509705,
      "log_odds_ratio": -0.6551839709281921,
      "logits/chosen": 229.7410430908203,
      "logits/rejected": 233.87667846679688,
      "logps/chosen": -0.9139402508735657,
      "logps/rejected": -1.0807549953460693,
      "loss": 43.7064,
      "nll_loss": 1.2109730243682861,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.45697012543678284,
      "rewards/margins": 0.08340738713741302,
      "rewards/rejected": -0.5403774976730347,
      "step": 130
    },
    {
      "epoch": 1.28,
      "grad_norm": 107.5,
      "learning_rate": 3.5362420368134356e-05,
      "log_odds_chosen": 0.39430850744247437,
      "log_odds_ratio": -0.6013480424880981,
      "logits/chosen": 232.01220703125,
      "logits/rejected": 235.47061157226562,
      "logps/chosen": -0.8469200134277344,
      "logps/rejected": -1.0819423198699951,
      "loss": 44.477,
      "nll_loss": 1.1646451950073242,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.4234600067138672,
      "rewards/margins": 0.11751119047403336,
      "rewards/rejected": -0.5409711599349976,
      "step": 135
    },
    {
      "epoch": 1.3274074074074074,
      "grad_norm": 153.0,
      "learning_rate": 3.408429731701635e-05,
      "log_odds_chosen": 0.3346176743507385,
      "log_odds_ratio": -0.6571810245513916,
      "logits/chosen": 231.61083984375,
      "logits/rejected": 233.050537109375,
      "logps/chosen": -0.9530075192451477,
      "logps/rejected": -1.1791385412216187,
      "loss": 45.6471,
      "nll_loss": 1.2547054290771484,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47650375962257385,
      "rewards/margins": 0.11306555569171906,
      "rewards/rejected": -0.5895692706108093,
      "step": 140
    },
    {
      "epoch": 1.374814814814815,
      "grad_norm": 77.0,
      "learning_rate": 3.2778194329621104e-05,
      "log_odds_chosen": 0.3642648160457611,
      "log_odds_ratio": -0.61054927110672,
      "logits/chosen": 226.9085235595703,
      "logits/rejected": 232.42965698242188,
      "logps/chosen": -0.924142062664032,
      "logps/rejected": -1.1682136058807373,
      "loss": 44.7382,
      "nll_loss": 1.1921000480651855,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.462071031332016,
      "rewards/margins": 0.12203577905893326,
      "rewards/rejected": -0.5841068029403687,
      "step": 145
    },
    {
      "epoch": 1.4222222222222223,
      "grad_norm": 55.25,
      "learning_rate": 3.144813424636031e-05,
      "log_odds_chosen": 0.282584011554718,
      "log_odds_ratio": -0.6607434153556824,
      "logits/chosen": 225.26840209960938,
      "logits/rejected": 226.21768188476562,
      "logps/chosen": -0.8751763105392456,
      "logps/rejected": -1.040479302406311,
      "loss": 42.9438,
      "nll_loss": 1.1665369272232056,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.4375881552696228,
      "rewards/margins": 0.08265147358179092,
      "rewards/rejected": -0.5202396512031555,
      "step": 150
    },
    {
      "epoch": 1.4696296296296296,
      "grad_norm": 86.5,
      "learning_rate": 3.0098213696293542e-05,
      "log_odds_chosen": 0.28907060623168945,
      "log_odds_ratio": -0.6529395580291748,
      "logits/chosen": 224.1173553466797,
      "logits/rejected": 232.13858032226562,
      "logps/chosen": -0.8760883212089539,
      "logps/rejected": -1.0371713638305664,
      "loss": 44.3679,
      "nll_loss": 1.1804320812225342,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.43804416060447693,
      "rewards/margins": 0.08054153621196747,
      "rewards/rejected": -0.5185856819152832,
      "step": 155
    },
    {
      "epoch": 1.5170370370370372,
      "grad_norm": 68.0,
      "learning_rate": 2.8732590479375165e-05,
      "log_odds_chosen": 0.32775264978408813,
      "log_odds_ratio": -0.612945020198822,
      "logits/chosen": 230.9765625,
      "logits/rejected": 230.38125610351562,
      "logps/chosen": -0.951357364654541,
      "logps/rejected": -1.1643283367156982,
      "loss": 45.2404,
      "nll_loss": 1.2451446056365967,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4756786823272705,
      "rewards/margins": 0.10648546367883682,
      "rewards/rejected": -0.5821641683578491,
      "step": 160
    },
    {
      "epoch": 1.5644444444444443,
      "grad_norm": 38.75,
      "learning_rate": 2.7355470760292956e-05,
      "log_odds_chosen": 0.4192623198032379,
      "log_odds_ratio": -0.5958751440048218,
      "logits/chosen": 225.8223876953125,
      "logits/rejected": 225.8138427734375,
      "logps/chosen": -0.9021142721176147,
      "logps/rejected": -1.1618568897247314,
      "loss": 43.8924,
      "nll_loss": 1.1659859418869019,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4510571360588074,
      "rewards/margins": 0.12987132370471954,
      "rewards/rejected": -0.5809284448623657,
      "step": 165
    },
    {
      "epoch": 1.6118518518518519,
      "grad_norm": 82.5,
      "learning_rate": 2.597109611334169e-05,
      "log_odds_chosen": 0.43345552682876587,
      "log_odds_ratio": -0.5985883474349976,
      "logits/chosen": 222.00634765625,
      "logits/rejected": 224.83993530273438,
      "logps/chosen": -0.878454864025116,
      "logps/rejected": -1.1517555713653564,
      "loss": 44.3274,
      "nll_loss": 1.1605513095855713,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.439227432012558,
      "rewards/margins": 0.13665039837360382,
      "rewards/rejected": -0.5758777856826782,
      "step": 170
    },
    {
      "epoch": 1.6592592592592592,
      "grad_norm": 84.5,
      "learning_rate": 2.458373045823404e-05,
      "log_odds_chosen": 0.30794280767440796,
      "log_odds_ratio": -0.6498476266860962,
      "logits/chosen": 230.5714874267578,
      "logits/rejected": 233.70053100585938,
      "logps/chosen": -0.8978121876716614,
      "logps/rejected": -1.0880495309829712,
      "loss": 43.0845,
      "nll_loss": 1.1957765817642212,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4489060938358307,
      "rewards/margins": 0.0951187014579773,
      "rewards/rejected": -0.5440247654914856,
      "step": 175
    },
    {
      "epoch": 1.7066666666666666,
      "grad_norm": 90.0,
      "learning_rate": 2.3197646927086697e-05,
      "log_odds_chosen": 0.13764269649982452,
      "log_odds_ratio": -0.7279932498931885,
      "logits/chosen": 232.72415161132812,
      "logits/rejected": 232.6797637939453,
      "logps/chosen": -0.9545130729675293,
      "logps/rejected": -1.035469651222229,
      "loss": 44.9809,
      "nll_loss": 1.2627184391021729,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.47725653648376465,
      "rewards/margins": 0.04047833010554314,
      "rewards/rejected": -0.5177348256111145,
      "step": 180
    },
    {
      "epoch": 1.7540740740740741,
      "grad_norm": 34.75,
      "learning_rate": 2.1817114703032176e-05,
      "log_odds_chosen": 0.345479816198349,
      "log_odds_ratio": -0.6043921709060669,
      "logits/chosen": 228.2840118408203,
      "logits/rejected": 230.993408203125,
      "logps/chosen": -0.8777548670768738,
      "logps/rejected": -1.0991740226745605,
      "loss": 43.1118,
      "nll_loss": 1.1345906257629395,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.4388774335384369,
      "rewards/margins": 0.11070958524942398,
      "rewards/rejected": -0.5495870113372803,
      "step": 185
    },
    {
      "epoch": 1.8014814814814815,
      "grad_norm": 56.0,
      "learning_rate": 2.0446385870993467e-05,
      "log_odds_chosen": 0.4094289243221283,
      "log_odds_ratio": -0.6055964231491089,
      "logits/chosen": 224.13839721679688,
      "logits/rejected": 230.32461547851562,
      "logps/chosen": -0.9269938468933105,
      "logps/rejected": -1.1887009143829346,
      "loss": 44.9884,
      "nll_loss": 1.2038931846618652,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4634969234466553,
      "rewards/margins": 0.1308535933494568,
      "rewards/rejected": -0.5943504571914673,
      "step": 190
    },
    {
      "epoch": 1.8488888888888888,
      "grad_norm": 54.75,
      "learning_rate": 1.9089682321121834e-05,
      "log_odds_chosen": 0.3977203965187073,
      "log_odds_ratio": -0.6004412174224854,
      "logits/chosen": 223.8681640625,
      "logits/rejected": 231.8001251220703,
      "logps/chosen": -0.8826181292533875,
      "logps/rejected": -1.1249583959579468,
      "loss": 43.2424,
      "nll_loss": 1.154944658279419,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.4413090646266937,
      "rewards/margins": 0.12117014825344086,
      "rewards/rejected": -0.5624791979789734,
      "step": 195
    },
    {
      "epoch": 1.8962962962962964,
      "grad_norm": 39.25,
      "learning_rate": 1.775118274523545e-05,
      "log_odds_chosen": 0.29216670989990234,
      "log_odds_ratio": -0.6725496053695679,
      "logits/chosen": 224.38119506835938,
      "logits/rejected": 227.98068237304688,
      "logps/chosen": -0.9147941470146179,
      "logps/rejected": -1.0938390493392944,
      "loss": 45.3255,
      "nll_loss": 1.1992260217666626,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.45739707350730896,
      "rewards/margins": 0.08952243626117706,
      "rewards/rejected": -0.5469195246696472,
      "step": 200
    },
    {
      "epoch": 1.9437037037037037,
      "grad_norm": 40.0,
      "learning_rate": 1.643500976631037e-05,
      "log_odds_chosen": 0.339089572429657,
      "log_odds_ratio": -0.6290854215621948,
      "logits/chosen": 223.801513671875,
      "logits/rejected": 232.53506469726562,
      "logps/chosen": -0.8788326382637024,
      "logps/rejected": -1.1053264141082764,
      "loss": 41.223,
      "nll_loss": 1.1145111322402954,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.4394163191318512,
      "rewards/margins": 0.11324682086706161,
      "rewards/rejected": -0.5526632070541382,
      "step": 205
    },
    {
      "epoch": 1.991111111111111,
      "grad_norm": 57.25,
      "learning_rate": 1.514521724066537e-05,
      "log_odds_chosen": 0.2665654718875885,
      "log_odds_ratio": -0.656215250492096,
      "logits/chosen": 226.04345703125,
      "logits/rejected": 222.84518432617188,
      "logps/chosen": -0.8619743585586548,
      "logps/rejected": -1.0236845016479492,
      "loss": 41.679,
      "nll_loss": 1.1403412818908691,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.4309871792793274,
      "rewards/margins": 0.08085508644580841,
      "rewards/rejected": -0.5118422508239746,
      "step": 210
    },
    {
      "epoch": 2.0385185185185186,
      "grad_norm": 41.0,
      "learning_rate": 1.3885777771950348e-05,
      "log_odds_chosen": 0.4965452253818512,
      "log_odds_ratio": -0.5693584084510803,
      "logits/chosen": 222.06100463867188,
      "logits/rejected": 227.81369018554688,
      "logps/chosen": -0.7932162284851074,
      "logps/rejected": -1.053929328918457,
      "loss": 38.5686,
      "nll_loss": 1.0264673233032227,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.3966081142425537,
      "rewards/margins": 0.13035652041435242,
      "rewards/rejected": -0.5269646644592285,
      "step": 215
    },
    {
      "epoch": 2.0859259259259257,
      "grad_norm": 61.0,
      "learning_rate": 1.2660570475395683e-05,
      "log_odds_chosen": 0.6846104860305786,
      "log_odds_ratio": -0.5175246596336365,
      "logits/chosen": 214.78866577148438,
      "logits/rejected": 215.9294891357422,
      "logps/chosen": -0.695962131023407,
      "logps/rejected": -1.07391357421875,
      "loss": 36.2322,
      "nll_loss": 0.9726701974868774,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3479810655117035,
      "rewards/margins": 0.1889757215976715,
      "rewards/rejected": -0.536956787109375,
      "step": 220
    },
    {
      "epoch": 2.1333333333333333,
      "grad_norm": 41.5,
      "learning_rate": 1.1473369030008974e-05,
      "log_odds_chosen": 0.4971061646938324,
      "log_odds_ratio": -0.5805861353874207,
      "logits/chosen": 220.0359649658203,
      "logits/rejected": 221.62265014648438,
      "logps/chosen": -0.6966139674186707,
      "logps/rejected": -0.9382969737052917,
      "loss": 36.4938,
      "nll_loss": 0.9666304588317871,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3483069837093353,
      "rewards/margins": 0.12084150314331055,
      "rewards/rejected": -0.4691484868526459,
      "step": 225
    },
    {
      "epoch": 2.180740740740741,
      "grad_norm": 46.5,
      "learning_rate": 1.0327830055518842e-05,
      "log_odds_chosen": 0.578362762928009,
      "log_odds_ratio": -0.5245636105537415,
      "logits/chosen": 215.7959747314453,
      "logits/rejected": 220.932861328125,
      "logps/chosen": -0.6989654302597046,
      "logps/rejected": -1.0136926174163818,
      "loss": 35.8101,
      "nll_loss": 0.9475434422492981,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.3494827151298523,
      "rewards/margins": 0.1573636829853058,
      "rewards/rejected": -0.5068463087081909,
      "step": 230
    },
    {
      "epoch": 2.228148148148148,
      "grad_norm": 48.0,
      "learning_rate": 9.227481849865235e-06,
      "log_odds_chosen": 0.46400395035743713,
      "log_odds_ratio": -0.5828996896743774,
      "logits/chosen": 215.46646118164062,
      "logits/rejected": 216.53781127929688,
      "logps/chosen": -0.7348538041114807,
      "logps/rejected": -0.9607747197151184,
      "loss": 36.3354,
      "nll_loss": 0.9704949259757996,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.36742690205574036,
      "rewards/margins": 0.11296045780181885,
      "rewards/rejected": -0.4803873598575592,
      "step": 235
    },
    {
      "epoch": 2.2755555555555556,
      "grad_norm": 52.0,
      "learning_rate": 8.175713521924978e-06,
      "log_odds_chosen": 0.608235776424408,
      "log_odds_ratio": -0.5281478762626648,
      "logits/chosen": 214.62643432617188,
      "logits/rejected": 220.86477661132812,
      "logps/chosen": -0.6918534636497498,
      "logps/rejected": -0.998008131980896,
      "loss": 36.3286,
      "nll_loss": 0.984667956829071,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3459267318248749,
      "rewards/margins": 0.15307730436325073,
      "rewards/rejected": -0.499004065990448,
      "step": 240
    },
    {
      "epoch": 2.322962962962963,
      "grad_norm": 56.0,
      "learning_rate": 7.1757645529443665e-06,
      "log_odds_chosen": 0.579681396484375,
      "log_odds_ratio": -0.5280523896217346,
      "logits/chosen": 211.4954071044922,
      "logits/rejected": 222.38168334960938,
      "logps/chosen": -0.7150880098342896,
      "logps/rejected": -1.0089267492294312,
      "loss": 35.879,
      "nll_loss": 0.9763582944869995,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3575440049171448,
      "rewards/margins": 0.1469193994998932,
      "rewards/rejected": -0.5044633746147156,
      "step": 245
    },
    {
      "epoch": 2.3703703703703702,
      "grad_norm": 43.25,
      "learning_rate": 6.230714818829733e-06,
      "log_odds_chosen": 0.6855599880218506,
      "log_odds_ratio": -0.5006572604179382,
      "logits/chosen": 212.50741577148438,
      "logits/rejected": 216.546630859375,
      "logps/chosen": -0.6900970339775085,
      "logps/rejected": -1.0552127361297607,
      "loss": 35.5148,
      "nll_loss": 0.922285258769989,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3450485169887543,
      "rewards/margins": 0.1825578510761261,
      "rewards/rejected": -0.5276063680648804,
      "step": 250
    },
    {
      "epoch": 2.417777777777778,
      "grad_norm": 39.25,
      "learning_rate": 5.343475104027743e-06,
      "log_odds_chosen": 0.6516860723495483,
      "log_odds_ratio": -0.5264917612075806,
      "logits/chosen": 208.18154907226562,
      "logits/rejected": 220.366943359375,
      "logps/chosen": -0.6484566926956177,
      "logps/rejected": -0.9873029589653015,
      "loss": 34.7539,
      "nll_loss": 0.9242755770683289,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32422834634780884,
      "rewards/margins": 0.16942311823368073,
      "rewards/rejected": -0.49365147948265076,
      "step": 255
    },
    {
      "epoch": 2.4651851851851854,
      "grad_norm": 43.75,
      "learning_rate": 4.516778136213037e-06,
      "log_odds_chosen": 0.649712085723877,
      "log_odds_ratio": -0.532970130443573,
      "logits/chosen": 206.3728485107422,
      "logits/rejected": 213.7884063720703,
      "logps/chosen": -0.6736012697219849,
      "logps/rejected": -1.0033750534057617,
      "loss": 35.165,
      "nll_loss": 0.9300982356071472,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.33680063486099243,
      "rewards/margins": 0.16488692164421082,
      "rewards/rejected": -0.5016875267028809,
      "step": 260
    },
    {
      "epoch": 2.5125925925925925,
      "grad_norm": 47.25,
      "learning_rate": 3.7531701693965554e-06,
      "log_odds_chosen": 0.6154508590698242,
      "log_odds_ratio": -0.5541368722915649,
      "logits/chosen": 212.042724609375,
      "logits/rejected": 211.31857299804688,
      "logps/chosen": -0.6950085163116455,
      "logps/rejected": -1.034769892692566,
      "loss": 35.0564,
      "nll_loss": 0.9271542429924011,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.34750425815582275,
      "rewards/margins": 0.1698807030916214,
      "rewards/rejected": -0.517384946346283,
      "step": 265
    },
    {
      "epoch": 2.56,
      "grad_norm": 46.75,
      "learning_rate": 3.055003141378948e-06,
      "log_odds_chosen": 0.6779940128326416,
      "log_odds_ratio": -0.49747544527053833,
      "logits/chosen": 207.754638671875,
      "logits/rejected": 214.58056640625,
      "logps/chosen": -0.7059646844863892,
      "logps/rejected": -1.07295560836792,
      "loss": 36.0368,
      "nll_loss": 0.9621596336364746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3529823422431946,
      "rewards/margins": 0.1834954470396042,
      "rewards/rejected": -0.53647780418396,
      "step": 270
    },
    {
      "epoch": 2.6074074074074076,
      "grad_norm": 44.0,
      "learning_rate": 2.424427429704365e-06,
      "log_odds_chosen": 0.5993890166282654,
      "log_odds_ratio": -0.5276089310646057,
      "logits/chosen": 209.30874633789062,
      "logits/rejected": 223.237548828125,
      "logps/chosen": -0.713978111743927,
      "logps/rejected": -1.013167142868042,
      "loss": 36.1104,
      "nll_loss": 0.9636555910110474,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3569890558719635,
      "rewards/margins": 0.14959454536437988,
      "rewards/rejected": -0.506583571434021,
      "step": 275
    },
    {
      "epoch": 2.6548148148148147,
      "grad_norm": 46.5,
      "learning_rate": 1.8633852284264508e-06,
      "log_odds_chosen": 0.641398549079895,
      "log_odds_ratio": -0.5212709307670593,
      "logits/chosen": 213.2962188720703,
      "logits/rejected": 211.81527709960938,
      "logps/chosen": -0.6817741394042969,
      "logps/rejected": -1.0385987758636475,
      "loss": 34.4299,
      "nll_loss": 0.8972374796867371,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.34088706970214844,
      "rewards/margins": 0.1784123331308365,
      "rewards/rejected": -0.5192993879318237,
      "step": 280
    },
    {
      "epoch": 2.7022222222222223,
      "grad_norm": 44.0,
      "learning_rate": 1.3736045660864034e-06,
      "log_odds_chosen": 0.5530254244804382,
      "log_odds_ratio": -0.5443213582038879,
      "logits/chosen": 210.8925323486328,
      "logits/rejected": 220.40585327148438,
      "logps/chosen": -0.6845903396606445,
      "logps/rejected": -0.9716375470161438,
      "loss": 35.8726,
      "nll_loss": 0.9356240034103394,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.34229516983032227,
      "rewards/margins": 0.14352357387542725,
      "rewards/rejected": -0.4858187735080719,
      "step": 285
    },
    {
      "epoch": 2.74962962962963,
      "grad_norm": 41.5,
      "learning_rate": 9.565939833279192e-07,
      "log_odds_chosen": 0.6134527325630188,
      "log_odds_ratio": -0.5205898284912109,
      "logits/chosen": 213.05337524414062,
      "logits/rejected": 214.83544921875,
      "logps/chosen": -0.7018261551856995,
      "logps/rejected": -1.010362982749939,
      "loss": 36.1954,
      "nll_loss": 0.976243793964386,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.35091307759284973,
      "rewards/margins": 0.15426845848560333,
      "rewards/rejected": -0.5051814913749695,
      "step": 290
    },
    {
      "epoch": 2.797037037037037,
      "grad_norm": 107.5,
      "learning_rate": 6.136378865420872e-07,
      "log_odds_chosen": 0.6149747967720032,
      "log_odds_ratio": -0.5573899745941162,
      "logits/chosen": 213.50210571289062,
      "logits/rejected": 216.36660766601562,
      "logps/chosen": -0.7356334924697876,
      "logps/rejected": -1.0573605298995972,
      "loss": 35.9121,
      "nll_loss": 0.977961540222168,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3678167462348938,
      "rewards/margins": 0.16086354851722717,
      "rewards/rejected": -0.5286802649497986,
      "step": 295
    },
    {
      "epoch": 2.8444444444444446,
      "grad_norm": 42.0,
      "learning_rate": 3.45792591853214e-07,
      "log_odds_chosen": 0.5418564081192017,
      "log_odds_ratio": -0.5683308839797974,
      "logits/chosen": 211.27420043945312,
      "logits/rejected": 218.8232879638672,
      "logps/chosen": -0.7445929646492004,
      "logps/rejected": -1.0526930093765259,
      "loss": 36.5699,
      "nll_loss": 1.0055818557739258,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3722964823246002,
      "rewards/margins": 0.15405002236366272,
      "rewards/rejected": -0.5263465046882629,
      "step": 300
    },
    {
      "epoch": 2.891851851851852,
      "grad_norm": 40.0,
      "learning_rate": 1.538830716302092e-07,
      "log_odds_chosen": 0.5894609093666077,
      "log_odds_ratio": -0.5367568731307983,
      "logits/chosen": 209.3128662109375,
      "logits/rejected": 217.3281707763672,
      "logps/chosen": -0.7301949262619019,
      "logps/rejected": -1.0283689498901367,
      "loss": 34.3942,
      "nll_loss": 0.911605954170227,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3650974631309509,
      "rewards/margins": 0.14908698201179504,
      "rewards/rejected": -0.5141844749450684,
      "step": 305
    },
    {
      "epoch": 2.9392592592592592,
      "grad_norm": 43.75,
      "learning_rate": 3.8500413544415025e-08,
      "log_odds_chosen": 0.626187801361084,
      "log_odds_ratio": -0.5338219404220581,
      "logits/chosen": 211.9689178466797,
      "logits/rejected": 221.7113800048828,
      "logps/chosen": -0.7173446416854858,
      "logps/rejected": -1.0214028358459473,
      "loss": 36.2169,
      "nll_loss": 0.935465931892395,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3586723208427429,
      "rewards/margins": 0.15202903747558594,
      "rewards/rejected": -0.5107014179229736,
      "step": 310
    },
    {
      "epoch": 2.986666666666667,
      "grad_norm": 46.75,
      "learning_rate": 0.0,
      "log_odds_chosen": 0.6916165947914124,
      "log_odds_ratio": -0.48836082220077515,
      "logits/chosen": 211.38467407226562,
      "logits/rejected": 218.0712890625,
      "logps/chosen": -0.6407631039619446,
      "logps/rejected": -0.9831441044807434,
      "loss": 34.8711,
      "nll_loss": 0.9277550578117371,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.3203815519809723,
      "rewards/margins": 0.1711905300617218,
      "rewards/rejected": -0.4915720522403717,
      "step": 315
    },
    {
      "epoch": 2.986666666666667,
      "step": 315,
      "total_flos": 0.0,
      "train_loss": 254.74021465967573,
      "train_runtime": 6927.757,
      "train_samples_per_second": 2.923,
      "train_steps_per_second": 0.045
    }
  ],
  "logging_steps": 5,
  "max_steps": 315,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}