{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.999253545658124,
  "eval_steps": 100,
  "global_step": 753,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0013270299411130464,
      "grad_norm": 1.7421875,
      "learning_rate": 6.578947368421053e-08,
      "logits/chosen": -3.0980052947998047,
      "logits/rejected": -3.127007007598877,
      "logps/chosen": -425.274169921875,
      "logps/rejected": -373.2780456542969,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.013270299411130464,
      "grad_norm": 1.765625,
      "learning_rate": 6.578947368421053e-07,
      "logits/chosen": -3.072850227355957,
      "logits/rejected": -3.0915706157684326,
      "logps/chosen": -421.3843994140625,
      "logps/rejected": -350.215087890625,
      "loss": 0.6919,
      "rewards/accuracies": 0.6041666865348816,
      "rewards/chosen": 0.0024290236178785563,
      "rewards/margins": 0.0028885826468467712,
      "rewards/rejected": -0.00045955937821418047,
      "step": 10
    },
    {
      "epoch": 0.026540598822260928,
      "grad_norm": 1.6875,
      "learning_rate": 1.3157894736842106e-06,
      "logits/chosen": -3.0872464179992676,
      "logits/rejected": -3.0978829860687256,
      "logps/chosen": -352.447021484375,
      "logps/rejected": -364.5257873535156,
      "loss": 0.6816,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.015185330994427204,
      "rewards/margins": 0.021979082375764847,
      "rewards/rejected": -0.006793751381337643,
      "step": 20
    },
    {
      "epoch": 0.039810898233391394,
      "grad_norm": 1.53125,
      "learning_rate": 1.973684210526316e-06,
      "logits/chosen": -3.055328130722046,
      "logits/rejected": -3.0872576236724854,
      "logps/chosen": -374.4645080566406,
      "logps/rejected": -364.3621520996094,
      "loss": 0.6547,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": 0.05001773685216904,
      "rewards/margins": 0.07953666895627975,
      "rewards/rejected": -0.029518935829401016,
      "step": 30
    },
    {
      "epoch": 0.053081197644521856,
      "grad_norm": 1.6640625,
      "learning_rate": 2.631578947368421e-06,
      "logits/chosen": -3.054077625274658,
      "logits/rejected": -3.090534210205078,
      "logps/chosen": -394.09552001953125,
      "logps/rejected": -394.32818603515625,
      "loss": 0.611,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.0969473272562027,
      "rewards/margins": 0.17953996360301971,
      "rewards/rejected": -0.08259265124797821,
      "step": 40
    },
    {
      "epoch": 0.06635149705565231,
      "grad_norm": 1.796875,
      "learning_rate": 3.289473684210527e-06,
      "logits/chosen": -3.0039563179016113,
      "logits/rejected": -3.0520451068878174,
      "logps/chosen": -376.13897705078125,
      "logps/rejected": -405.67303466796875,
      "loss": 0.5412,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.12179754674434662,
      "rewards/margins": 0.3453168272972107,
      "rewards/rejected": -0.22351928055286407,
      "step": 50
    },
    {
      "epoch": 0.07962179646678279,
      "grad_norm": 1.7890625,
      "learning_rate": 3.947368421052632e-06,
      "logits/chosen": -2.974642515182495,
      "logits/rejected": -3.0201048851013184,
      "logps/chosen": -418.08135986328125,
      "logps/rejected": -455.23846435546875,
      "loss": 0.4179,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.048082564026117325,
      "rewards/margins": 0.6676187515258789,
      "rewards/rejected": -0.6195362210273743,
      "step": 60
    },
    {
      "epoch": 0.09289209587791325,
      "grad_norm": 1.34375,
      "learning_rate": 4.605263157894737e-06,
      "logits/chosen": -2.86934757232666,
      "logits/rejected": -2.9434354305267334,
      "logps/chosen": -400.07568359375,
      "logps/rejected": -482.0399475097656,
      "loss": 0.3225,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.1442013680934906,
      "rewards/margins": 1.0631463527679443,
      "rewards/rejected": -1.2073477506637573,
      "step": 70
    },
    {
      "epoch": 0.10616239528904371,
      "grad_norm": 1.359375,
      "learning_rate": 4.9995693346469565e-06,
      "logits/chosen": -2.6817970275878906,
      "logits/rejected": -2.778237819671631,
      "logps/chosen": -460.60394287109375,
      "logps/rejected": -621.35498046875,
      "loss": 0.2331,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.620966374874115,
      "rewards/margins": 1.6919386386871338,
      "rewards/rejected": -2.3129050731658936,
      "step": 80
    },
    {
      "epoch": 0.11943269470017417,
      "grad_norm": 1.8671875,
      "learning_rate": 4.994726053293703e-06,
      "logits/chosen": -2.5432372093200684,
      "logits/rejected": -2.636983633041382,
      "logps/chosen": -504.8564453125,
      "logps/rejected": -734.1746826171875,
      "loss": 0.1809,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.099843978881836,
      "rewards/margins": 2.6429855823516846,
      "rewards/rejected": -3.7428295612335205,
      "step": 90
    },
    {
      "epoch": 0.13270299411130462,
      "grad_norm": 1.140625,
      "learning_rate": 4.984511621268103e-06,
      "logits/chosen": -2.420698642730713,
      "logits/rejected": -2.541872501373291,
      "logps/chosen": -544.0470581054688,
      "logps/rejected": -866.2991943359375,
      "loss": 0.1029,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5227975845336914,
      "rewards/margins": 3.385746479034424,
      "rewards/rejected": -4.908544063568115,
      "step": 100
    },
    {
      "epoch": 0.13270299411130462,
      "eval_logits/chosen": -2.3704655170440674,
      "eval_logits/rejected": -2.4775025844573975,
      "eval_logps/chosen": -579.9114990234375,
      "eval_logps/rejected": -892.3373413085938,
      "eval_loss": 0.10992002487182617,
      "eval_rewards/accuracies": 0.9679104685783386,
      "eval_rewards/chosen": -1.8067275285720825,
      "eval_rewards/margins": 3.5615758895874023,
      "eval_rewards/rejected": -5.368303298950195,
      "eval_runtime": 828.869,
      "eval_samples_per_second": 6.465,
      "eval_steps_per_second": 1.617,
      "step": 100
    },
    {
      "epoch": 0.1459732935224351,
      "grad_norm": 1.2421875,
      "learning_rate": 4.968948030264743e-06,
      "logits/chosen": -2.349208354949951,
      "logits/rejected": -2.439763069152832,
      "logps/chosen": -572.8814697265625,
      "logps/rejected": -938.94091796875,
      "loss": 0.085,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6822025775909424,
      "rewards/margins": 4.082047939300537,
      "rewards/rejected": -5.764250755310059,
      "step": 110
    },
    {
      "epoch": 0.15924359293356558,
      "grad_norm": 2.046875,
      "learning_rate": 4.948068788729238e-06,
      "logits/chosen": -2.156416177749634,
      "logits/rejected": -2.215620756149292,
      "logps/chosen": -626.0386962890625,
      "logps/rejected": -1036.8035888671875,
      "loss": 0.0874,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.1521973609924316,
      "rewards/margins": 4.6609344482421875,
      "rewards/rejected": -6.813131809234619,
      "step": 120
    },
    {
      "epoch": 0.17251389234469602,
      "grad_norm": 1.8515625,
      "learning_rate": 4.921918849714475e-06,
      "logits/chosen": -2.1903834342956543,
      "logits/rejected": -2.2045347690582275,
      "logps/chosen": -673.7794189453125,
      "logps/rejected": -1226.3961181640625,
      "loss": 0.0559,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5693655014038086,
      "rewards/margins": 5.772281646728516,
      "rewards/rejected": -8.341647148132324,
      "step": 130
    },
    {
      "epoch": 0.1857841917558265,
      "grad_norm": 1.0078125,
      "learning_rate": 4.890554514096592e-06,
      "logits/chosen": -2.1850454807281494,
      "logits/rejected": -2.19503116607666,
      "logps/chosen": -688.4675903320312,
      "logps/rejected": -1202.0238037109375,
      "loss": 0.0699,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.7659270763397217,
      "rewards/margins": 5.731719970703125,
      "rewards/rejected": -8.497647285461426,
      "step": 140
    },
    {
      "epoch": 0.19905449116695695,
      "grad_norm": 4.96875,
      "learning_rate": 4.854043309359063e-06,
      "logits/chosen": -2.045557737350464,
      "logits/rejected": -2.022369146347046,
      "logps/chosen": -797.4768676757812,
      "logps/rejected": -1449.0230712890625,
      "loss": 0.0569,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.9922683238983154,
      "rewards/margins": 6.881680488586426,
      "rewards/rejected": -10.87394905090332,
      "step": 150
    },
    {
      "epoch": 0.21232479057808742,
      "grad_norm": 2.5,
      "learning_rate": 4.8124638442058856e-06,
      "logits/chosen": -2.0948424339294434,
      "logits/rejected": -2.136207103729248,
      "logps/chosen": -731.327392578125,
      "logps/rejected": -1373.2447509765625,
      "loss": 0.05,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.1236228942871094,
      "rewards/margins": 6.655406951904297,
      "rewards/rejected": -9.77902889251709,
      "step": 160
    },
    {
      "epoch": 0.22559508998921787,
      "grad_norm": 1.265625,
      "learning_rate": 4.765905639316861e-06,
      "logits/chosen": -1.9219977855682373,
      "logits/rejected": -1.9046119451522827,
      "logps/chosen": -720.5792236328125,
      "logps/rejected": -1352.9737548828125,
      "loss": 0.0472,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.072010040283203,
      "rewards/margins": 6.776447296142578,
      "rewards/rejected": -9.848457336425781,
      "step": 170
    },
    {
      "epoch": 0.23886538940034835,
      "grad_norm": 2.234375,
      "learning_rate": 4.7144689346093814e-06,
      "logits/chosen": -1.9168952703475952,
      "logits/rejected": -1.8547157049179077,
      "logps/chosen": -825.1336059570312,
      "logps/rejected": -1616.734375,
      "loss": 0.0578,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.402952671051025,
      "rewards/margins": 8.200037956237793,
      "rewards/rejected": -12.602991104125977,
      "step": 180
    },
    {
      "epoch": 0.2521356888114788,
      "grad_norm": 2.296875,
      "learning_rate": 4.65826447342166e-06,
      "logits/chosen": -1.9839226007461548,
      "logits/rejected": -1.9680023193359375,
      "logps/chosen": -803.8746948242188,
      "logps/rejected": -1493.1898193359375,
      "loss": 0.0497,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.822723865509033,
      "rewards/margins": 7.312263488769531,
      "rewards/rejected": -11.134986877441406,
      "step": 190
    },
    {
      "epoch": 0.26540598822260925,
      "grad_norm": 8.8125,
      "learning_rate": 4.597413264082086e-06,
      "logits/chosen": -1.9621204137802124,
      "logits/rejected": -1.9449526071548462,
      "logps/chosen": -757.6912841796875,
      "logps/rejected": -1455.258544921875,
      "loss": 0.042,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.6233069896698,
      "rewards/margins": 7.341341495513916,
      "rewards/rejected": -10.964648246765137,
      "step": 200
    },
    {
      "epoch": 0.26540598822260925,
      "eval_logits/chosen": -1.9894516468048096,
      "eval_logits/rejected": -1.9743856191635132,
      "eval_logps/chosen": -750.5289306640625,
      "eval_logps/rejected": -1423.288330078125,
      "eval_loss": 0.04295578598976135,
      "eval_rewards/accuracies": 0.9828358292579651,
      "eval_rewards/chosen": -3.5129029750823975,
      "eval_rewards/margins": 7.164910793304443,
      "eval_rewards/rejected": -10.677813529968262,
      "eval_runtime": 830.8607,
      "eval_samples_per_second": 6.45,
      "eval_steps_per_second": 1.613,
      "step": 200
    },
    {
      "epoch": 0.27867628763373975,
      "grad_norm": 9.6875,
      "learning_rate": 4.5320463193780265e-06,
      "logits/chosen": -1.9735777378082275,
      "logits/rejected": -1.9742670059204102,
      "logps/chosen": -789.7581176757812,
      "logps/rejected": -1441.5927734375,
      "loss": 0.0416,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.7721946239471436,
      "rewards/margins": 6.881577968597412,
      "rewards/rejected": -10.653772354125977,
      "step": 210
    },
    {
      "epoch": 0.2919465870448702,
      "grad_norm": 2.71875,
      "learning_rate": 4.462304374485005e-06,
      "logits/chosen": -1.910851240158081,
      "logits/rejected": -1.8877220153808594,
      "logps/chosen": -795.4382934570312,
      "logps/rejected": -1552.471923828125,
      "loss": 0.0435,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.075817584991455,
      "rewards/margins": 7.9385833740234375,
      "rewards/rejected": -12.014400482177734,
      "step": 220
    },
    {
      "epoch": 0.30521688645600065,
      "grad_norm": 0.9296875,
      "learning_rate": 4.388337583963563e-06,
      "logits/chosen": -1.8420207500457764,
      "logits/rejected": -1.7979652881622314,
      "logps/chosen": -831.4200439453125,
      "logps/rejected": -1719.750244140625,
      "loss": 0.0253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.229403018951416,
      "rewards/margins": 9.30135440826416,
      "rewards/rejected": -13.530756950378418,
      "step": 230
    },
    {
      "epoch": 0.31848718586713115,
      "grad_norm": 1.15625,
      "learning_rate": 4.310305198476161e-06,
      "logits/chosen": -1.813542366027832,
      "logits/rejected": -1.7659708261489868,
      "logps/chosen": -770.3541259765625,
      "logps/rejected": -1698.886474609375,
      "loss": 0.0336,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.893308162689209,
      "rewards/margins": 9.639238357543945,
      "rewards/rejected": -13.532546997070312,
      "step": 240
    },
    {
      "epoch": 0.3317574852782616,
      "grad_norm": 2.03125,
      "learning_rate": 4.228375221920147e-06,
      "logits/chosen": -1.8265107870101929,
      "logits/rejected": -1.765747308731079,
      "logps/chosen": -721.0618896484375,
      "logps/rejected": -1561.09033203125,
      "loss": 0.0269,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.446256160736084,
      "rewards/margins": 8.8670015335083,
      "rewards/rejected": -12.313258171081543,
      "step": 250
    },
    {
      "epoch": 0.34502778468939205,
      "grad_norm": 3.875,
      "learning_rate": 4.142724049715005e-06,
      "logits/chosen": -1.7385963201522827,
      "logits/rejected": -1.5829213857650757,
      "logps/chosen": -922.6043701171875,
      "logps/rejected": -2136.38525390625,
      "loss": 0.0346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.981525897979736,
      "rewards/margins": 11.962621688842773,
      "rewards/rejected": -16.94414710998535,
      "step": 260
    },
    {
      "epoch": 0.3582980841005225,
      "grad_norm": 5.5,
      "learning_rate": 4.053536089022624e-06,
      "logits/chosen": -1.841082215309143,
      "logits/rejected": -1.7555363178253174,
      "logps/chosen": -756.6123657226562,
      "logps/rejected": -1619.0562744140625,
      "loss": 0.0434,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.6720402240753174,
      "rewards/margins": 8.794347763061523,
      "rewards/rejected": -12.466386795043945,
      "step": 270
    },
    {
      "epoch": 0.371568383511653,
      "grad_norm": 3.1875,
      "learning_rate": 3.961003361718272e-06,
      "logits/chosen": -1.8635780811309814,
      "logits/rejected": -1.7543712854385376,
      "logps/chosen": -745.083251953125,
      "logps/rejected": -1585.915283203125,
      "loss": 0.0333,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.244401216506958,
      "rewards/margins": 8.643343925476074,
      "rewards/rejected": -11.887744903564453,
      "step": 280
    },
    {
      "epoch": 0.38483868292278345,
      "grad_norm": 16.5,
      "learning_rate": 3.8653250909670815e-06,
      "logits/chosen": -1.6911243200302124,
      "logits/rejected": -1.550756573677063,
      "logps/chosen": -840.0848388671875,
      "logps/rejected": -1973.161376953125,
      "loss": 0.0493,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.447511196136475,
      "rewards/margins": 11.780040740966797,
      "rewards/rejected": -16.227550506591797,
      "step": 290
    },
    {
      "epoch": 0.3981089823339139,
      "grad_norm": 1.5859375,
      "learning_rate": 3.7667072722961363e-06,
      "logits/chosen": -1.8192800283432007,
      "logits/rejected": -1.7625439167022705,
      "logps/chosen": -772.19189453125,
      "logps/rejected": -1698.5767822265625,
      "loss": 0.0278,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.4313864707946777,
      "rewards/margins": 9.797091484069824,
      "rewards/rejected": -13.228475570678711,
      "step": 300
    },
    {
      "epoch": 0.3981089823339139,
      "eval_logits/chosen": -1.8191460371017456,
      "eval_logits/rejected": -1.7454465627670288,
      "eval_logps/chosen": -772.5892944335938,
      "eval_logps/rejected": -1707.0360107421875,
      "eval_loss": 0.03441401198506355,
      "eval_rewards/accuracies": 0.9828358292579651,
      "eval_rewards/chosen": -3.733506202697754,
      "eval_rewards/margins": 9.781785011291504,
      "eval_rewards/rejected": -13.51529312133789,
      "eval_runtime": 829.9568,
      "eval_samples_per_second": 6.457,
      "eval_steps_per_second": 1.615,
      "step": 300
    },
    {
      "epoch": 0.41137928174504435,
      "grad_norm": 6.5625,
      "learning_rate": 3.665362230085646e-06,
      "logits/chosen": -1.755239725112915,
      "logits/rejected": -1.6191673278808594,
      "logps/chosen": -831.6585693359375,
      "logps/rejected": -1947.4847412109375,
      "loss": 0.0316,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.389147758483887,
      "rewards/margins": 11.429269790649414,
      "rewards/rejected": -15.8184175491333,
      "step": 310
    },
    {
      "epoch": 0.42464958115617485,
      "grad_norm": 2.9375,
      "learning_rate": 3.5615081604340905e-06,
      "logits/chosen": -1.812110185623169,
      "logits/rejected": -1.6317228078842163,
      "logps/chosen": -835.9284057617188,
      "logps/rejected": -2101.62646484375,
      "loss": 0.04,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.403895378112793,
      "rewards/margins": 12.521451950073242,
      "rewards/rejected": -16.92534828186035,
      "step": 320
    },
    {
      "epoch": 0.4379198805673053,
      "grad_norm": 1.3125,
      "learning_rate": 3.4553686613815436e-06,
      "logits/chosen": -1.9260807037353516,
      "logits/rejected": -1.830836534500122,
      "logps/chosen": -688.6129760742188,
      "logps/rejected": -1593.781982421875,
      "loss": 0.0215,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.1708381175994873,
      "rewards/margins": 8.699010848999023,
      "rewards/rejected": -11.869850158691406,
      "step": 330
    },
    {
      "epoch": 0.45119017997843575,
      "grad_norm": 1.8515625,
      "learning_rate": 3.3471722515025986e-06,
      "logits/chosen": -1.8425785303115845,
      "logits/rejected": -1.7366511821746826,
      "logps/chosen": -812.2515258789062,
      "logps/rejected": -1760.351806640625,
      "loss": 0.0311,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.064314842224121,
      "rewards/margins": 10.192426681518555,
      "rewards/rejected": -14.256741523742676,
      "step": 340
    },
    {
      "epoch": 0.46446047938956625,
      "grad_norm": 1.25,
      "learning_rate": 3.2371518779053744e-06,
      "logits/chosen": -1.9980113506317139,
      "logits/rejected": -1.9302339553833008,
      "logps/chosen": -760.2137451171875,
      "logps/rejected": -1788.198486328125,
      "loss": 0.0314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4233691692352295,
      "rewards/margins": 10.155590057373047,
      "rewards/rejected": -13.578959465026855,
      "step": 350
    },
    {
      "epoch": 0.4777307788006967,
      "grad_norm": 2.125,
      "learning_rate": 3.1255444146958845e-06,
      "logits/chosen": -1.8775854110717773,
      "logits/rejected": -1.7533352375030518,
      "logps/chosen": -726.8655395507812,
      "logps/rejected": -1770.5924072265625,
      "loss": 0.0329,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.306746244430542,
      "rewards/margins": 10.716817855834961,
      "rewards/rejected": -14.023564338684082,
      "step": 360
    },
    {
      "epoch": 0.49100107821182715,
      "grad_norm": 2.578125,
      "learning_rate": 3.0125901529875612e-06,
      "logits/chosen": -1.854692816734314,
      "logits/rejected": -1.7414394617080688,
      "logps/chosen": -820.2403564453125,
      "logps/rejected": -1938.7720947265625,
      "loss": 0.0401,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.7310116291046143,
      "rewards/margins": 12.33505630493164,
      "rewards/rejected": -16.06606674194336,
      "step": 370
    },
    {
      "epoch": 0.5042713776229576,
      "grad_norm": 1.4765625,
      "learning_rate": 2.898532283553963e-06,
      "logits/chosen": -1.8446468114852905,
      "logits/rejected": -1.6768661737442017,
      "logps/chosen": -750.1494750976562,
      "logps/rejected": -1983.754638671875,
      "loss": 0.0151,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.6851119995117188,
      "rewards/margins": 12.596293449401855,
      "rewards/rejected": -16.281402587890625,
      "step": 380
    },
    {
      "epoch": 0.517541677034088,
      "grad_norm": 3.78125,
      "learning_rate": 2.783616373238507e-06,
      "logits/chosen": -1.808468222618103,
      "logits/rejected": -1.6193923950195312,
      "logps/chosen": -817.4727783203125,
      "logps/rejected": -1965.7611083984375,
      "loss": 0.0232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.224337577819824,
      "rewards/margins": 11.751169204711914,
      "rewards/rejected": -15.975506782531738,
      "step": 390
    },
    {
      "epoch": 0.5308119764452185,
      "grad_norm": 1.3359375,
      "learning_rate": 2.6680898362485126e-06,
      "logits/chosen": -1.9052226543426514,
      "logits/rejected": -1.790858507156372,
      "logps/chosen": -760.7432861328125,
      "logps/rejected": -1788.7734375,
      "loss": 0.0223,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.564972400665283,
      "rewards/margins": 10.65664291381836,
      "rewards/rejected": -14.2216157913208,
      "step": 400
    },
    {
      "epoch": 0.5308119764452185,
      "eval_logits/chosen": -1.918375015258789,
      "eval_logits/rejected": -1.8019860982894897,
      "eval_logps/chosen": -764.7831420898438,
      "eval_logps/rejected": -1732.62890625,
      "eval_loss": 0.030813412740826607,
      "eval_rewards/accuracies": 0.9858208894729614,
      "eval_rewards/chosen": -3.655445098876953,
      "eval_rewards/margins": 10.11577320098877,
      "eval_rewards/rejected": -13.771217346191406,
      "eval_runtime": 830.1888,
      "eval_samples_per_second": 6.455,
      "eval_steps_per_second": 1.614,
      "step": 400
    },
    {
      "epoch": 0.544082275856349,
      "grad_norm": 0.76953125,
      "learning_rate": 2.55220140147187e-06,
      "logits/chosen": -1.919586420059204,
      "logits/rejected": -1.729554533958435,
      "logps/chosen": -761.3343505859375,
      "logps/rejected": -1788.455322265625,
      "loss": 0.0345,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.7783074378967285,
      "rewards/margins": 10.44767951965332,
      "rewards/rejected": -14.225985527038574,
      "step": 410
    },
    {
      "epoch": 0.5573525752674795,
      "grad_norm": 0.7890625,
      "learning_rate": 2.4362005769631985e-06,
      "logits/chosen": -1.8321539163589478,
      "logits/rejected": -1.664629340171814,
      "logps/chosen": -864.9075317382812,
      "logps/rejected": -2111.5986328125,
      "loss": 0.0269,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.227338790893555,
      "rewards/margins": 13.148414611816406,
      "rewards/rejected": -17.375751495361328,
      "step": 420
    },
    {
      "epoch": 0.57062287467861,
      "grad_norm": 0.6796875,
      "learning_rate": 2.320337112752459e-06,
      "logits/chosen": -1.767311692237854,
      "logits/rejected": -1.6259944438934326,
      "logps/chosen": -846.6857299804688,
      "logps/rejected": -1985.771240234375,
      "loss": 0.0251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.594606399536133,
      "rewards/margins": 11.99472427368164,
      "rewards/rejected": -16.58932876586914,
      "step": 430
    },
    {
      "epoch": 0.5838931740897404,
      "grad_norm": 2.21875,
      "learning_rate": 2.2048604631325896e-06,
      "logits/chosen": -1.9071296453475952,
      "logits/rejected": -1.7615699768066406,
      "logps/chosen": -740.1785888671875,
      "logps/rejected": -1816.4349365234375,
      "loss": 0.0218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5860729217529297,
      "rewards/margins": 11.016485214233398,
      "rewards/rejected": -14.602559089660645,
      "step": 440
    },
    {
      "epoch": 0.5971634735008708,
      "grad_norm": 1.3359375,
      "learning_rate": 2.0900192495838617e-06,
      "logits/chosen": -1.8986167907714844,
      "logits/rejected": -1.749355673789978,
      "logps/chosen": -749.77294921875,
      "logps/rejected": -1782.9183349609375,
      "loss": 0.0303,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.5921592712402344,
      "rewards/margins": 10.537958145141602,
      "rewards/rejected": -14.130119323730469,
      "step": 450
    },
    {
      "epoch": 0.6104337729120013,
      "grad_norm": 5.6875,
      "learning_rate": 1.976060725491293e-06,
      "logits/chosen": -1.8696537017822266,
      "logits/rejected": -1.6489883661270142,
      "logps/chosen": -817.318359375,
      "logps/rejected": -2079.50732421875,
      "loss": 0.0169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.171055316925049,
      "rewards/margins": 12.784004211425781,
      "rewards/rejected": -16.955059051513672,
      "step": 460
    },
    {
      "epoch": 0.6237040723231317,
      "grad_norm": 0.466796875,
      "learning_rate": 1.8632302438075618e-06,
      "logits/chosen": -1.7880550622940063,
      "logits/rejected": -1.5018467903137207,
      "logps/chosen": -904.4384765625,
      "logps/rejected": -2307.56103515625,
      "loss": 0.0164,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -4.95499324798584,
      "rewards/margins": 14.273486137390137,
      "rewards/rejected": -19.22848129272461,
      "step": 470
    },
    {
      "epoch": 0.6369743717342623,
      "grad_norm": 0.8046875,
      "learning_rate": 1.7517707288075617e-06,
      "logits/chosen": -1.7752134799957275,
      "logits/rejected": -1.5753581523895264,
      "logps/chosen": -829.3978271484375,
      "logps/rejected": -2190.319091796875,
      "loss": 0.0281,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.3248748779296875,
      "rewards/margins": 13.842320442199707,
      "rewards/rejected": -18.16719627380371,
      "step": 480
    },
    {
      "epoch": 0.6502446711453927,
      "grad_norm": 5.03125,
      "learning_rate": 1.6419221530719062e-06,
      "logits/chosen": -1.8694307804107666,
      "logits/rejected": -1.6351697444915771,
      "logps/chosen": -807.4894409179688,
      "logps/rejected": -2210.259765625,
      "loss": 0.0224,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.219082832336426,
      "rewards/margins": 13.85973834991455,
      "rewards/rejected": -18.078821182250977,
      "step": 490
    },
    {
      "epoch": 0.6635149705565232,
      "grad_norm": 5.25,
      "learning_rate": 1.5339210208254345e-06,
      "logits/chosen": -1.796460509300232,
      "logits/rejected": -1.5865790843963623,
      "logps/chosen": -825.5130615234375,
      "logps/rejected": -2155.32177734375,
      "loss": 0.0378,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.26907205581665,
      "rewards/margins": 13.452142715454102,
      "rewards/rejected": -17.721214294433594,
      "step": 500
    },
    {
      "epoch": 0.6635149705565232,
      "eval_logits/chosen": -1.8649603128433228,
      "eval_logits/rejected": -1.6923589706420898,
      "eval_logps/chosen": -799.422119140625,
      "eval_logps/rejected": -1988.354248046875,
      "eval_loss": 0.029703186824917793,
      "eval_rewards/accuracies": 0.9850746393203735,
      "eval_rewards/chosen": -4.001834392547607,
      "eval_rewards/margins": 12.326638221740723,
      "eval_rewards/rejected": -16.328474044799805,
      "eval_runtime": 830.0926,
      "eval_samples_per_second": 6.456,
      "eval_steps_per_second": 1.614,
      "step": 500
    },
    {
      "epoch": 0.6767852699676536,
      "grad_norm": 2.53125,
      "learning_rate": 1.4279998587430944e-06,
      "logits/chosen": -1.9209420680999756,
      "logits/rejected": -1.7078396081924438,
      "logps/chosen": -800.6217041015625,
      "logps/rejected": -2088.8896484375,
      "loss": 0.0344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.650468349456787,
      "rewards/margins": 13.276763916015625,
      "rewards/rejected": -16.927234649658203,
      "step": 510
    },
    {
      "epoch": 0.6900555693787841,
      "grad_norm": 0.44921875,
      "learning_rate": 1.3243867153195033e-06,
      "logits/chosen": -1.9058411121368408,
      "logits/rejected": -1.7081434726715088,
      "logps/chosen": -756.5093994140625,
      "logps/rejected": -1919.1995849609375,
      "loss": 0.0323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.5202860832214355,
      "rewards/margins": 11.93966007232666,
      "rewards/rejected": -15.459945678710938,
      "step": 520
    },
    {
      "epoch": 0.7033258687899145,
      "grad_norm": 10.4375,
      "learning_rate": 1.2233046698800343e-06,
      "logits/chosen": -1.9104827642440796,
      "logits/rejected": -1.6314716339111328,
      "logps/chosen": -794.9130859375,
      "logps/rejected": -2090.015380859375,
      "loss": 0.0287,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.8667774200439453,
      "rewards/margins": 13.207371711730957,
      "rewards/rejected": -17.074146270751953,
      "step": 530
    },
    {
      "epoch": 0.716596168201045,
      "grad_norm": 1.3125,
      "learning_rate": 1.124971352290545e-06,
      "logits/chosen": -1.8662798404693604,
      "logits/rejected": -1.6738961935043335,
      "logps/chosen": -800.027587890625,
      "logps/rejected": -2160.337158203125,
      "loss": 0.0261,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -4.005196571350098,
      "rewards/margins": 13.762578964233398,
      "rewards/rejected": -17.76777458190918,
      "step": 540
    },
    {
      "epoch": 0.7298664676121756,
      "grad_norm": 1.3046875,
      "learning_rate": 1.0295984743997911e-06,
      "logits/chosen": -1.8652830123901367,
      "logits/rejected": -1.6665403842926025,
      "logps/chosen": -794.8709106445312,
      "logps/rejected": -1995.255859375,
      "loss": 0.03,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.9191575050354004,
      "rewards/margins": 12.423823356628418,
      "rewards/rejected": -16.34298324584961,
      "step": 550
    },
    {
      "epoch": 0.743136767023306,
      "grad_norm": 0.470703125,
      "learning_rate": 9.37391374223355e-07,
      "logits/chosen": -1.8799976110458374,
      "logits/rejected": -1.6955276727676392,
      "logps/chosen": -788.1275634765625,
      "logps/rejected": -1998.5726318359375,
      "loss": 0.0196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.821469783782959,
      "rewards/margins": 12.33929443359375,
      "rewards/rejected": -16.160762786865234,
      "step": 560
    },
    {
      "epoch": 0.7564070664344364,
      "grad_norm": 1.6171875,
      "learning_rate": 8.48548573850449e-07,
      "logits/chosen": -1.8499343395233154,
      "logits/rejected": -1.6717488765716553,
      "logps/chosen": -778.8970947265625,
      "logps/rejected": -1945.9202880859375,
      "loss": 0.0362,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.6539416313171387,
      "rewards/margins": 12.185694694519043,
      "rewards/rejected": -15.839635848999023,
      "step": 570
    },
    {
      "epoch": 0.7696773658455669,
      "grad_norm": 2.46875,
      "learning_rate": 7.632613520254159e-07,
      "logits/chosen": -1.9424989223480225,
      "logits/rejected": -1.8162052631378174,
      "logps/chosen": -791.0227661132812,
      "logps/rejected": -1805.901123046875,
      "loss": 0.035,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.653287410736084,
      "rewards/margins": 11.017667770385742,
      "rewards/rejected": -14.670953750610352,
      "step": 580
    },
    {
      "epoch": 0.7829476652566973,
      "grad_norm": 0.94921875,
      "learning_rate": 6.817133323241757e-07,
      "logits/chosen": -1.9590984582901,
      "logits/rejected": -1.754612922668457,
      "logps/chosen": -768.0120239257812,
      "logps/rejected": -2016.432373046875,
      "loss": 0.0217,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.4281082153320312,
      "rewards/margins": 12.803730964660645,
      "rewards/rejected": -16.23183822631836,
      "step": 590
    },
    {
      "epoch": 0.7962179646678278,
      "grad_norm": 2.203125,
      "learning_rate": 6.040800878122655e-07,
      "logits/chosen": -1.9362680912017822,
      "logits/rejected": -1.8174184560775757,
      "logps/chosen": -807.0682373046875,
      "logps/rejected": -1890.33984375,
      "loss": 0.0352,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6727454662323,
      "rewards/margins": 11.275753021240234,
      "rewards/rejected": -14.948498725891113,
      "step": 600
    },
    {
      "epoch": 0.7962179646678278,
      "eval_logits/chosen": -1.8977892398834229,
      "eval_logits/rejected": -1.7437201738357544,
      "eval_logps/chosen": -780.2752075195312,
      "eval_logps/rejected": -1919.8118896484375,
      "eval_loss": 0.02784702554345131,
      "eval_rewards/accuracies": 0.983582079410553,
      "eval_rewards/chosen": -3.810366153717041,
      "eval_rewards/margins": 11.832680702209473,
      "eval_rewards/rejected": -15.643047332763672,
      "eval_runtime": 829.9624,
      "eval_samples_per_second": 6.457,
      "eval_steps_per_second": 1.615,
      "step": 600
    },
    {
      "epoch": 0.8094882640789582,
      "grad_norm": 1.171875,
      "learning_rate": 5.305287630356363e-07,
      "logits/chosen": -1.8633983135223389,
      "logits/rejected": -1.6604722738265991,
      "logps/chosen": -751.8145141601562,
      "logps/rejected": -1894.828857421875,
      "loss": 0.0299,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.7778866291046143,
      "rewards/margins": 11.620625495910645,
      "rewards/rejected": -15.39851188659668,
      "step": 610
    },
    {
      "epoch": 0.8227585634900887,
      "grad_norm": 0.37109375,
      "learning_rate": 4.612177141580876e-07,
      "logits/chosen": -1.8507238626480103,
      "logits/rejected": -1.6502704620361328,
      "logps/chosen": -739.475341796875,
      "logps/rejected": -1800.6304931640625,
      "loss": 0.028,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.580106735229492,
      "rewards/margins": 10.93178939819336,
      "rewards/rejected": -14.511896133422852,
      "step": 620
    },
    {
      "epoch": 0.8360288629012192,
      "grad_norm": 2.375,
      "learning_rate": 3.962961680200927e-07,
      "logits/chosen": -1.9084770679473877,
      "logits/rejected": -1.7507511377334595,
      "logps/chosen": -785.728515625,
      "logps/rejected": -1896.435791015625,
      "loss": 0.0222,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.7701239585876465,
      "rewards/margins": 11.418277740478516,
      "rewards/rejected": -15.18840217590332,
      "step": 630
    },
    {
      "epoch": 0.8492991623123497,
      "grad_norm": 1.96875,
      "learning_rate": 3.3590390085308457e-07,
      "logits/chosen": -1.9369093179702759,
      "logits/rejected": -1.782634973526001,
      "logps/chosen": -792.2256469726562,
      "logps/rejected": -1962.2239990234375,
      "loss": 0.0246,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.8103549480438232,
      "rewards/margins": 11.93663501739502,
      "rewards/rejected": -15.746989250183105,
      "step": 640
    },
    {
      "epoch": 0.8625694617234801,
      "grad_norm": 0.373046875,
      "learning_rate": 2.801709373409248e-07,
      "logits/chosen": -1.8602193593978882,
      "logits/rejected": -1.7032759189605713,
      "logps/chosen": -804.9444580078125,
      "logps/rejected": -2019.756103515625,
      "loss": 0.0177,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.675445556640625,
      "rewards/margins": 12.645570755004883,
      "rewards/rejected": -16.321016311645508,
      "step": 650
    },
    {
      "epoch": 0.8758397611346106,
      "grad_norm": 1.90625,
      "learning_rate": 2.2921727067647032e-07,
      "logits/chosen": -1.9662708044052124,
      "logits/rejected": -1.7979133129119873,
      "logps/chosen": -745.5474243164062,
      "logps/rejected": -1877.61328125,
      "loss": 0.0226,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.5280399322509766,
      "rewards/margins": 11.784358024597168,
      "rewards/rejected": -15.312397956848145,
      "step": 660
    },
    {
      "epoch": 0.889110060545741,
      "grad_norm": 15.375,
      "learning_rate": 1.8315260421596925e-07,
      "logits/chosen": -1.9315414428710938,
      "logits/rejected": -1.7226520776748657,
      "logps/chosen": -760.8677978515625,
      "logps/rejected": -1978.8160400390625,
      "loss": 0.0338,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.70927357673645,
      "rewards/margins": 12.64477252960205,
      "rewards/rejected": -16.35404396057129,
      "step": 670
    },
    {
      "epoch": 0.9023803599568715,
      "grad_norm": 0.412109375,
      "learning_rate": 1.4207611528749e-07,
      "logits/chosen": -1.9155975580215454,
      "logits/rejected": -1.6965796947479248,
      "logps/chosen": -769.4285278320312,
      "logps/rejected": -1877.641845703125,
      "loss": 0.0319,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.767559766769409,
      "rewards/margins": 11.334938049316406,
      "rewards/rejected": -15.102499008178711,
      "step": 680
    },
    {
      "epoch": 0.9156506593680019,
      "grad_norm": 0.56640625,
      "learning_rate": 1.060762416619196e-07,
      "logits/chosen": -1.946947693824768,
      "logits/rejected": -1.7013956308364868,
      "logps/chosen": -765.5987548828125,
      "logps/rejected": -2080.13134765625,
      "loss": 0.0215,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.7093491554260254,
      "rewards/margins": 12.91864013671875,
      "rewards/rejected": -16.627988815307617,
      "step": 690
    },
    {
      "epoch": 0.9289209587791325,
      "grad_norm": 0.515625,
      "learning_rate": 7.523049114624647e-08,
      "logits/chosen": -1.9617208242416382,
      "logits/rejected": -1.8319323062896729,
      "logps/chosen": -802.1334838867188,
      "logps/rejected": -1892.474609375,
      "loss": 0.0238,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.8938992023468018,
      "rewards/margins": 11.566261291503906,
      "rewards/rejected": -15.460162162780762,
      "step": 700
    },
    {
      "epoch": 0.9289209587791325,
      "eval_logits/chosen": -1.8936866521835327,
      "eval_logits/rejected": -1.737060785293579,
      "eval_logps/chosen": -788.9779663085938,
      "eval_logps/rejected": -1951.9310302734375,
      "eval_loss": 0.027877720072865486,
      "eval_rewards/accuracies": 0.9828358292579651,
      "eval_rewards/chosen": -3.897392511367798,
      "eval_rewards/margins": 12.06684684753418,
      "eval_rewards/rejected": -15.964240074157715,
      "eval_runtime": 830.3344,
      "eval_samples_per_second": 6.454,
      "eval_steps_per_second": 1.614,
      "step": 700
    },
    {
      "epoch": 0.942191258190263,
      "grad_norm": 0.92578125,
      "learning_rate": 4.9605274709082774e-08,
      "logits/chosen": -1.883724570274353,
      "logits/rejected": -1.7111318111419678,
      "logps/chosen": -796.9892578125,
      "logps/rejected": -2012.552001953125,
      "loss": 0.0267,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.85164213180542,
      "rewards/margins": 12.14155101776123,
      "rewards/rejected": -15.993194580078125,
      "step": 710
    },
    {
      "epoch": 0.9554615576013934,
      "grad_norm": 6.9375,
      "learning_rate": 2.9255763497703373e-08,
      "logits/chosen": -1.8911021947860718,
      "logits/rejected": -1.6832084655761719,
      "logps/chosen": -790.8480224609375,
      "logps/rejected": -2087.356689453125,
      "loss": 0.0314,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.734605312347412,
      "rewards/margins": 13.477132797241211,
      "rewards/rejected": -17.21173858642578,
      "step": 720
    },
    {
      "epoch": 0.9687318570125238,
      "grad_norm": 3.46875,
      "learning_rate": 1.42257700544432e-08,
      "logits/chosen": -1.9454247951507568,
      "logits/rejected": -1.7627389430999756,
      "logps/chosen": -774.6414794921875,
      "logps/rejected": -1878.0687255859375,
      "loss": 0.0266,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.7591774463653564,
      "rewards/margins": 11.635783195495605,
      "rewards/rejected": -15.3949613571167,
      "step": 730
    },
    {
      "epoch": 0.9820021564236543,
      "grad_norm": 0.6640625,
      "learning_rate": 4.547653988198619e-09,
      "logits/chosen": -1.8585050106048584,
      "logits/rejected": -1.6734154224395752,
      "logps/chosen": -771.71044921875,
      "logps/rejected": -1891.1732177734375,
      "loss": 0.0211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.8070855140686035,
      "rewards/margins": 11.580972671508789,
      "rewards/rejected": -15.38805866241455,
      "step": 740
    },
    {
      "epoch": 0.9952724558347847,
      "grad_norm": 0.46875,
      "learning_rate": 2.422523041178959e-10,
      "logits/chosen": -1.9344615936279297,
      "logits/rejected": -1.733974814414978,
      "logps/chosen": -803.4793090820312,
      "logps/rejected": -2031.9625244140625,
      "loss": 0.0295,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.7710418701171875,
      "rewards/margins": 12.417600631713867,
      "rewards/rejected": -16.188644409179688,
      "step": 750
    },
    {
      "epoch": 0.999253545658124,
      "step": 753,
      "total_flos": 0.0,
      "train_loss": 0.0882907345950366,
      "train_runtime": 20220.3954,
      "train_samples_per_second": 2.385,
      "train_steps_per_second": 0.037
    }
  ],
  "logging_steps": 10,
  "max_steps": 753,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}