{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.999297541394882,
  "eval_steps": 400,
  "global_step": 5604,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002676032781401572,
      "grad_norm": 2.5959486156333185,
      "learning_rate": 8.9126559714795e-09,
      "logits/chosen": -0.056712377816438675,
      "logits/rejected": 0.15198665857315063,
      "logps/chosen": -1.7157800197601318,
      "logps/rejected": -1.8902485370635986,
      "loss": 0.188,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.7157800197601318,
      "rewards/margins": 0.17446839809417725,
      "rewards/rejected": -1.8902485370635986,
      "step": 5
    },
    {
      "epoch": 0.005352065562803144,
      "grad_norm": 1.7278464279967156,
      "learning_rate": 1.7825311942959e-08,
      "logits/chosen": 0.019519444555044174,
      "logits/rejected": 0.14068982005119324,
      "logps/chosen": -1.8027324676513672,
      "logps/rejected": -1.845827341079712,
      "loss": 0.1929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.8027324676513672,
      "rewards/margins": 0.04309467226266861,
      "rewards/rejected": -1.845827341079712,
      "step": 10
    },
    {
      "epoch": 0.008028098344204716,
      "grad_norm": 2.1616665254530463,
      "learning_rate": 2.67379679144385e-08,
      "logits/chosen": -0.034807801246643066,
      "logits/rejected": 0.061377476900815964,
      "logps/chosen": -1.6351745128631592,
      "logps/rejected": -1.7653728723526,
      "loss": 0.2178,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.6351745128631592,
      "rewards/margins": 0.1301981806755066,
      "rewards/rejected": -1.7653728723526,
      "step": 15
    },
    {
      "epoch": 0.010704131125606288,
      "grad_norm": 2.683201559728177,
      "learning_rate": 3.5650623885918e-08,
      "logits/chosen": -0.05545501783490181,
      "logits/rejected": 0.027817878872156143,
      "logps/chosen": -1.7257553339004517,
      "logps/rejected": -1.8060623407363892,
      "loss": 0.2129,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.7257553339004517,
      "rewards/margins": 0.08030694723129272,
      "rewards/rejected": -1.8060623407363892,
      "step": 20
    },
    {
      "epoch": 0.013380163907007862,
      "grad_norm": 2.779637232276281,
      "learning_rate": 4.45632798573975e-08,
      "logits/chosen": -0.06745022535324097,
      "logits/rejected": 0.015730513259768486,
      "logps/chosen": -1.87118661403656,
      "logps/rejected": -1.780146598815918,
      "loss": 0.2407,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -1.87118661403656,
      "rewards/margins": -0.09104005247354507,
      "rewards/rejected": -1.780146598815918,
      "step": 25
    },
    {
      "epoch": 0.016056196688409432,
      "grad_norm": 2.103022389924566,
      "learning_rate": 5.3475935828877e-08,
      "logits/chosen": -0.10801396518945694,
      "logits/rejected": -0.018065670505166054,
      "logps/chosen": -1.9097427129745483,
      "logps/rejected": -1.8332710266113281,
      "loss": 0.1915,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -1.9097427129745483,
      "rewards/margins": -0.0764719545841217,
      "rewards/rejected": -1.8332710266113281,
      "step": 30
    },
    {
      "epoch": 0.018732229469811006,
      "grad_norm": 2.7583793657792124,
      "learning_rate": 6.23885918003565e-08,
      "logits/chosen": -0.042829036712646484,
      "logits/rejected": 0.11742081493139267,
      "logps/chosen": -1.847778081893921,
      "logps/rejected": -1.998828649520874,
      "loss": 0.2082,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.847778081893921,
      "rewards/margins": 0.15105068683624268,
      "rewards/rejected": -1.998828649520874,
      "step": 35
    },
    {
      "epoch": 0.021408262251212576,
      "grad_norm": 2.093754406965642,
      "learning_rate": 7.1301247771836e-08,
      "logits/chosen": 0.04008164256811142,
      "logits/rejected": 0.21402421593666077,
      "logps/chosen": -1.8867143392562866,
      "logps/rejected": -1.7465845346450806,
      "loss": 0.2136,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -1.8867143392562866,
      "rewards/margins": -0.14013007283210754,
      "rewards/rejected": -1.7465845346450806,
      "step": 40
    },
    {
      "epoch": 0.02408429503261415,
      "grad_norm": 2.5242789182895855,
      "learning_rate": 8.021390374331551e-08,
      "logits/chosen": 0.019323470070958138,
      "logits/rejected": 0.21447435021400452,
      "logps/chosen": -1.841912865638733,
      "logps/rejected": -1.875614881515503,
      "loss": 0.2061,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -1.841912865638733,
      "rewards/margins": 0.03370223939418793,
      "rewards/rejected": -1.875614881515503,
      "step": 45
    },
    {
      "epoch": 0.026760327814015723,
      "grad_norm": 2.7028642694529057,
      "learning_rate": 8.9126559714795e-08,
      "logits/chosen": -0.04440991207957268,
      "logits/rejected": 0.10613987594842911,
      "logps/chosen": -1.905246376991272,
      "logps/rejected": -1.7834599018096924,
      "loss": 0.206,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.905246376991272,
      "rewards/margins": -0.1217864379286766,
      "rewards/rejected": -1.7834599018096924,
      "step": 50
    },
    {
      "epoch": 0.029436360595417294,
      "grad_norm": 2.335790543868567,
      "learning_rate": 9.80392156862745e-08,
      "logits/chosen": -0.11254777759313583,
      "logits/rejected": 0.10788735002279282,
      "logps/chosen": -1.8472318649291992,
      "logps/rejected": -1.8795379400253296,
      "loss": 0.1967,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.8472318649291992,
      "rewards/margins": 0.03230590373277664,
      "rewards/rejected": -1.8795379400253296,
      "step": 55
    },
    {
      "epoch": 0.032112393376818864,
      "grad_norm": 2.407426326767885,
      "learning_rate": 1.06951871657754e-07,
      "logits/chosen": -0.08842127025127411,
      "logits/rejected": 0.0990448147058487,
      "logps/chosen": -1.805034875869751,
      "logps/rejected": -1.9109728336334229,
      "loss": 0.1883,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.805034875869751,
      "rewards/margins": 0.10593798011541367,
      "rewards/rejected": -1.9109728336334229,
      "step": 60
    },
    {
      "epoch": 0.03478842615822044,
      "grad_norm": 1.8396252486517852,
      "learning_rate": 1.158645276292335e-07,
      "logits/chosen": -0.025138963013887405,
      "logits/rejected": 0.11898098886013031,
      "logps/chosen": -1.654987096786499,
      "logps/rejected": -1.7878128290176392,
      "loss": 0.2043,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.654987096786499,
      "rewards/margins": 0.13282574713230133,
      "rewards/rejected": -1.7878128290176392,
      "step": 65
    },
    {
      "epoch": 0.03746445893962201,
      "grad_norm": 3.476242594450433,
      "learning_rate": 1.24777183600713e-07,
      "logits/chosen": -0.07213078439235687,
      "logits/rejected": 0.08099976181983948,
      "logps/chosen": -1.7887370586395264,
      "logps/rejected": -1.8376140594482422,
      "loss": 0.2107,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -1.7887370586395264,
      "rewards/margins": 0.04887682944536209,
      "rewards/rejected": -1.8376140594482422,
      "step": 70
    },
    {
      "epoch": 0.04014049172102358,
      "grad_norm": 2.0424059273070054,
      "learning_rate": 1.3368983957219251e-07,
      "logits/chosen": -0.03479757905006409,
      "logits/rejected": 0.148690328001976,
      "logps/chosen": -1.8188259601593018,
      "logps/rejected": -2.085181951522827,
      "loss": 0.1871,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.8188259601593018,
      "rewards/margins": 0.2663561701774597,
      "rewards/rejected": -2.085181951522827,
      "step": 75
    },
    {
      "epoch": 0.04281652450242515,
      "grad_norm": 2.0921873249681804,
      "learning_rate": 1.42602495543672e-07,
      "logits/chosen": -0.007114878389984369,
      "logits/rejected": 0.09492503106594086,
      "logps/chosen": -1.7679901123046875,
      "logps/rejected": -1.800490379333496,
      "loss": 0.2077,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.7679901123046875,
      "rewards/margins": 0.032500259578228,
      "rewards/rejected": -1.800490379333496,
      "step": 80
    },
    {
      "epoch": 0.04549255728382673,
      "grad_norm": 1.949535019842773,
      "learning_rate": 1.5151515151515152e-07,
      "logits/chosen": -0.12747876346111298,
      "logits/rejected": 0.12648716568946838,
      "logps/chosen": -1.8510005474090576,
      "logps/rejected": -2.0407958030700684,
      "loss": 0.201,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.8510005474090576,
      "rewards/margins": 0.18979501724243164,
      "rewards/rejected": -2.0407958030700684,
      "step": 85
    },
    {
      "epoch": 0.0481685900652283,
      "grad_norm": 1.854198134694878,
      "learning_rate": 1.6042780748663102e-07,
      "logits/chosen": 0.08328904956579208,
      "logits/rejected": 0.043371789157390594,
      "logps/chosen": -1.8253381252288818,
      "logps/rejected": -1.8358964920043945,
      "loss": 0.2133,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.8253381252288818,
      "rewards/margins": 0.010558363981544971,
      "rewards/rejected": -1.8358964920043945,
      "step": 90
    },
    {
      "epoch": 0.05084462284662987,
      "grad_norm": 1.5999637803097684,
      "learning_rate": 1.693404634581105e-07,
      "logits/chosen": -0.06051814556121826,
      "logits/rejected": 0.08952151238918304,
      "logps/chosen": -1.910400390625,
      "logps/rejected": -1.9918123483657837,
      "loss": 0.1958,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.910400390625,
      "rewards/margins": 0.08141212910413742,
      "rewards/rejected": -1.9918123483657837,
      "step": 95
    },
    {
      "epoch": 0.05352065562803145,
      "grad_norm": 1.7371485443949752,
      "learning_rate": 1.7825311942959e-07,
      "logits/chosen": -0.03409487381577492,
      "logits/rejected": 0.028276998549699783,
      "logps/chosen": -1.7833763360977173,
      "logps/rejected": -1.8959996700286865,
      "loss": 0.1937,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.7833763360977173,
      "rewards/margins": 0.11262327432632446,
      "rewards/rejected": -1.8959996700286865,
      "step": 100
    },
    {
      "epoch": 0.05619668840943302,
      "grad_norm": 1.5280103227404247,
      "learning_rate": 1.8716577540106952e-07,
      "logits/chosen": 0.0694635882973671,
      "logits/rejected": 0.0944514125585556,
      "logps/chosen": -1.7633424997329712,
      "logps/rejected": -1.9262104034423828,
      "loss": 0.1901,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.7633424997329712,
      "rewards/margins": 0.16286788880825043,
      "rewards/rejected": -1.9262104034423828,
      "step": 105
    },
    {
      "epoch": 0.05887272119083459,
      "grad_norm": 1.8031386925804367,
      "learning_rate": 1.96078431372549e-07,
      "logits/chosen": 0.03330797702074051,
      "logits/rejected": 0.12829729914665222,
      "logps/chosen": -1.8492472171783447,
      "logps/rejected": -1.9097312688827515,
      "loss": 0.2055,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.8492472171783447,
      "rewards/margins": 0.060484059154987335,
      "rewards/rejected": -1.9097312688827515,
      "step": 110
    },
    {
      "epoch": 0.06154875397223616,
      "grad_norm": 1.8724864864146538,
      "learning_rate": 2.049910873440285e-07,
      "logits/chosen": 0.05002112314105034,
      "logits/rejected": 0.2617008090019226,
      "logps/chosen": -1.8411915302276611,
      "logps/rejected": -2.172214984893799,
      "loss": 0.1737,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.8411915302276611,
      "rewards/margins": 0.33102303743362427,
      "rewards/rejected": -2.172214984893799,
      "step": 115
    },
    {
      "epoch": 0.06422478675363773,
      "grad_norm": 1.362757099940896,
      "learning_rate": 2.13903743315508e-07,
      "logits/chosen": -0.06960698217153549,
      "logits/rejected": 0.10502330958843231,
      "logps/chosen": -1.9873268604278564,
      "logps/rejected": -2.132310390472412,
      "loss": 0.1782,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -1.9873268604278564,
      "rewards/margins": 0.1449836790561676,
      "rewards/rejected": -2.132310390472412,
      "step": 120
    },
    {
      "epoch": 0.0669008195350393,
      "grad_norm": 2.048266976521782,
      "learning_rate": 2.2281639928698751e-07,
      "logits/chosen": -0.07030968368053436,
      "logits/rejected": 0.060926832258701324,
      "logps/chosen": -1.8836619853973389,
      "logps/rejected": -1.800286054611206,
      "loss": 0.2116,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -1.8836619853973389,
      "rewards/margins": -0.08337591588497162,
      "rewards/rejected": -1.800286054611206,
      "step": 125
    },
    {
      "epoch": 0.06957685231644088,
      "grad_norm": 1.749990112935248,
      "learning_rate": 2.31729055258467e-07,
      "logits/chosen": 0.05183602124452591,
      "logits/rejected": 0.18980661034584045,
      "logps/chosen": -1.9521589279174805,
      "logps/rejected": -2.0761661529541016,
      "loss": 0.1788,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -1.9521589279174805,
      "rewards/margins": 0.12400729954242706,
      "rewards/rejected": -2.0761661529541016,
      "step": 130
    },
    {
      "epoch": 0.07225288509784245,
      "grad_norm": 1.6913584996696023,
      "learning_rate": 2.406417112299465e-07,
      "logits/chosen": -0.046922482550144196,
      "logits/rejected": 0.07465045154094696,
      "logps/chosen": -2.034792423248291,
      "logps/rejected": -2.0069899559020996,
      "loss": 0.1912,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -2.034792423248291,
      "rewards/margins": -0.027802636846899986,
      "rewards/rejected": -2.0069899559020996,
      "step": 135
    },
    {
      "epoch": 0.07492891787924402,
      "grad_norm": 2.280572609769781,
      "learning_rate": 2.49554367201426e-07,
      "logits/chosen": -0.01147941779345274,
      "logits/rejected": 0.16297344863414764,
      "logps/chosen": -2.000673770904541,
      "logps/rejected": -2.258969783782959,
      "loss": 0.1671,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -2.000673770904541,
      "rewards/margins": 0.2582961916923523,
      "rewards/rejected": -2.258969783782959,
      "step": 140
    },
    {
      "epoch": 0.0776049506606456,
      "grad_norm": 1.5178086160841753,
      "learning_rate": 2.5846702317290554e-07,
      "logits/chosen": -0.003580691758543253,
      "logits/rejected": 0.1548602283000946,
      "logps/chosen": -1.9716379642486572,
      "logps/rejected": -2.120543956756592,
      "loss": 0.1783,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -1.9716379642486572,
      "rewards/margins": 0.14890597760677338,
      "rewards/rejected": -2.120543956756592,
      "step": 145
    },
    {
      "epoch": 0.08028098344204716,
      "grad_norm": 1.7757255592279313,
      "learning_rate": 2.6737967914438503e-07,
      "logits/chosen": -0.03968437761068344,
      "logits/rejected": 0.1305098533630371,
      "logps/chosen": -1.9659814834594727,
      "logps/rejected": -1.9681564569473267,
      "loss": 0.1953,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.9659814834594727,
      "rewards/margins": 0.0021751702297478914,
      "rewards/rejected": -1.9681564569473267,
      "step": 150
    },
    {
      "epoch": 0.08295701622344874,
      "grad_norm": 1.7507504907848528,
      "learning_rate": 2.762923351158645e-07,
      "logits/chosen": -0.04737339913845062,
      "logits/rejected": -0.0013456568121910095,
      "logps/chosen": -2.086106300354004,
      "logps/rejected": -2.132288932800293,
      "loss": 0.1858,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -2.086106300354004,
      "rewards/margins": 0.046182699501514435,
      "rewards/rejected": -2.132288932800293,
      "step": 155
    },
    {
      "epoch": 0.0856330490048503,
      "grad_norm": 1.7014065369103604,
      "learning_rate": 2.85204991087344e-07,
      "logits/chosen": -0.1297018975019455,
      "logits/rejected": 0.014775288291275501,
      "logps/chosen": -2.2923808097839355,
      "logps/rejected": -2.267503499984741,
      "loss": 0.1836,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.2923808097839355,
      "rewards/margins": -0.024877404794096947,
      "rewards/rejected": -2.267503499984741,
      "step": 160
    },
    {
      "epoch": 0.08830908178625188,
      "grad_norm": 1.584650297113137,
      "learning_rate": 2.941176470588235e-07,
      "logits/chosen": -0.014486071653664112,
      "logits/rejected": 0.16885574162006378,
      "logps/chosen": -2.024024486541748,
      "logps/rejected": -2.3356518745422363,
      "loss": 0.1802,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -2.024024486541748,
      "rewards/margins": 0.31162726879119873,
      "rewards/rejected": -2.3356518745422363,
      "step": 165
    },
    {
      "epoch": 0.09098511456765346,
      "grad_norm": 1.6317465000662204,
      "learning_rate": 3.0303030303030305e-07,
      "logits/chosen": -0.06774498522281647,
      "logits/rejected": -0.014745334163308144,
      "logps/chosen": -2.3060860633850098,
      "logps/rejected": -2.280571699142456,
      "loss": 0.1741,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -2.3060860633850098,
      "rewards/margins": -0.02551431395113468,
      "rewards/rejected": -2.280571699142456,
      "step": 170
    },
    {
      "epoch": 0.09366114734905502,
      "grad_norm": 1.6000700197652378,
      "learning_rate": 3.1194295900178254e-07,
      "logits/chosen": 0.07237660139799118,
      "logits/rejected": 0.07158859819173813,
      "logps/chosen": -2.187448024749756,
      "logps/rejected": -2.2516541481018066,
      "loss": 0.1977,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -2.187448024749756,
      "rewards/margins": 0.06420598179101944,
      "rewards/rejected": -2.2516541481018066,
      "step": 175
    },
    {
      "epoch": 0.0963371801304566,
      "grad_norm": 1.2204538547136514,
      "learning_rate": 3.2085561497326203e-07,
      "logits/chosen": 0.05055837705731392,
      "logits/rejected": 0.0520298071205616,
      "logps/chosen": -2.2838351726531982,
      "logps/rejected": -2.2445807456970215,
      "loss": 0.1732,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.2838351726531982,
      "rewards/margins": -0.039254408329725266,
      "rewards/rejected": -2.2445807456970215,
      "step": 180
    },
    {
      "epoch": 0.09901321291185818,
      "grad_norm": 1.648651303663527,
      "learning_rate": 3.297682709447415e-07,
      "logits/chosen": -0.093844935297966,
      "logits/rejected": -0.0025992332957684994,
      "logps/chosen": -2.218714714050293,
      "logps/rejected": -2.309197425842285,
      "loss": 0.1887,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -2.218714714050293,
      "rewards/margins": 0.09048263728618622,
      "rewards/rejected": -2.309197425842285,
      "step": 185
    },
    {
      "epoch": 0.10168924569325974,
      "grad_norm": 1.89288906666452,
      "learning_rate": 3.38680926916221e-07,
      "logits/chosen": -0.012668641284108162,
      "logits/rejected": 0.12098245322704315,
      "logps/chosen": -2.7527928352355957,
      "logps/rejected": -2.600078582763672,
      "loss": 0.1585,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -2.7527928352355957,
      "rewards/margins": -0.1527143120765686,
      "rewards/rejected": -2.600078582763672,
      "step": 190
    },
    {
      "epoch": 0.10436527847466132,
      "grad_norm": 1.337326192763444,
      "learning_rate": 3.475935828877005e-07,
      "logits/chosen": 0.07230935245752335,
      "logits/rejected": 0.2421010434627533,
      "logps/chosen": -2.1195826530456543,
      "logps/rejected": -2.1807503700256348,
      "loss": 0.1768,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.1195826530456543,
      "rewards/margins": 0.061167918145656586,
      "rewards/rejected": -2.1807503700256348,
      "step": 195
    },
    {
      "epoch": 0.1070413112560629,
      "grad_norm": 1.5859879530568681,
      "learning_rate": 3.5650623885918e-07,
      "logits/chosen": -0.02128932811319828,
      "logits/rejected": 0.12987074255943298,
      "logps/chosen": -2.5602498054504395,
      "logps/rejected": -2.2940516471862793,
      "loss": 0.1745,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.5602498054504395,
      "rewards/margins": -0.26619845628738403,
      "rewards/rejected": -2.2940516471862793,
      "step": 200
    },
    {
      "epoch": 0.10971734403746446,
      "grad_norm": 2.6271898499509536,
      "learning_rate": 3.654188948306595e-07,
      "logits/chosen": -0.031300414353609085,
      "logits/rejected": 0.12799985706806183,
      "logps/chosen": -2.8899550437927246,
      "logps/rejected": -2.5607776641845703,
      "loss": 0.1598,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -2.8899550437927246,
      "rewards/margins": -0.3291778266429901,
      "rewards/rejected": -2.5607776641845703,
      "step": 205
    },
    {
      "epoch": 0.11239337681886603,
      "grad_norm": 1.9812876259484598,
      "learning_rate": 3.7433155080213904e-07,
      "logits/chosen": -0.12196006625890732,
      "logits/rejected": 0.09021838754415512,
      "logps/chosen": -2.774871587753296,
      "logps/rejected": -3.1648194789886475,
      "loss": 0.1291,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.774871587753296,
      "rewards/margins": 0.3899478316307068,
      "rewards/rejected": -3.1648194789886475,
      "step": 210
    },
    {
      "epoch": 0.1150694096002676,
      "grad_norm": 1.619323912196597,
      "learning_rate": 3.8324420677361853e-07,
      "logits/chosen": -0.11946575343608856,
      "logits/rejected": 0.15606310963630676,
      "logps/chosen": -2.615541458129883,
      "logps/rejected": -2.746011257171631,
      "loss": 0.1402,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.615541458129883,
      "rewards/margins": 0.13046978414058685,
      "rewards/rejected": -2.746011257171631,
      "step": 215
    },
    {
      "epoch": 0.11774544238166917,
      "grad_norm": 1.7545624795316679,
      "learning_rate": 3.92156862745098e-07,
      "logits/chosen": 0.09455358982086182,
      "logits/rejected": 0.2060193121433258,
      "logps/chosen": -3.1222751140594482,
      "logps/rejected": -3.587467908859253,
      "loss": 0.1215,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -3.1222751140594482,
      "rewards/margins": 0.4651927947998047,
      "rewards/rejected": -3.587467908859253,
      "step": 220
    },
    {
      "epoch": 0.12042147516307075,
      "grad_norm": 1.7595224375241685,
      "learning_rate": 4.010695187165775e-07,
      "logits/chosen": -0.08301651477813721,
      "logits/rejected": 0.10286612808704376,
      "logps/chosen": -2.9543631076812744,
      "logps/rejected": -3.0319743156433105,
      "loss": 0.1262,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.9543631076812744,
      "rewards/margins": 0.0776112824678421,
      "rewards/rejected": -3.0319743156433105,
      "step": 225
    },
    {
      "epoch": 0.12309750794447231,
      "grad_norm": 1.6447537520832118,
      "learning_rate": 4.09982174688057e-07,
      "logits/chosen": 0.05024641007184982,
      "logits/rejected": 0.13864049315452576,
      "logps/chosen": -3.6502983570098877,
      "logps/rejected": -3.639857053756714,
      "loss": 0.1238,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -3.6502983570098877,
      "rewards/margins": -0.01044134795665741,
      "rewards/rejected": -3.639857053756714,
      "step": 230
    },
    {
      "epoch": 0.1257735407258739,
      "grad_norm": 1.351707798714646,
      "learning_rate": 4.188948306595365e-07,
      "logits/chosen": 0.046742282807826996,
      "logits/rejected": 0.20803876221179962,
      "logps/chosen": -3.4221158027648926,
      "logps/rejected": -3.7459359169006348,
      "loss": 0.1047,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -3.4221158027648926,
      "rewards/margins": 0.323820024728775,
      "rewards/rejected": -3.7459359169006348,
      "step": 235
    },
    {
      "epoch": 0.12844957350727546,
      "grad_norm": 1.5281577741062018,
      "learning_rate": 4.27807486631016e-07,
      "logits/chosen": 0.02192387916147709,
      "logits/rejected": 0.15917302668094635,
      "logps/chosen": -3.6928000450134277,
      "logps/rejected": -3.7299110889434814,
      "loss": 0.1152,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -3.6928000450134277,
      "rewards/margins": 0.03711073473095894,
      "rewards/rejected": -3.7299110889434814,
      "step": 240
    },
    {
      "epoch": 0.13112560628867703,
      "grad_norm": 1.928017248720081,
      "learning_rate": 4.3672014260249554e-07,
      "logits/chosen": 0.10645352303981781,
      "logits/rejected": 0.24784211814403534,
      "logps/chosen": -3.362978458404541,
      "logps/rejected": -4.105316162109375,
      "loss": 0.1078,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -3.362978458404541,
      "rewards/margins": 0.7423376441001892,
      "rewards/rejected": -4.105316162109375,
      "step": 245
    },
    {
      "epoch": 0.1338016390700786,
      "grad_norm": 1.978575993475755,
      "learning_rate": 4.4563279857397503e-07,
      "logits/chosen": 0.03877265378832817,
      "logits/rejected": 0.22483864426612854,
      "logps/chosen": -5.149932384490967,
      "logps/rejected": -5.004909038543701,
      "loss": 0.1067,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -5.149932384490967,
      "rewards/margins": -0.14502307772636414,
      "rewards/rejected": -5.004909038543701,
      "step": 250
    },
    {
      "epoch": 0.1364776718514802,
      "grad_norm": 1.049973795532614,
      "learning_rate": 4.545454545454545e-07,
      "logits/chosen": 0.06372581422328949,
      "logits/rejected": 0.23879358172416687,
      "logps/chosen": -3.7124056816101074,
      "logps/rejected": -4.460031986236572,
      "loss": 0.1006,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -3.7124056816101074,
      "rewards/margins": 0.7476261854171753,
      "rewards/rejected": -4.460031986236572,
      "step": 255
    },
    {
      "epoch": 0.13915370463288176,
      "grad_norm": 0.9587092439633599,
      "learning_rate": 4.63458110516934e-07,
      "logits/chosen": -0.09507884085178375,
      "logits/rejected": 0.026835089549422264,
      "logps/chosen": -5.046217918395996,
      "logps/rejected": -4.29465389251709,
      "loss": 0.0801,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -5.046217918395996,
      "rewards/margins": -0.751563549041748,
      "rewards/rejected": -4.29465389251709,
      "step": 260
    },
    {
      "epoch": 0.1418297374142833,
      "grad_norm": 0.8547779761112152,
      "learning_rate": 4.723707664884135e-07,
      "logits/chosen": 0.10267798602581024,
      "logits/rejected": 0.19742190837860107,
      "logps/chosen": -6.0519118309021,
      "logps/rejected": -4.779889106750488,
      "loss": 0.0907,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -6.0519118309021,
      "rewards/margins": -1.2720227241516113,
      "rewards/rejected": -4.779889106750488,
      "step": 265
    },
    {
      "epoch": 0.1445057701956849,
      "grad_norm": 0.7538486426367985,
      "learning_rate": 4.81283422459893e-07,
      "logits/chosen": 0.07596492767333984,
      "logits/rejected": 0.24630455672740936,
      "logps/chosen": -4.7757248878479,
      "logps/rejected": -5.148313045501709,
      "loss": 0.0871,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -4.7757248878479,
      "rewards/margins": 0.3725886344909668,
      "rewards/rejected": -5.148313045501709,
      "step": 270
    },
    {
      "epoch": 0.14718180297708647,
      "grad_norm": 1.110027495676636,
      "learning_rate": 4.901960784313725e-07,
      "logits/chosen": 0.16651324927806854,
      "logits/rejected": 0.2681465744972229,
      "logps/chosen": -5.388066291809082,
      "logps/rejected": -6.059487819671631,
      "loss": 0.0967,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -5.388066291809082,
      "rewards/margins": 0.6714224815368652,
      "rewards/rejected": -6.059487819671631,
      "step": 275
    },
    {
      "epoch": 0.14985783575848804,
      "grad_norm": 0.8323309877454357,
      "learning_rate": 4.99108734402852e-07,
      "logits/chosen": 0.03581435605883598,
      "logits/rejected": 0.23406556248664856,
      "logps/chosen": -6.224254608154297,
      "logps/rejected": -5.9006781578063965,
      "loss": 0.0712,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -6.224254608154297,
      "rewards/margins": -0.32357633113861084,
      "rewards/rejected": -5.9006781578063965,
      "step": 280
    },
    {
      "epoch": 0.15253386853988962,
      "grad_norm": 1.1225734080196366,
      "learning_rate": 5.080213903743315e-07,
      "logits/chosen": 0.07181982696056366,
      "logits/rejected": 0.24139073491096497,
      "logps/chosen": -5.71528434753418,
      "logps/rejected": -5.607803821563721,
      "loss": 0.0865,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -5.71528434753418,
      "rewards/margins": -0.10748078674077988,
      "rewards/rejected": -5.607803821563721,
      "step": 285
    },
    {
      "epoch": 0.1552099013212912,
      "grad_norm": 0.7380070746796342,
      "learning_rate": 5.169340463458111e-07,
      "logits/chosen": 0.05033759027719498,
      "logits/rejected": 0.4079322814941406,
      "logps/chosen": -4.654083251953125,
      "logps/rejected": -5.902131080627441,
      "loss": 0.0531,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -4.654083251953125,
      "rewards/margins": 1.248047947883606,
      "rewards/rejected": -5.902131080627441,
      "step": 290
    },
    {
      "epoch": 0.15788593410269275,
      "grad_norm": 0.9651271623075317,
      "learning_rate": 5.258467023172905e-07,
      "logits/chosen": 0.14832785725593567,
      "logits/rejected": 0.21768374741077423,
      "logps/chosen": -6.472908020019531,
      "logps/rejected": -5.77554988861084,
      "loss": 0.071,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -6.472908020019531,
      "rewards/margins": -0.6973584294319153,
      "rewards/rejected": -5.77554988861084,
      "step": 295
    },
    {
      "epoch": 0.16056196688409433,
      "grad_norm": 1.7798721883884965,
      "learning_rate": 5.347593582887701e-07,
      "logits/chosen": 0.1505071222782135,
      "logits/rejected": 0.35654035210609436,
      "logps/chosen": -7.263270378112793,
      "logps/rejected": -7.17093563079834,
      "loss": 0.0664,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -7.263270378112793,
      "rewards/margins": -0.0923335999250412,
      "rewards/rejected": -7.17093563079834,
      "step": 300
    },
    {
      "epoch": 0.1632379996654959,
      "grad_norm": 4.972200794839469,
      "learning_rate": 5.436720142602496e-07,
      "logits/chosen": 0.29129844903945923,
      "logits/rejected": 0.37213748693466187,
      "logps/chosen": -6.570043087005615,
      "logps/rejected": -6.351378440856934,
      "loss": 0.0564,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -6.570043087005615,
      "rewards/margins": -0.21866460144519806,
      "rewards/rejected": -6.351378440856934,
      "step": 305
    },
    {
      "epoch": 0.16591403244689748,
      "grad_norm": 2.769377848703755,
      "learning_rate": 5.52584670231729e-07,
      "logits/chosen": 0.40942397713661194,
      "logits/rejected": 0.528160810470581,
      "logps/chosen": -8.790095329284668,
      "logps/rejected": -8.929426193237305,
      "loss": 0.0251,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -8.790095329284668,
      "rewards/margins": 0.13933177292346954,
      "rewards/rejected": -8.929426193237305,
      "step": 310
    },
    {
      "epoch": 0.16859006522829906,
      "grad_norm": 1.276617643202457,
      "learning_rate": 5.614973262032086e-07,
      "logits/chosen": 1.0396369695663452,
      "logits/rejected": 1.2752702236175537,
      "logps/chosen": -11.090681076049805,
      "logps/rejected": -11.999565124511719,
      "loss": 0.0103,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -11.090681076049805,
      "rewards/margins": 0.9088830947875977,
      "rewards/rejected": -11.999565124511719,
      "step": 315
    },
    {
      "epoch": 0.1712660980097006,
      "grad_norm": 0.5601596778795463,
      "learning_rate": 5.70409982174688e-07,
      "logits/chosen": 1.4934452772140503,
      "logits/rejected": 1.6631572246551514,
      "logps/chosen": -13.802734375,
      "logps/rejected": -13.695340156555176,
      "loss": 0.0034,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -13.802734375,
      "rewards/margins": -0.10739493370056152,
      "rewards/rejected": -13.695340156555176,
      "step": 320
    },
    {
      "epoch": 0.17394213079110218,
      "grad_norm": 0.20184763189353147,
      "learning_rate": 5.793226381461676e-07,
      "logits/chosen": 2.1631858348846436,
      "logits/rejected": 2.3616809844970703,
      "logps/chosen": -18.422916412353516,
      "logps/rejected": -20.24477767944336,
      "loss": 0.0014,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -18.422916412353516,
      "rewards/margins": 1.821862816810608,
      "rewards/rejected": -20.24477767944336,
      "step": 325
    },
    {
      "epoch": 0.17661816357250376,
      "grad_norm": 0.0995784931963209,
      "learning_rate": 5.88235294117647e-07,
      "logits/chosen": 2.7550110816955566,
      "logits/rejected": 2.7731616497039795,
      "logps/chosen": -19.130725860595703,
      "logps/rejected": -22.46588706970215,
      "loss": 0.0012,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -19.130725860595703,
      "rewards/margins": 3.335160732269287,
      "rewards/rejected": -22.46588706970215,
      "step": 330
    },
    {
      "epoch": 0.17929419635390534,
      "grad_norm": 1.1455800197469745,
      "learning_rate": 5.971479500891266e-07,
      "logits/chosen": 3.486672878265381,
      "logits/rejected": 3.5515098571777344,
      "logps/chosen": -23.925006866455078,
      "logps/rejected": -24.635570526123047,
      "loss": 0.001,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -23.925006866455078,
      "rewards/margins": 0.7105626463890076,
      "rewards/rejected": -24.635570526123047,
      "step": 335
    },
    {
      "epoch": 0.18197022913530692,
      "grad_norm": 0.04100025582306579,
      "learning_rate": 6.060606060606061e-07,
      "logits/chosen": 3.870731830596924,
      "logits/rejected": 3.9299209117889404,
      "logps/chosen": -24.719100952148438,
      "logps/rejected": -27.2982234954834,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -24.719100952148438,
      "rewards/margins": 2.5791244506835938,
      "rewards/rejected": -27.2982234954834,
      "step": 340
    },
    {
      "epoch": 0.1846462619167085,
      "grad_norm": 0.051020900747332926,
      "learning_rate": 6.149732620320855e-07,
      "logits/chosen": 4.12877082824707,
      "logits/rejected": 4.255687713623047,
      "logps/chosen": -25.529422760009766,
      "logps/rejected": -26.956287384033203,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -25.529422760009766,
      "rewards/margins": 1.4268611669540405,
      "rewards/rejected": -26.956287384033203,
      "step": 345
    },
    {
      "epoch": 0.18732229469811004,
      "grad_norm": 0.0007457271947595468,
      "learning_rate": 6.238859180035651e-07,
      "logits/chosen": 4.296173095703125,
      "logits/rejected": 4.500910758972168,
      "logps/chosen": -26.123943328857422,
      "logps/rejected": -28.987768173217773,
      "loss": 0.001,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -26.123943328857422,
      "rewards/margins": 2.8638224601745605,
      "rewards/rejected": -28.987768173217773,
      "step": 350
    },
    {
      "epoch": 0.18999832747951162,
      "grad_norm": 0.05024276051322549,
      "learning_rate": 6.327985739750445e-07,
      "logits/chosen": 4.909904956817627,
      "logits/rejected": 4.810797214508057,
      "logps/chosen": -28.670263290405273,
      "logps/rejected": -30.414974212646484,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -28.670263290405273,
      "rewards/margins": 1.7447071075439453,
      "rewards/rejected": -30.414974212646484,
      "step": 355
    },
    {
      "epoch": 0.1926743602609132,
      "grad_norm": 0.0001298293902042708,
      "learning_rate": 6.417112299465241e-07,
      "logits/chosen": 5.005853652954102,
      "logits/rejected": 5.016454696655273,
      "logps/chosen": -30.7520694732666,
      "logps/rejected": -31.67702293395996,
      "loss": 0.0002,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -30.7520694732666,
      "rewards/margins": 0.9249529838562012,
      "rewards/rejected": -31.67702293395996,
      "step": 360
    },
    {
      "epoch": 0.19535039304231477,
      "grad_norm": 0.0032856705929756665,
      "learning_rate": 6.506238859180035e-07,
      "logits/chosen": 4.896598815917969,
      "logits/rejected": 4.941209316253662,
      "logps/chosen": -29.46160316467285,
      "logps/rejected": -30.0184383392334,
      "loss": 0.0004,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -29.46160316467285,
      "rewards/margins": 0.5568350553512573,
      "rewards/rejected": -30.0184383392334,
      "step": 365
    },
    {
      "epoch": 0.19802642582371635,
      "grad_norm": 0.014872173339104884,
      "learning_rate": 6.59536541889483e-07,
      "logits/chosen": 5.059857368469238,
      "logits/rejected": 4.976897239685059,
      "logps/chosen": -29.063770294189453,
      "logps/rejected": -29.823429107666016,
      "loss": 0.0001,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -29.063770294189453,
      "rewards/margins": 0.7596587538719177,
      "rewards/rejected": -29.823429107666016,
      "step": 370
    },
    {
      "epoch": 0.2007024586051179,
      "grad_norm": 0.04583182594767222,
      "learning_rate": 6.684491978609626e-07,
      "logits/chosen": 5.03874397277832,
      "logits/rejected": 4.995621681213379,
      "logps/chosen": -29.2627010345459,
      "logps/rejected": -31.88068199157715,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -29.2627010345459,
      "rewards/margins": 2.617982864379883,
      "rewards/rejected": -31.88068199157715,
      "step": 375
    },
    {
      "epoch": 0.20337849138651948,
      "grad_norm": 1.274954313595139e-05,
      "learning_rate": 6.77361853832442e-07,
      "logits/chosen": 5.426454067230225,
      "logits/rejected": 5.617036819458008,
      "logps/chosen": -31.229167938232422,
      "logps/rejected": -34.28889846801758,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -31.229167938232422,
      "rewards/margins": 3.059727907180786,
      "rewards/rejected": -34.28889846801758,
      "step": 380
    },
    {
      "epoch": 0.20605452416792105,
      "grad_norm": 3.0701830683835347e-05,
      "learning_rate": 6.862745098039216e-07,
      "logits/chosen": 5.503627777099609,
      "logits/rejected": 5.433765888214111,
      "logps/chosen": -30.88620376586914,
      "logps/rejected": -33.45307159423828,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -30.88620376586914,
      "rewards/margins": 2.5668678283691406,
      "rewards/rejected": -33.45307159423828,
      "step": 385
    },
    {
      "epoch": 0.20873055694932263,
      "grad_norm": 0.005973549099023764,
      "learning_rate": 6.95187165775401e-07,
      "logits/chosen": 5.093874931335449,
      "logits/rejected": 4.7967753410339355,
      "logps/chosen": -31.51506996154785,
      "logps/rejected": -33.51251220703125,
      "loss": 0.0017,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -31.51506996154785,
      "rewards/margins": 1.9974451065063477,
      "rewards/rejected": -33.51251220703125,
      "step": 390
    },
    {
      "epoch": 0.2114065897307242,
      "grad_norm": 0.002799700748698269,
      "learning_rate": 7.040998217468806e-07,
      "logits/chosen": 5.599579811096191,
      "logits/rejected": 5.566360950469971,
      "logps/chosen": -31.227807998657227,
      "logps/rejected": -33.56954574584961,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -31.227807998657227,
      "rewards/margins": 2.341738700866699,
      "rewards/rejected": -33.56954574584961,
      "step": 395
    },
    {
      "epoch": 0.2140826225121258,
      "grad_norm": 0.07792558659998519,
      "learning_rate": 7.1301247771836e-07,
      "logits/chosen": 5.242839336395264,
      "logits/rejected": 5.270083427429199,
      "logps/chosen": -30.75326919555664,
      "logps/rejected": -33.046470642089844,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -30.75326919555664,
      "rewards/margins": 2.293198823928833,
      "rewards/rejected": -33.046470642089844,
      "step": 400
    },
    {
      "epoch": 0.2140826225121258,
      "eval_logits/chosen": 4.526402473449707,
      "eval_logits/rejected": 4.529143810272217,
      "eval_logps/chosen": -30.934097290039062,
      "eval_logps/rejected": -32.71940231323242,
      "eval_loss": 0.0001360369351459667,
      "eval_rewards/accuracies": 0.5697329640388489,
      "eval_rewards/chosen": -30.934097290039062,
      "eval_rewards/margins": 1.7853031158447266,
      "eval_rewards/rejected": -32.71940231323242,
      "eval_runtime": 41.3229,
      "eval_samples_per_second": 32.549,
      "eval_steps_per_second": 8.155,
      "step": 400
    },
    {
      "epoch": 0.21675865529352734,
      "grad_norm": 1.9254525758027886e-07,
      "learning_rate": 7.219251336898395e-07,
      "logits/chosen": 5.443844795227051,
      "logits/rejected": 5.441424369812012,
      "logps/chosen": -33.176361083984375,
      "logps/rejected": -34.68500900268555,
      "loss": 0.0001,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -33.176361083984375,
      "rewards/margins": 1.5086463689804077,
      "rewards/rejected": -34.68500900268555,
      "step": 405
    },
    {
      "epoch": 0.2194346880749289,
      "grad_norm": 0.0007020919320299974,
      "learning_rate": 7.30837789661319e-07,
      "logits/chosen": 5.657177448272705,
      "logits/rejected": 5.550604820251465,
      "logps/chosen": -32.41621780395508,
      "logps/rejected": -35.28019332885742,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -32.41621780395508,
      "rewards/margins": 2.8639800548553467,
      "rewards/rejected": -35.28019332885742,
      "step": 410
    },
    {
      "epoch": 0.2221107208563305,
      "grad_norm": 2.8201998189865507e-07,
      "learning_rate": 7.397504456327985e-07,
      "logits/chosen": 5.725876808166504,
      "logits/rejected": 5.796631336212158,
      "logps/chosen": -33.93684005737305,
      "logps/rejected": -34.73693084716797,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -33.93684005737305,
      "rewards/margins": 0.8000904321670532,
      "rewards/rejected": -34.73693084716797,
      "step": 415
    },
    {
      "epoch": 0.22478675363773207,
      "grad_norm": 0.02077114021177462,
      "learning_rate": 7.486631016042781e-07,
      "logits/chosen": 5.470408916473389,
      "logits/rejected": 5.250080585479736,
      "logps/chosen": -32.31761932373047,
      "logps/rejected": -35.006935119628906,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -32.31761932373047,
      "rewards/margins": 2.6893131732940674,
      "rewards/rejected": -35.006935119628906,
      "step": 420
    },
    {
      "epoch": 0.22746278641913364,
      "grad_norm": 0.0005105256889512131,
      "learning_rate": 7.575757575757575e-07,
      "logits/chosen": 5.305121898651123,
      "logits/rejected": 5.3682451248168945,
      "logps/chosen": -32.294410705566406,
      "logps/rejected": -35.14203643798828,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -32.294410705566406,
      "rewards/margins": 2.847626209259033,
      "rewards/rejected": -35.14203643798828,
      "step": 425
    },
    {
      "epoch": 0.2301388192005352,
      "grad_norm": 0.005656459646702251,
      "learning_rate": 7.664884135472371e-07,
      "logits/chosen": 5.671530723571777,
      "logits/rejected": 5.511305809020996,
      "logps/chosen": -30.075464248657227,
      "logps/rejected": -34.07829666137695,
      "loss": 0.0,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -30.075464248657227,
      "rewards/margins": 4.002833366394043,
      "rewards/rejected": -34.07829666137695,
      "step": 430
    },
    {
      "epoch": 0.23281485198193677,
      "grad_norm": 5.8596539532771534e-05,
      "learning_rate": 7.754010695187165e-07,
      "logits/chosen": 5.692728042602539,
      "logits/rejected": 5.715226173400879,
      "logps/chosen": -32.5540657043457,
      "logps/rejected": -33.38398742675781,
      "loss": 0.0001,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -32.5540657043457,
      "rewards/margins": 0.8299189805984497,
      "rewards/rejected": -33.38398742675781,
      "step": 435
    },
    {
      "epoch": 0.23549088476333835,
      "grad_norm": 0.00022384455796583787,
      "learning_rate": 7.84313725490196e-07,
      "logits/chosen": 5.3080339431762695,
      "logits/rejected": 5.231874942779541,
      "logps/chosen": -30.1303768157959,
      "logps/rejected": -31.926959991455078,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -30.1303768157959,
      "rewards/margins": 1.7965834140777588,
      "rewards/rejected": -31.926959991455078,
      "step": 440
    },
    {
      "epoch": 0.23816691754473993,
      "grad_norm": 0.03193044644782529,
      "learning_rate": 7.932263814616755e-07,
      "logits/chosen": 5.448077201843262,
      "logits/rejected": 5.429624080657959,
      "logps/chosen": -31.28450584411621,
      "logps/rejected": -33.11250686645508,
      "loss": 0.0001,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -31.28450584411621,
      "rewards/margins": 1.828005075454712,
      "rewards/rejected": -33.11250686645508,
      "step": 445
    },
    {
      "epoch": 0.2408429503261415,
      "grad_norm": 0.0002920685384981377,
      "learning_rate": 8.02139037433155e-07,
      "logits/chosen": 5.754220008850098,
      "logits/rejected": 5.815160751342773,
      "logps/chosen": -32.84495162963867,
      "logps/rejected": -34.813167572021484,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -32.84495162963867,
      "rewards/margins": 1.968212366104126,
      "rewards/rejected": -34.813167572021484,
      "step": 450
    },
    {
      "epoch": 0.24351898310754308,
      "grad_norm": 0.00016037192952240452,
      "learning_rate": 8.110516934046346e-07,
      "logits/chosen": 5.854935169219971,
      "logits/rejected": 5.885193824768066,
      "logps/chosen": -31.178136825561523,
      "logps/rejected": -33.45836639404297,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -31.178136825561523,
      "rewards/margins": 2.2802300453186035,
      "rewards/rejected": -33.45836639404297,
      "step": 455
    },
    {
      "epoch": 0.24619501588894463,
      "grad_norm": 0.0004142962921022656,
      "learning_rate": 8.19964349376114e-07,
      "logits/chosen": 5.814682960510254,
      "logits/rejected": 5.778790473937988,
      "logps/chosen": -31.413259506225586,
      "logps/rejected": -33.53882598876953,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -31.413259506225586,
      "rewards/margins": 2.125563859939575,
      "rewards/rejected": -33.53882598876953,
      "step": 460
    },
    {
      "epoch": 0.2488710486703462,
      "grad_norm": 0.0014625833247701478,
      "learning_rate": 8.288770053475936e-07,
      "logits/chosen": 5.884660720825195,
      "logits/rejected": 5.809075355529785,
      "logps/chosen": -33.626068115234375,
      "logps/rejected": -34.831302642822266,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -33.626068115234375,
      "rewards/margins": 1.2052299976348877,
      "rewards/rejected": -34.831302642822266,
      "step": 465
    },
    {
      "epoch": 0.2515470814517478,
      "grad_norm": 0.0066164164003505825,
      "learning_rate": 8.37789661319073e-07,
      "logits/chosen": 6.057608604431152,
      "logits/rejected": 6.203212261199951,
      "logps/chosen": -35.04254150390625,
      "logps/rejected": -34.7065315246582,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -35.04254150390625,
      "rewards/margins": -0.33600565791130066,
      "rewards/rejected": -34.7065315246582,
      "step": 470
    },
    {
      "epoch": 0.25422311423314936,
      "grad_norm": 0.005282456856397385,
      "learning_rate": 8.467023172905525e-07,
      "logits/chosen": 6.180428504943848,
      "logits/rejected": 6.111413955688477,
      "logps/chosen": -33.236610412597656,
      "logps/rejected": -37.252593994140625,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -33.236610412597656,
      "rewards/margins": 4.015982627868652,
      "rewards/rejected": -37.252593994140625,
      "step": 475
    },
    {
      "epoch": 0.2568991470145509,
      "grad_norm": 0.18653210021766384,
      "learning_rate": 8.55614973262032e-07,
      "logits/chosen": 6.247392177581787,
      "logits/rejected": 6.122766017913818,
      "logps/chosen": -33.51787567138672,
      "logps/rejected": -36.1046257019043,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -33.51787567138672,
      "rewards/margins": 2.5867512226104736,
      "rewards/rejected": -36.1046257019043,
      "step": 480
    },
    {
      "epoch": 0.2595751797959525,
      "grad_norm": 0.00033331207777263316,
      "learning_rate": 8.645276292335115e-07,
      "logits/chosen": 6.566782474517822,
      "logits/rejected": 6.646945953369141,
      "logps/chosen": -34.993953704833984,
      "logps/rejected": -37.12001037597656,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -34.993953704833984,
      "rewards/margins": 2.126059055328369,
      "rewards/rejected": -37.12001037597656,
      "step": 485
    },
    {
      "epoch": 0.26225121257735406,
      "grad_norm": 0.0007288719209561518,
      "learning_rate": 8.734402852049911e-07,
      "logits/chosen": 6.060391426086426,
      "logits/rejected": 6.139369964599609,
      "logps/chosen": -35.0052604675293,
      "logps/rejected": -36.17188262939453,
      "loss": 0.0002,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -35.0052604675293,
      "rewards/margins": 1.1666234731674194,
      "rewards/rejected": -36.17188262939453,
      "step": 490
    },
    {
      "epoch": 0.26492724535875567,
      "grad_norm": 0.00022275326383988783,
      "learning_rate": 8.823529411764705e-07,
      "logits/chosen": 6.39509916305542,
      "logits/rejected": 6.392638683319092,
      "logps/chosen": -36.97453689575195,
      "logps/rejected": -35.98308181762695,
      "loss": 0.0,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -36.97453689575195,
      "rewards/margins": -0.9914531707763672,
      "rewards/rejected": -35.98308181762695,
      "step": 495
    },
    {
      "epoch": 0.2676032781401572,
      "grad_norm": 8.439080279748877e-05,
      "learning_rate": 8.912655971479501e-07,
      "logits/chosen": 6.375731945037842,
      "logits/rejected": 6.280875205993652,
      "logps/chosen": -35.84641647338867,
      "logps/rejected": -36.8585205078125,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -35.84641647338867,
      "rewards/margins": 1.0121053457260132,
      "rewards/rejected": -36.8585205078125,
      "step": 500
    },
    {
      "epoch": 0.27027931092155877,
      "grad_norm": 0.08333674244371847,
      "learning_rate": 9.001782531194295e-07,
      "logits/chosen": 6.321547508239746,
      "logits/rejected": 6.214648246765137,
      "logps/chosen": -34.122711181640625,
      "logps/rejected": -35.329795837402344,
      "loss": 0.0012,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -34.122711181640625,
      "rewards/margins": 1.2070846557617188,
      "rewards/rejected": -35.329795837402344,
      "step": 505
    },
    {
      "epoch": 0.2729553437029604,
      "grad_norm": 8.87406572537442e-05,
      "learning_rate": 9.09090909090909e-07,
      "logits/chosen": 6.118102073669434,
      "logits/rejected": 6.064526081085205,
      "logps/chosen": -35.52603530883789,
      "logps/rejected": -36.2239875793457,
      "loss": 0.0001,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -35.52603530883789,
      "rewards/margins": 0.6979531049728394,
      "rewards/rejected": -36.2239875793457,
      "step": 510
    },
    {
      "epoch": 0.2756313764843619,
      "grad_norm": 0.0011493852245518313,
      "learning_rate": 9.180035650623885e-07,
      "logits/chosen": 5.755102157592773,
      "logits/rejected": 5.8633222579956055,
      "logps/chosen": -32.83620834350586,
      "logps/rejected": -34.56470489501953,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -32.83620834350586,
      "rewards/margins": 1.7284952402114868,
      "rewards/rejected": -34.56470489501953,
      "step": 515
    },
    {
      "epoch": 0.27830740926576353,
      "grad_norm": 5.041611698120519e-05,
      "learning_rate": 9.26916221033868e-07,
      "logits/chosen": 6.290412902832031,
      "logits/rejected": 6.041254997253418,
      "logps/chosen": -35.81562042236328,
      "logps/rejected": -36.734764099121094,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -35.81562042236328,
      "rewards/margins": 0.9191436767578125,
      "rewards/rejected": -36.734764099121094,
      "step": 520
    },
    {
      "epoch": 0.2809834420471651,
      "grad_norm": 0.006131159413679776,
      "learning_rate": 9.358288770053476e-07,
      "logits/chosen": 5.8230390548706055,
      "logits/rejected": 5.748825550079346,
      "logps/chosen": -34.16297912597656,
      "logps/rejected": -35.419288635253906,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -34.16297912597656,
      "rewards/margins": 1.2563061714172363,
      "rewards/rejected": -35.419288635253906,
      "step": 525
    },
    {
      "epoch": 0.2836594748285666,
      "grad_norm": 0.05630429039298123,
      "learning_rate": 9.44741532976827e-07,
      "logits/chosen": 5.912903785705566,
      "logits/rejected": 6.099035739898682,
      "logps/chosen": -32.822418212890625,
      "logps/rejected": -33.937232971191406,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -32.822418212890625,
      "rewards/margins": 1.1148147583007812,
      "rewards/rejected": -33.937232971191406,
      "step": 530
    },
    {
      "epoch": 0.28633550760996823,
      "grad_norm": 1.1529432084581304e-05,
      "learning_rate": 9.536541889483066e-07,
      "logits/chosen": 5.906094551086426,
      "logits/rejected": 5.595171928405762,
      "logps/chosen": -32.46159744262695,
      "logps/rejected": -36.4743537902832,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -32.46159744262695,
      "rewards/margins": 4.012753963470459,
      "rewards/rejected": -36.4743537902832,
      "step": 535
    },
    {
      "epoch": 0.2890115403913698,
      "grad_norm": 0.002945561638762314,
      "learning_rate": 9.62566844919786e-07,
      "logits/chosen": 5.9652814865112305,
      "logits/rejected": 5.922352313995361,
      "logps/chosen": -32.957305908203125,
      "logps/rejected": -34.000389099121094,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -32.957305908203125,
      "rewards/margins": 1.0430856943130493,
      "rewards/rejected": -34.000389099121094,
      "step": 540
    },
    {
      "epoch": 0.2916875731727714,
      "grad_norm": 0.015155948396947559,
      "learning_rate": 9.714795008912655e-07,
      "logits/chosen": 5.913393974304199,
      "logits/rejected": 5.796065330505371,
      "logps/chosen": -33.27630615234375,
      "logps/rejected": -36.43669509887695,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -33.27630615234375,
      "rewards/margins": 3.1603920459747314,
      "rewards/rejected": -36.43669509887695,
      "step": 545
    },
    {
      "epoch": 0.29436360595417294,
      "grad_norm": 0.0002522111178388746,
      "learning_rate": 9.80392156862745e-07,
      "logits/chosen": 6.193312168121338,
      "logits/rejected": 6.141693115234375,
      "logps/chosen": -34.021278381347656,
      "logps/rejected": -35.327903747558594,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -34.021278381347656,
      "rewards/margins": 1.306623101234436,
      "rewards/rejected": -35.327903747558594,
      "step": 550
    },
    {
      "epoch": 0.2970396387355745,
      "grad_norm": 0.010712043187526058,
      "learning_rate": 9.893048128342244e-07,
      "logits/chosen": 6.200936794281006,
      "logits/rejected": 5.888747215270996,
      "logps/chosen": -35.00139617919922,
      "logps/rejected": -35.022987365722656,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -35.00139617919922,
      "rewards/margins": 0.0215925220400095,
      "rewards/rejected": -35.022987365722656,
      "step": 555
    },
    {
      "epoch": 0.2997156715169761,
      "grad_norm": 0.015186372594368142,
      "learning_rate": 9.98217468805704e-07,
      "logits/chosen": 6.27797794342041,
      "logits/rejected": 6.344517707824707,
      "logps/chosen": -35.294925689697266,
      "logps/rejected": -35.53615188598633,
      "loss": 0.0,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -35.294925689697266,
      "rewards/margins": 0.24121804535388947,
      "rewards/rejected": -35.53615188598633,
      "step": 560
    },
    {
      "epoch": 0.30239170429837764,
      "grad_norm": 0.0003237423807535268,
      "learning_rate": 9.999984476788462e-07,
      "logits/chosen": 6.507061004638672,
      "logits/rejected": 6.4309587478637695,
      "logps/chosen": -34.80686950683594,
      "logps/rejected": -36.58795166015625,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -34.80686950683594,
      "rewards/margins": 1.7810804843902588,
      "rewards/rejected": -36.58795166015625,
      "step": 565
    },
    {
      "epoch": 0.30506773707977924,
      "grad_norm": 5.815543791446198e-06,
      "learning_rate": 9.999921413906797e-07,
      "logits/chosen": 6.260038375854492,
      "logits/rejected": 6.169661998748779,
      "logps/chosen": -34.07218551635742,
      "logps/rejected": -36.635677337646484,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -34.07218551635742,
      "rewards/margins": 2.563491106033325,
      "rewards/rejected": -36.635677337646484,
      "step": 570
    },
    {
      "epoch": 0.3077437698611808,
      "grad_norm": 1.325352692946733e-06,
      "learning_rate": 9.999809841765644e-07,
      "logits/chosen": 6.912776947021484,
      "logits/rejected": 7.2961626052856445,
      "logps/chosen": -36.65069580078125,
      "logps/rejected": -38.139320373535156,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -36.65069580078125,
      "rewards/margins": 1.4886243343353271,
      "rewards/rejected": -38.139320373535156,
      "step": 575
    },
    {
      "epoch": 0.3104198026425824,
      "grad_norm": 5.074484014465836e-06,
      "learning_rate": 9.999649761447477e-07,
      "logits/chosen": 7.057467460632324,
      "logits/rejected": 6.7020158767700195,
      "logps/chosen": -36.71874237060547,
      "logps/rejected": -37.977630615234375,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -36.71874237060547,
      "rewards/margins": 1.258882999420166,
      "rewards/rejected": -37.977630615234375,
      "step": 580
    },
    {
      "epoch": 0.31309583542398395,
      "grad_norm": 0.0008223583693317254,
      "learning_rate": 9.999441174505398e-07,
      "logits/chosen": 6.75624942779541,
      "logits/rejected": 6.8660078048706055,
      "logps/chosen": -36.46296310424805,
      "logps/rejected": -38.49901580810547,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -36.46296310424805,
      "rewards/margins": 2.036057233810425,
      "rewards/rejected": -38.49901580810547,
      "step": 585
    },
    {
      "epoch": 0.3157718682053855,
      "grad_norm": 2.5039444521803124e-07,
      "learning_rate": 9.999184082963116e-07,
      "logits/chosen": 6.8557329177856445,
      "logits/rejected": 6.9033637046813965,
      "logps/chosen": -36.50396728515625,
      "logps/rejected": -38.169029235839844,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -36.50396728515625,
      "rewards/margins": 1.6650577783584595,
      "rewards/rejected": -38.169029235839844,
      "step": 590
    },
    {
      "epoch": 0.3184479009867871,
      "grad_norm": 0.0005401199248432682,
      "learning_rate": 9.998878489314937e-07,
      "logits/chosen": 7.008899688720703,
      "logits/rejected": 6.642820835113525,
      "logps/chosen": -36.62739181518555,
      "logps/rejected": -38.907012939453125,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -36.62739181518555,
      "rewards/margins": 2.279622793197632,
      "rewards/rejected": -38.907012939453125,
      "step": 595
    },
    {
      "epoch": 0.32112393376818865,
      "grad_norm": 0.0006512484096722162,
      "learning_rate": 9.99852439652573e-07,
      "logits/chosen": 6.869595527648926,
      "logits/rejected": 6.742593288421631,
      "logps/chosen": -35.97058868408203,
      "logps/rejected": -37.555999755859375,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -35.97058868408203,
      "rewards/margins": 1.5854099988937378,
      "rewards/rejected": -37.555999755859375,
      "step": 600
    },
    {
      "epoch": 0.32379996654959026,
      "grad_norm": 2.7183653573716168e-06,
      "learning_rate": 9.998121808030904e-07,
      "logits/chosen": 6.894481658935547,
      "logits/rejected": 6.905220031738281,
      "logps/chosen": -36.660003662109375,
      "logps/rejected": -37.91825485229492,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -36.660003662109375,
      "rewards/margins": 1.2582588195800781,
      "rewards/rejected": -37.91825485229492,
      "step": 605
    },
    {
      "epoch": 0.3264759993309918,
      "grad_norm": 0.0014474865256437017,
      "learning_rate": 9.997670727736379e-07,
      "logits/chosen": 6.804381370544434,
      "logits/rejected": 6.628779411315918,
      "logps/chosen": -36.24687194824219,
      "logps/rejected": -38.64838409423828,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -36.24687194824219,
      "rewards/margins": 2.401505947113037,
      "rewards/rejected": -38.64838409423828,
      "step": 610
    },
    {
      "epoch": 0.32915203211239336,
      "grad_norm": 2.6013778944738846e-05,
      "learning_rate": 9.99717116001853e-07,
      "logits/chosen": 6.911631107330322,
      "logits/rejected": 6.937872409820557,
      "logps/chosen": -35.35862350463867,
      "logps/rejected": -37.873538970947266,
      "loss": 0.0001,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -35.35862350463867,
      "rewards/margins": 2.514913558959961,
      "rewards/rejected": -37.873538970947266,
      "step": 615
    },
    {
      "epoch": 0.33182806489379496,
      "grad_norm": 9.826303694149804e-08,
      "learning_rate": 9.996623109724173e-07,
      "logits/chosen": 6.775407314300537,
      "logits/rejected": 6.680213928222656,
      "logps/chosen": -36.26496124267578,
      "logps/rejected": -37.71826171875,
      "loss": 0.0001,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -36.26496124267578,
      "rewards/margins": 1.4533039331436157,
      "rewards/rejected": -37.71826171875,
      "step": 620
    },
    {
      "epoch": 0.3345040976751965,
      "grad_norm": 1.1634211304529595e-06,
      "learning_rate": 9.996026582170488e-07,
      "logits/chosen": 6.944779396057129,
      "logits/rejected": 6.652417182922363,
      "logps/chosen": -36.578887939453125,
      "logps/rejected": -38.96196746826172,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -36.578887939453125,
      "rewards/margins": 2.383082866668701,
      "rewards/rejected": -38.96196746826172,
      "step": 625
    },
    {
      "epoch": 0.3371801304565981,
      "grad_norm": 1.1117895999656755e-06,
      "learning_rate": 9.995381583144996e-07,
      "logits/chosen": 6.875652313232422,
      "logits/rejected": 6.8520612716674805,
      "logps/chosen": -35.62145233154297,
      "logps/rejected": -37.62517547607422,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -35.62145233154297,
      "rewards/margins": 2.0037238597869873,
      "rewards/rejected": -37.62517547607422,
      "step": 630
    },
    {
      "epoch": 0.33985616323799966,
      "grad_norm": 0.00015628279586919798,
      "learning_rate": 9.994688118905471e-07,
      "logits/chosen": 7.339330196380615,
      "logits/rejected": 6.943647861480713,
      "logps/chosen": -36.8288459777832,
      "logps/rejected": -39.714012145996094,
      "loss": 0.0,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -36.8288459777832,
      "rewards/margins": 2.8851709365844727,
      "rewards/rejected": -39.714012145996094,
      "step": 635
    },
    {
      "epoch": 0.3425321960194012,
      "grad_norm": 2.7911399881969935e-05,
      "learning_rate": 9.993946196179912e-07,
      "logits/chosen": 7.054766654968262,
      "logits/rejected": 6.722894191741943,
      "logps/chosen": -37.25520706176758,
      "logps/rejected": -38.761775970458984,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -37.25520706176758,
      "rewards/margins": 1.5065667629241943,
      "rewards/rejected": -38.761775970458984,
      "step": 640
    },
    {
      "epoch": 0.3452082288008028,
      "grad_norm": 1.324854965434706e-06,
      "learning_rate": 9.993155822166455e-07,
      "logits/chosen": 7.476956367492676,
      "logits/rejected": 7.256749153137207,
      "logps/chosen": -37.280494689941406,
      "logps/rejected": -38.97529983520508,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -37.280494689941406,
      "rewards/margins": 1.6948060989379883,
      "rewards/rejected": -38.97529983520508,
      "step": 645
    },
    {
      "epoch": 0.34788426158220437,
      "grad_norm": 1.999433267198434e-06,
      "learning_rate": 9.992317004533313e-07,
      "logits/chosen": 7.322606086730957,
      "logits/rejected": 7.418975830078125,
      "logps/chosen": -37.304710388183594,
      "logps/rejected": -38.7960090637207,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -37.304710388183594,
      "rewards/margins": 1.491295576095581,
      "rewards/rejected": -38.7960090637207,
      "step": 650
    },
    {
      "epoch": 0.350560294363606,
      "grad_norm": 4.870940438851254e-05,
      "learning_rate": 9.991429751418696e-07,
      "logits/chosen": 7.15334415435791,
      "logits/rejected": 7.352728366851807,
      "logps/chosen": -38.16060256958008,
      "logps/rejected": -38.95783996582031,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -38.16060256958008,
      "rewards/margins": 0.7972370386123657,
      "rewards/rejected": -38.95783996582031,
      "step": 655
    },
    {
      "epoch": 0.3532363271450075,
      "grad_norm": 0.001327098594245767,
      "learning_rate": 9.99049407143074e-07,
      "logits/chosen": 7.489438056945801,
      "logits/rejected": 7.206766605377197,
      "logps/chosen": -38.31577682495117,
      "logps/rejected": -39.319854736328125,
      "loss": 0.0,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -38.31577682495117,
      "rewards/margins": 1.0040757656097412,
      "rewards/rejected": -39.319854736328125,
      "step": 660
    },
    {
      "epoch": 0.35591235992640907,
      "grad_norm": 1.296402505263512e-07,
      "learning_rate": 9.989509973647416e-07,
      "logits/chosen": 7.1884870529174805,
      "logits/rejected": 7.007891654968262,
      "logps/chosen": -37.89320755004883,
      "logps/rejected": -38.987327575683594,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -37.89320755004883,
      "rewards/margins": 1.0941221714019775,
      "rewards/rejected": -38.987327575683594,
      "step": 665
    },
    {
      "epoch": 0.3585883927078107,
      "grad_norm": 1.2752147877231267e-05,
      "learning_rate": 9.988477467616445e-07,
      "logits/chosen": 7.596221923828125,
      "logits/rejected": 7.293318271636963,
      "logps/chosen": -39.22991943359375,
      "logps/rejected": -41.35580825805664,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -39.22991943359375,
      "rewards/margins": 2.1258859634399414,
      "rewards/rejected": -41.35580825805664,
      "step": 670
    },
    {
      "epoch": 0.3612644254892122,
      "grad_norm": 0.005616316911493808,
      "learning_rate": 9.987396563355205e-07,
      "logits/chosen": 7.4965715408325195,
      "logits/rejected": 7.409778594970703,
      "logps/chosen": -38.291465759277344,
      "logps/rejected": -39.631126403808594,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -38.291465759277344,
      "rewards/margins": 1.3396642208099365,
      "rewards/rejected": -39.631126403808594,
      "step": 675
    },
    {
      "epoch": 0.36394045827061383,
      "grad_norm": 0.0010070305166587808,
      "learning_rate": 9.986267271350631e-07,
      "logits/chosen": 7.268815517425537,
      "logits/rejected": 7.02053689956665,
      "logps/chosen": -38.00390625,
      "logps/rejected": -39.53660202026367,
      "loss": 0.0008,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -38.00390625,
      "rewards/margins": 1.5326961278915405,
      "rewards/rejected": -39.53660202026367,
      "step": 680
    },
    {
      "epoch": 0.3666164910520154,
      "grad_norm": 1.1784406355122806e-07,
      "learning_rate": 9.985089602559123e-07,
      "logits/chosen": 7.47506046295166,
      "logits/rejected": 7.009593963623047,
      "logps/chosen": -37.1651725769043,
      "logps/rejected": -39.48485565185547,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -37.1651725769043,
      "rewards/margins": 2.3196816444396973,
      "rewards/rejected": -39.48485565185547,
      "step": 685
    },
    {
      "epoch": 0.369292523833417,
      "grad_norm": 5.7397758747787465e-05,
      "learning_rate": 9.983863568406428e-07,
      "logits/chosen": 7.1291632652282715,
      "logits/rejected": 7.154665946960449,
      "logps/chosen": -37.958702087402344,
      "logps/rejected": -38.281982421875,
      "loss": 0.0003,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -37.958702087402344,
      "rewards/margins": 0.32327860593795776,
      "rewards/rejected": -38.281982421875,
      "step": 690
    },
    {
      "epoch": 0.37196855661481854,
      "grad_norm": 0.001137490490594298,
      "learning_rate": 9.982589180787532e-07,
      "logits/chosen": 7.346639156341553,
      "logits/rejected": 7.143630027770996,
      "logps/chosen": -39.02979278564453,
      "logps/rejected": -39.376888275146484,
      "loss": 0.0,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -39.02979278564453,
      "rewards/margins": 0.3470901846885681,
      "rewards/rejected": -39.376888275146484,
      "step": 695
    },
    {
      "epoch": 0.3746445893962201,
      "grad_norm": 3.7717142908208344e-06,
      "learning_rate": 9.981266452066553e-07,
      "logits/chosen": 7.674281120300293,
      "logits/rejected": 7.2975568771362305,
      "logps/chosen": -37.99858856201172,
      "logps/rejected": -39.65293502807617,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -37.99858856201172,
      "rewards/margins": 1.6543452739715576,
      "rewards/rejected": -39.65293502807617,
      "step": 700
    },
    {
      "epoch": 0.3773206221776217,
      "grad_norm": 0.023676507269363073,
      "learning_rate": 9.979895395076608e-07,
      "logits/chosen": 7.7849016189575195,
      "logits/rejected": 7.240628719329834,
      "logps/chosen": -39.166927337646484,
      "logps/rejected": -40.50483322143555,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.166927337646484,
      "rewards/margins": 1.3379062414169312,
      "rewards/rejected": -40.50483322143555,
      "step": 705
    },
    {
      "epoch": 0.37999665495902324,
      "grad_norm": 5.585276207180237e-06,
      "learning_rate": 9.9784760231197e-07,
      "logits/chosen": 7.600384712219238,
      "logits/rejected": 7.647927284240723,
      "logps/chosen": -39.21758270263672,
      "logps/rejected": -41.73616409301758,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -39.21758270263672,
      "rewards/margins": 2.5185837745666504,
      "rewards/rejected": -41.73616409301758,
      "step": 710
    },
    {
      "epoch": 0.38267268774042484,
      "grad_norm": 0.028625952654836223,
      "learning_rate": 9.97700834996658e-07,
      "logits/chosen": 7.289938926696777,
      "logits/rejected": 7.025412559509277,
      "logps/chosen": -38.9752311706543,
      "logps/rejected": -40.6868782043457,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -38.9752311706543,
      "rewards/margins": 1.7116445302963257,
      "rewards/rejected": -40.6868782043457,
      "step": 715
    },
    {
      "epoch": 0.3853487205218264,
      "grad_norm": 4.456570489655624e-06,
      "learning_rate": 9.97549238985662e-07,
      "logits/chosen": 7.541611671447754,
      "logits/rejected": 7.062547206878662,
      "logps/chosen": -38.65824890136719,
      "logps/rejected": -40.66016387939453,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -38.65824890136719,
      "rewards/margins": 2.0019166469573975,
      "rewards/rejected": -40.66016387939453,
      "step": 720
    },
    {
      "epoch": 0.38802475330322794,
      "grad_norm": 4.812314844934354e-08,
      "learning_rate": 9.973928157497674e-07,
      "logits/chosen": 7.8245439529418945,
      "logits/rejected": 7.413116455078125,
      "logps/chosen": -38.85778045654297,
      "logps/rejected": -40.64165115356445,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -38.85778045654297,
      "rewards/margins": 1.7838714122772217,
      "rewards/rejected": -40.64165115356445,
      "step": 725
    },
    {
      "epoch": 0.39070078608462955,
      "grad_norm": 0.00013872073437120846,
      "learning_rate": 9.972315668065927e-07,
      "logits/chosen": 7.691391944885254,
      "logits/rejected": 7.472957611083984,
      "logps/chosen": -40.77444076538086,
      "logps/rejected": -41.11881637573242,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -40.77444076538086,
      "rewards/margins": 0.34437984228134155,
      "rewards/rejected": -41.11881637573242,
      "step": 730
    },
    {
      "epoch": 0.3933768188660311,
      "grad_norm": 0.0008878885392751228,
      "learning_rate": 9.97065493720576e-07,
      "logits/chosen": 7.598130702972412,
      "logits/rejected": 7.447795867919922,
      "logps/chosen": -37.833473205566406,
      "logps/rejected": -38.955223083496094,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -37.833473205566406,
      "rewards/margins": 1.121758222579956,
      "rewards/rejected": -38.955223083496094,
      "step": 735
    },
    {
      "epoch": 0.3960528516474327,
      "grad_norm": 1.375875786483902e-07,
      "learning_rate": 9.968945981029594e-07,
      "logits/chosen": 7.650731563568115,
      "logits/rejected": 7.378519535064697,
      "logps/chosen": -37.884193420410156,
      "logps/rejected": -40.358375549316406,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -37.884193420410156,
      "rewards/margins": 2.4741861820220947,
      "rewards/rejected": -40.358375549316406,
      "step": 740
    },
    {
      "epoch": 0.39872888442883425,
      "grad_norm": 1.903459202430746e-06,
      "learning_rate": 9.967188816117726e-07,
      "logits/chosen": 7.605856895446777,
      "logits/rejected": 7.532936096191406,
      "logps/chosen": -38.794044494628906,
      "logps/rejected": -42.311805725097656,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -38.794044494628906,
      "rewards/margins": 3.517763614654541,
      "rewards/rejected": -42.311805725097656,
      "step": 745
    },
    {
      "epoch": 0.4014049172102358,
      "grad_norm": 2.2909880648621647e-05,
      "learning_rate": 9.965383459518179e-07,
      "logits/chosen": 7.405412197113037,
      "logits/rejected": 7.279669761657715,
      "logps/chosen": -37.94025421142578,
      "logps/rejected": -40.590606689453125,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -37.94025421142578,
      "rewards/margins": 2.6503536701202393,
      "rewards/rejected": -40.590606689453125,
      "step": 750
    },
    {
      "epoch": 0.4040809499916374,
      "grad_norm": 4.309641032744545e-08,
      "learning_rate": 9.963529928746533e-07,
      "logits/chosen": 7.635949611663818,
      "logits/rejected": 7.175973415374756,
      "logps/chosen": -37.675777435302734,
      "logps/rejected": -39.88976287841797,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -37.675777435302734,
      "rewards/margins": 2.213982105255127,
      "rewards/rejected": -39.88976287841797,
      "step": 755
    },
    {
      "epoch": 0.40675698277303896,
      "grad_norm": 0.00017627563731633056,
      "learning_rate": 9.961628241785746e-07,
      "logits/chosen": 7.488482475280762,
      "logits/rejected": 7.366297245025635,
      "logps/chosen": -38.159339904785156,
      "logps/rejected": -39.05756759643555,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -38.159339904785156,
      "rewards/margins": 0.898226261138916,
      "rewards/rejected": -39.05756759643555,
      "step": 760
    },
    {
      "epoch": 0.40943301555444056,
      "grad_norm": 0.001246448017112297,
      "learning_rate": 9.959678417085998e-07,
      "logits/chosen": 7.340538024902344,
      "logits/rejected": 7.280026435852051,
      "logps/chosen": -38.07993698120117,
      "logps/rejected": -39.67035675048828,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -38.07993698120117,
      "rewards/margins": 1.5904178619384766,
      "rewards/rejected": -39.67035675048828,
      "step": 765
    },
    {
      "epoch": 0.4121090483358421,
      "grad_norm": 3.1148264388655873e-08,
      "learning_rate": 9.957680473564493e-07,
      "logits/chosen": 7.929078102111816,
      "logits/rejected": 7.6863861083984375,
      "logps/chosen": -40.12217330932617,
      "logps/rejected": -42.769569396972656,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -40.12217330932617,
      "rewards/margins": 2.6473898887634277,
      "rewards/rejected": -42.769569396972656,
      "step": 770
    },
    {
      "epoch": 0.41478508111724366,
      "grad_norm": 0.001449580236483943,
      "learning_rate": 9.95563443060529e-07,
      "logits/chosen": 7.885899543762207,
      "logits/rejected": 7.405299186706543,
      "logps/chosen": -38.058937072753906,
      "logps/rejected": -40.24571990966797,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -38.058937072753906,
      "rewards/margins": 2.186777353286743,
      "rewards/rejected": -40.24571990966797,
      "step": 775
    },
    {
      "epoch": 0.41746111389864526,
      "grad_norm": 8.127610342179473e-06,
      "learning_rate": 9.95354030805911e-07,
      "logits/chosen": 7.651318550109863,
      "logits/rejected": 7.276161193847656,
      "logps/chosen": -40.11216354370117,
      "logps/rejected": -41.697166442871094,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -40.11216354370117,
      "rewards/margins": 1.5850043296813965,
      "rewards/rejected": -41.697166442871094,
      "step": 780
    },
    {
      "epoch": 0.4201371466800468,
      "grad_norm": 6.568960805962362e-05,
      "learning_rate": 9.951398126243133e-07,
      "logits/chosen": 7.567418575286865,
      "logits/rejected": 7.433699607849121,
      "logps/chosen": -39.13808822631836,
      "logps/rejected": -40.28246307373047,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -39.13808822631836,
      "rewards/margins": 1.1443712711334229,
      "rewards/rejected": -40.28246307373047,
      "step": 785
    },
    {
      "epoch": 0.4228131794614484,
      "grad_norm": 0.00010688381123525779,
      "learning_rate": 9.94920790594082e-07,
      "logits/chosen": 7.681375980377197,
      "logits/rejected": 7.550932884216309,
      "logps/chosen": -39.68568801879883,
      "logps/rejected": -41.98265838623047,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.68568801879883,
      "rewards/margins": 2.296964168548584,
      "rewards/rejected": -41.98265838623047,
      "step": 790
    },
    {
      "epoch": 0.42548921224284997,
      "grad_norm": 2.0419807470476298e-06,
      "learning_rate": 9.946969668401696e-07,
      "logits/chosen": 7.568524360656738,
      "logits/rejected": 7.054306983947754,
      "logps/chosen": -38.40340042114258,
      "logps/rejected": -39.81110763549805,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -38.40340042114258,
      "rewards/margins": 1.4077152013778687,
      "rewards/rejected": -39.81110763549805,
      "step": 795
    },
    {
      "epoch": 0.4281652450242516,
      "grad_norm": 1.1969880987285123e-10,
      "learning_rate": 9.944683435341155e-07,
      "logits/chosen": 7.648104190826416,
      "logits/rejected": 7.492392539978027,
      "logps/chosen": -40.38782501220703,
      "logps/rejected": -41.098594665527344,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -40.38782501220703,
      "rewards/margins": 0.7107691168785095,
      "rewards/rejected": -41.098594665527344,
      "step": 800
    },
    {
      "epoch": 0.4281652450242516,
      "eval_logits/chosen": 6.387713432312012,
      "eval_logits/rejected": 6.234112739562988,
      "eval_logps/chosen": -38.75341033935547,
      "eval_logps/rejected": -40.20651626586914,
      "eval_loss": 8.201554919651244e-06,
      "eval_rewards/accuracies": 0.5593471527099609,
      "eval_rewards/chosen": -38.75341033935547,
      "eval_rewards/margins": 1.453106164932251,
      "eval_rewards/rejected": -40.20651626586914,
      "eval_runtime": 39.9831,
      "eval_samples_per_second": 33.639,
      "eval_steps_per_second": 8.429,
      "step": 800
    },
    {
      "epoch": 0.4308412778056531,
      "grad_norm": 0.00214393589125904,
      "learning_rate": 9.942349228940236e-07,
      "logits/chosen": 7.632666110992432,
      "logits/rejected": 7.378471374511719,
      "logps/chosen": -37.2575798034668,
      "logps/rejected": -41.00340270996094,
      "loss": 0.0,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -37.2575798034668,
      "rewards/margins": 3.7458271980285645,
      "rewards/rejected": -41.00340270996094,
      "step": 805
    },
    {
      "epoch": 0.43351731058705467,
      "grad_norm": 3.483609076622263e-10,
      "learning_rate": 9.939967071845424e-07,
      "logits/chosen": 7.6920166015625,
      "logits/rejected": 7.619635581970215,
      "logps/chosen": -39.262939453125,
      "logps/rejected": -40.47943878173828,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -39.262939453125,
      "rewards/margins": 1.216496229171753,
      "rewards/rejected": -40.47943878173828,
      "step": 810
    },
    {
      "epoch": 0.4361933433684563,
      "grad_norm": 1.9992802857725255e-08,
      "learning_rate": 9.937536987168413e-07,
      "logits/chosen": 7.390772342681885,
      "logits/rejected": 7.155269622802734,
      "logps/chosen": -40.3416862487793,
      "logps/rejected": -42.98020935058594,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.3416862487793,
      "rewards/margins": 2.638521671295166,
      "rewards/rejected": -42.98020935058594,
      "step": 815
    },
    {
      "epoch": 0.4388693761498578,
      "grad_norm": 2.5784795679496942e-06,
      "learning_rate": 9.935058998485896e-07,
      "logits/chosen": 7.640423774719238,
      "logits/rejected": 7.675130367279053,
      "logps/chosen": -41.21143341064453,
      "logps/rejected": -41.54305648803711,
      "loss": 0.0,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -41.21143341064453,
      "rewards/margins": 0.33162397146224976,
      "rewards/rejected": -41.54305648803711,
      "step": 820
    },
    {
      "epoch": 0.44154540893125943,
      "grad_norm": 6.1167614225078975e-06,
      "learning_rate": 9.932533129839333e-07,
      "logits/chosen": 8.059869766235352,
      "logits/rejected": 7.694645881652832,
      "logps/chosen": -40.091739654541016,
      "logps/rejected": -41.312564849853516,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -40.091739654541016,
      "rewards/margins": 1.2208276987075806,
      "rewards/rejected": -41.312564849853516,
      "step": 825
    },
    {
      "epoch": 0.444221441712661,
      "grad_norm": 0.005694760619745585,
      "learning_rate": 9.929959405734711e-07,
      "logits/chosen": 7.445316314697266,
      "logits/rejected": 7.297863960266113,
      "logps/chosen": -37.77349090576172,
      "logps/rejected": -39.21232223510742,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -37.77349090576172,
      "rewards/margins": 1.4388357400894165,
      "rewards/rejected": -39.21232223510742,
      "step": 830
    },
    {
      "epoch": 0.44689747449406253,
      "grad_norm": 5.9970507754520306e-06,
      "learning_rate": 9.927337851142314e-07,
      "logits/chosen": 7.840022087097168,
      "logits/rejected": 7.5967512130737305,
      "logps/chosen": -39.84214401245117,
      "logps/rejected": -40.91309356689453,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -39.84214401245117,
      "rewards/margins": 1.0709518194198608,
      "rewards/rejected": -40.91309356689453,
      "step": 835
    },
    {
      "epoch": 0.44957350727546413,
      "grad_norm": 3.0843714277592874e-09,
      "learning_rate": 9.924668491496474e-07,
      "logits/chosen": 7.4397172927856445,
      "logits/rejected": 6.968742370605469,
      "logps/chosen": -39.052711486816406,
      "logps/rejected": -41.229530334472656,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -39.052711486816406,
      "rewards/margins": 2.176821231842041,
      "rewards/rejected": -41.229530334472656,
      "step": 840
    },
    {
      "epoch": 0.4522495400568657,
      "grad_norm": 1.4827005506824818e-05,
      "learning_rate": 9.92195135269533e-07,
      "logits/chosen": 7.5393242835998535,
      "logits/rejected": 7.576888084411621,
      "logps/chosen": -39.29246139526367,
      "logps/rejected": -41.3503303527832,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -39.29246139526367,
      "rewards/margins": 2.057870626449585,
      "rewards/rejected": -41.3503303527832,
      "step": 845
    },
    {
      "epoch": 0.4549255728382673,
      "grad_norm": 4.145789039731052e-07,
      "learning_rate": 9.919186461100574e-07,
      "logits/chosen": 7.906198024749756,
      "logits/rejected": 7.591418266296387,
      "logps/chosen": -39.675418853759766,
      "logps/rejected": -41.53825759887695,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -39.675418853759766,
      "rewards/margins": 1.862841248512268,
      "rewards/rejected": -41.53825759887695,
      "step": 850
    },
    {
      "epoch": 0.45760160561966884,
      "grad_norm": 0.15688934539151994,
      "learning_rate": 9.9163738435372e-07,
      "logits/chosen": 7.2153449058532715,
      "logits/rejected": 7.003320217132568,
      "logps/chosen": -37.39909744262695,
      "logps/rejected": -40.19093704223633,
      "loss": 0.0007,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -37.39909744262695,
      "rewards/margins": 2.791842222213745,
      "rewards/rejected": -40.19093704223633,
      "step": 855
    },
    {
      "epoch": 0.4602776384010704,
      "grad_norm": 1.55863922182974e-05,
      "learning_rate": 9.913513527293234e-07,
      "logits/chosen": 7.881860256195068,
      "logits/rejected": 7.732068061828613,
      "logps/chosen": -38.326210021972656,
      "logps/rejected": -40.46181106567383,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -38.326210021972656,
      "rewards/margins": 2.135598659515381,
      "rewards/rejected": -40.46181106567383,
      "step": 860
    },
    {
      "epoch": 0.462953671182472,
      "grad_norm": 2.5832084342473995e-08,
      "learning_rate": 9.910605540119474e-07,
      "logits/chosen": 7.957991600036621,
      "logits/rejected": 7.52456521987915,
      "logps/chosen": -41.4791145324707,
      "logps/rejected": -42.26188659667969,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -41.4791145324707,
      "rewards/margins": 0.7827737331390381,
      "rewards/rejected": -42.26188659667969,
      "step": 865
    },
    {
      "epoch": 0.46562970396387354,
      "grad_norm": 1.243215861243637e-07,
      "learning_rate": 9.907649910229227e-07,
      "logits/chosen": 7.661241054534912,
      "logits/rejected": 7.384352684020996,
      "logps/chosen": -38.367706298828125,
      "logps/rejected": -41.336151123046875,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -38.367706298828125,
      "rewards/margins": 2.9684455394744873,
      "rewards/rejected": -41.336151123046875,
      "step": 870
    },
    {
      "epoch": 0.46830573674527515,
      "grad_norm": 2.0580533028112043e-06,
      "learning_rate": 9.90464666629803e-07,
      "logits/chosen": 7.8262434005737305,
      "logits/rejected": 7.625802040100098,
      "logps/chosen": -39.67253875732422,
      "logps/rejected": -40.37131118774414,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -39.67253875732422,
      "rewards/margins": 0.6987723112106323,
      "rewards/rejected": -40.37131118774414,
      "step": 875
    },
    {
      "epoch": 0.4709817695266767,
      "grad_norm": 5.187332082791037e-05,
      "learning_rate": 9.901595837463363e-07,
      "logits/chosen": 7.366092681884766,
      "logits/rejected": 7.230430603027344,
      "logps/chosen": -37.39326858520508,
      "logps/rejected": -39.85638427734375,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -37.39326858520508,
      "rewards/margins": 2.463115692138672,
      "rewards/rejected": -39.85638427734375,
      "step": 880
    },
    {
      "epoch": 0.47365780230807825,
      "grad_norm": 1.4995196630825657e-07,
      "learning_rate": 9.898497453324384e-07,
      "logits/chosen": 7.840872287750244,
      "logits/rejected": 7.809952735900879,
      "logps/chosen": -40.72235870361328,
      "logps/rejected": -42.15755081176758,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.72235870361328,
      "rewards/margins": 1.4351966381072998,
      "rewards/rejected": -42.15755081176758,
      "step": 885
    },
    {
      "epoch": 0.47633383508947985,
      "grad_norm": 3.38311064842543e-06,
      "learning_rate": 9.895351543941628e-07,
      "logits/chosen": 7.9592180252075195,
      "logits/rejected": 7.718374729156494,
      "logps/chosen": -39.50871658325195,
      "logps/rejected": -41.48107147216797,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -39.50871658325195,
      "rewards/margins": 1.9723562002182007,
      "rewards/rejected": -41.48107147216797,
      "step": 890
    },
    {
      "epoch": 0.4790098678708814,
      "grad_norm": 6.420253214415502e-09,
      "learning_rate": 9.892158139836724e-07,
      "logits/chosen": 7.805755615234375,
      "logits/rejected": 7.631558418273926,
      "logps/chosen": -40.610618591308594,
      "logps/rejected": -41.32982635498047,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -40.610618591308594,
      "rewards/margins": 0.7192095518112183,
      "rewards/rejected": -41.32982635498047,
      "step": 895
    },
    {
      "epoch": 0.481685900652283,
      "grad_norm": 9.858749346963943e-06,
      "learning_rate": 9.88891727199209e-07,
      "logits/chosen": 7.969757080078125,
      "logits/rejected": 7.848060607910156,
      "logps/chosen": -41.07607650756836,
      "logps/rejected": -43.005699157714844,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -41.07607650756836,
      "rewards/margins": 1.9296247959136963,
      "rewards/rejected": -43.005699157714844,
      "step": 900
    },
    {
      "epoch": 0.48436193343368455,
      "grad_norm": 0.0005228483338045444,
      "learning_rate": 9.885628971850641e-07,
      "logits/chosen": 7.909764766693115,
      "logits/rejected": 7.381610870361328,
      "logps/chosen": -39.787025451660156,
      "logps/rejected": -41.198184967041016,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -39.787025451660156,
      "rewards/margins": 1.4111547470092773,
      "rewards/rejected": -41.198184967041016,
      "step": 905
    },
    {
      "epoch": 0.48703796621508616,
      "grad_norm": 6.583192239489562e-10,
      "learning_rate": 9.882293271315481e-07,
      "logits/chosen": 7.572544097900391,
      "logits/rejected": 7.360169887542725,
      "logps/chosen": -39.28374481201172,
      "logps/rejected": -40.5693359375,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -39.28374481201172,
      "rewards/margins": 1.2855923175811768,
      "rewards/rejected": -40.5693359375,
      "step": 910
    },
    {
      "epoch": 0.4897139989964877,
      "grad_norm": 0.0005621920168361672,
      "learning_rate": 9.878910202749589e-07,
      "logits/chosen": 7.92510461807251,
      "logits/rejected": 7.582913398742676,
      "logps/chosen": -38.3166618347168,
      "logps/rejected": -40.33130645751953,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -38.3166618347168,
      "rewards/margins": 2.0146424770355225,
      "rewards/rejected": -40.33130645751953,
      "step": 915
    },
    {
      "epoch": 0.49239003177788926,
      "grad_norm": 3.108711492851123e-05,
      "learning_rate": 9.875479798975512e-07,
      "logits/chosen": 7.561453342437744,
      "logits/rejected": 7.001914978027344,
      "logps/chosen": -40.34862518310547,
      "logps/rejected": -42.33332061767578,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.34862518310547,
      "rewards/margins": 1.984697699546814,
      "rewards/rejected": -42.33332061767578,
      "step": 920
    },
    {
      "epoch": 0.49506606455929086,
      "grad_norm": 4.3137686508013296e-08,
      "learning_rate": 9.87200209327504e-07,
      "logits/chosen": 7.756928443908691,
      "logits/rejected": 7.319848537445068,
      "logps/chosen": -39.95264434814453,
      "logps/rejected": -41.978492736816406,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -39.95264434814453,
      "rewards/margins": 2.0258522033691406,
      "rewards/rejected": -41.978492736816406,
      "step": 925
    },
    {
      "epoch": 0.4977420973406924,
      "grad_norm": 7.884714156512496e-11,
      "learning_rate": 9.868477119388894e-07,
      "logits/chosen": 7.5992255210876465,
      "logits/rejected": 7.4619340896606445,
      "logps/chosen": -39.18184280395508,
      "logps/rejected": -40.127540588378906,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -39.18184280395508,
      "rewards/margins": 0.9456941485404968,
      "rewards/rejected": -40.127540588378906,
      "step": 930
    },
    {
      "epoch": 0.500418130122094,
      "grad_norm": 1.7873872620721694e-05,
      "learning_rate": 9.864904911516383e-07,
      "logits/chosen": 8.143692016601562,
      "logits/rejected": 7.941512107849121,
      "logps/chosen": -40.5237922668457,
      "logps/rejected": -42.04798126220703,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -40.5237922668457,
      "rewards/margins": 1.5241888761520386,
      "rewards/rejected": -42.04798126220703,
      "step": 935
    },
    {
      "epoch": 0.5030941629034956,
      "grad_norm": 0.010245692881493693,
      "learning_rate": 9.861285504315084e-07,
      "logits/chosen": 7.608607292175293,
      "logits/rejected": 7.429617404937744,
      "logps/chosen": -40.089881896972656,
      "logps/rejected": -41.561378479003906,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -40.089881896972656,
      "rewards/margins": 1.4714927673339844,
      "rewards/rejected": -41.561378479003906,
      "step": 940
    },
    {
      "epoch": 0.5057701956848971,
      "grad_norm": 4.448162057980245e-06,
      "learning_rate": 9.857618932900502e-07,
      "logits/chosen": 8.184416770935059,
      "logits/rejected": 7.763899326324463,
      "logps/chosen": -40.53358840942383,
      "logps/rejected": -42.84503936767578,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -40.53358840942383,
      "rewards/margins": 2.3114514350891113,
      "rewards/rejected": -42.84503936767578,
      "step": 945
    },
    {
      "epoch": 0.5084462284662987,
      "grad_norm": 0.002655095801335887,
      "learning_rate": 9.853905232845727e-07,
      "logits/chosen": 7.779921054840088,
      "logits/rejected": 7.576930046081543,
      "logps/chosen": -39.01407241821289,
      "logps/rejected": -41.28191375732422,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -39.01407241821289,
      "rewards/margins": 2.2678425312042236,
      "rewards/rejected": -41.28191375732422,
      "step": 950
    },
    {
      "epoch": 0.5111222612477003,
      "grad_norm": 2.688067734787641e-07,
      "learning_rate": 9.850144440181095e-07,
      "logits/chosen": 7.699735164642334,
      "logits/rejected": 7.403003692626953,
      "logps/chosen": -38.741798400878906,
      "logps/rejected": -41.49602127075195,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -38.741798400878906,
      "rewards/margins": 2.7542200088500977,
      "rewards/rejected": -41.49602127075195,
      "step": 955
    },
    {
      "epoch": 0.5137982940291018,
      "grad_norm": 1.0423539830007132e-05,
      "learning_rate": 9.846336591393832e-07,
      "logits/chosen": 7.773317813873291,
      "logits/rejected": 7.4539947509765625,
      "logps/chosen": -39.26493453979492,
      "logps/rejected": -40.9766845703125,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.26493453979492,
      "rewards/margins": 1.7117456197738647,
      "rewards/rejected": -40.9766845703125,
      "step": 960
    },
    {
      "epoch": 0.5164743268105034,
      "grad_norm": 5.929284977300872e-09,
      "learning_rate": 9.842481723427704e-07,
      "logits/chosen": 7.601541996002197,
      "logits/rejected": 7.587291717529297,
      "logps/chosen": -38.767662048339844,
      "logps/rejected": -40.391136169433594,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -38.767662048339844,
      "rewards/margins": 1.6234767436981201,
      "rewards/rejected": -40.391136169433594,
      "step": 965
    },
    {
      "epoch": 0.519150359591905,
      "grad_norm": 0.00012789742107946277,
      "learning_rate": 9.838579873682658e-07,
      "logits/chosen": 7.497960090637207,
      "logits/rejected": 7.488619804382324,
      "logps/chosen": -38.4771728515625,
      "logps/rejected": -39.034366607666016,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -38.4771728515625,
      "rewards/margins": 0.5571956038475037,
      "rewards/rejected": -39.034366607666016,
      "step": 970
    },
    {
      "epoch": 0.5218263923733065,
      "grad_norm": 8.22329849715888e-07,
      "learning_rate": 9.834631080014457e-07,
      "logits/chosen": 8.051076889038086,
      "logits/rejected": 7.555508613586426,
      "logps/chosen": -39.85700607299805,
      "logps/rejected": -40.69036865234375,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -39.85700607299805,
      "rewards/margins": 0.833358883857727,
      "rewards/rejected": -40.69036865234375,
      "step": 975
    },
    {
      "epoch": 0.5245024251547081,
      "grad_norm": 1.152067121332151e-05,
      "learning_rate": 9.830635380734312e-07,
      "logits/chosen": 8.047284126281738,
      "logits/rejected": 7.717115879058838,
      "logps/chosen": -39.85627746582031,
      "logps/rejected": -42.298194885253906,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -39.85627746582031,
      "rewards/margins": 2.441920042037964,
      "rewards/rejected": -42.298194885253906,
      "step": 980
    },
    {
      "epoch": 0.5271784579361097,
      "grad_norm": 0.0003358713395731031,
      "learning_rate": 9.826592814608517e-07,
      "logits/chosen": 8.105627059936523,
      "logits/rejected": 7.619837760925293,
      "logps/chosen": -41.140541076660156,
      "logps/rejected": -42.03690719604492,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -41.140541076660156,
      "rewards/margins": 0.8963673710823059,
      "rewards/rejected": -42.03690719604492,
      "step": 985
    },
    {
      "epoch": 0.5298544907175113,
      "grad_norm": 5.507470109533997e-10,
      "learning_rate": 9.822503420858067e-07,
      "logits/chosen": 7.712205410003662,
      "logits/rejected": 7.816235542297363,
      "logps/chosen": -40.780723571777344,
      "logps/rejected": -40.14775848388672,
      "loss": 0.0,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -40.780723571777344,
      "rewards/margins": -0.632966160774231,
      "rewards/rejected": -40.14775848388672,
      "step": 990
    },
    {
      "epoch": 0.5325305234989128,
      "grad_norm": 3.911270487481136e-06,
      "learning_rate": 9.818367239158277e-07,
      "logits/chosen": 7.550829887390137,
      "logits/rejected": 7.4847893714904785,
      "logps/chosen": -40.165138244628906,
      "logps/rejected": -40.12540817260742,
      "loss": 0.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -40.165138244628906,
      "rewards/margins": -0.03972914069890976,
      "rewards/rejected": -40.12540817260742,
      "step": 995
    },
    {
      "epoch": 0.5352065562803144,
      "grad_norm": 2.4753534141633786e-07,
      "learning_rate": 9.8141843096384e-07,
      "logits/chosen": 7.896836757659912,
      "logits/rejected": 7.35595703125,
      "logps/chosen": -39.200294494628906,
      "logps/rejected": -41.045631408691406,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -39.200294494628906,
      "rewards/margins": 1.8453350067138672,
      "rewards/rejected": -41.045631408691406,
      "step": 1000
    },
    {
      "epoch": 0.537882589061716,
      "grad_norm": 0.007206850558714663,
      "learning_rate": 9.809954672881237e-07,
      "logits/chosen": 7.7587175369262695,
      "logits/rejected": 7.365087032318115,
      "logps/chosen": -40.33055114746094,
      "logps/rejected": -40.973854064941406,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -40.33055114746094,
      "rewards/margins": 0.6433035135269165,
      "rewards/rejected": -40.973854064941406,
      "step": 1005
    },
    {
      "epoch": 0.5405586218431175,
      "grad_norm": 2.39664881314602e-05,
      "learning_rate": 9.80567836992274e-07,
      "logits/chosen": 8.1408052444458,
      "logits/rejected": 7.902805328369141,
      "logps/chosen": -39.11881637573242,
      "logps/rejected": -41.7402458190918,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -39.11881637573242,
      "rewards/margins": 2.6214280128479004,
      "rewards/rejected": -41.7402458190918,
      "step": 1010
    },
    {
      "epoch": 0.5432346546245191,
      "grad_norm": 8.667854131788611e-07,
      "learning_rate": 9.801355442251625e-07,
      "logits/chosen": 8.337836265563965,
      "logits/rejected": 7.815174102783203,
      "logps/chosen": -39.21738815307617,
      "logps/rejected": -40.245689392089844,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -39.21738815307617,
      "rewards/margins": 1.028298258781433,
      "rewards/rejected": -40.245689392089844,
      "step": 1015
    },
    {
      "epoch": 0.5459106874059207,
      "grad_norm": 1.0404455785104034e-06,
      "learning_rate": 9.796985931808949e-07,
      "logits/chosen": 8.048584938049316,
      "logits/rejected": 7.759082794189453,
      "logps/chosen": -39.96399688720703,
      "logps/rejected": -42.24923324584961,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.96399688720703,
      "rewards/margins": 2.2852401733398438,
      "rewards/rejected": -42.24923324584961,
      "step": 1020
    },
    {
      "epoch": 0.5485867201873222,
      "grad_norm": 0.00010432664694382001,
      "learning_rate": 9.792569880987724e-07,
      "logits/chosen": 8.046297073364258,
      "logits/rejected": 7.577686309814453,
      "logps/chosen": -40.54745101928711,
      "logps/rejected": -41.47987365722656,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -40.54745101928711,
      "rewards/margins": 0.932418704032898,
      "rewards/rejected": -41.47987365722656,
      "step": 1025
    },
    {
      "epoch": 0.5512627529687238,
      "grad_norm": 0.28943102559906186,
      "learning_rate": 9.788107332632493e-07,
      "logits/chosen": 7.854372501373291,
      "logits/rejected": 8.16811752319336,
      "logps/chosen": -40.36214828491211,
      "logps/rejected": -42.3426628112793,
      "loss": 0.0017,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -40.36214828491211,
      "rewards/margins": 1.9805189371109009,
      "rewards/rejected": -42.3426628112793,
      "step": 1030
    },
    {
      "epoch": 0.5539387857501255,
      "grad_norm": 0.000248098949503976,
      "learning_rate": 9.783598330038924e-07,
      "logits/chosen": 8.188563346862793,
      "logits/rejected": 7.961118221282959,
      "logps/chosen": -39.57174301147461,
      "logps/rejected": -41.35323715209961,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -39.57174301147461,
      "rewards/margins": 1.781496286392212,
      "rewards/rejected": -41.35323715209961,
      "step": 1035
    },
    {
      "epoch": 0.5566148185315271,
      "grad_norm": 1.5239409872294668e-09,
      "learning_rate": 9.779042916953376e-07,
      "logits/chosen": 7.630479335784912,
      "logits/rejected": 7.0815324783325195,
      "logps/chosen": -39.128868103027344,
      "logps/rejected": -40.38596725463867,
      "loss": 0.0001,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -39.128868103027344,
      "rewards/margins": 1.2571032047271729,
      "rewards/rejected": -40.38596725463867,
      "step": 1040
    },
    {
      "epoch": 0.5592908513129285,
      "grad_norm": 1.1725176273331583e-06,
      "learning_rate": 9.774441137572487e-07,
      "logits/chosen": 7.857902526855469,
      "logits/rejected": 7.599123954772949,
      "logps/chosen": -40.07386016845703,
      "logps/rejected": -42.12800216674805,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -40.07386016845703,
      "rewards/margins": 2.0541446208953857,
      "rewards/rejected": -42.12800216674805,
      "step": 1045
    },
    {
      "epoch": 0.5619668840943302,
      "grad_norm": 0.00017924127321663645,
      "learning_rate": 9.76979303654274e-07,
      "logits/chosen": 7.688004493713379,
      "logits/rejected": 7.365685939788818,
      "logps/chosen": -39.148128509521484,
      "logps/rejected": -40.32774353027344,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -39.148128509521484,
      "rewards/margins": 1.1796153783798218,
      "rewards/rejected": -40.32774353027344,
      "step": 1050
    },
    {
      "epoch": 0.5646429168757318,
      "grad_norm": 5.29270178941127e-10,
      "learning_rate": 9.765098658960035e-07,
      "logits/chosen": 8.039278030395508,
      "logits/rejected": 8.221173286437988,
      "logps/chosen": -40.57965087890625,
      "logps/rejected": -42.73926544189453,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.57965087890625,
      "rewards/margins": 2.159611463546753,
      "rewards/rejected": -42.73926544189453,
      "step": 1055
    },
    {
      "epoch": 0.5673189496571333,
      "grad_norm": 0.11308364236274115,
      "learning_rate": 9.76035805036924e-07,
      "logits/chosen": 8.069418907165527,
      "logits/rejected": 7.796140193939209,
      "logps/chosen": -40.13853454589844,
      "logps/rejected": -42.0114860534668,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -40.13853454589844,
      "rewards/margins": 1.8729581832885742,
      "rewards/rejected": -42.0114860534668,
      "step": 1060
    },
    {
      "epoch": 0.5699949824385349,
      "grad_norm": 0.00023643621203627515,
      "learning_rate": 9.755571256763764e-07,
      "logits/chosen": 8.094942092895508,
      "logits/rejected": 7.95784854888916,
      "logps/chosen": -39.6126594543457,
      "logps/rejected": -41.992401123046875,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -39.6126594543457,
      "rewards/margins": 2.3797473907470703,
      "rewards/rejected": -41.992401123046875,
      "step": 1065
    },
    {
      "epoch": 0.5726710152199365,
      "grad_norm": 9.138497492606896e-08,
      "learning_rate": 9.750738324585097e-07,
      "logits/chosen": 7.91768741607666,
      "logits/rejected": 7.458584785461426,
      "logps/chosen": -39.10829544067383,
      "logps/rejected": -41.588218688964844,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -39.10829544067383,
      "rewards/margins": 2.4799249172210693,
      "rewards/rejected": -41.588218688964844,
      "step": 1070
    },
    {
      "epoch": 0.5753470480013381,
      "grad_norm": 1.3606130571200045e-07,
      "learning_rate": 9.74585930072237e-07,
      "logits/chosen": 8.199902534484863,
      "logits/rejected": 7.93454647064209,
      "logps/chosen": -41.86429214477539,
      "logps/rejected": -44.158424377441406,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -41.86429214477539,
      "rewards/margins": 2.2941322326660156,
      "rewards/rejected": -44.158424377441406,
      "step": 1075
    },
    {
      "epoch": 0.5780230807827396,
      "grad_norm": 0.004733455076170493,
      "learning_rate": 9.740934232511892e-07,
      "logits/chosen": 7.963520050048828,
      "logits/rejected": 7.732601165771484,
      "logps/chosen": -40.29644012451172,
      "logps/rejected": -41.31216812133789,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -40.29644012451172,
      "rewards/margins": 1.0157251358032227,
      "rewards/rejected": -41.31216812133789,
      "step": 1080
    },
    {
      "epoch": 0.5806991135641412,
      "grad_norm": 4.311333610936151e-08,
      "learning_rate": 9.735963167736698e-07,
      "logits/chosen": 8.359457015991211,
      "logits/rejected": 8.06285572052002,
      "logps/chosen": -41.02088165283203,
      "logps/rejected": -42.714111328125,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -41.02088165283203,
      "rewards/margins": 1.6932319402694702,
      "rewards/rejected": -42.714111328125,
      "step": 1085
    },
    {
      "epoch": 0.5833751463455428,
      "grad_norm": 8.884636640611937e-08,
      "learning_rate": 9.730946154626078e-07,
      "logits/chosen": 8.117534637451172,
      "logits/rejected": 7.6370673179626465,
      "logps/chosen": -40.9040641784668,
      "logps/rejected": -41.58605194091797,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -40.9040641784668,
      "rewards/margins": 0.6819964647293091,
      "rewards/rejected": -41.58605194091797,
      "step": 1090
    },
    {
      "epoch": 0.5860511791269443,
      "grad_norm": 9.497692060367303e-07,
      "learning_rate": 9.725883241855117e-07,
      "logits/chosen": 8.558547973632812,
      "logits/rejected": 8.142523765563965,
      "logps/chosen": -41.9089469909668,
      "logps/rejected": -43.25872039794922,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.9089469909668,
      "rewards/margins": 1.3497748374938965,
      "rewards/rejected": -43.25872039794922,
      "step": 1095
    },
    {
      "epoch": 0.5887272119083459,
      "grad_norm": 0.6910948754256523,
      "learning_rate": 9.720774478544218e-07,
      "logits/chosen": 8.081572532653809,
      "logits/rejected": 7.713902950286865,
      "logps/chosen": -41.470645904541016,
      "logps/rejected": -41.90363693237305,
      "loss": 0.0002,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -41.470645904541016,
      "rewards/margins": 0.4329954981803894,
      "rewards/rejected": -41.90363693237305,
      "step": 1100
    },
    {
      "epoch": 0.5914032446897475,
      "grad_norm": 5.2846744522499125e-06,
      "learning_rate": 9.715619914258624e-07,
      "logits/chosen": 8.419504165649414,
      "logits/rejected": 8.27764892578125,
      "logps/chosen": -41.308631896972656,
      "logps/rejected": -41.972557067871094,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.308631896972656,
      "rewards/margins": 0.6639317274093628,
      "rewards/rejected": -41.972557067871094,
      "step": 1105
    },
    {
      "epoch": 0.594079277471149,
      "grad_norm": 1.3241557852868006e-05,
      "learning_rate": 9.710419599007937e-07,
      "logits/chosen": 7.523890495300293,
      "logits/rejected": 7.138746738433838,
      "logps/chosen": -39.12733459472656,
      "logps/rejected": -39.64752197265625,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -39.12733459472656,
      "rewards/margins": 0.5201890468597412,
      "rewards/rejected": -39.64752197265625,
      "step": 1110
    },
    {
      "epoch": 0.5967553102525506,
      "grad_norm": 2.491677341054301e-05,
      "learning_rate": 9.705173583245643e-07,
      "logits/chosen": 7.876932621002197,
      "logits/rejected": 7.516255855560303,
      "logps/chosen": -39.9119873046875,
      "logps/rejected": -42.42407989501953,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -39.9119873046875,
      "rewards/margins": 2.5120937824249268,
      "rewards/rejected": -42.42407989501953,
      "step": 1115
    },
    {
      "epoch": 0.5994313430339522,
      "grad_norm": 1.2308072415891968e-05,
      "learning_rate": 9.699881917868609e-07,
      "logits/chosen": 7.379809379577637,
      "logits/rejected": 7.197291374206543,
      "logps/chosen": -36.56322479248047,
      "logps/rejected": -38.758880615234375,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -36.56322479248047,
      "rewards/margins": 2.1956562995910645,
      "rewards/rejected": -38.758880615234375,
      "step": 1120
    },
    {
      "epoch": 0.6021073758153538,
      "grad_norm": 0.0011634921608723325,
      "learning_rate": 9.694544654216594e-07,
      "logits/chosen": 6.991583824157715,
      "logits/rejected": 6.620404243469238,
      "logps/chosen": -38.428184509277344,
      "logps/rejected": -40.102535247802734,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -38.428184509277344,
      "rewards/margins": 1.6743491888046265,
      "rewards/rejected": -40.102535247802734,
      "step": 1125
    },
    {
      "epoch": 0.6047834085967553,
      "grad_norm": 0.007819407186863227,
      "learning_rate": 9.689161844071755e-07,
      "logits/chosen": 6.7202911376953125,
      "logits/rejected": 6.669839382171631,
      "logps/chosen": -37.00651931762695,
      "logps/rejected": -39.640045166015625,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -37.00651931762695,
      "rewards/margins": 2.6335272789001465,
      "rewards/rejected": -39.640045166015625,
      "step": 1130
    },
    {
      "epoch": 0.6074594413781569,
      "grad_norm": 9.630439144913395e-05,
      "learning_rate": 9.683733539658138e-07,
      "logits/chosen": 7.106778621673584,
      "logits/rejected": 6.804064750671387,
      "logps/chosen": -36.53816604614258,
      "logps/rejected": -39.43313980102539,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -36.53816604614258,
      "rewards/margins": 2.8949732780456543,
      "rewards/rejected": -39.43313980102539,
      "step": 1135
    },
    {
      "epoch": 0.6101354741595585,
      "grad_norm": 4.2663898268019295e-05,
      "learning_rate": 9.678259793641178e-07,
      "logits/chosen": 7.419839382171631,
      "logits/rejected": 7.381341457366943,
      "logps/chosen": -39.218299865722656,
      "logps/rejected": -40.048805236816406,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -39.218299865722656,
      "rewards/margins": 0.8305082321166992,
      "rewards/rejected": -40.048805236816406,
      "step": 1140
    },
    {
      "epoch": 0.61281150694096,
      "grad_norm": 1.387342249726653e-08,
      "learning_rate": 9.672740659127183e-07,
      "logits/chosen": 7.587912082672119,
      "logits/rejected": 7.224678039550781,
      "logps/chosen": -38.409088134765625,
      "logps/rejected": -39.208412170410156,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -38.409088134765625,
      "rewards/margins": 0.7993215322494507,
      "rewards/rejected": -39.208412170410156,
      "step": 1145
    },
    {
      "epoch": 0.6154875397223616,
      "grad_norm": 1.8868692864864813e-09,
      "learning_rate": 9.667176189662818e-07,
      "logits/chosen": 7.54209041595459,
      "logits/rejected": 7.219571113586426,
      "logps/chosen": -39.008811950683594,
      "logps/rejected": -40.15082550048828,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -39.008811950683594,
      "rewards/margins": 1.1420154571533203,
      "rewards/rejected": -40.15082550048828,
      "step": 1150
    },
    {
      "epoch": 0.6181635725037632,
      "grad_norm": 5.624134539927692e-05,
      "learning_rate": 9.661566439234592e-07,
      "logits/chosen": 7.303518772125244,
      "logits/rejected": 7.275427341461182,
      "logps/chosen": -38.004661560058594,
      "logps/rejected": -39.50519943237305,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.004661560058594,
      "rewards/margins": 1.5005372762680054,
      "rewards/rejected": -39.50519943237305,
      "step": 1155
    },
    {
      "epoch": 0.6208396052851648,
      "grad_norm": 2.340282285696317e-06,
      "learning_rate": 9.655911462268327e-07,
      "logits/chosen": 7.251467704772949,
      "logits/rejected": 7.144508361816406,
      "logps/chosen": -37.6694221496582,
      "logps/rejected": -40.294578552246094,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -37.6694221496582,
      "rewards/margins": 2.6251559257507324,
      "rewards/rejected": -40.294578552246094,
      "step": 1160
    },
    {
      "epoch": 0.6235156380665663,
      "grad_norm": 3.130506465211095e-07,
      "learning_rate": 9.650211313628636e-07,
      "logits/chosen": 7.5035271644592285,
      "logits/rejected": 7.495081424713135,
      "logps/chosen": -38.59466552734375,
      "logps/rejected": -39.94190216064453,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -38.59466552734375,
      "rewards/margins": 1.3472362756729126,
      "rewards/rejected": -39.94190216064453,
      "step": 1165
    },
    {
      "epoch": 0.6261916708479679,
      "grad_norm": 0.0004657473116419142,
      "learning_rate": 9.644466048618386e-07,
      "logits/chosen": 7.282487392425537,
      "logits/rejected": 6.958624839782715,
      "logps/chosen": -37.960105895996094,
      "logps/rejected": -38.776588439941406,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -37.960105895996094,
      "rewards/margins": 0.8164798021316528,
      "rewards/rejected": -38.776588439941406,
      "step": 1170
    },
    {
      "epoch": 0.6288677036293695,
      "grad_norm": 7.183839328914674e-07,
      "learning_rate": 9.63867572297816e-07,
      "logits/chosen": 7.779725551605225,
      "logits/rejected": 7.4187517166137695,
      "logps/chosen": -39.71879577636719,
      "logps/rejected": -41.02433776855469,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -39.71879577636719,
      "rewards/margins": 1.3055438995361328,
      "rewards/rejected": -41.02433776855469,
      "step": 1175
    },
    {
      "epoch": 0.631543736410771,
      "grad_norm": 2.710140316776621e-08,
      "learning_rate": 9.632840392885727e-07,
      "logits/chosen": 7.609490871429443,
      "logits/rejected": 7.224745273590088,
      "logps/chosen": -37.97541427612305,
      "logps/rejected": -40.044315338134766,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -37.97541427612305,
      "rewards/margins": 2.068896770477295,
      "rewards/rejected": -40.044315338134766,
      "step": 1180
    },
    {
      "epoch": 0.6342197691921726,
      "grad_norm": 0.004658030739156868,
      "learning_rate": 9.626960114955483e-07,
      "logits/chosen": 7.618990898132324,
      "logits/rejected": 7.236300468444824,
      "logps/chosen": -37.04119873046875,
      "logps/rejected": -39.97250747680664,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -37.04119873046875,
      "rewards/margins": 2.9313063621520996,
      "rewards/rejected": -39.97250747680664,
      "step": 1185
    },
    {
      "epoch": 0.6368958019735742,
      "grad_norm": 1.8639922655114676e-08,
      "learning_rate": 9.621034946237909e-07,
      "logits/chosen": 7.523528099060059,
      "logits/rejected": 7.188287258148193,
      "logps/chosen": -39.09798812866211,
      "logps/rejected": -40.832359313964844,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -39.09798812866211,
      "rewards/margins": 1.7343683242797852,
      "rewards/rejected": -40.832359313964844,
      "step": 1190
    },
    {
      "epoch": 0.6395718347549757,
      "grad_norm": 0.0002711038897834181,
      "learning_rate": 9.615064944219021e-07,
      "logits/chosen": 7.693884372711182,
      "logits/rejected": 7.346987247467041,
      "logps/chosen": -40.755252838134766,
      "logps/rejected": -42.5510139465332,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -40.755252838134766,
      "rewards/margins": 1.795758843421936,
      "rewards/rejected": -42.5510139465332,
      "step": 1195
    },
    {
      "epoch": 0.6422478675363773,
      "grad_norm": 2.3621369059552778e-07,
      "learning_rate": 9.609050166819803e-07,
      "logits/chosen": 7.598134517669678,
      "logits/rejected": 7.5568108558654785,
      "logps/chosen": -39.770469665527344,
      "logps/rejected": -40.04588317871094,
      "loss": 0.0009,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -39.770469665527344,
      "rewards/margins": 0.27541404962539673,
      "rewards/rejected": -40.04588317871094,
      "step": 1200
    },
    {
      "epoch": 0.6422478675363773,
      "eval_logits/chosen": 6.277879238128662,
      "eval_logits/rejected": 6.1244282722473145,
      "eval_logps/chosen": -38.545963287353516,
      "eval_logps/rejected": -39.95781707763672,
      "eval_loss": 2.102980033669155e-06,
      "eval_rewards/accuracies": 0.5511869192123413,
      "eval_rewards/chosen": -38.545963287353516,
      "eval_rewards/margins": 1.4118520021438599,
      "eval_rewards/rejected": -39.95781707763672,
      "eval_runtime": 40.1588,
      "eval_samples_per_second": 33.492,
      "eval_steps_per_second": 8.392,
      "step": 1200
    },
    {
      "epoch": 0.6449239003177789,
      "grad_norm": 2.2578066542529478e-06,
      "learning_rate": 9.602990672395653e-07,
      "logits/chosen": 7.663047790527344,
      "logits/rejected": 7.214613914489746,
      "logps/chosen": -39.425392150878906,
      "logps/rejected": -41.066856384277344,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -39.425392150878906,
      "rewards/margins": 1.6414620876312256,
      "rewards/rejected": -41.066856384277344,
      "step": 1205
    },
    {
      "epoch": 0.6475999330991805,
      "grad_norm": 3.559003006923499e-09,
      "learning_rate": 9.59688651973581e-07,
      "logits/chosen": 7.73367166519165,
      "logits/rejected": 7.2976579666137695,
      "logps/chosen": -38.8815803527832,
      "logps/rejected": -39.933563232421875,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -38.8815803527832,
      "rewards/margins": 1.0519788265228271,
      "rewards/rejected": -39.933563232421875,
      "step": 1210
    },
    {
      "epoch": 0.650275965880582,
      "grad_norm": 0.0019705371527585914,
      "learning_rate": 9.590737768062792e-07,
      "logits/chosen": 7.577195644378662,
      "logits/rejected": 7.211702823638916,
      "logps/chosen": -38.491214752197266,
      "logps/rejected": -40.38583755493164,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -38.491214752197266,
      "rewards/margins": 1.894627332687378,
      "rewards/rejected": -40.38583755493164,
      "step": 1215
    },
    {
      "epoch": 0.6529519986619836,
      "grad_norm": 0.29718665995505644,
      "learning_rate": 9.584544477031816e-07,
      "logits/chosen": 7.411413669586182,
      "logits/rejected": 7.279717922210693,
      "logps/chosen": -39.75653839111328,
      "logps/rejected": -42.37934494018555,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -39.75653839111328,
      "rewards/margins": 2.622805118560791,
      "rewards/rejected": -42.37934494018555,
      "step": 1220
    },
    {
      "epoch": 0.6556280314433852,
      "grad_norm": 3.376380021263883e-05,
      "learning_rate": 9.578306706730215e-07,
      "logits/chosen": 7.971227169036865,
      "logits/rejected": 7.566740989685059,
      "logps/chosen": -40.2457389831543,
      "logps/rejected": -42.09596633911133,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -40.2457389831543,
      "rewards/margins": 1.8502248525619507,
      "rewards/rejected": -42.09596633911133,
      "step": 1225
    },
    {
      "epoch": 0.6583040642247867,
      "grad_norm": 2.2352952764670997e-07,
      "learning_rate": 9.572024517676865e-07,
      "logits/chosen": 8.149125099182129,
      "logits/rejected": 7.896838188171387,
      "logps/chosen": -40.58433151245117,
      "logps/rejected": -41.86503601074219,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -40.58433151245117,
      "rewards/margins": 1.2807040214538574,
      "rewards/rejected": -41.86503601074219,
      "step": 1230
    },
    {
      "epoch": 0.6609800970061883,
      "grad_norm": 1.8628154936616201e-06,
      "learning_rate": 9.565697970821593e-07,
      "logits/chosen": 8.205276489257812,
      "logits/rejected": 7.75179386138916,
      "logps/chosen": -40.08845138549805,
      "logps/rejected": -41.72840118408203,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -40.08845138549805,
      "rewards/margins": 1.6399505138397217,
      "rewards/rejected": -41.72840118408203,
      "step": 1235
    },
    {
      "epoch": 0.6636561297875899,
      "grad_norm": 2.9394820539864427e-06,
      "learning_rate": 9.559327127544585e-07,
      "logits/chosen": 8.448633193969727,
      "logits/rejected": 8.346811294555664,
      "logps/chosen": -41.940582275390625,
      "logps/rejected": -43.5684814453125,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.940582275390625,
      "rewards/margins": 1.6278976202011108,
      "rewards/rejected": -43.5684814453125,
      "step": 1240
    },
    {
      "epoch": 0.6663321625689914,
      "grad_norm": 6.675774092292334e-10,
      "learning_rate": 9.552912049655789e-07,
      "logits/chosen": 8.406103134155273,
      "logits/rejected": 8.038366317749023,
      "logps/chosen": -41.683712005615234,
      "logps/rejected": -43.56096267700195,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.683712005615234,
      "rewards/margins": 1.877249002456665,
      "rewards/rejected": -43.56096267700195,
      "step": 1245
    },
    {
      "epoch": 0.669008195350393,
      "grad_norm": 0.005707848503850519,
      "learning_rate": 9.546452799394315e-07,
      "logits/chosen": 8.447736740112305,
      "logits/rejected": 7.929117679595947,
      "logps/chosen": -41.303218841552734,
      "logps/rejected": -42.93030548095703,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.303218841552734,
      "rewards/margins": 1.6270862817764282,
      "rewards/rejected": -42.93030548095703,
      "step": 1250
    },
    {
      "epoch": 0.6716842281317946,
      "grad_norm": 3.721585863572294e-08,
      "learning_rate": 9.539949439427846e-07,
      "logits/chosen": 8.136041641235352,
      "logits/rejected": 7.983328819274902,
      "logps/chosen": -40.66224670410156,
      "logps/rejected": -41.81073760986328,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -40.66224670410156,
      "rewards/margins": 1.1484874486923218,
      "rewards/rejected": -41.81073760986328,
      "step": 1255
    },
    {
      "epoch": 0.6743602609131962,
      "grad_norm": 3.9589879683131836e-09,
      "learning_rate": 9.533402032852002e-07,
      "logits/chosen": 8.099668502807617,
      "logits/rejected": 7.7415666580200195,
      "logps/chosen": -39.847991943359375,
      "logps/rejected": -42.599281311035156,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -39.847991943359375,
      "rewards/margins": 2.7512881755828857,
      "rewards/rejected": -42.599281311035156,
      "step": 1260
    },
    {
      "epoch": 0.6770362936945977,
      "grad_norm": 1.9255604745871127e-09,
      "learning_rate": 9.526810643189754e-07,
      "logits/chosen": 8.33497142791748,
      "logits/rejected": 7.557356357574463,
      "logps/chosen": -40.18687057495117,
      "logps/rejected": -42.38703536987305,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -40.18687057495117,
      "rewards/margins": 2.2001659870147705,
      "rewards/rejected": -42.38703536987305,
      "step": 1265
    },
    {
      "epoch": 0.6797123264759993,
      "grad_norm": 1.8625382014199978e-06,
      "learning_rate": 9.52017533439079e-07,
      "logits/chosen": 8.354607582092285,
      "logits/rejected": 8.240533828735352,
      "logps/chosen": -41.47136306762695,
      "logps/rejected": -42.389617919921875,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -41.47136306762695,
      "rewards/margins": 0.9182575345039368,
      "rewards/rejected": -42.389617919921875,
      "step": 1270
    },
    {
      "epoch": 0.6823883592574009,
      "grad_norm": 1.398237045952001e-06,
      "learning_rate": 9.513496170830909e-07,
      "logits/chosen": 8.16751480102539,
      "logits/rejected": 8.006908416748047,
      "logps/chosen": -40.472530364990234,
      "logps/rejected": -41.67839431762695,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.472530364990234,
      "rewards/margins": 1.205863118171692,
      "rewards/rejected": -41.67839431762695,
      "step": 1275
    },
    {
      "epoch": 0.6850643920388024,
      "grad_norm": 1.3718018119272273e-06,
      "learning_rate": 9.506773217311382e-07,
      "logits/chosen": 7.865796089172363,
      "logits/rejected": 7.4651780128479,
      "logps/chosen": -40.82707595825195,
      "logps/rejected": -42.023006439208984,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -40.82707595825195,
      "rewards/margins": 1.1959372758865356,
      "rewards/rejected": -42.023006439208984,
      "step": 1280
    },
    {
      "epoch": 0.687740424820204,
      "grad_norm": 1.581561156285125e-07,
      "learning_rate": 9.500006539058334e-07,
      "logits/chosen": 8.35900592803955,
      "logits/rejected": 7.867652893066406,
      "logps/chosen": -41.17774963378906,
      "logps/rejected": -42.7989387512207,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -41.17774963378906,
      "rewards/margins": 1.6211884021759033,
      "rewards/rejected": -42.7989387512207,
      "step": 1285
    },
    {
      "epoch": 0.6904164576016056,
      "grad_norm": 5.405402932995935e-06,
      "learning_rate": 9.493196201722109e-07,
      "logits/chosen": 8.292418479919434,
      "logits/rejected": 7.9591240882873535,
      "logps/chosen": -40.387630462646484,
      "logps/rejected": -42.18065643310547,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -40.387630462646484,
      "rewards/margins": 1.7930271625518799,
      "rewards/rejected": -42.18065643310547,
      "step": 1290
    },
    {
      "epoch": 0.6930924903830072,
      "grad_norm": 1.7560905466936283e-10,
      "learning_rate": 9.486342271376628e-07,
      "logits/chosen": 8.256156921386719,
      "logits/rejected": 8.552331924438477,
      "logps/chosen": -40.69643783569336,
      "logps/rejected": -42.06412124633789,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.69643783569336,
      "rewards/margins": 1.3676789999008179,
      "rewards/rejected": -42.06412124633789,
      "step": 1295
    },
    {
      "epoch": 0.6957685231644087,
      "grad_norm": 3.1909789734095277e-07,
      "learning_rate": 9.479444814518755e-07,
      "logits/chosen": 8.402515411376953,
      "logits/rejected": 7.637152194976807,
      "logps/chosen": -40.322975158691406,
      "logps/rejected": -42.68449401855469,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -40.322975158691406,
      "rewards/margins": 2.361518144607544,
      "rewards/rejected": -42.68449401855469,
      "step": 1300
    },
    {
      "epoch": 0.6984445559458103,
      "grad_norm": 1.8473003163472545e-06,
      "learning_rate": 9.472503898067645e-07,
      "logits/chosen": 7.935676574707031,
      "logits/rejected": 7.927350044250488,
      "logps/chosen": -41.35259246826172,
      "logps/rejected": -42.002933502197266,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.35259246826172,
      "rewards/margins": 0.650338888168335,
      "rewards/rejected": -42.002933502197266,
      "step": 1305
    },
    {
      "epoch": 0.701120588727212,
      "grad_norm": 2.4217720473108985e-09,
      "learning_rate": 9.465519589364099e-07,
      "logits/chosen": 7.781865119934082,
      "logits/rejected": 7.673886775970459,
      "logps/chosen": -39.51872253417969,
      "logps/rejected": -41.09035873413086,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -39.51872253417969,
      "rewards/margins": 1.5716428756713867,
      "rewards/rejected": -41.09035873413086,
      "step": 1310
    },
    {
      "epoch": 0.7037966215086134,
      "grad_norm": 3.291452146690667e-06,
      "learning_rate": 9.458491956169914e-07,
      "logits/chosen": 8.48716926574707,
      "logits/rejected": 8.060966491699219,
      "logps/chosen": -39.346370697021484,
      "logps/rejected": -41.73097229003906,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -39.346370697021484,
      "rewards/margins": 2.384601354598999,
      "rewards/rejected": -41.73097229003906,
      "step": 1315
    },
    {
      "epoch": 0.706472654290015,
      "grad_norm": 3.871015614000862e-07,
      "learning_rate": 9.451421066667215e-07,
      "logits/chosen": 8.446621894836426,
      "logits/rejected": 8.100547790527344,
      "logps/chosen": -39.652591705322266,
      "logps/rejected": -41.96711349487305,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -39.652591705322266,
      "rewards/margins": 2.3145267963409424,
      "rewards/rejected": -41.96711349487305,
      "step": 1320
    },
    {
      "epoch": 0.7091486870714167,
      "grad_norm": 0.0001376592426154493,
      "learning_rate": 9.444306989457805e-07,
      "logits/chosen": 8.023016929626465,
      "logits/rejected": 7.898001194000244,
      "logps/chosen": -40.59425735473633,
      "logps/rejected": -42.078521728515625,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -40.59425735473633,
      "rewards/margins": 1.4842627048492432,
      "rewards/rejected": -42.078521728515625,
      "step": 1325
    },
    {
      "epoch": 0.7118247198528181,
      "grad_norm": 1.2934921082349808e-08,
      "learning_rate": 9.437149793562489e-07,
      "logits/chosen": 7.9620161056518555,
      "logits/rejected": 7.7126359939575195,
      "logps/chosen": -39.53632354736328,
      "logps/rejected": -40.03577423095703,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -39.53632354736328,
      "rewards/margins": 0.49944964051246643,
      "rewards/rejected": -40.03577423095703,
      "step": 1330
    },
    {
      "epoch": 0.7145007526342197,
      "grad_norm": 2.3959626822076756e-05,
      "learning_rate": 9.429949548420417e-07,
      "logits/chosen": 8.325448989868164,
      "logits/rejected": 8.049530029296875,
      "logps/chosen": -40.465431213378906,
      "logps/rejected": -41.5115852355957,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -40.465431213378906,
      "rewards/margins": 1.0461547374725342,
      "rewards/rejected": -41.5115852355957,
      "step": 1335
    },
    {
      "epoch": 0.7171767854156214,
      "grad_norm": 1.99120013977766,
      "learning_rate": 9.422706323888396e-07,
      "logits/chosen": 8.247047424316406,
      "logits/rejected": 8.281377792358398,
      "logps/chosen": -39.6649284362793,
      "logps/rejected": -40.74580764770508,
      "loss": 0.0002,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -39.6649284362793,
      "rewards/margins": 1.0808765888214111,
      "rewards/rejected": -40.74580764770508,
      "step": 1340
    },
    {
      "epoch": 0.719852818197023,
      "grad_norm": 6.4104760820790105e-06,
      "learning_rate": 9.415420190240225e-07,
      "logits/chosen": 8.322467803955078,
      "logits/rejected": 7.8098955154418945,
      "logps/chosen": -39.618202209472656,
      "logps/rejected": -43.106807708740234,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -39.618202209472656,
      "rewards/margins": 3.4886040687561035,
      "rewards/rejected": -43.106807708740234,
      "step": 1345
    },
    {
      "epoch": 0.7225288509784245,
      "grad_norm": 0.5341358021074406,
      "learning_rate": 9.408091218166002e-07,
      "logits/chosen": 6.268855094909668,
      "logits/rejected": 6.186150550842285,
      "logps/chosen": -34.201576232910156,
      "logps/rejected": -34.833396911621094,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -34.201576232910156,
      "rewards/margins": 0.6318244934082031,
      "rewards/rejected": -34.833396911621094,
      "step": 1350
    },
    {
      "epoch": 0.7252048837598261,
      "grad_norm": 0.00046878423352249266,
      "learning_rate": 9.400719478771449e-07,
      "logits/chosen": 6.748335838317871,
      "logits/rejected": 6.308575630187988,
      "logps/chosen": -35.94706344604492,
      "logps/rejected": -39.0224609375,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -35.94706344604492,
      "rewards/margins": 3.075397491455078,
      "rewards/rejected": -39.0224609375,
      "step": 1355
    },
    {
      "epoch": 0.7278809165412277,
      "grad_norm": 2.8833659068652375e-05,
      "learning_rate": 9.393305043577209e-07,
      "logits/chosen": 7.55231237411499,
      "logits/rejected": 7.3341546058654785,
      "logps/chosen": -38.76103973388672,
      "logps/rejected": -39.605979919433594,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -38.76103973388672,
      "rewards/margins": 0.8449405431747437,
      "rewards/rejected": -39.605979919433594,
      "step": 1360
    },
    {
      "epoch": 0.7305569493226292,
      "grad_norm": 0.00046315135994724315,
      "learning_rate": 9.38584798451817e-07,
      "logits/chosen": 7.632124900817871,
      "logits/rejected": 7.35461950302124,
      "logps/chosen": -37.470726013183594,
      "logps/rejected": -39.73419189453125,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -37.470726013183594,
      "rewards/margins": 2.2634730339050293,
      "rewards/rejected": -39.73419189453125,
      "step": 1365
    },
    {
      "epoch": 0.7332329821040308,
      "grad_norm": 2.2788179584964154e-07,
      "learning_rate": 9.37834837394275e-07,
      "logits/chosen": 7.586686134338379,
      "logits/rejected": 7.300710201263428,
      "logps/chosen": -38.65477752685547,
      "logps/rejected": -40.73984146118164,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -38.65477752685547,
      "rewards/margins": 2.085059642791748,
      "rewards/rejected": -40.73984146118164,
      "step": 1370
    },
    {
      "epoch": 0.7359090148854324,
      "grad_norm": 1.3637033624443518e-05,
      "learning_rate": 9.370806284612203e-07,
      "logits/chosen": 7.811972141265869,
      "logits/rejected": 7.576725959777832,
      "logps/chosen": -37.89400100708008,
      "logps/rejected": -40.05268096923828,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -37.89400100708008,
      "rewards/margins": 2.1586759090423584,
      "rewards/rejected": -40.05268096923828,
      "step": 1375
    },
    {
      "epoch": 0.738585047666834,
      "grad_norm": 7.211660710516863e-06,
      "learning_rate": 9.363221789699912e-07,
      "logits/chosen": 8.053945541381836,
      "logits/rejected": 7.638498783111572,
      "logps/chosen": -38.376888275146484,
      "logps/rejected": -40.625587463378906,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -38.376888275146484,
      "rewards/margins": 2.2486977577209473,
      "rewards/rejected": -40.625587463378906,
      "step": 1380
    },
    {
      "epoch": 0.7412610804482355,
      "grad_norm": 0.0001323628547332212,
      "learning_rate": 9.355594962790682e-07,
      "logits/chosen": 7.778914451599121,
      "logits/rejected": 7.3721184730529785,
      "logps/chosen": -39.056419372558594,
      "logps/rejected": -39.898841857910156,
      "loss": 0.0,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -39.056419372558594,
      "rewards/margins": 0.8424198031425476,
      "rewards/rejected": -39.898841857910156,
      "step": 1385
    },
    {
      "epoch": 0.7439371132296371,
      "grad_norm": 1.0478518665091114e-05,
      "learning_rate": 9.34792587788002e-07,
      "logits/chosen": 7.712989807128906,
      "logits/rejected": 7.353451728820801,
      "logps/chosen": -39.175472259521484,
      "logps/rejected": -40.94335174560547,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -39.175472259521484,
      "rewards/margins": 1.7678802013397217,
      "rewards/rejected": -40.94335174560547,
      "step": 1390
    },
    {
      "epoch": 0.7466131460110387,
      "grad_norm": 0.0003283560466566843,
      "learning_rate": 9.34021460937342e-07,
      "logits/chosen": 7.43767786026001,
      "logits/rejected": 7.37692928314209,
      "logps/chosen": -39.12328338623047,
      "logps/rejected": -39.51963424682617,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -39.12328338623047,
      "rewards/margins": 0.3963487446308136,
      "rewards/rejected": -39.51963424682617,
      "step": 1395
    },
    {
      "epoch": 0.7492891787924402,
      "grad_norm": 7.02459898767118e-09,
      "learning_rate": 9.332461232085646e-07,
      "logits/chosen": 7.841141700744629,
      "logits/rejected": 7.301096439361572,
      "logps/chosen": -38.66679000854492,
      "logps/rejected": -39.682777404785156,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -38.66679000854492,
      "rewards/margins": 1.015986680984497,
      "rewards/rejected": -39.682777404785156,
      "step": 1400
    },
    {
      "epoch": 0.7519652115738418,
      "grad_norm": 8.985466043604728e-07,
      "learning_rate": 9.324665821239998e-07,
      "logits/chosen": 7.725131034851074,
      "logits/rejected": 7.3171210289001465,
      "logps/chosen": -38.61504364013672,
      "logps/rejected": -40.03667449951172,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -38.61504364013672,
      "rewards/margins": 1.4216245412826538,
      "rewards/rejected": -40.03667449951172,
      "step": 1405
    },
    {
      "epoch": 0.7546412443552434,
      "grad_norm": 6.725606617823982e-08,
      "learning_rate": 9.316828452467583e-07,
      "logits/chosen": 7.756146430969238,
      "logits/rejected": 7.518537998199463,
      "logps/chosen": -39.640899658203125,
      "logps/rejected": -41.82984161376953,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -39.640899658203125,
      "rewards/margins": 2.188947916030884,
      "rewards/rejected": -41.82984161376953,
      "step": 1410
    },
    {
      "epoch": 0.7573172771366449,
      "grad_norm": 7.314386309273594e-06,
      "learning_rate": 9.30894920180659e-07,
      "logits/chosen": 7.588531494140625,
      "logits/rejected": 7.469851493835449,
      "logps/chosen": -38.47243118286133,
      "logps/rejected": -40.729740142822266,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -38.47243118286133,
      "rewards/margins": 2.2573134899139404,
      "rewards/rejected": -40.729740142822266,
      "step": 1415
    },
    {
      "epoch": 0.7599933099180465,
      "grad_norm": 4.138645901621825e-05,
      "learning_rate": 9.301028145701543e-07,
      "logits/chosen": 7.515957832336426,
      "logits/rejected": 7.183793544769287,
      "logps/chosen": -40.00874328613281,
      "logps/rejected": -41.86363983154297,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.00874328613281,
      "rewards/margins": 1.8549010753631592,
      "rewards/rejected": -41.86363983154297,
      "step": 1420
    },
    {
      "epoch": 0.7626693426994481,
      "grad_norm": 0.00013488060141971276,
      "learning_rate": 9.293065361002563e-07,
      "logits/chosen": 7.7457098960876465,
      "logits/rejected": 7.228750705718994,
      "logps/chosen": -39.105873107910156,
      "logps/rejected": -39.65242004394531,
      "loss": 0.0018,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -39.105873107910156,
      "rewards/margins": 0.546552836894989,
      "rewards/rejected": -39.65242004394531,
      "step": 1425
    },
    {
      "epoch": 0.7653453754808497,
      "grad_norm": 1.4598105360134454e-08,
      "learning_rate": 9.285060924964622e-07,
      "logits/chosen": 7.878639221191406,
      "logits/rejected": 7.701871395111084,
      "logps/chosen": -39.983192443847656,
      "logps/rejected": -41.359954833984375,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -39.983192443847656,
      "rewards/margins": 1.3767613172531128,
      "rewards/rejected": -41.359954833984375,
      "step": 1430
    },
    {
      "epoch": 0.7680214082622512,
      "grad_norm": 9.13776185283731e-06,
      "learning_rate": 9.277014915246792e-07,
      "logits/chosen": 7.722917079925537,
      "logits/rejected": 7.634284019470215,
      "logps/chosen": -39.978981018066406,
      "logps/rejected": -40.567649841308594,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.978981018066406,
      "rewards/margins": 0.5886683464050293,
      "rewards/rejected": -40.567649841308594,
      "step": 1435
    },
    {
      "epoch": 0.7706974410436528,
      "grad_norm": 1.1446054988823118e-05,
      "learning_rate": 9.268927409911498e-07,
      "logits/chosen": 7.750887870788574,
      "logits/rejected": 7.445874214172363,
      "logps/chosen": -38.838775634765625,
      "logps/rejected": -39.77943801879883,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -38.838775634765625,
      "rewards/margins": 0.9406577944755554,
      "rewards/rejected": -39.77943801879883,
      "step": 1440
    },
    {
      "epoch": 0.7733734738250544,
      "grad_norm": 2.520721235009491e-07,
      "learning_rate": 9.260798487423749e-07,
      "logits/chosen": 7.914003849029541,
      "logits/rejected": 7.370151519775391,
      "logps/chosen": -38.39636993408203,
      "logps/rejected": -40.597511291503906,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -38.39636993408203,
      "rewards/margins": 2.201145648956299,
      "rewards/rejected": -40.597511291503906,
      "step": 1445
    },
    {
      "epoch": 0.7760495066064559,
      "grad_norm": 1.7684568575055986e-05,
      "learning_rate": 9.252628226650389e-07,
      "logits/chosen": 7.578721523284912,
      "logits/rejected": 7.619417667388916,
      "logps/chosen": -39.15451431274414,
      "logps/rejected": -40.19221878051758,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -39.15451431274414,
      "rewards/margins": 1.0377026796340942,
      "rewards/rejected": -40.19221878051758,
      "step": 1450
    },
    {
      "epoch": 0.7787255393878575,
      "grad_norm": 6.044782616648161e-07,
      "learning_rate": 9.244416706859321e-07,
      "logits/chosen": 8.035229682922363,
      "logits/rejected": 7.653140068054199,
      "logps/chosen": -39.565879821777344,
      "logps/rejected": -41.63979721069336,
      "loss": 0.0003,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -39.565879821777344,
      "rewards/margins": 2.073922634124756,
      "rewards/rejected": -41.63979721069336,
      "step": 1455
    },
    {
      "epoch": 0.7814015721692591,
      "grad_norm": 5.332723509690697e-07,
      "learning_rate": 9.23616400771875e-07,
      "logits/chosen": 8.152082443237305,
      "logits/rejected": 7.573739528656006,
      "logps/chosen": -39.73722457885742,
      "logps/rejected": -42.264190673828125,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -39.73722457885742,
      "rewards/margins": 2.5269672870635986,
      "rewards/rejected": -42.264190673828125,
      "step": 1460
    },
    {
      "epoch": 0.7840776049506607,
      "grad_norm": 1.2581672327586524e-08,
      "learning_rate": 9.227870209296395e-07,
      "logits/chosen": 7.694755554199219,
      "logits/rejected": 7.399949073791504,
      "logps/chosen": -38.500831604003906,
      "logps/rejected": -40.089698791503906,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -38.500831604003906,
      "rewards/margins": 1.5888638496398926,
      "rewards/rejected": -40.089698791503906,
      "step": 1465
    },
    {
      "epoch": 0.7867536377320622,
      "grad_norm": 5.338673208290298e-08,
      "learning_rate": 9.219535392058728e-07,
      "logits/chosen": 7.544592380523682,
      "logits/rejected": 7.643113613128662,
      "logps/chosen": -39.783199310302734,
      "logps/rejected": -40.92317199707031,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -39.783199310302734,
      "rewards/margins": 1.1399691104888916,
      "rewards/rejected": -40.92317199707031,
      "step": 1470
    },
    {
      "epoch": 0.7894296705134638,
      "grad_norm": 0.00020652353878615633,
      "learning_rate": 9.211159636870181e-07,
      "logits/chosen": 7.866264343261719,
      "logits/rejected": 7.275368690490723,
      "logps/chosen": -38.486698150634766,
      "logps/rejected": -40.86024856567383,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -38.486698150634766,
      "rewards/margins": 2.373549222946167,
      "rewards/rejected": -40.86024856567383,
      "step": 1475
    },
    {
      "epoch": 0.7921057032948654,
      "grad_norm": 1.4952879819594262e-05,
      "learning_rate": 9.202743024992367e-07,
      "logits/chosen": 7.568036079406738,
      "logits/rejected": 7.459419250488281,
      "logps/chosen": -39.39044952392578,
      "logps/rejected": -41.20718002319336,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -39.39044952392578,
      "rewards/margins": 1.8167299032211304,
      "rewards/rejected": -41.20718002319336,
      "step": 1480
    },
    {
      "epoch": 0.7947817360762669,
      "grad_norm": 1.400216617912237e-05,
      "learning_rate": 9.194285638083293e-07,
      "logits/chosen": 7.644814968109131,
      "logits/rejected": 7.262423515319824,
      "logps/chosen": -38.38935089111328,
      "logps/rejected": -40.437599182128906,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -38.38935089111328,
      "rewards/margins": 2.0482475757598877,
      "rewards/rejected": -40.437599182128906,
      "step": 1485
    },
    {
      "epoch": 0.7974577688576685,
      "grad_norm": 0.0001420045544757509,
      "learning_rate": 9.185787558196562e-07,
      "logits/chosen": 7.929560661315918,
      "logits/rejected": 7.786153316497803,
      "logps/chosen": -39.58082962036133,
      "logps/rejected": -41.307106018066406,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -39.58082962036133,
      "rewards/margins": 1.7262766361236572,
      "rewards/rejected": -41.307106018066406,
      "step": 1490
    },
    {
      "epoch": 0.8001338016390701,
      "grad_norm": 7.753742345711771e-10,
      "learning_rate": 9.177248867780583e-07,
      "logits/chosen": 7.826272487640381,
      "logits/rejected": 7.752885341644287,
      "logps/chosen": -39.452388763427734,
      "logps/rejected": -40.267921447753906,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -39.452388763427734,
      "rewards/margins": 0.8155332803726196,
      "rewards/rejected": -40.267921447753906,
      "step": 1495
    },
    {
      "epoch": 0.8028098344204716,
      "grad_norm": 0.21380885712339692,
      "learning_rate": 9.168669649677769e-07,
      "logits/chosen": 7.830197334289551,
      "logits/rejected": 7.682694911956787,
      "logps/chosen": -39.503658294677734,
      "logps/rejected": -40.941436767578125,
      "loss": 0.0009,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -39.503658294677734,
      "rewards/margins": 1.4377771615982056,
      "rewards/rejected": -40.941436767578125,
      "step": 1500
    },
    {
      "epoch": 0.8054858672018732,
      "grad_norm": 2.4568952689561324e-06,
      "learning_rate": 9.16004998712373e-07,
      "logits/chosen": 7.836570739746094,
      "logits/rejected": 7.652500152587891,
      "logps/chosen": -38.87848663330078,
      "logps/rejected": -40.60747146606445,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -38.87848663330078,
      "rewards/margins": 1.7289817333221436,
      "rewards/rejected": -40.60747146606445,
      "step": 1505
    },
    {
      "epoch": 0.8081618999832748,
      "grad_norm": 6.346042898843261e-07,
      "learning_rate": 9.151389963746472e-07,
      "logits/chosen": 7.843935489654541,
      "logits/rejected": 6.99429988861084,
      "logps/chosen": -38.70612335205078,
      "logps/rejected": -41.376678466796875,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -38.70612335205078,
      "rewards/margins": 2.670555591583252,
      "rewards/rejected": -41.376678466796875,
      "step": 1510
    },
    {
      "epoch": 0.8108379327646764,
      "grad_norm": 7.145917140560798e-07,
      "learning_rate": 9.142689663565577e-07,
      "logits/chosen": 8.017499923706055,
      "logits/rejected": 7.976357936859131,
      "logps/chosen": -40.212562561035156,
      "logps/rejected": -40.88829803466797,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -40.212562561035156,
      "rewards/margins": 0.6757341623306274,
      "rewards/rejected": -40.88829803466797,
      "step": 1515
    },
    {
      "epoch": 0.8135139655460779,
      "grad_norm": 2.676125289845692e-06,
      "learning_rate": 9.133949170991397e-07,
      "logits/chosen": 7.681761741638184,
      "logits/rejected": 7.592918395996094,
      "logps/chosen": -40.51093673706055,
      "logps/rejected": -41.224449157714844,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -40.51093673706055,
      "rewards/margins": 0.7135159969329834,
      "rewards/rejected": -41.224449157714844,
      "step": 1520
    },
    {
      "epoch": 0.8161899983274795,
      "grad_norm": 3.976583391185811e-05,
      "learning_rate": 9.125168570824231e-07,
      "logits/chosen": 8.363571166992188,
      "logits/rejected": 7.7566819190979,
      "logps/chosen": -40.30500411987305,
      "logps/rejected": -41.10575866699219,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -40.30500411987305,
      "rewards/margins": 0.8007561564445496,
      "rewards/rejected": -41.10575866699219,
      "step": 1525
    },
    {
      "epoch": 0.8188660311088811,
      "grad_norm": 1.6103254134052267e-06,
      "learning_rate": 9.116347948253496e-07,
      "logits/chosen": 7.757523536682129,
      "logits/rejected": 7.353224277496338,
      "logps/chosen": -37.689659118652344,
      "logps/rejected": -39.65977096557617,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -37.689659118652344,
      "rewards/margins": 1.9701191186904907,
      "rewards/rejected": -39.65977096557617,
      "step": 1530
    },
    {
      "epoch": 0.8215420638902826,
      "grad_norm": 5.853808682716718e-06,
      "learning_rate": 9.107487388856916e-07,
      "logits/chosen": 7.816054344177246,
      "logits/rejected": 7.2476396560668945,
      "logps/chosen": -39.557884216308594,
      "logps/rejected": -41.22559356689453,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -39.557884216308594,
      "rewards/margins": 1.6677097082138062,
      "rewards/rejected": -41.22559356689453,
      "step": 1535
    },
    {
      "epoch": 0.8242180966716842,
      "grad_norm": 2.473959686560337e-07,
      "learning_rate": 9.098586978599673e-07,
      "logits/chosen": 8.023536682128906,
      "logits/rejected": 7.619076728820801,
      "logps/chosen": -40.832489013671875,
      "logps/rejected": -41.5240364074707,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -40.832489013671875,
      "rewards/margins": 0.6915456056594849,
      "rewards/rejected": -41.5240364074707,
      "step": 1540
    },
    {
      "epoch": 0.8268941294530858,
      "grad_norm": 2.27095246052323e-11,
      "learning_rate": 9.089646803833588e-07,
      "logits/chosen": 7.904210090637207,
      "logits/rejected": 7.4583563804626465,
      "logps/chosen": -38.71350860595703,
      "logps/rejected": -40.64717483520508,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -38.71350860595703,
      "rewards/margins": 1.9336620569229126,
      "rewards/rejected": -40.64717483520508,
      "step": 1545
    },
    {
      "epoch": 0.8295701622344873,
      "grad_norm": 3.3819161249539896e-10,
      "learning_rate": 9.080666951296276e-07,
      "logits/chosen": 8.2002534866333,
      "logits/rejected": 7.529571533203125,
      "logps/chosen": -39.46220397949219,
      "logps/rejected": -41.51323699951172,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -39.46220397949219,
      "rewards/margins": 2.0510334968566895,
      "rewards/rejected": -41.51323699951172,
      "step": 1550
    },
    {
      "epoch": 0.8322461950158889,
      "grad_norm": 3.067420680656231e-08,
      "learning_rate": 9.071647508110305e-07,
      "logits/chosen": 8.213407516479492,
      "logits/rejected": 7.441866874694824,
      "logps/chosen": -38.236305236816406,
      "logps/rejected": -41.46370315551758,
      "loss": 0.0,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -38.236305236816406,
      "rewards/margins": 3.227395534515381,
      "rewards/rejected": -41.46370315551758,
      "step": 1555
    },
    {
      "epoch": 0.8349222277972905,
      "grad_norm": 2.95969486826896e-09,
      "learning_rate": 9.062588561782354e-07,
      "logits/chosen": 7.764901161193848,
      "logits/rejected": 7.799078464508057,
      "logps/chosen": -41.49794387817383,
      "logps/rejected": -42.57024383544922,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -41.49794387817383,
      "rewards/margins": 1.0723036527633667,
      "rewards/rejected": -42.57024383544922,
      "step": 1560
    },
    {
      "epoch": 0.8375982605786921,
      "grad_norm": 1.4152617212499327e-05,
      "learning_rate": 9.053490200202358e-07,
      "logits/chosen": 8.034921646118164,
      "logits/rejected": 8.120762825012207,
      "logps/chosen": -40.08647537231445,
      "logps/rejected": -40.591224670410156,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -40.08647537231445,
      "rewards/margins": 0.5047468543052673,
      "rewards/rejected": -40.591224670410156,
      "step": 1565
    },
    {
      "epoch": 0.8402742933600936,
      "grad_norm": 1.5601640832831647e-08,
      "learning_rate": 9.044352511642661e-07,
      "logits/chosen": 8.210533142089844,
      "logits/rejected": 8.029157638549805,
      "logps/chosen": -41.439979553222656,
      "logps/rejected": -41.70062255859375,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -41.439979553222656,
      "rewards/margins": 0.260641872882843,
      "rewards/rejected": -41.70062255859375,
      "step": 1570
    },
    {
      "epoch": 0.8429503261414952,
      "grad_norm": 0.37196646025378177,
      "learning_rate": 9.03517558475716e-07,
      "logits/chosen": 7.8325066566467285,
      "logits/rejected": 7.576259613037109,
      "logps/chosen": -39.38726043701172,
      "logps/rejected": -39.933921813964844,
      "loss": 0.0014,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -39.38726043701172,
      "rewards/margins": 0.5466631054878235,
      "rewards/rejected": -39.933921813964844,
      "step": 1575
    },
    {
      "epoch": 0.8456263589228968,
      "grad_norm": 5.642810232770941e-09,
      "learning_rate": 9.025959508580436e-07,
      "logits/chosen": 7.975683689117432,
      "logits/rejected": 7.219354152679443,
      "logps/chosen": -39.45317840576172,
      "logps/rejected": -41.32285690307617,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -39.45317840576172,
      "rewards/margins": 1.869680404663086,
      "rewards/rejected": -41.32285690307617,
      "step": 1580
    },
    {
      "epoch": 0.8483023917042983,
      "grad_norm": 0.000395155671439392,
      "learning_rate": 9.016704372526905e-07,
      "logits/chosen": 8.072778701782227,
      "logits/rejected": 7.618019104003906,
      "logps/chosen": -39.28447723388672,
      "logps/rejected": -41.08146667480469,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -39.28447723388672,
      "rewards/margins": 1.7969859838485718,
      "rewards/rejected": -41.08146667480469,
      "step": 1585
    },
    {
      "epoch": 0.8509784244856999,
      "grad_norm": 1.0011542098540318e-06,
      "learning_rate": 9.007410266389934e-07,
      "logits/chosen": 8.216329574584961,
      "logits/rejected": 8.265836715698242,
      "logps/chosen": -40.846519470214844,
      "logps/rejected": -42.641273498535156,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.846519470214844,
      "rewards/margins": 1.794745683670044,
      "rewards/rejected": -42.641273498535156,
      "step": 1590
    },
    {
      "epoch": 0.8536544572671015,
      "grad_norm": 1.4455226025171557e-05,
      "learning_rate": 8.998077280340981e-07,
      "logits/chosen": 7.888262748718262,
      "logits/rejected": 7.80618143081665,
      "logps/chosen": -40.44511795043945,
      "logps/rejected": -41.439693450927734,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -40.44511795043945,
      "rewards/margins": 0.994576096534729,
      "rewards/rejected": -41.439693450927734,
      "step": 1595
    },
    {
      "epoch": 0.8563304900485031,
      "grad_norm": 6.423137034301042e-06,
      "learning_rate": 8.988705504928722e-07,
      "logits/chosen": 8.276338577270508,
      "logits/rejected": 7.813223361968994,
      "logps/chosen": -39.72477722167969,
      "logps/rejected": -41.605533599853516,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -39.72477722167969,
      "rewards/margins": 1.8807570934295654,
      "rewards/rejected": -41.605533599853516,
      "step": 1600
    },
    {
      "epoch": 0.8563304900485031,
      "eval_logits/chosen": 6.734649658203125,
      "eval_logits/rejected": 6.54942512512207,
      "eval_logps/chosen": -40.022151947021484,
      "eval_logps/rejected": -41.41145324707031,
      "eval_loss": 7.622449516020424e-07,
      "eval_rewards/accuracies": 0.5689911246299744,
      "eval_rewards/chosen": -40.022151947021484,
      "eval_rewards/margins": 1.3893060684204102,
      "eval_rewards/rejected": -41.41145324707031,
      "eval_runtime": 40.1221,
      "eval_samples_per_second": 33.523,
      "eval_steps_per_second": 8.399,
      "step": 1600
    },
    {
      "epoch": 0.8590065228299046,
      "grad_norm": 0.002761385120373784,
      "learning_rate": 8.979295031078157e-07,
      "logits/chosen": 8.065135955810547,
      "logits/rejected": 7.577500343322754,
      "logps/chosen": -38.22058868408203,
      "logps/rejected": -41.09245300292969,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -38.22058868408203,
      "rewards/margins": 2.8718667030334473,
      "rewards/rejected": -41.09245300292969,
      "step": 1605
    },
    {
      "epoch": 0.8616825556113062,
      "grad_norm": 4.236119688925486e-08,
      "learning_rate": 8.969845950089751e-07,
      "logits/chosen": 8.205350875854492,
      "logits/rejected": 7.829659461975098,
      "logps/chosen": -40.17414855957031,
      "logps/rejected": -42.43650436401367,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.17414855957031,
      "rewards/margins": 2.2623534202575684,
      "rewards/rejected": -42.43650436401367,
      "step": 1610
    },
    {
      "epoch": 0.8643585883927078,
      "grad_norm": 7.896108283628366e-06,
      "learning_rate": 8.960358353638526e-07,
      "logits/chosen": 8.38143539428711,
      "logits/rejected": 8.025314331054688,
      "logps/chosen": -41.28398132324219,
      "logps/rejected": -42.69435501098633,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.28398132324219,
      "rewards/margins": 1.410369634628296,
      "rewards/rejected": -42.69435501098633,
      "step": 1615
    },
    {
      "epoch": 0.8670346211741093,
      "grad_norm": 6.247935836688208e-07,
      "learning_rate": 8.950832333773184e-07,
      "logits/chosen": 8.18684196472168,
      "logits/rejected": 7.746510982513428,
      "logps/chosen": -42.06782531738281,
      "logps/rejected": -43.7523307800293,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.06782531738281,
      "rewards/margins": 1.6845118999481201,
      "rewards/rejected": -43.7523307800293,
      "step": 1620
    },
    {
      "epoch": 0.869710653955511,
      "grad_norm": 0.0015260435121433147,
      "learning_rate": 8.941267982915213e-07,
      "logits/chosen": 8.26119613647461,
      "logits/rejected": 7.8717193603515625,
      "logps/chosen": -41.67167282104492,
      "logps/rejected": -40.71431350708008,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -41.67167282104492,
      "rewards/margins": -0.9573593139648438,
      "rewards/rejected": -40.71431350708008,
      "step": 1625
    },
    {
      "epoch": 0.8723866867369126,
      "grad_norm": 1.4332950627250909e-06,
      "learning_rate": 8.931665393857983e-07,
      "logits/chosen": 8.437042236328125,
      "logits/rejected": 8.065763473510742,
      "logps/chosen": -41.52033615112305,
      "logps/rejected": -43.22309112548828,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.52033615112305,
      "rewards/margins": 1.702749490737915,
      "rewards/rejected": -43.22309112548828,
      "step": 1630
    },
    {
      "epoch": 0.875062719518314,
      "grad_norm": 1.2675251887812716e-11,
      "learning_rate": 8.922024659765861e-07,
      "logits/chosen": 8.694414138793945,
      "logits/rejected": 8.221012115478516,
      "logps/chosen": -41.88777542114258,
      "logps/rejected": -43.541622161865234,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -41.88777542114258,
      "rewards/margins": 1.653850793838501,
      "rewards/rejected": -43.541622161865234,
      "step": 1635
    },
    {
      "epoch": 0.8777387522997157,
      "grad_norm": 1.6969471886560934e-10,
      "learning_rate": 8.912345874173288e-07,
      "logits/chosen": 8.443890571594238,
      "logits/rejected": 8.079405784606934,
      "logps/chosen": -41.49980163574219,
      "logps/rejected": -43.46422576904297,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -41.49980163574219,
      "rewards/margins": 1.9644253253936768,
      "rewards/rejected": -43.46422576904297,
      "step": 1640
    },
    {
      "epoch": 0.8804147850811173,
      "grad_norm": 5.832806245550386e-07,
      "learning_rate": 8.902629130983885e-07,
      "logits/chosen": 8.289989471435547,
      "logits/rejected": 8.217000961303711,
      "logps/chosen": -41.150543212890625,
      "logps/rejected": -42.07365036010742,
      "loss": 0.0005,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.150543212890625,
      "rewards/margins": 0.9231061935424805,
      "rewards/rejected": -42.07365036010742,
      "step": 1645
    },
    {
      "epoch": 0.8830908178625189,
      "grad_norm": 5.842372342736316e-06,
      "learning_rate": 8.892874524469537e-07,
      "logits/chosen": 8.605502128601074,
      "logits/rejected": 8.349967956542969,
      "logps/chosen": -40.33566665649414,
      "logps/rejected": -40.847625732421875,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -40.33566665649414,
      "rewards/margins": 0.5119602680206299,
      "rewards/rejected": -40.847625732421875,
      "step": 1650
    },
    {
      "epoch": 0.8857668506439204,
      "grad_norm": 8.671475551452057e-08,
      "learning_rate": 8.883082149269478e-07,
      "logits/chosen": 8.478113174438477,
      "logits/rejected": 8.237870216369629,
      "logps/chosen": -41.74320983886719,
      "logps/rejected": -42.48762893676758,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.74320983886719,
      "rewards/margins": 0.7444199323654175,
      "rewards/rejected": -42.48762893676758,
      "step": 1655
    },
    {
      "epoch": 0.888442883425322,
      "grad_norm": 6.057990790215548e-07,
      "learning_rate": 8.873252100389377e-07,
      "logits/chosen": 8.566787719726562,
      "logits/rejected": 8.392692565917969,
      "logps/chosen": -40.782020568847656,
      "logps/rejected": -41.765872955322266,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -40.782020568847656,
      "rewards/margins": 0.9838517904281616,
      "rewards/rejected": -41.765872955322266,
      "step": 1660
    },
    {
      "epoch": 0.8911189162067236,
      "grad_norm": 2.1331299903138964e-05,
      "learning_rate": 8.863384473200411e-07,
      "logits/chosen": 7.973538398742676,
      "logits/rejected": 7.838541507720947,
      "logps/chosen": -41.561458587646484,
      "logps/rejected": -42.49988555908203,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -41.561458587646484,
      "rewards/margins": 0.9384317398071289,
      "rewards/rejected": -42.49988555908203,
      "step": 1665
    },
    {
      "epoch": 0.8937949489881251,
      "grad_norm": 3.572792957846333e-07,
      "learning_rate": 8.853479363438342e-07,
      "logits/chosen": 8.695677757263184,
      "logits/rejected": 8.124799728393555,
      "logps/chosen": -42.88197708129883,
      "logps/rejected": -44.08369064331055,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.88197708129883,
      "rewards/margins": 1.2017176151275635,
      "rewards/rejected": -44.08369064331055,
      "step": 1670
    },
    {
      "epoch": 0.8964709817695267,
      "grad_norm": 1.7620065196211736e-07,
      "learning_rate": 8.843536867202588e-07,
      "logits/chosen": 8.643811225891113,
      "logits/rejected": 8.183439254760742,
      "logps/chosen": -42.746726989746094,
      "logps/rejected": -44.14118194580078,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -42.746726989746094,
      "rewards/margins": 1.3944545984268188,
      "rewards/rejected": -44.14118194580078,
      "step": 1675
    },
    {
      "epoch": 0.8991470145509283,
      "grad_norm": 9.881708567393247e-07,
      "learning_rate": 8.833557080955292e-07,
      "logits/chosen": 8.322452545166016,
      "logits/rejected": 7.871786594390869,
      "logps/chosen": -41.227455139160156,
      "logps/rejected": -41.42321014404297,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -41.227455139160156,
      "rewards/margins": 0.19575843214988708,
      "rewards/rejected": -41.42321014404297,
      "step": 1680
    },
    {
      "epoch": 0.9018230473323299,
      "grad_norm": 5.034545029088386e-06,
      "learning_rate": 8.823540101520381e-07,
      "logits/chosen": 8.499072074890137,
      "logits/rejected": 7.848641872406006,
      "logps/chosen": -41.52507400512695,
      "logps/rejected": -43.086971282958984,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -41.52507400512695,
      "rewards/margins": 1.5618988275527954,
      "rewards/rejected": -43.086971282958984,
      "step": 1685
    },
    {
      "epoch": 0.9044990801137314,
      "grad_norm": 0.0003232667420100884,
      "learning_rate": 8.813486026082637e-07,
      "logits/chosen": 8.496286392211914,
      "logits/rejected": 8.1013765335083,
      "logps/chosen": -41.6943244934082,
      "logps/rejected": -43.588951110839844,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -41.6943244934082,
      "rewards/margins": 1.894622802734375,
      "rewards/rejected": -43.588951110839844,
      "step": 1690
    },
    {
      "epoch": 0.907175112895133,
      "grad_norm": 2.3577975539690234e-09,
      "learning_rate": 8.803394952186742e-07,
      "logits/chosen": 8.580137252807617,
      "logits/rejected": 8.167448043823242,
      "logps/chosen": -41.94511795043945,
      "logps/rejected": -44.74840545654297,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -41.94511795043945,
      "rewards/margins": 2.8032901287078857,
      "rewards/rejected": -44.74840545654297,
      "step": 1695
    },
    {
      "epoch": 0.9098511456765346,
      "grad_norm": 2.916944669111609e-11,
      "learning_rate": 8.793266977736342e-07,
      "logits/chosen": 8.197355270385742,
      "logits/rejected": 8.480428695678711,
      "logps/chosen": -42.772789001464844,
      "logps/rejected": -42.657264709472656,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -42.772789001464844,
      "rewards/margins": -0.11552252620458603,
      "rewards/rejected": -42.657264709472656,
      "step": 1700
    },
    {
      "epoch": 0.9125271784579361,
      "grad_norm": 6.668823122537201e-12,
      "learning_rate": 8.783102200993085e-07,
      "logits/chosen": 8.413911819458008,
      "logits/rejected": 8.282989501953125,
      "logps/chosen": -40.90471649169922,
      "logps/rejected": -42.81175994873047,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -40.90471649169922,
      "rewards/margins": 1.9070361852645874,
      "rewards/rejected": -42.81175994873047,
      "step": 1705
    },
    {
      "epoch": 0.9152032112393377,
      "grad_norm": 0.0027435313202987706,
      "learning_rate": 8.772900720575683e-07,
      "logits/chosen": 8.541772842407227,
      "logits/rejected": 8.33812427520752,
      "logps/chosen": -41.10717010498047,
      "logps/rejected": -42.386558532714844,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.10717010498047,
      "rewards/margins": 1.2793904542922974,
      "rewards/rejected": -42.386558532714844,
      "step": 1710
    },
    {
      "epoch": 0.9178792440207393,
      "grad_norm": 2.094359371452155e-08,
      "learning_rate": 8.762662635458944e-07,
      "logits/chosen": 8.611839294433594,
      "logits/rejected": 8.148812294006348,
      "logps/chosen": -42.44575500488281,
      "logps/rejected": -43.40314483642578,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.44575500488281,
      "rewards/margins": 0.9573896527290344,
      "rewards/rejected": -43.40314483642578,
      "step": 1715
    },
    {
      "epoch": 0.9205552768021408,
      "grad_norm": 2.259701601208213e-06,
      "learning_rate": 8.752388044972811e-07,
      "logits/chosen": 8.005902290344238,
      "logits/rejected": 7.8360185623168945,
      "logps/chosen": -40.55109405517578,
      "logps/rejected": -42.002342224121094,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.55109405517578,
      "rewards/margins": 1.4512544870376587,
      "rewards/rejected": -42.002342224121094,
      "step": 1720
    },
    {
      "epoch": 0.9232313095835424,
      "grad_norm": 2.9083126470412582e-08,
      "learning_rate": 8.74207704880141e-07,
      "logits/chosen": 7.996532440185547,
      "logits/rejected": 7.73723840713501,
      "logps/chosen": -40.723236083984375,
      "logps/rejected": -42.35182571411133,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -40.723236083984375,
      "rewards/margins": 1.628584861755371,
      "rewards/rejected": -42.35182571411133,
      "step": 1725
    },
    {
      "epoch": 0.925907342364944,
      "grad_norm": 1.369811063774902e-08,
      "learning_rate": 8.731729746982068e-07,
      "logits/chosen": 8.13209342956543,
      "logits/rejected": 8.156176567077637,
      "logps/chosen": -41.320064544677734,
      "logps/rejected": -42.317283630371094,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.320064544677734,
      "rewards/margins": 0.9972155690193176,
      "rewards/rejected": -42.317283630371094,
      "step": 1730
    },
    {
      "epoch": 0.9285833751463456,
      "grad_norm": 9.839647887252709e-10,
      "learning_rate": 8.721346239904355e-07,
      "logits/chosen": 8.967312812805176,
      "logits/rejected": 8.168137550354004,
      "logps/chosen": -41.162635803222656,
      "logps/rejected": -43.78205108642578,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -41.162635803222656,
      "rewards/margins": 2.619415283203125,
      "rewards/rejected": -43.78205108642578,
      "step": 1735
    },
    {
      "epoch": 0.9312594079277471,
      "grad_norm": 4.3799880594479626e-07,
      "learning_rate": 8.710926628309101e-07,
      "logits/chosen": 8.808586120605469,
      "logits/rejected": 8.132209777832031,
      "logps/chosen": -41.76424789428711,
      "logps/rejected": -43.739479064941406,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -41.76424789428711,
      "rewards/margins": 1.9752365350723267,
      "rewards/rejected": -43.739479064941406,
      "step": 1740
    },
    {
      "epoch": 0.9339354407091487,
      "grad_norm": 5.1354793163340565e-08,
      "learning_rate": 8.700471013287424e-07,
      "logits/chosen": 8.201939582824707,
      "logits/rejected": 8.200143814086914,
      "logps/chosen": -43.27826690673828,
      "logps/rejected": -43.340965270996094,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -43.27826690673828,
      "rewards/margins": 0.06269676983356476,
      "rewards/rejected": -43.340965270996094,
      "step": 1745
    },
    {
      "epoch": 0.9366114734905503,
      "grad_norm": 3.520086445710192e-08,
      "learning_rate": 8.689979496279746e-07,
      "logits/chosen": 8.340746879577637,
      "logits/rejected": 8.356642723083496,
      "logps/chosen": -42.59680938720703,
      "logps/rejected": -44.01886749267578,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -42.59680938720703,
      "rewards/margins": 1.4220478534698486,
      "rewards/rejected": -44.01886749267578,
      "step": 1750
    },
    {
      "epoch": 0.9392875062719518,
      "grad_norm": 2.784015669411274e-11,
      "learning_rate": 8.679452179074811e-07,
      "logits/chosen": 8.585580825805664,
      "logits/rejected": 8.09564208984375,
      "logps/chosen": -41.592613220214844,
      "logps/rejected": -42.21293258666992,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.592613220214844,
      "rewards/margins": 0.6203219294548035,
      "rewards/rejected": -42.21293258666992,
      "step": 1755
    },
    {
      "epoch": 0.9419635390533534,
      "grad_norm": 7.407367241557675e-06,
      "learning_rate": 8.668889163808698e-07,
      "logits/chosen": 8.71497917175293,
      "logits/rejected": 8.257665634155273,
      "logps/chosen": -41.678829193115234,
      "logps/rejected": -43.082374572753906,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -41.678829193115234,
      "rewards/margins": 1.403543472290039,
      "rewards/rejected": -43.082374572753906,
      "step": 1760
    },
    {
      "epoch": 0.944639571834755,
      "grad_norm": 3.471891219487546e-07,
      "learning_rate": 8.658290552963827e-07,
      "logits/chosen": 8.433521270751953,
      "logits/rejected": 8.206440925598145,
      "logps/chosen": -42.357627868652344,
      "logps/rejected": -42.70856857299805,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -42.357627868652344,
      "rewards/margins": 0.35094431042671204,
      "rewards/rejected": -42.70856857299805,
      "step": 1765
    },
    {
      "epoch": 0.9473156046161565,
      "grad_norm": 0.002201539058723332,
      "learning_rate": 8.647656449367966e-07,
      "logits/chosen": 8.694360733032227,
      "logits/rejected": 8.334745407104492,
      "logps/chosen": -42.95388412475586,
      "logps/rejected": -43.68437957763672,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.95388412475586,
      "rewards/margins": 0.7304926514625549,
      "rewards/rejected": -43.68437957763672,
      "step": 1770
    },
    {
      "epoch": 0.9499916373975581,
      "grad_norm": 0.0014708195083644588,
      "learning_rate": 8.636986956193235e-07,
      "logits/chosen": 8.296953201293945,
      "logits/rejected": 7.818070888519287,
      "logps/chosen": -42.06635665893555,
      "logps/rejected": -43.17518615722656,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.06635665893555,
      "rewards/margins": 1.1088327169418335,
      "rewards/rejected": -43.17518615722656,
      "step": 1775
    },
    {
      "epoch": 0.9526676701789597,
      "grad_norm": 3.420495542446005e-09,
      "learning_rate": 8.626282176955104e-07,
      "logits/chosen": 8.911827087402344,
      "logits/rejected": 8.519376754760742,
      "logps/chosen": -42.86060333251953,
      "logps/rejected": -44.744632720947266,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -42.86060333251953,
      "rewards/margins": 1.8840324878692627,
      "rewards/rejected": -44.744632720947266,
      "step": 1780
    },
    {
      "epoch": 0.9553437029603613,
      "grad_norm": 7.776688257675224e-12,
      "learning_rate": 8.615542215511389e-07,
      "logits/chosen": 8.58808708190918,
      "logits/rejected": 8.394648551940918,
      "logps/chosen": -42.973960876464844,
      "logps/rejected": -44.11064529418945,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -42.973960876464844,
      "rewards/margins": 1.1366803646087646,
      "rewards/rejected": -44.11064529418945,
      "step": 1785
    },
    {
      "epoch": 0.9580197357417628,
      "grad_norm": 5.5938860306159576e-06,
      "learning_rate": 8.604767176061241e-07,
      "logits/chosen": 8.352361679077148,
      "logits/rejected": 7.854604244232178,
      "logps/chosen": -42.853572845458984,
      "logps/rejected": -43.37690734863281,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -42.853572845458984,
      "rewards/margins": 0.5233327150344849,
      "rewards/rejected": -43.37690734863281,
      "step": 1790
    },
    {
      "epoch": 0.9606957685231644,
      "grad_norm": 1.3661032576296858e-10,
      "learning_rate": 8.593957163144141e-07,
      "logits/chosen": 8.70111083984375,
      "logits/rejected": 8.260234832763672,
      "logps/chosen": -43.15165710449219,
      "logps/rejected": -44.842491149902344,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.15165710449219,
      "rewards/margins": 1.6908363103866577,
      "rewards/rejected": -44.842491149902344,
      "step": 1795
    },
    {
      "epoch": 0.963371801304566,
      "grad_norm": 8.032786151297712e-06,
      "learning_rate": 8.58311228163888e-07,
      "logits/chosen": 9.00760555267334,
      "logits/rejected": 8.793514251708984,
      "logps/chosen": -43.318546295166016,
      "logps/rejected": -44.29603958129883,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.318546295166016,
      "rewards/margins": 0.9774940609931946,
      "rewards/rejected": -44.29603958129883,
      "step": 1800
    },
    {
      "epoch": 0.9660478340859675,
      "grad_norm": 1.694038949436402e-05,
      "learning_rate": 8.57223263676255e-07,
      "logits/chosen": 9.294004440307617,
      "logits/rejected": 8.8113431930542,
      "logps/chosen": -43.03097915649414,
      "logps/rejected": -44.179664611816406,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.03097915649414,
      "rewards/margins": 1.1486834287643433,
      "rewards/rejected": -44.179664611816406,
      "step": 1805
    },
    {
      "epoch": 0.9687238668673691,
      "grad_norm": 1.4750523337676962e-07,
      "learning_rate": 8.561318334069511e-07,
      "logits/chosen": 8.735937118530273,
      "logits/rejected": 8.429232597351074,
      "logps/chosen": -43.47926330566406,
      "logps/rejected": -45.0063362121582,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.47926330566406,
      "rewards/margins": 1.5270713567733765,
      "rewards/rejected": -45.0063362121582,
      "step": 1810
    },
    {
      "epoch": 0.9713998996487707,
      "grad_norm": 3.8724096524068797e-11,
      "learning_rate": 8.550369479450375e-07,
      "logits/chosen": 8.787284851074219,
      "logits/rejected": 8.19578742980957,
      "logps/chosen": -41.862693786621094,
      "logps/rejected": -43.49113082885742,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -41.862693786621094,
      "rewards/margins": 1.6284383535385132,
      "rewards/rejected": -43.49113082885742,
      "step": 1815
    },
    {
      "epoch": 0.9740759324301723,
      "grad_norm": 1.3347005599297042e-05,
      "learning_rate": 8.539386179130977e-07,
      "logits/chosen": 8.54887866973877,
      "logits/rejected": 8.226997375488281,
      "logps/chosen": -42.793678283691406,
      "logps/rejected": -43.489524841308594,
      "loss": 0.0001,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -42.793678283691406,
      "rewards/margins": 0.6958504915237427,
      "rewards/rejected": -43.489524841308594,
      "step": 1820
    },
    {
      "epoch": 0.9767519652115738,
      "grad_norm": 5.965655549840825e-05,
      "learning_rate": 8.528368539671347e-07,
      "logits/chosen": 8.820348739624023,
      "logits/rejected": 8.11103630065918,
      "logps/chosen": -44.245567321777344,
      "logps/rejected": -45.766029357910156,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.245567321777344,
      "rewards/margins": 1.5204589366912842,
      "rewards/rejected": -45.766029357910156,
      "step": 1825
    },
    {
      "epoch": 0.9794279979929754,
      "grad_norm": 1.4419617290907867e-05,
      "learning_rate": 8.51731666796467e-07,
      "logits/chosen": 8.005538940429688,
      "logits/rejected": 7.946224212646484,
      "logps/chosen": -41.70168685913086,
      "logps/rejected": -42.295692443847656,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -41.70168685913086,
      "rewards/margins": 0.5940046310424805,
      "rewards/rejected": -42.295692443847656,
      "step": 1830
    },
    {
      "epoch": 0.982104030774377,
      "grad_norm": 2.22547397420859e-07,
      "learning_rate": 8.506230671236254e-07,
      "logits/chosen": 8.346652030944824,
      "logits/rejected": 8.038139343261719,
      "logps/chosen": -42.232234954833984,
      "logps/rejected": -44.02997970581055,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.232234954833984,
      "rewards/margins": 1.7977468967437744,
      "rewards/rejected": -44.02997970581055,
      "step": 1835
    },
    {
      "epoch": 0.9847800635557785,
      "grad_norm": 6.583176202841572e-08,
      "learning_rate": 8.495110657042488e-07,
      "logits/chosen": 8.591679573059082,
      "logits/rejected": 8.006250381469727,
      "logps/chosen": -40.95270538330078,
      "logps/rejected": -42.5833625793457,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -40.95270538330078,
      "rewards/margins": 1.6306545734405518,
      "rewards/rejected": -42.5833625793457,
      "step": 1840
    },
    {
      "epoch": 0.9874560963371801,
      "grad_norm": 4.531156271153425e-11,
      "learning_rate": 8.483956733269799e-07,
      "logits/chosen": 8.36819839477539,
      "logits/rejected": 8.24220085144043,
      "logps/chosen": -43.28996658325195,
      "logps/rejected": -44.041107177734375,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.28996658325195,
      "rewards/margins": 0.7511435747146606,
      "rewards/rejected": -44.041107177734375,
      "step": 1845
    },
    {
      "epoch": 0.9901321291185817,
      "grad_norm": 1.1804644662256964e-09,
      "learning_rate": 8.472769008133602e-07,
      "logits/chosen": 8.548955917358398,
      "logits/rejected": 8.488530158996582,
      "logps/chosen": -40.545005798339844,
      "logps/rejected": -42.96592330932617,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -40.545005798339844,
      "rewards/margins": 2.4209156036376953,
      "rewards/rejected": -42.96592330932617,
      "step": 1850
    },
    {
      "epoch": 0.9928081618999832,
      "grad_norm": 5.399983327161223e-11,
      "learning_rate": 8.461547590177259e-07,
      "logits/chosen": 8.739306449890137,
      "logits/rejected": 8.291426658630371,
      "logps/chosen": -43.02908706665039,
      "logps/rejected": -44.632598876953125,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.02908706665039,
      "rewards/margins": 1.603514313697815,
      "rewards/rejected": -44.632598876953125,
      "step": 1855
    },
    {
      "epoch": 0.9954841946813848,
      "grad_norm": 8.881935411136834e-12,
      "learning_rate": 8.450292588271014e-07,
      "logits/chosen": 8.667684555053711,
      "logits/rejected": 8.301679611206055,
      "logps/chosen": -42.72836685180664,
      "logps/rejected": -43.61931610107422,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.72836685180664,
      "rewards/margins": 0.8909528851509094,
      "rewards/rejected": -43.61931610107422,
      "step": 1860
    },
    {
      "epoch": 0.9981602274627864,
      "grad_norm": 0.08753328229400195,
      "learning_rate": 8.439004111610945e-07,
      "logits/chosen": 8.803364753723145,
      "logits/rejected": 8.488096237182617,
      "logps/chosen": -43.48000717163086,
      "logps/rejected": -43.44244384765625,
      "loss": 0.0003,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -43.48000717163086,
      "rewards/margins": -0.03756345435976982,
      "rewards/rejected": -43.44244384765625,
      "step": 1865
    },
    {
      "epoch": 1.000836260244188,
      "grad_norm": 8.943847208218013e-09,
      "learning_rate": 8.427682269717901e-07,
      "logits/chosen": 8.532743453979492,
      "logits/rejected": 8.1969633102417,
      "logps/chosen": -41.90042495727539,
      "logps/rejected": -43.24674987792969,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -41.90042495727539,
      "rewards/margins": 1.3463269472122192,
      "rewards/rejected": -43.24674987792969,
      "step": 1870
    },
    {
      "epoch": 1.0035122930255895,
      "grad_norm": 6.66371362037926e-06,
      "learning_rate": 8.416327172436446e-07,
      "logits/chosen": 9.156952857971191,
      "logits/rejected": 8.5985689163208,
      "logps/chosen": -43.20523452758789,
      "logps/rejected": -44.43049621582031,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.20523452758789,
      "rewards/margins": 1.2252585887908936,
      "rewards/rejected": -44.43049621582031,
      "step": 1875
    },
    {
      "epoch": 1.0061883258069912,
      "grad_norm": 8.51858248294637e-11,
      "learning_rate": 8.404938929933778e-07,
      "logits/chosen": 8.590893745422363,
      "logits/rejected": 8.292744636535645,
      "logps/chosen": -43.576141357421875,
      "logps/rejected": -44.4603157043457,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -43.576141357421875,
      "rewards/margins": 0.8841696977615356,
      "rewards/rejected": -44.4603157043457,
      "step": 1880
    },
    {
      "epoch": 1.0088643585883927,
      "grad_norm": 9.627666083431892e-08,
      "learning_rate": 8.39351765269868e-07,
      "logits/chosen": 8.743722915649414,
      "logits/rejected": 8.70246410369873,
      "logps/chosen": -43.842132568359375,
      "logps/rejected": -44.154640197753906,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.842132568359375,
      "rewards/margins": 0.31250327825546265,
      "rewards/rejected": -44.154640197753906,
      "step": 1885
    },
    {
      "epoch": 1.0115403913697942,
      "grad_norm": 1.13900078844664e-09,
      "learning_rate": 8.382063451540431e-07,
      "logits/chosen": 8.592141151428223,
      "logits/rejected": 8.086342811584473,
      "logps/chosen": -41.795047760009766,
      "logps/rejected": -43.09931182861328,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -41.795047760009766,
      "rewards/margins": 1.3042659759521484,
      "rewards/rejected": -43.09931182861328,
      "step": 1890
    },
    {
      "epoch": 1.014216424151196,
      "grad_norm": 0.00011627941488685385,
      "learning_rate": 8.370576437587742e-07,
      "logits/chosen": 8.390515327453613,
      "logits/rejected": 8.421978950500488,
      "logps/chosen": -41.48577117919922,
      "logps/rejected": -42.77922439575195,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.48577117919922,
      "rewards/margins": 1.293454647064209,
      "rewards/rejected": -42.77922439575195,
      "step": 1895
    },
    {
      "epoch": 1.0168924569325974,
      "grad_norm": 5.104927511708731e-06,
      "learning_rate": 8.359056722287674e-07,
      "logits/chosen": 9.309017181396484,
      "logits/rejected": 8.46671199798584,
      "logps/chosen": -42.21105194091797,
      "logps/rejected": -43.988563537597656,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.21105194091797,
      "rewards/margins": 1.7775068283081055,
      "rewards/rejected": -43.988563537597656,
      "step": 1900
    },
    {
      "epoch": 1.019568489713999,
      "grad_norm": 4.43969952570509e-07,
      "learning_rate": 8.347504417404553e-07,
      "logits/chosen": 8.61168098449707,
      "logits/rejected": 8.234295845031738,
      "logps/chosen": -42.109764099121094,
      "logps/rejected": -43.71144485473633,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.109764099121094,
      "rewards/margins": 1.6016775369644165,
      "rewards/rejected": -43.71144485473633,
      "step": 1905
    },
    {
      "epoch": 1.0222445224954007,
      "grad_norm": 3.3723417147981277e-07,
      "learning_rate": 8.335919635018893e-07,
      "logits/chosen": 8.798188209533691,
      "logits/rejected": 8.498754501342773,
      "logps/chosen": -43.32455062866211,
      "logps/rejected": -43.65833282470703,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.32455062866211,
      "rewards/margins": 0.33378100395202637,
      "rewards/rejected": -43.65833282470703,
      "step": 1910
    },
    {
      "epoch": 1.0249205552768021,
      "grad_norm": 2.7087568949305054e-08,
      "learning_rate": 8.324302487526303e-07,
      "logits/chosen": 8.762731552124023,
      "logits/rejected": 8.394326210021973,
      "logps/chosen": -43.267677307128906,
      "logps/rejected": -44.11416244506836,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.267677307128906,
      "rewards/margins": 0.8464804887771606,
      "rewards/rejected": -44.11416244506836,
      "step": 1915
    },
    {
      "epoch": 1.0275965880582036,
      "grad_norm": 4.3477684432625015e-08,
      "learning_rate": 8.312653087636398e-07,
      "logits/chosen": 9.015066146850586,
      "logits/rejected": 8.743660926818848,
      "logps/chosen": -43.62114715576172,
      "logps/rejected": -44.832603454589844,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.62114715576172,
      "rewards/margins": 1.211453914642334,
      "rewards/rejected": -44.832603454589844,
      "step": 1920
    },
    {
      "epoch": 1.0302726208396054,
      "grad_norm": 8.510747455909584e-09,
      "learning_rate": 8.300971548371711e-07,
      "logits/chosen": 9.043620109558105,
      "logits/rejected": 8.34809684753418,
      "logps/chosen": -42.1926383972168,
      "logps/rejected": -44.4886360168457,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -42.1926383972168,
      "rewards/margins": 2.2959978580474854,
      "rewards/rejected": -44.4886360168457,
      "step": 1925
    },
    {
      "epoch": 1.0329486536210069,
      "grad_norm": 1.423995685948287e-10,
      "learning_rate": 8.289257983066582e-07,
      "logits/chosen": 8.903180122375488,
      "logits/rejected": 8.467575073242188,
      "logps/chosen": -42.621944427490234,
      "logps/rejected": -44.510459899902344,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.621944427490234,
      "rewards/margins": 1.8885166645050049,
      "rewards/rejected": -44.510459899902344,
      "step": 1930
    },
    {
      "epoch": 1.0356246864024083,
      "grad_norm": 1.6484924734862472e-06,
      "learning_rate": 8.277512505366077e-07,
      "logits/chosen": 8.71349048614502,
      "logits/rejected": 8.1802339553833,
      "logps/chosen": -41.685386657714844,
      "logps/rejected": -44.19927215576172,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -41.685386657714844,
      "rewards/margins": 2.5138864517211914,
      "rewards/rejected": -44.19927215576172,
      "step": 1935
    },
    {
      "epoch": 1.03830071918381,
      "grad_norm": 2.441391793810888e-09,
      "learning_rate": 8.265735229224868e-07,
      "logits/chosen": 8.782456398010254,
      "logits/rejected": 8.615585327148438,
      "logps/chosen": -42.2185173034668,
      "logps/rejected": -44.476715087890625,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.2185173034668,
      "rewards/margins": 2.258202314376831,
      "rewards/rejected": -44.476715087890625,
      "step": 1940
    },
    {
      "epoch": 1.0409767519652116,
      "grad_norm": 2.8980246280541657e-10,
      "learning_rate": 8.253926268906144e-07,
      "logits/chosen": 8.691670417785645,
      "logits/rejected": 8.372140884399414,
      "logps/chosen": -42.080787658691406,
      "logps/rejected": -44.23410415649414,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.080787658691406,
      "rewards/margins": 2.1533143520355225,
      "rewards/rejected": -44.23410415649414,
      "step": 1945
    },
    {
      "epoch": 1.043652784746613,
      "grad_norm": 0.1897265726307852,
      "learning_rate": 8.242085738980487e-07,
      "logits/chosen": 8.874157905578613,
      "logits/rejected": 8.133414268493652,
      "logps/chosen": -42.67548751831055,
      "logps/rejected": -44.448326110839844,
      "loss": 0.0008,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.67548751831055,
      "rewards/margins": 1.7728404998779297,
      "rewards/rejected": -44.448326110839844,
      "step": 1950
    },
    {
      "epoch": 1.0463288175280148,
      "grad_norm": 1.2954100944548133e-08,
      "learning_rate": 8.230213754324772e-07,
      "logits/chosen": 9.036465644836426,
      "logits/rejected": 8.703609466552734,
      "logps/chosen": -42.643943786621094,
      "logps/rejected": -43.67623519897461,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.643943786621094,
      "rewards/margins": 1.0322908163070679,
      "rewards/rejected": -43.67623519897461,
      "step": 1955
    },
    {
      "epoch": 1.0490048503094163,
      "grad_norm": 4.972616134113379e-09,
      "learning_rate": 8.218310430121045e-07,
      "logits/chosen": 8.691082954406738,
      "logits/rejected": 8.539655685424805,
      "logps/chosen": -43.29679870605469,
      "logps/rejected": -43.7542610168457,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.29679870605469,
      "rewards/margins": 0.4574567675590515,
      "rewards/rejected": -43.7542610168457,
      "step": 1960
    },
    {
      "epoch": 1.051680883090818,
      "grad_norm": 3.885143336983548e-10,
      "learning_rate": 8.20637588185541e-07,
      "logits/chosen": 8.450057983398438,
      "logits/rejected": 8.270268440246582,
      "logps/chosen": -41.76343536376953,
      "logps/rejected": -43.358177185058594,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -41.76343536376953,
      "rewards/margins": 1.5947455167770386,
      "rewards/rejected": -43.358177185058594,
      "step": 1965
    },
    {
      "epoch": 1.0543569158722195,
      "grad_norm": 7.989290451155311e-07,
      "learning_rate": 8.194410225316906e-07,
      "logits/chosen": 8.867195129394531,
      "logits/rejected": 8.251626014709473,
      "logps/chosen": -42.74251174926758,
      "logps/rejected": -44.1317024230957,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -42.74251174926758,
      "rewards/margins": 1.3891897201538086,
      "rewards/rejected": -44.1317024230957,
      "step": 1970
    },
    {
      "epoch": 1.057032948653621,
      "grad_norm": 1.845538809860964e-07,
      "learning_rate": 8.182413576596385e-07,
      "logits/chosen": 8.691468238830566,
      "logits/rejected": 8.621297836303711,
      "logps/chosen": -43.08960723876953,
      "logps/rejected": -44.57876968383789,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.08960723876953,
      "rewards/margins": 1.4891583919525146,
      "rewards/rejected": -44.57876968383789,
      "step": 1975
    },
    {
      "epoch": 1.0597089814350227,
      "grad_norm": 3.74488640941994e-07,
      "learning_rate": 8.170386052085389e-07,
      "logits/chosen": 8.653565406799316,
      "logits/rejected": 8.229268074035645,
      "logps/chosen": -42.642669677734375,
      "logps/rejected": -43.07680892944336,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -42.642669677734375,
      "rewards/margins": 0.43413639068603516,
      "rewards/rejected": -43.07680892944336,
      "step": 1980
    },
    {
      "epoch": 1.0623850142164242,
      "grad_norm": 6.504029972656149e-06,
      "learning_rate": 8.158327768475008e-07,
      "logits/chosen": 8.6459379196167,
      "logits/rejected": 8.188133239746094,
      "logps/chosen": -42.11989212036133,
      "logps/rejected": -43.46466064453125,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.11989212036133,
      "rewards/margins": 1.3447706699371338,
      "rewards/rejected": -43.46466064453125,
      "step": 1985
    },
    {
      "epoch": 1.0650610469978257,
      "grad_norm": 1.3778157037892374e-08,
      "learning_rate": 8.146238842754767e-07,
      "logits/chosen": 8.956924438476562,
      "logits/rejected": 8.432344436645508,
      "logps/chosen": -43.13521957397461,
      "logps/rejected": -43.81230926513672,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -43.13521957397461,
      "rewards/margins": 0.6770875453948975,
      "rewards/rejected": -43.81230926513672,
      "step": 1990
    },
    {
      "epoch": 1.0677370797792274,
      "grad_norm": 2.489851318412953e-05,
      "learning_rate": 8.134119392211476e-07,
      "logits/chosen": 8.880282402038574,
      "logits/rejected": 8.309147834777832,
      "logps/chosen": -43.41967010498047,
      "logps/rejected": -44.22102355957031,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.41967010498047,
      "rewards/margins": 0.8013540506362915,
      "rewards/rejected": -44.22102355957031,
      "step": 1995
    },
    {
      "epoch": 1.0704131125606289,
      "grad_norm": 6.41679976594431e-07,
      "learning_rate": 8.121969534428094e-07,
      "logits/chosen": 8.959689140319824,
      "logits/rejected": 8.50571060180664,
      "logps/chosen": -44.57155990600586,
      "logps/rejected": -45.72116470336914,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.57155990600586,
      "rewards/margins": 1.149599313735962,
      "rewards/rejected": -45.72116470336914,
      "step": 2000
    },
    {
      "epoch": 1.0704131125606289,
      "eval_logits/chosen": 7.350369453430176,
      "eval_logits/rejected": 7.081839084625244,
      "eval_logps/chosen": -43.05656051635742,
      "eval_logps/rejected": -44.22747039794922,
      "eval_loss": 3.2351957202081394e-07,
      "eval_rewards/accuracies": 0.5652819275856018,
      "eval_rewards/chosen": -43.05656051635742,
      "eval_rewards/margins": 1.1709110736846924,
      "eval_rewards/rejected": -44.22747039794922,
      "eval_runtime": 40.0925,
      "eval_samples_per_second": 33.547,
      "eval_steps_per_second": 8.406,
      "step": 2000
    },
    {
      "epoch": 1.0730891453420304,
      "grad_norm": 7.419936058124262e-08,
      "learning_rate": 8.109789387282599e-07,
      "logits/chosen": 8.68393611907959,
      "logits/rejected": 8.521844863891602,
      "logps/chosen": -43.422325134277344,
      "logps/rejected": -44.33806610107422,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.422325134277344,
      "rewards/margins": 0.915744423866272,
      "rewards/rejected": -44.33806610107422,
      "step": 2005
    },
    {
      "epoch": 1.075765178123432,
      "grad_norm": 5.292042013430684e-12,
      "learning_rate": 8.097579068946827e-07,
      "logits/chosen": 8.659852027893066,
      "logits/rejected": 8.207812309265137,
      "logps/chosen": -43.48197555541992,
      "logps/rejected": -44.778053283691406,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.48197555541992,
      "rewards/margins": 1.2960784435272217,
      "rewards/rejected": -44.778053283691406,
      "step": 2010
    },
    {
      "epoch": 1.0784412109048336,
      "grad_norm": 2.4506657573406684e-09,
      "learning_rate": 8.085338697885344e-07,
      "logits/chosen": 8.508906364440918,
      "logits/rejected": 8.130454063415527,
      "logps/chosen": -43.33389663696289,
      "logps/rejected": -44.075950622558594,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.33389663696289,
      "rewards/margins": 0.7420555353164673,
      "rewards/rejected": -44.075950622558594,
      "step": 2015
    },
    {
      "epoch": 1.081117243686235,
      "grad_norm": 1.8282395133667655e-10,
      "learning_rate": 8.073068392854282e-07,
      "logits/chosen": 9.17225456237793,
      "logits/rejected": 8.343960762023926,
      "logps/chosen": -42.68581008911133,
      "logps/rejected": -44.159629821777344,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.68581008911133,
      "rewards/margins": 1.4738194942474365,
      "rewards/rejected": -44.159629821777344,
      "step": 2020
    },
    {
      "epoch": 1.0837932764676368,
      "grad_norm": 1.3308253770411174e-05,
      "learning_rate": 8.060768272900193e-07,
      "logits/chosen": 9.22791862487793,
      "logits/rejected": 8.561975479125977,
      "logps/chosen": -44.48027420043945,
      "logps/rejected": -45.280784606933594,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.48027420043945,
      "rewards/margins": 0.8005092740058899,
      "rewards/rejected": -45.280784606933594,
      "step": 2025
    },
    {
      "epoch": 1.0864693092490383,
      "grad_norm": 3.494895663333807e-05,
      "learning_rate": 8.0484384573589e-07,
      "logits/chosen": 8.314882278442383,
      "logits/rejected": 8.426496505737305,
      "logps/chosen": -42.596534729003906,
      "logps/rejected": -43.365299224853516,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -42.596534729003906,
      "rewards/margins": 0.7687662243843079,
      "rewards/rejected": -43.365299224853516,
      "step": 2030
    },
    {
      "epoch": 1.0891453420304398,
      "grad_norm": 2.128392332226667e-05,
      "learning_rate": 8.03607906585432e-07,
      "logits/chosen": 8.7073974609375,
      "logits/rejected": 8.39579963684082,
      "logps/chosen": -43.47890090942383,
      "logps/rejected": -45.63812255859375,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.47890090942383,
      "rewards/margins": 2.1592299938201904,
      "rewards/rejected": -45.63812255859375,
      "step": 2035
    },
    {
      "epoch": 1.0918213748118415,
      "grad_norm": 4.906351350662181e-07,
      "learning_rate": 8.023690218297329e-07,
      "logits/chosen": 8.71317195892334,
      "logits/rejected": 8.738039016723633,
      "logps/chosen": -43.52259063720703,
      "logps/rejected": -44.76307678222656,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.52259063720703,
      "rewards/margins": 1.2404820919036865,
      "rewards/rejected": -44.76307678222656,
      "step": 2040
    },
    {
      "epoch": 1.094497407593243,
      "grad_norm": 5.001953503703762e-08,
      "learning_rate": 8.01127203488458e-07,
      "logits/chosen": 8.33745002746582,
      "logits/rejected": 8.217546463012695,
      "logps/chosen": -42.32817077636719,
      "logps/rejected": -43.400917053222656,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -42.32817077636719,
      "rewards/margins": 1.0727466344833374,
      "rewards/rejected": -43.400917053222656,
      "step": 2045
    },
    {
      "epoch": 1.0971734403746445,
      "grad_norm": 1.3691370863215083e-05,
      "learning_rate": 7.998824636097339e-07,
      "logits/chosen": 9.19202709197998,
      "logits/rejected": 8.742021560668945,
      "logps/chosen": -42.797508239746094,
      "logps/rejected": -44.608482360839844,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.797508239746094,
      "rewards/margins": 1.810969352722168,
      "rewards/rejected": -44.608482360839844,
      "step": 2050
    },
    {
      "epoch": 1.0998494731560462,
      "grad_norm": 1.502348162764091e-10,
      "learning_rate": 7.986348142700328e-07,
      "logits/chosen": 8.75763988494873,
      "logits/rejected": 8.437353134155273,
      "logps/chosen": -42.772300720214844,
      "logps/rejected": -44.720699310302734,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.772300720214844,
      "rewards/margins": 1.948394775390625,
      "rewards/rejected": -44.720699310302734,
      "step": 2055
    },
    {
      "epoch": 1.1025255059374477,
      "grad_norm": 4.4335426342294256e-08,
      "learning_rate": 7.973842675740539e-07,
      "logits/chosen": 8.502705574035645,
      "logits/rejected": 8.494972229003906,
      "logps/chosen": -43.58302688598633,
      "logps/rejected": -44.24788284301758,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -43.58302688598633,
      "rewards/margins": 0.6648537516593933,
      "rewards/rejected": -44.24788284301758,
      "step": 2060
    },
    {
      "epoch": 1.1052015387188494,
      "grad_norm": 7.273470078185088e-06,
      "learning_rate": 7.961308356546066e-07,
      "logits/chosen": 8.698301315307617,
      "logits/rejected": 8.425902366638184,
      "logps/chosen": -42.487144470214844,
      "logps/rejected": -43.998748779296875,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.487144470214844,
      "rewards/margins": 1.5116064548492432,
      "rewards/rejected": -43.998748779296875,
      "step": 2065
    },
    {
      "epoch": 1.107877571500251,
      "grad_norm": 1.0763330454527172e-07,
      "learning_rate": 7.948745306724931e-07,
      "logits/chosen": 8.871599197387695,
      "logits/rejected": 8.478073120117188,
      "logps/chosen": -43.731361389160156,
      "logps/rejected": -44.89190673828125,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.731361389160156,
      "rewards/margins": 1.16054105758667,
      "rewards/rejected": -44.89190673828125,
      "step": 2070
    },
    {
      "epoch": 1.1105536042816524,
      "grad_norm": 4.777698843109935e-10,
      "learning_rate": 7.936153648163897e-07,
      "logits/chosen": 8.939006805419922,
      "logits/rejected": 8.702670097351074,
      "logps/chosen": -43.34833526611328,
      "logps/rejected": -45.311004638671875,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.34833526611328,
      "rewards/margins": 1.9626678228378296,
      "rewards/rejected": -45.311004638671875,
      "step": 2075
    },
    {
      "epoch": 1.1132296370630541,
      "grad_norm": 2.7705505180250193e-06,
      "learning_rate": 7.92353350302729e-07,
      "logits/chosen": 8.565095901489258,
      "logits/rejected": 8.232255935668945,
      "logps/chosen": -41.69127655029297,
      "logps/rejected": -43.322635650634766,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -41.69127655029297,
      "rewards/margins": 1.6313543319702148,
      "rewards/rejected": -43.322635650634766,
      "step": 2080
    },
    {
      "epoch": 1.1159056698444556,
      "grad_norm": 6.03941347691768e-08,
      "learning_rate": 7.910884993755816e-07,
      "logits/chosen": 8.783143997192383,
      "logits/rejected": 8.513270378112793,
      "logps/chosen": -42.798423767089844,
      "logps/rejected": -44.42816925048828,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.798423767089844,
      "rewards/margins": 1.629741907119751,
      "rewards/rejected": -44.42816925048828,
      "step": 2085
    },
    {
      "epoch": 1.118581702625857,
      "grad_norm": 4.5450576336810844e-09,
      "learning_rate": 7.898208243065367e-07,
      "logits/chosen": 8.803214073181152,
      "logits/rejected": 8.838974952697754,
      "logps/chosen": -43.979095458984375,
      "logps/rejected": -44.18968200683594,
      "loss": 0.0001,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.979095458984375,
      "rewards/margins": 0.21058520674705505,
      "rewards/rejected": -44.18968200683594,
      "step": 2090
    },
    {
      "epoch": 1.1212577354072588,
      "grad_norm": 1.4167286784948548e-09,
      "learning_rate": 7.88550337394583e-07,
      "logits/chosen": 9.222752571105957,
      "logits/rejected": 8.78388500213623,
      "logps/chosen": -42.92869186401367,
      "logps/rejected": -44.4694709777832,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.92869186401367,
      "rewards/margins": 1.540779709815979,
      "rewards/rejected": -44.4694709777832,
      "step": 2095
    },
    {
      "epoch": 1.1239337681886603,
      "grad_norm": 1.93585309105478e-07,
      "learning_rate": 7.872770509659905e-07,
      "logits/chosen": 8.536057472229004,
      "logits/rejected": 8.395170211791992,
      "logps/chosen": -42.78935623168945,
      "logps/rejected": -44.01171112060547,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.78935623168945,
      "rewards/margins": 1.222355604171753,
      "rewards/rejected": -44.01171112060547,
      "step": 2100
    },
    {
      "epoch": 1.1266098009700618,
      "grad_norm": 9.349478340294662e-06,
      "learning_rate": 7.860009773741896e-07,
      "logits/chosen": 8.915497779846191,
      "logits/rejected": 8.378942489624023,
      "logps/chosen": -42.82252883911133,
      "logps/rejected": -44.62291717529297,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.82252883911133,
      "rewards/margins": 1.8003942966461182,
      "rewards/rejected": -44.62291717529297,
      "step": 2105
    },
    {
      "epoch": 1.1292858337514635,
      "grad_norm": 2.2421274145840705e-06,
      "learning_rate": 7.84722128999652e-07,
      "logits/chosen": 8.817072868347168,
      "logits/rejected": 8.509749412536621,
      "logps/chosen": -43.40343475341797,
      "logps/rejected": -44.535362243652344,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.40343475341797,
      "rewards/margins": 1.131927728652954,
      "rewards/rejected": -44.535362243652344,
      "step": 2110
    },
    {
      "epoch": 1.131961866532865,
      "grad_norm": 0.0010395379711368071,
      "learning_rate": 7.834405182497699e-07,
      "logits/chosen": 8.531498908996582,
      "logits/rejected": 8.352632522583008,
      "logps/chosen": -42.895790100097656,
      "logps/rejected": -44.186378479003906,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.895790100097656,
      "rewards/margins": 1.290583848953247,
      "rewards/rejected": -44.186378479003906,
      "step": 2115
    },
    {
      "epoch": 1.1346378993142665,
      "grad_norm": 1.5123108087074543e-07,
      "learning_rate": 7.821561575587368e-07,
      "logits/chosen": 8.779542922973633,
      "logits/rejected": 8.61886978149414,
      "logps/chosen": -42.46742630004883,
      "logps/rejected": -43.34770965576172,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.46742630004883,
      "rewards/margins": 0.880290687084198,
      "rewards/rejected": -43.34770965576172,
      "step": 2120
    },
    {
      "epoch": 1.1373139320956682,
      "grad_norm": 1.2609906288717107e-07,
      "learning_rate": 7.808690593874254e-07,
      "logits/chosen": 8.832437515258789,
      "logits/rejected": 8.350345611572266,
      "logps/chosen": -43.25499725341797,
      "logps/rejected": -44.13262176513672,
      "loss": 0.0015,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -43.25499725341797,
      "rewards/margins": 0.8776203393936157,
      "rewards/rejected": -44.13262176513672,
      "step": 2125
    },
    {
      "epoch": 1.1399899648770697,
      "grad_norm": 0.00012310032314017053,
      "learning_rate": 7.79579236223268e-07,
      "logits/chosen": 8.714537620544434,
      "logits/rejected": 7.991166591644287,
      "logps/chosen": -42.105796813964844,
      "logps/rejected": -45.18198013305664,
      "loss": 0.0,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -42.105796813964844,
      "rewards/margins": 3.07619047164917,
      "rewards/rejected": -45.18198013305664,
      "step": 2130
    },
    {
      "epoch": 1.1426659976584714,
      "grad_norm": 6.749196725696971e-08,
      "learning_rate": 7.782867005801346e-07,
      "logits/chosen": 8.968340873718262,
      "logits/rejected": 8.366634368896484,
      "logps/chosen": -42.5718879699707,
      "logps/rejected": -44.91071319580078,
      "loss": 0.0,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -42.5718879699707,
      "rewards/margins": 2.338827133178711,
      "rewards/rejected": -44.91071319580078,
      "step": 2135
    },
    {
      "epoch": 1.145342030439873,
      "grad_norm": 7.306954643610832e-07,
      "learning_rate": 7.769914649982117e-07,
      "logits/chosen": 8.708084106445312,
      "logits/rejected": 8.260618209838867,
      "logps/chosen": -42.67002868652344,
      "logps/rejected": -44.796661376953125,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -42.67002868652344,
      "rewards/margins": 2.126635789871216,
      "rewards/rejected": -44.796661376953125,
      "step": 2140
    },
    {
      "epoch": 1.1480180632212744,
      "grad_norm": 8.335905616073768e-10,
      "learning_rate": 7.756935420438803e-07,
      "logits/chosen": 8.6829195022583,
      "logits/rejected": 8.422986030578613,
      "logps/chosen": -43.164031982421875,
      "logps/rejected": -44.609073638916016,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.164031982421875,
      "rewards/margins": 1.44503915309906,
      "rewards/rejected": -44.609073638916016,
      "step": 2145
    },
    {
      "epoch": 1.1506940960026761,
      "grad_norm": 5.377015746272716e-05,
      "learning_rate": 7.743929443095951e-07,
      "logits/chosen": 8.756189346313477,
      "logits/rejected": 8.58474349975586,
      "logps/chosen": -43.47377014160156,
      "logps/rejected": -44.60169982910156,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.47377014160156,
      "rewards/margins": 1.1279295682907104,
      "rewards/rejected": -44.60169982910156,
      "step": 2150
    },
    {
      "epoch": 1.1533701287840776,
      "grad_norm": 1.978335163378929e-05,
      "learning_rate": 7.730896844137609e-07,
      "logits/chosen": 8.597490310668945,
      "logits/rejected": 8.439139366149902,
      "logps/chosen": -43.48359298706055,
      "logps/rejected": -44.268943786621094,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -43.48359298706055,
      "rewards/margins": 0.7853471040725708,
      "rewards/rejected": -44.268943786621094,
      "step": 2155
    },
    {
      "epoch": 1.1560461615654791,
      "grad_norm": 3.3369911166403113e-09,
      "learning_rate": 7.717837750006106e-07,
      "logits/chosen": 9.033609390258789,
      "logits/rejected": 8.801519393920898,
      "logps/chosen": -44.42205047607422,
      "logps/rejected": -45.103248596191406,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -44.42205047607422,
      "rewards/margins": 0.6811951994895935,
      "rewards/rejected": -45.103248596191406,
      "step": 2160
    },
    {
      "epoch": 1.1587221943468808,
      "grad_norm": 9.081359635269009e-09,
      "learning_rate": 7.704752287400832e-07,
      "logits/chosen": 8.743975639343262,
      "logits/rejected": 8.270899772644043,
      "logps/chosen": -44.27429962158203,
      "logps/rejected": -44.66236114501953,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -44.27429962158203,
      "rewards/margins": 0.38805803656578064,
      "rewards/rejected": -44.66236114501953,
      "step": 2165
    },
    {
      "epoch": 1.1613982271282823,
      "grad_norm": 2.3119281825544007e-07,
      "learning_rate": 7.691640583277004e-07,
      "logits/chosen": 8.958982467651367,
      "logits/rejected": 8.352134704589844,
      "logps/chosen": -43.95872116088867,
      "logps/rejected": -45.59920120239258,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -43.95872116088867,
      "rewards/margins": 1.640470266342163,
      "rewards/rejected": -45.59920120239258,
      "step": 2170
    },
    {
      "epoch": 1.1640742599096838,
      "grad_norm": 3.6453222033040424e-11,
      "learning_rate": 7.678502764844433e-07,
      "logits/chosen": 9.203370094299316,
      "logits/rejected": 8.44577407836914,
      "logps/chosen": -44.5631217956543,
      "logps/rejected": -44.42543029785156,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -44.5631217956543,
      "rewards/margins": -0.13769233226776123,
      "rewards/rejected": -44.42543029785156,
      "step": 2175
    },
    {
      "epoch": 1.1667502926910855,
      "grad_norm": 1.2005559821026968e-06,
      "learning_rate": 7.665338959566288e-07,
      "logits/chosen": 8.911245346069336,
      "logits/rejected": 8.687984466552734,
      "logps/chosen": -44.21027755737305,
      "logps/rejected": -45.16591262817383,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.21027755737305,
      "rewards/margins": 0.9556310772895813,
      "rewards/rejected": -45.16591262817383,
      "step": 2180
    },
    {
      "epoch": 1.169426325472487,
      "grad_norm": 5.044568057483656e-07,
      "learning_rate": 7.652149295157868e-07,
      "logits/chosen": 8.676664352416992,
      "logits/rejected": 8.143527030944824,
      "logps/chosen": -43.6501350402832,
      "logps/rejected": -44.1270866394043,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.6501350402832,
      "rewards/margins": 0.47695106267929077,
      "rewards/rejected": -44.1270866394043,
      "step": 2185
    },
    {
      "epoch": 1.1721023582538885,
      "grad_norm": 0.0005459047106061931,
      "learning_rate": 7.638933899585354e-07,
      "logits/chosen": 8.363175392150879,
      "logits/rejected": 8.322877883911133,
      "logps/chosen": -43.31201934814453,
      "logps/rejected": -44.193031311035156,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -43.31201934814453,
      "rewards/margins": 0.8810040354728699,
      "rewards/rejected": -44.193031311035156,
      "step": 2190
    },
    {
      "epoch": 1.1747783910352902,
      "grad_norm": 0.03197545417001345,
      "learning_rate": 7.625692901064573e-07,
      "logits/chosen": 8.460412979125977,
      "logits/rejected": 8.334818840026855,
      "logps/chosen": -43.237464904785156,
      "logps/rejected": -43.879539489746094,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -43.237464904785156,
      "rewards/margins": 0.6420688629150391,
      "rewards/rejected": -43.879539489746094,
      "step": 2195
    },
    {
      "epoch": 1.1774544238166917,
      "grad_norm": 4.7282263063464206e-07,
      "learning_rate": 7.61242642805975e-07,
      "logits/chosen": 8.820535659790039,
      "logits/rejected": 8.900537490844727,
      "logps/chosen": -44.11952590942383,
      "logps/rejected": -44.774879455566406,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -44.11952590942383,
      "rewards/margins": 0.6553515195846558,
      "rewards/rejected": -44.774879455566406,
      "step": 2200
    },
    {
      "epoch": 1.1801304565980932,
      "grad_norm": 1.3543907861855837e-07,
      "learning_rate": 7.599134609282266e-07,
      "logits/chosen": 9.069814682006836,
      "logits/rejected": 8.347467422485352,
      "logps/chosen": -43.64423751831055,
      "logps/rejected": -45.304840087890625,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.64423751831055,
      "rewards/margins": 1.6606051921844482,
      "rewards/rejected": -45.304840087890625,
      "step": 2205
    },
    {
      "epoch": 1.182806489379495,
      "grad_norm": 1.391530060961859e-10,
      "learning_rate": 7.585817573689402e-07,
      "logits/chosen": 8.809715270996094,
      "logits/rejected": 8.632291793823242,
      "logps/chosen": -43.1541862487793,
      "logps/rejected": -44.686058044433594,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.1541862487793,
      "rewards/margins": 1.5318666696548462,
      "rewards/rejected": -44.686058044433594,
      "step": 2210
    },
    {
      "epoch": 1.1854825221608964,
      "grad_norm": 7.989380104934349e-09,
      "learning_rate": 7.572475450483098e-07,
      "logits/chosen": 8.617358207702637,
      "logits/rejected": 8.26659870147705,
      "logps/chosen": -42.30150604248047,
      "logps/rejected": -43.67645263671875,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.30150604248047,
      "rewards/margins": 1.3749443292617798,
      "rewards/rejected": -43.67645263671875,
      "step": 2215
    },
    {
      "epoch": 1.188158554942298,
      "grad_norm": 6.040415262505287e-07,
      "learning_rate": 7.559108369108689e-07,
      "logits/chosen": 8.645989418029785,
      "logits/rejected": 8.325116157531738,
      "logps/chosen": -41.88881301879883,
      "logps/rejected": -42.61737060546875,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.88881301879883,
      "rewards/margins": 0.7285608053207397,
      "rewards/rejected": -42.61737060546875,
      "step": 2220
    },
    {
      "epoch": 1.1908345877236997,
      "grad_norm": 1.037140802023574e-07,
      "learning_rate": 7.54571645925366e-07,
      "logits/chosen": 9.216730117797852,
      "logits/rejected": 8.472485542297363,
      "logps/chosen": -42.10993576049805,
      "logps/rejected": -44.74381637573242,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -42.10993576049805,
      "rewards/margins": 2.6338820457458496,
      "rewards/rejected": -44.74381637573242,
      "step": 2225
    },
    {
      "epoch": 1.1935106205051011,
      "grad_norm": 6.743224421814544e-09,
      "learning_rate": 7.532299850846378e-07,
      "logits/chosen": 8.87272834777832,
      "logits/rejected": 8.299006462097168,
      "logps/chosen": -43.306480407714844,
      "logps/rejected": -44.64222717285156,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.306480407714844,
      "rewards/margins": 1.335749626159668,
      "rewards/rejected": -44.64222717285156,
      "step": 2230
    },
    {
      "epoch": 1.1961866532865026,
      "grad_norm": 5.863272763906507e-08,
      "learning_rate": 7.518858674054838e-07,
      "logits/chosen": 8.889973640441895,
      "logits/rejected": 8.310088157653809,
      "logps/chosen": -43.003299713134766,
      "logps/rejected": -43.6577262878418,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.003299713134766,
      "rewards/margins": 0.6544263958930969,
      "rewards/rejected": -43.6577262878418,
      "step": 2235
    },
    {
      "epoch": 1.1988626860679044,
      "grad_norm": 3.4524407474040006e-10,
      "learning_rate": 7.505393059285394e-07,
      "logits/chosen": 8.675268173217773,
      "logits/rejected": 8.224763870239258,
      "logps/chosen": -43.433998107910156,
      "logps/rejected": -44.526084899902344,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.433998107910156,
      "rewards/margins": 1.0920863151550293,
      "rewards/rejected": -44.526084899902344,
      "step": 2240
    },
    {
      "epoch": 1.2015387188493059,
      "grad_norm": 1.131601181800809e-06,
      "learning_rate": 7.491903137181501e-07,
      "logits/chosen": 8.70720386505127,
      "logits/rejected": 8.709518432617188,
      "logps/chosen": -43.72024917602539,
      "logps/rejected": -44.94924545288086,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.72024917602539,
      "rewards/margins": 1.2290009260177612,
      "rewards/rejected": -44.94924545288086,
      "step": 2245
    },
    {
      "epoch": 1.2042147516307076,
      "grad_norm": 4.366510134152807e-08,
      "learning_rate": 7.478389038622441e-07,
      "logits/chosen": 8.659378051757812,
      "logits/rejected": 8.567684173583984,
      "logps/chosen": -43.549903869628906,
      "logps/rejected": -44.28810501098633,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.549903869628906,
      "rewards/margins": 0.7381996512413025,
      "rewards/rejected": -44.28810501098633,
      "step": 2250
    },
    {
      "epoch": 1.206890784412109,
      "grad_norm": 5.2489041180993564e-05,
      "learning_rate": 7.46485089472206e-07,
      "logits/chosen": 8.799867630004883,
      "logits/rejected": 8.735482215881348,
      "logps/chosen": -44.50786590576172,
      "logps/rejected": -45.25513458251953,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.50786590576172,
      "rewards/margins": 0.7472699880599976,
      "rewards/rejected": -45.25513458251953,
      "step": 2255
    },
    {
      "epoch": 1.2095668171935106,
      "grad_norm": 2.585228059071912e-05,
      "learning_rate": 7.451288836827487e-07,
      "logits/chosen": 8.703425407409668,
      "logits/rejected": 8.84393310546875,
      "logps/chosen": -43.92967987060547,
      "logps/rejected": -44.20005416870117,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.92967987060547,
      "rewards/margins": 0.2703762948513031,
      "rewards/rejected": -44.20005416870117,
      "step": 2260
    },
    {
      "epoch": 1.2122428499749123,
      "grad_norm": 3.61697464633231e-11,
      "learning_rate": 7.437702996517869e-07,
      "logits/chosen": 8.96699047088623,
      "logits/rejected": 8.777639389038086,
      "logps/chosen": -42.558631896972656,
      "logps/rejected": -44.17717361450195,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.558631896972656,
      "rewards/margins": 1.6185423135757446,
      "rewards/rejected": -44.17717361450195,
      "step": 2265
    },
    {
      "epoch": 1.2149188827563138,
      "grad_norm": 9.930724478064595e-08,
      "learning_rate": 7.424093505603087e-07,
      "logits/chosen": 8.984644889831543,
      "logits/rejected": 8.553757667541504,
      "logps/chosen": -42.68560028076172,
      "logps/rejected": -44.547889709472656,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.68560028076172,
      "rewards/margins": 1.862290620803833,
      "rewards/rejected": -44.547889709472656,
      "step": 2270
    },
    {
      "epoch": 1.2175949155377153,
      "grad_norm": 4.3700272076706105e-07,
      "learning_rate": 7.410460496122482e-07,
      "logits/chosen": 8.7372407913208,
      "logits/rejected": 8.254986763000488,
      "logps/chosen": -44.240814208984375,
      "logps/rejected": -45.06039810180664,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.240814208984375,
      "rewards/margins": 0.819582462310791,
      "rewards/rejected": -45.06039810180664,
      "step": 2275
    },
    {
      "epoch": 1.220270948319117,
      "grad_norm": 1.3876647635144054e-05,
      "learning_rate": 7.396804100343572e-07,
      "logits/chosen": 9.157417297363281,
      "logits/rejected": 8.389032363891602,
      "logps/chosen": -43.11670684814453,
      "logps/rejected": -44.49758529663086,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.11670684814453,
      "rewards/margins": 1.3808748722076416,
      "rewards/rejected": -44.49758529663086,
      "step": 2280
    },
    {
      "epoch": 1.2229469811005185,
      "grad_norm": 0.0001533827689804894,
      "learning_rate": 7.383124450760768e-07,
      "logits/chosen": 9.31833267211914,
      "logits/rejected": 8.729985237121582,
      "logps/chosen": -43.43534469604492,
      "logps/rejected": -45.07497024536133,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.43534469604492,
      "rewards/margins": 1.6396201848983765,
      "rewards/rejected": -45.07497024536133,
      "step": 2285
    },
    {
      "epoch": 1.22562301388192,
      "grad_norm": 5.024761482366787e-12,
      "learning_rate": 7.369421680094091e-07,
      "logits/chosen": 9.373112678527832,
      "logits/rejected": 8.902400016784668,
      "logps/chosen": -43.8685417175293,
      "logps/rejected": -45.79479217529297,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -43.8685417175293,
      "rewards/margins": 1.9262548685073853,
      "rewards/rejected": -45.79479217529297,
      "step": 2290
    },
    {
      "epoch": 1.2282990466633217,
      "grad_norm": 2.3092658677215708e-05,
      "learning_rate": 7.355695921287881e-07,
      "logits/chosen": 9.006020545959473,
      "logits/rejected": 8.578181266784668,
      "logps/chosen": -43.67167282104492,
      "logps/rejected": -43.86157989501953,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -43.67167282104492,
      "rewards/margins": 0.1899031698703766,
      "rewards/rejected": -43.86157989501953,
      "step": 2295
    },
    {
      "epoch": 1.2309750794447232,
      "grad_norm": 1.610477390641295e-08,
      "learning_rate": 7.341947307509513e-07,
      "logits/chosen": 8.980446815490723,
      "logits/rejected": 8.67359733581543,
      "logps/chosen": -43.82781219482422,
      "logps/rejected": -45.37762451171875,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -43.82781219482422,
      "rewards/margins": 1.5498144626617432,
      "rewards/rejected": -45.37762451171875,
      "step": 2300
    },
    {
      "epoch": 1.233651112226125,
      "grad_norm": 6.677480956465622e-12,
      "learning_rate": 7.328175972148094e-07,
      "logits/chosen": 8.974390983581543,
      "logits/rejected": 8.484504699707031,
      "logps/chosen": -42.899269104003906,
      "logps/rejected": -44.76365280151367,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -42.899269104003906,
      "rewards/margins": 1.8643907308578491,
      "rewards/rejected": -44.76365280151367,
      "step": 2305
    },
    {
      "epoch": 1.2363271450075264,
      "grad_norm": 7.944484534807101e-11,
      "learning_rate": 7.314382048813185e-07,
      "logits/chosen": 8.714093208312988,
      "logits/rejected": 8.0159330368042,
      "logps/chosen": -42.637882232666016,
      "logps/rejected": -45.641231536865234,
      "loss": 0.0002,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -42.637882232666016,
      "rewards/margins": 3.0033535957336426,
      "rewards/rejected": -45.641231536865234,
      "step": 2310
    },
    {
      "epoch": 1.2390031777889279,
      "grad_norm": 9.541153509141635e-09,
      "learning_rate": 7.300565671333486e-07,
      "logits/chosen": 8.986320495605469,
      "logits/rejected": 8.41690731048584,
      "logps/chosen": -43.528892517089844,
      "logps/rejected": -44.78378677368164,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.528892517089844,
      "rewards/margins": 1.2548954486846924,
      "rewards/rejected": -44.78378677368164,
      "step": 2315
    },
    {
      "epoch": 1.2416792105703296,
      "grad_norm": 2.0988436898504333e-05,
      "learning_rate": 7.286726973755554e-07,
      "logits/chosen": 8.793320655822754,
      "logits/rejected": 8.506603240966797,
      "logps/chosen": -43.76630401611328,
      "logps/rejected": -43.853004455566406,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.76630401611328,
      "rewards/margins": 0.08669869601726532,
      "rewards/rejected": -43.853004455566406,
      "step": 2320
    },
    {
      "epoch": 1.244355243351731,
      "grad_norm": 8.695936275281853e-06,
      "learning_rate": 7.272866090342493e-07,
      "logits/chosen": 8.417692184448242,
      "logits/rejected": 8.149435043334961,
      "logps/chosen": -42.13292694091797,
      "logps/rejected": -43.943782806396484,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -42.13292694091797,
      "rewards/margins": 1.8108575344085693,
      "rewards/rejected": -43.943782806396484,
      "step": 2325
    },
    {
      "epoch": 1.2470312761331326,
      "grad_norm": 2.8364611259261004e-07,
      "learning_rate": 7.258983155572656e-07,
      "logits/chosen": 9.112143516540527,
      "logits/rejected": 8.815134048461914,
      "logps/chosen": -44.718345642089844,
      "logps/rejected": -44.523929595947266,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.718345642089844,
      "rewards/margins": -0.19441905617713928,
      "rewards/rejected": -44.523929595947266,
      "step": 2330
    },
    {
      "epoch": 1.2497073089145343,
      "grad_norm": 8.73399943343062e-05,
      "learning_rate": 7.245078304138335e-07,
      "logits/chosen": 8.605252265930176,
      "logits/rejected": 8.403312683105469,
      "logps/chosen": -42.78369903564453,
      "logps/rejected": -43.851531982421875,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.78369903564453,
      "rewards/margins": 1.0678341388702393,
      "rewards/rejected": -43.851531982421875,
      "step": 2335
    },
    {
      "epoch": 1.2523833416959358,
      "grad_norm": 9.896938237971726e-10,
      "learning_rate": 7.231151670944462e-07,
      "logits/chosen": 9.050447463989258,
      "logits/rejected": 8.163823127746582,
      "logps/chosen": -43.93961715698242,
      "logps/rejected": -44.41556167602539,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -43.93961715698242,
      "rewards/margins": 0.4759446978569031,
      "rewards/rejected": -44.41556167602539,
      "step": 2340
    },
    {
      "epoch": 1.2550593744773373,
      "grad_norm": 1.4409456006423543e-05,
      "learning_rate": 7.217203391107291e-07,
      "logits/chosen": 8.652153968811035,
      "logits/rejected": 8.203964233398438,
      "logps/chosen": -43.7486572265625,
      "logps/rejected": -44.235958099365234,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -43.7486572265625,
      "rewards/margins": 0.48729753494262695,
      "rewards/rejected": -44.235958099365234,
      "step": 2345
    },
    {
      "epoch": 1.257735407258739,
      "grad_norm": 0.2561621154484161,
      "learning_rate": 7.203233599953096e-07,
      "logits/chosen": 8.513713836669922,
      "logits/rejected": 8.106762886047363,
      "logps/chosen": -41.624420166015625,
      "logps/rejected": -42.31261444091797,
      "loss": 0.0006,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -41.624420166015625,
      "rewards/margins": 0.688197910785675,
      "rewards/rejected": -42.31261444091797,
      "step": 2350
    },
    {
      "epoch": 1.2604114400401405,
      "grad_norm": 4.313674170677161e-05,
      "learning_rate": 7.189242433016852e-07,
      "logits/chosen": 8.514612197875977,
      "logits/rejected": 8.192414283752441,
      "logps/chosen": -43.90970230102539,
      "logps/rejected": -45.3636360168457,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.90970230102539,
      "rewards/margins": 1.4539411067962646,
      "rewards/rejected": -45.3636360168457,
      "step": 2355
    },
    {
      "epoch": 1.263087472821542,
      "grad_norm": 6.56132162179694e-12,
      "learning_rate": 7.17523002604092e-07,
      "logits/chosen": 9.072576522827148,
      "logits/rejected": 8.553306579589844,
      "logps/chosen": -44.00154495239258,
      "logps/rejected": -45.31732177734375,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.00154495239258,
      "rewards/margins": 1.3157778978347778,
      "rewards/rejected": -45.31732177734375,
      "step": 2360
    },
    {
      "epoch": 1.2657635056029437,
      "grad_norm": 1.2787554382427445e-07,
      "learning_rate": 7.161196514973734e-07,
      "logits/chosen": 8.704858779907227,
      "logits/rejected": 8.082131385803223,
      "logps/chosen": -42.36492919921875,
      "logps/rejected": -44.36050033569336,
      "loss": 0.0014,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.36492919921875,
      "rewards/margins": 1.99557363986969,
      "rewards/rejected": -44.36050033569336,
      "step": 2365
    },
    {
      "epoch": 1.2684395383843452,
      "grad_norm": 1.7438793497296593e-07,
      "learning_rate": 7.147142035968483e-07,
      "logits/chosen": 8.870776176452637,
      "logits/rejected": 8.340474128723145,
      "logps/chosen": -43.07660675048828,
      "logps/rejected": -44.705753326416016,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -43.07660675048828,
      "rewards/margins": 1.6291462182998657,
      "rewards/rejected": -44.705753326416016,
      "step": 2370
    },
    {
      "epoch": 1.2711155711657467,
      "grad_norm": 3.97598527107857e-06,
      "learning_rate": 7.133066725381781e-07,
      "logits/chosen": 9.06567096710205,
      "logits/rejected": 8.432821273803711,
      "logps/chosen": -44.099647521972656,
      "logps/rejected": -45.33367919921875,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.099647521972656,
      "rewards/margins": 1.2340365648269653,
      "rewards/rejected": -45.33367919921875,
      "step": 2375
    },
    {
      "epoch": 1.2737916039471484,
      "grad_norm": 5.1451180874242726e-09,
      "learning_rate": 7.118970719772354e-07,
      "logits/chosen": 8.83424186706543,
      "logits/rejected": 8.410809516906738,
      "logps/chosen": -43.32245635986328,
      "logps/rejected": -45.31129455566406,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -43.32245635986328,
      "rewards/margins": 1.9888372421264648,
      "rewards/rejected": -45.31129455566406,
      "step": 2380
    },
    {
      "epoch": 1.27646763672855,
      "grad_norm": 4.1589378771657374e-05,
      "learning_rate": 7.104854155899711e-07,
      "logits/chosen": 8.954573631286621,
      "logits/rejected": 8.612079620361328,
      "logps/chosen": -44.549110412597656,
      "logps/rejected": -45.16191101074219,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -44.549110412597656,
      "rewards/margins": 0.6128073930740356,
      "rewards/rejected": -45.16191101074219,
      "step": 2385
    },
    {
      "epoch": 1.2791436695099514,
      "grad_norm": 2.9976621221034792e-12,
      "learning_rate": 7.090717170722817e-07,
      "logits/chosen": 8.636760711669922,
      "logits/rejected": 8.668012619018555,
      "logps/chosen": -44.68073272705078,
      "logps/rejected": -45.909400939941406,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.68073272705078,
      "rewards/margins": 1.2286657094955444,
      "rewards/rejected": -45.909400939941406,
      "step": 2390
    },
    {
      "epoch": 1.2818197022913531,
      "grad_norm": 7.366918243774153e-07,
      "learning_rate": 7.076559901398762e-07,
      "logits/chosen": 9.457544326782227,
      "logits/rejected": 8.78124713897705,
      "logps/chosen": -43.03073501586914,
      "logps/rejected": -45.13961410522461,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.03073501586914,
      "rewards/margins": 2.108881711959839,
      "rewards/rejected": -45.13961410522461,
      "step": 2395
    },
    {
      "epoch": 1.2844957350727546,
      "grad_norm": 1.315252847106035e-06,
      "learning_rate": 7.062382485281436e-07,
      "logits/chosen": 8.982632637023926,
      "logits/rejected": 8.425972938537598,
      "logps/chosen": -42.52170181274414,
      "logps/rejected": -43.81257247924805,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -42.52170181274414,
      "rewards/margins": 1.290871024131775,
      "rewards/rejected": -43.81257247924805,
      "step": 2400
    },
    {
      "epoch": 1.2844957350727546,
      "eval_logits/chosen": 7.477530479431152,
      "eval_logits/rejected": 7.188204765319824,
      "eval_logps/chosen": -43.5288200378418,
      "eval_logps/rejected": -44.64771270751953,
      "eval_loss": 2.685528386336955e-07,
      "eval_rewards/accuracies": 0.5645400881767273,
      "eval_rewards/chosen": -43.5288200378418,
      "eval_rewards/margins": 1.1188887357711792,
      "eval_rewards/rejected": -44.64771270751953,
      "eval_runtime": 40.1039,
      "eval_samples_per_second": 33.538,
      "eval_steps_per_second": 8.403,
      "step": 2400
    },
    {
      "epoch": 1.287171767854156,
      "grad_norm": 1.2335533677398493e-10,
      "learning_rate": 7.048185059920193e-07,
      "logits/chosen": 8.967894554138184,
      "logits/rejected": 8.617827415466309,
      "logps/chosen": -44.100799560546875,
      "logps/rejected": -45.70526885986328,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.100799560546875,
      "rewards/margins": 1.6044723987579346,
      "rewards/rejected": -45.70526885986328,
      "step": 2405
    },
    {
      "epoch": 1.2898478006355578,
      "grad_norm": 1.6401369724201697e-07,
      "learning_rate": 7.033967763058516e-07,
      "logits/chosen": 8.8171968460083,
      "logits/rejected": 8.319070816040039,
      "logps/chosen": -42.73849105834961,
      "logps/rejected": -43.9620475769043,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -42.73849105834961,
      "rewards/margins": 1.2235503196716309,
      "rewards/rejected": -43.9620475769043,
      "step": 2410
    },
    {
      "epoch": 1.2925238334169593,
      "grad_norm": 2.518449417010279e-12,
      "learning_rate": 7.019730732632681e-07,
      "logits/chosen": 8.900062561035156,
      "logits/rejected": 8.395971298217773,
      "logps/chosen": -44.1333122253418,
      "logps/rejected": -45.22997283935547,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.1333122253418,
      "rewards/margins": 1.0966647863388062,
      "rewards/rejected": -45.22997283935547,
      "step": 2415
    },
    {
      "epoch": 1.2951998661983608,
      "grad_norm": 3.557650195818253e-09,
      "learning_rate": 7.005474106770418e-07,
      "logits/chosen": 8.817143440246582,
      "logits/rejected": 8.497065544128418,
      "logps/chosen": -43.52418518066406,
      "logps/rejected": -44.698753356933594,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.52418518066406,
      "rewards/margins": 1.1745717525482178,
      "rewards/rejected": -44.698753356933594,
      "step": 2420
    },
    {
      "epoch": 1.2978758989797625,
      "grad_norm": 8.209476944051782e-09,
      "learning_rate": 6.991198023789577e-07,
      "logits/chosen": 8.827470779418945,
      "logits/rejected": 8.705556869506836,
      "logps/chosen": -44.37858200073242,
      "logps/rejected": -44.643150329589844,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -44.37858200073242,
      "rewards/margins": 0.26456794142723083,
      "rewards/rejected": -44.643150329589844,
      "step": 2425
    },
    {
      "epoch": 1.300551931761164,
      "grad_norm": 3.84755517489469e-09,
      "learning_rate": 6.976902622196776e-07,
      "logits/chosen": 8.775060653686523,
      "logits/rejected": 8.53449821472168,
      "logps/chosen": -44.041221618652344,
      "logps/rejected": -44.72358322143555,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -44.041221618652344,
      "rewards/margins": 0.6823638081550598,
      "rewards/rejected": -44.72358322143555,
      "step": 2430
    },
    {
      "epoch": 1.3032279645425655,
      "grad_norm": 2.561094123140341e-12,
      "learning_rate": 6.962588040686064e-07,
      "logits/chosen": 9.047769546508789,
      "logits/rejected": 8.488920211791992,
      "logps/chosen": -43.698970794677734,
      "logps/rejected": -44.816505432128906,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.698970794677734,
      "rewards/margins": 1.1175332069396973,
      "rewards/rejected": -44.816505432128906,
      "step": 2435
    },
    {
      "epoch": 1.3059039973239672,
      "grad_norm": 3.0078243355442346e-10,
      "learning_rate": 6.948254418137573e-07,
      "logits/chosen": 8.797759056091309,
      "logits/rejected": 8.371089935302734,
      "logps/chosen": -43.231903076171875,
      "logps/rejected": -44.988426208496094,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -43.231903076171875,
      "rewards/margins": 1.7565221786499023,
      "rewards/rejected": -44.988426208496094,
      "step": 2440
    },
    {
      "epoch": 1.3085800301053687,
      "grad_norm": 0.007367476057309296,
      "learning_rate": 6.933901893616174e-07,
      "logits/chosen": 8.919111251831055,
      "logits/rejected": 8.538690567016602,
      "logps/chosen": -42.889930725097656,
      "logps/rejected": -44.163063049316406,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -42.889930725097656,
      "rewards/margins": 1.2731287479400635,
      "rewards/rejected": -44.163063049316406,
      "step": 2445
    },
    {
      "epoch": 1.3112560628867704,
      "grad_norm": 1.1738375477413586e-07,
      "learning_rate": 6.919530606370121e-07,
      "logits/chosen": 8.889062881469727,
      "logits/rejected": 8.299957275390625,
      "logps/chosen": -44.39117431640625,
      "logps/rejected": -45.030433654785156,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.39117431640625,
      "rewards/margins": 0.6392605304718018,
      "rewards/rejected": -45.030433654785156,
      "step": 2450
    },
    {
      "epoch": 1.313932095668172,
      "grad_norm": 7.657322078755612e-09,
      "learning_rate": 6.905140695829706e-07,
      "logits/chosen": 9.013423919677734,
      "logits/rejected": 8.15529727935791,
      "logps/chosen": -43.84007263183594,
      "logps/rejected": -45.195579528808594,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -43.84007263183594,
      "rewards/margins": 1.355499029159546,
      "rewards/rejected": -45.195579528808594,
      "step": 2455
    },
    {
      "epoch": 1.3166081284495736,
      "grad_norm": 1.2688314432103736e-14,
      "learning_rate": 6.890732301605904e-07,
      "logits/chosen": 8.528648376464844,
      "logits/rejected": 8.185813903808594,
      "logps/chosen": -44.281009674072266,
      "logps/rejected": -44.74654006958008,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.281009674072266,
      "rewards/margins": 0.4655328691005707,
      "rewards/rejected": -44.74654006958008,
      "step": 2460
    },
    {
      "epoch": 1.3192841612309751,
      "grad_norm": 1.2866627977846996e-08,
      "learning_rate": 6.876305563489021e-07,
      "logits/chosen": 9.166059494018555,
      "logits/rejected": 8.877128601074219,
      "logps/chosen": -43.18642044067383,
      "logps/rejected": -45.07292556762695,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.18642044067383,
      "rewards/margins": 1.8865007162094116,
      "rewards/rejected": -45.07292556762695,
      "step": 2465
    },
    {
      "epoch": 1.3219601940123766,
      "grad_norm": 5.376884314090088e-07,
      "learning_rate": 6.861860621447331e-07,
      "logits/chosen": 9.161568641662598,
      "logits/rejected": 8.700502395629883,
      "logps/chosen": -43.42621994018555,
      "logps/rejected": -44.045860290527344,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.42621994018555,
      "rewards/margins": 0.6196433305740356,
      "rewards/rejected": -44.045860290527344,
      "step": 2470
    },
    {
      "epoch": 1.3246362267937783,
      "grad_norm": 5.4993109002753744e-08,
      "learning_rate": 6.847397615625725e-07,
      "logits/chosen": 8.724710464477539,
      "logits/rejected": 8.599674224853516,
      "logps/chosen": -44.45661163330078,
      "logps/rejected": -44.86403274536133,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.45661163330078,
      "rewards/margins": 0.40742069482803345,
      "rewards/rejected": -44.86403274536133,
      "step": 2475
    },
    {
      "epoch": 1.3273122595751798,
      "grad_norm": 1.4514371056761866e-06,
      "learning_rate": 6.83291668634435e-07,
      "logits/chosen": 9.193476676940918,
      "logits/rejected": 8.687743186950684,
      "logps/chosen": -42.90135955810547,
      "logps/rejected": -45.41196060180664,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -42.90135955810547,
      "rewards/margins": 2.5105979442596436,
      "rewards/rejected": -45.41196060180664,
      "step": 2480
    },
    {
      "epoch": 1.3299882923565813,
      "grad_norm": 0.00011286911546544007,
      "learning_rate": 6.818417974097246e-07,
      "logits/chosen": 8.690770149230957,
      "logits/rejected": 8.27407455444336,
      "logps/chosen": -43.57719802856445,
      "logps/rejected": -44.149253845214844,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.57719802856445,
      "rewards/margins": 0.5720575451850891,
      "rewards/rejected": -44.149253845214844,
      "step": 2485
    },
    {
      "epoch": 1.332664325137983,
      "grad_norm": 3.68348000806125e-07,
      "learning_rate": 6.803901619550981e-07,
      "logits/chosen": 8.70672607421875,
      "logits/rejected": 8.554915428161621,
      "logps/chosen": -42.70306396484375,
      "logps/rejected": -43.507144927978516,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.70306396484375,
      "rewards/margins": 0.8040782809257507,
      "rewards/rejected": -43.507144927978516,
      "step": 2490
    },
    {
      "epoch": 1.3353403579193845,
      "grad_norm": 9.596568141462554e-06,
      "learning_rate": 6.789367763543292e-07,
      "logits/chosen": 8.967519760131836,
      "logits/rejected": 8.798467636108398,
      "logps/chosen": -44.8049201965332,
      "logps/rejected": -44.795013427734375,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -44.8049201965332,
      "rewards/margins": -0.009904563426971436,
      "rewards/rejected": -44.795013427734375,
      "step": 2495
    },
    {
      "epoch": 1.338016390700786,
      "grad_norm": 3.490397189203556e-08,
      "learning_rate": 6.774816547081714e-07,
      "logits/chosen": 8.61393928527832,
      "logits/rejected": 8.062116622924805,
      "logps/chosen": -43.432369232177734,
      "logps/rejected": -44.87363052368164,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -43.432369232177734,
      "rewards/margins": 1.4412589073181152,
      "rewards/rejected": -44.87363052368164,
      "step": 2500
    },
    {
      "epoch": 1.3406924234821878,
      "grad_norm": 1.436825495957708e-08,
      "learning_rate": 6.760248111342211e-07,
      "logits/chosen": 9.027219772338867,
      "logits/rejected": 8.503440856933594,
      "logps/chosen": -45.249107360839844,
      "logps/rejected": -46.699554443359375,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -45.249107360839844,
      "rewards/margins": 1.4504512548446655,
      "rewards/rejected": -46.699554443359375,
      "step": 2505
    },
    {
      "epoch": 1.3433684562635893,
      "grad_norm": 0.0003025260416835782,
      "learning_rate": 6.745662597667813e-07,
      "logits/chosen": 8.892065048217773,
      "logits/rejected": 8.44166088104248,
      "logps/chosen": -43.70719528198242,
      "logps/rejected": -44.52797317504883,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.70719528198242,
      "rewards/margins": 0.820775032043457,
      "rewards/rejected": -44.52797317504883,
      "step": 2510
    },
    {
      "epoch": 1.3460444890449907,
      "grad_norm": 3.9652636888014633e-10,
      "learning_rate": 6.731060147567236e-07,
      "logits/chosen": 8.894283294677734,
      "logits/rejected": 8.618927955627441,
      "logps/chosen": -44.151283264160156,
      "logps/rejected": -45.2234992980957,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.151283264160156,
      "rewards/margins": 1.072216510772705,
      "rewards/rejected": -45.2234992980957,
      "step": 2515
    },
    {
      "epoch": 1.3487205218263925,
      "grad_norm": 2.8062207882062855e-10,
      "learning_rate": 6.716440902713515e-07,
      "logits/chosen": 9.023406028747559,
      "logits/rejected": 8.769869804382324,
      "logps/chosen": -43.43317794799805,
      "logps/rejected": -44.1347770690918,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.43317794799805,
      "rewards/margins": 0.7015973329544067,
      "rewards/rejected": -44.1347770690918,
      "step": 2520
    },
    {
      "epoch": 1.351396554607794,
      "grad_norm": 1.324605280105238e-07,
      "learning_rate": 6.701805004942627e-07,
      "logits/chosen": 9.072699546813965,
      "logits/rejected": 8.673077583312988,
      "logps/chosen": -43.530433654785156,
      "logps/rejected": -44.5306396484375,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.530433654785156,
      "rewards/margins": 1.0002013444900513,
      "rewards/rejected": -44.5306396484375,
      "step": 2525
    },
    {
      "epoch": 1.3540725873891954,
      "grad_norm": 4.113064878281929e-07,
      "learning_rate": 6.687152596252119e-07,
      "logits/chosen": 8.847375869750977,
      "logits/rejected": 8.690252304077148,
      "logps/chosen": -43.82762908935547,
      "logps/rejected": -44.86438751220703,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.82762908935547,
      "rewards/margins": 1.0367536544799805,
      "rewards/rejected": -44.86438751220703,
      "step": 2530
    },
    {
      "epoch": 1.3567486201705972,
      "grad_norm": 1.110626827240476e-10,
      "learning_rate": 6.672483818799722e-07,
      "logits/chosen": 9.088361740112305,
      "logits/rejected": 8.60782241821289,
      "logps/chosen": -43.470611572265625,
      "logps/rejected": -44.7743034362793,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -43.470611572265625,
      "rewards/margins": 1.3036936521530151,
      "rewards/rejected": -44.7743034362793,
      "step": 2535
    },
    {
      "epoch": 1.3594246529519987,
      "grad_norm": 4.3460573505278765e-10,
      "learning_rate": 6.657798814901978e-07,
      "logits/chosen": 8.929586410522461,
      "logits/rejected": 8.211908340454102,
      "logps/chosen": -44.248111724853516,
      "logps/rejected": -44.60112380981445,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -44.248111724853516,
      "rewards/margins": 0.3530149459838867,
      "rewards/rejected": -44.60112380981445,
      "step": 2540
    },
    {
      "epoch": 1.3621006857334002,
      "grad_norm": 9.401099188188303e-07,
      "learning_rate": 6.643097727032863e-07,
      "logits/chosen": 9.192752838134766,
      "logits/rejected": 8.569879531860352,
      "logps/chosen": -44.52490234375,
      "logps/rejected": -45.78219223022461,
      "loss": 0.0002,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.52490234375,
      "rewards/margins": 1.2572908401489258,
      "rewards/rejected": -45.78219223022461,
      "step": 2545
    },
    {
      "epoch": 1.3647767185148019,
      "grad_norm": 1.861200495533287e-07,
      "learning_rate": 6.628380697822392e-07,
      "logits/chosen": 9.386123657226562,
      "logits/rejected": 8.828864097595215,
      "logps/chosen": -44.24148941040039,
      "logps/rejected": -44.93312454223633,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.24148941040039,
      "rewards/margins": 0.6916356682777405,
      "rewards/rejected": -44.93312454223633,
      "step": 2550
    },
    {
      "epoch": 1.3674527512962034,
      "grad_norm": 5.9414741203110414e-06,
      "learning_rate": 6.61364787005525e-07,
      "logits/chosen": 8.587624549865723,
      "logits/rejected": 8.26928997039795,
      "logps/chosen": -42.757999420166016,
      "logps/rejected": -44.27712631225586,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -42.757999420166016,
      "rewards/margins": 1.5191264152526855,
      "rewards/rejected": -44.27712631225586,
      "step": 2555
    },
    {
      "epoch": 1.3701287840776049,
      "grad_norm": 2.5096110748314925e-11,
      "learning_rate": 6.598899386669395e-07,
      "logits/chosen": 9.061269760131836,
      "logits/rejected": 8.639310836791992,
      "logps/chosen": -43.891273498535156,
      "logps/rejected": -45.08263397216797,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.891273498535156,
      "rewards/margins": 1.1913549900054932,
      "rewards/rejected": -45.08263397216797,
      "step": 2560
    },
    {
      "epoch": 1.3728048168590066,
      "grad_norm": 4.628468455953659e-13,
      "learning_rate": 6.584135390754679e-07,
      "logits/chosen": 8.935189247131348,
      "logits/rejected": 8.477245330810547,
      "logps/chosen": -44.158485412597656,
      "logps/rejected": -45.72394943237305,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.158485412597656,
      "rewards/margins": 1.565462589263916,
      "rewards/rejected": -45.72394943237305,
      "step": 2565
    },
    {
      "epoch": 1.375480849640408,
      "grad_norm": 5.4241685197929126e-08,
      "learning_rate": 6.569356025551454e-07,
      "logits/chosen": 8.837717056274414,
      "logits/rejected": 8.653925895690918,
      "logps/chosen": -44.34320068359375,
      "logps/rejected": -44.75142288208008,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.34320068359375,
      "rewards/margins": 0.40822634100914,
      "rewards/rejected": -44.75142288208008,
      "step": 2570
    },
    {
      "epoch": 1.3781568824218096,
      "grad_norm": 5.153854333664593e-07,
      "learning_rate": 6.554561434449186e-07,
      "logits/chosen": 9.111549377441406,
      "logits/rejected": 8.701370239257812,
      "logps/chosen": -44.06297302246094,
      "logps/rejected": -45.460174560546875,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.06297302246094,
      "rewards/margins": 1.3971984386444092,
      "rewards/rejected": -45.460174560546875,
      "step": 2575
    },
    {
      "epoch": 1.3808329152032113,
      "grad_norm": 8.084422246552633e-08,
      "learning_rate": 6.539751760985063e-07,
      "logits/chosen": 8.989825248718262,
      "logits/rejected": 8.662254333496094,
      "logps/chosen": -42.79070281982422,
      "logps/rejected": -43.82093811035156,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -42.79070281982422,
      "rewards/margins": 1.0302305221557617,
      "rewards/rejected": -43.82093811035156,
      "step": 2580
    },
    {
      "epoch": 1.3835089479846128,
      "grad_norm": 1.886252599913661e-08,
      "learning_rate": 6.524927148842602e-07,
      "logits/chosen": 8.793694496154785,
      "logits/rejected": 8.304144859313965,
      "logps/chosen": -44.783531188964844,
      "logps/rejected": -46.32426834106445,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.783531188964844,
      "rewards/margins": 1.5407402515411377,
      "rewards/rejected": -46.32426834106445,
      "step": 2585
    },
    {
      "epoch": 1.3861849807660143,
      "grad_norm": 5.55151736374725e-16,
      "learning_rate": 6.510087741850254e-07,
      "logits/chosen": 9.032625198364258,
      "logits/rejected": 8.751620292663574,
      "logps/chosen": -44.62474822998047,
      "logps/rejected": -46.43467330932617,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.62474822998047,
      "rewards/margins": 1.8099292516708374,
      "rewards/rejected": -46.43467330932617,
      "step": 2590
    },
    {
      "epoch": 1.388861013547416,
      "grad_norm": 7.898281466894075e-08,
      "learning_rate": 6.495233683980012e-07,
      "logits/chosen": 9.088022232055664,
      "logits/rejected": 8.775041580200195,
      "logps/chosen": -43.37187576293945,
      "logps/rejected": -44.22704315185547,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -43.37187576293945,
      "rewards/margins": 0.8551721572875977,
      "rewards/rejected": -44.22704315185547,
      "step": 2595
    },
    {
      "epoch": 1.3915370463288175,
      "grad_norm": 1.0602977930936094e-05,
      "learning_rate": 6.480365119346011e-07,
      "logits/chosen": 8.722105026245117,
      "logits/rejected": 8.226268768310547,
      "logps/chosen": -42.82024002075195,
      "logps/rejected": -43.6745491027832,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -42.82024002075195,
      "rewards/margins": 0.8543079495429993,
      "rewards/rejected": -43.6745491027832,
      "step": 2600
    },
    {
      "epoch": 1.394213079110219,
      "grad_norm": 2.2220990906469772e-07,
      "learning_rate": 6.465482192203129e-07,
      "logits/chosen": 8.41450309753418,
      "logits/rejected": 8.316696166992188,
      "logps/chosen": -43.53263854980469,
      "logps/rejected": -44.77671432495117,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.53263854980469,
      "rewards/margins": 1.2440794706344604,
      "rewards/rejected": -44.77671432495117,
      "step": 2605
    },
    {
      "epoch": 1.3968891118916207,
      "grad_norm": 4.594386746961978e-09,
      "learning_rate": 6.45058504694559e-07,
      "logits/chosen": 8.368760108947754,
      "logits/rejected": 8.414548873901367,
      "logps/chosen": -43.29320526123047,
      "logps/rejected": -45.40201187133789,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -43.29320526123047,
      "rewards/margins": 2.108802318572998,
      "rewards/rejected": -45.40201187133789,
      "step": 2610
    },
    {
      "epoch": 1.3995651446730222,
      "grad_norm": 1.3510738737277372e-05,
      "learning_rate": 6.435673828105564e-07,
      "logits/chosen": 8.851028442382812,
      "logits/rejected": 8.493639945983887,
      "logps/chosen": -42.133628845214844,
      "logps/rejected": -44.039939880371094,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -42.133628845214844,
      "rewards/margins": 1.9063007831573486,
      "rewards/rejected": -44.039939880371094,
      "step": 2615
    },
    {
      "epoch": 1.402241177454424,
      "grad_norm": 1.8570044425100403e-13,
      "learning_rate": 6.420748680351763e-07,
      "logits/chosen": 8.704290390014648,
      "logits/rejected": 8.760000228881836,
      "logps/chosen": -43.10923385620117,
      "logps/rejected": -44.30144119262695,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.10923385620117,
      "rewards/margins": 1.1922073364257812,
      "rewards/rejected": -44.30144119262695,
      "step": 2620
    },
    {
      "epoch": 1.4049172102358254,
      "grad_norm": 5.507780333984778e-10,
      "learning_rate": 6.405809748488032e-07,
      "logits/chosen": 9.032635688781738,
      "logits/rejected": 8.741667747497559,
      "logps/chosen": -44.94239807128906,
      "logps/rejected": -46.04739761352539,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.94239807128906,
      "rewards/margins": 1.1050008535385132,
      "rewards/rejected": -46.04739761352539,
      "step": 2625
    },
    {
      "epoch": 1.4075932430172269,
      "grad_norm": 4.957170113595558e-12,
      "learning_rate": 6.390857177451956e-07,
      "logits/chosen": 9.049455642700195,
      "logits/rejected": 8.504581451416016,
      "logps/chosen": -43.72017288208008,
      "logps/rejected": -45.2539176940918,
      "loss": 0.0007,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -43.72017288208008,
      "rewards/margins": 1.5337450504302979,
      "rewards/rejected": -45.2539176940918,
      "step": 2630
    },
    {
      "epoch": 1.4102692757986286,
      "grad_norm": 1.5679722816621783e-07,
      "learning_rate": 6.375891112313445e-07,
      "logits/chosen": 8.864026069641113,
      "logits/rejected": 8.517353057861328,
      "logps/chosen": -43.94837188720703,
      "logps/rejected": -45.54801559448242,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.94837188720703,
      "rewards/margins": 1.5996500253677368,
      "rewards/rejected": -45.54801559448242,
      "step": 2635
    },
    {
      "epoch": 1.41294530858003,
      "grad_norm": 4.17824535569652e-09,
      "learning_rate": 6.360911698273326e-07,
      "logits/chosen": 8.899682998657227,
      "logits/rejected": 8.48131275177002,
      "logps/chosen": -43.622947692871094,
      "logps/rejected": -44.64672088623047,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -43.622947692871094,
      "rewards/margins": 1.0237665176391602,
      "rewards/rejected": -44.64672088623047,
      "step": 2640
    },
    {
      "epoch": 1.4156213413614318,
      "grad_norm": 1.3777143779924243e-12,
      "learning_rate": 6.345919080661944e-07,
      "logits/chosen": 9.12095832824707,
      "logits/rejected": 8.803476333618164,
      "logps/chosen": -44.438777923583984,
      "logps/rejected": -45.3214225769043,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.438777923583984,
      "rewards/margins": 0.8826438188552856,
      "rewards/rejected": -45.3214225769043,
      "step": 2645
    },
    {
      "epoch": 1.4182973741428333,
      "grad_norm": 2.963740343563398e-11,
      "learning_rate": 6.330913404937737e-07,
      "logits/chosen": 9.228857040405273,
      "logits/rejected": 8.725469589233398,
      "logps/chosen": -44.851829528808594,
      "logps/rejected": -46.50203323364258,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -44.851829528808594,
      "rewards/margins": 1.650202989578247,
      "rewards/rejected": -46.50203323364258,
      "step": 2650
    },
    {
      "epoch": 1.4209734069242348,
      "grad_norm": 1.5627159630332862e-05,
      "learning_rate": 6.315894816685838e-07,
      "logits/chosen": 8.794290542602539,
      "logits/rejected": 8.38827896118164,
      "logps/chosen": -45.08928298950195,
      "logps/rejected": -45.71906661987305,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -45.08928298950195,
      "rewards/margins": 0.629784107208252,
      "rewards/rejected": -45.71906661987305,
      "step": 2655
    },
    {
      "epoch": 1.4236494397056365,
      "grad_norm": 6.801852675515726e-13,
      "learning_rate": 6.300863461616657e-07,
      "logits/chosen": 8.364763259887695,
      "logits/rejected": 8.048707008361816,
      "logps/chosen": -44.47334671020508,
      "logps/rejected": -45.635276794433594,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.47334671020508,
      "rewards/margins": 1.16193425655365,
      "rewards/rejected": -45.635276794433594,
      "step": 2660
    },
    {
      "epoch": 1.426325472487038,
      "grad_norm": 2.048719742690745e-10,
      "learning_rate": 6.285819485564465e-07,
      "logits/chosen": 9.320186614990234,
      "logits/rejected": 8.60809326171875,
      "logps/chosen": -44.33464813232422,
      "logps/rejected": -45.12282943725586,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.33464813232422,
      "rewards/margins": 0.7881826162338257,
      "rewards/rejected": -45.12282943725586,
      "step": 2665
    },
    {
      "epoch": 1.4290015052684395,
      "grad_norm": 5.51392834365607e-07,
      "learning_rate": 6.270763034485986e-07,
      "logits/chosen": 8.74808406829834,
      "logits/rejected": 8.420801162719727,
      "logps/chosen": -42.67164993286133,
      "logps/rejected": -43.8131103515625,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -42.67164993286133,
      "rewards/margins": 1.1414577960968018,
      "rewards/rejected": -43.8131103515625,
      "step": 2670
    },
    {
      "epoch": 1.4316775380498412,
      "grad_norm": 2.061855586444012e-06,
      "learning_rate": 6.255694254458972e-07,
      "logits/chosen": 8.81074333190918,
      "logits/rejected": 8.362446784973145,
      "logps/chosen": -42.71105194091797,
      "logps/rejected": -45.105682373046875,
      "loss": 0.0022,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -42.71105194091797,
      "rewards/margins": 2.3946311473846436,
      "rewards/rejected": -45.105682373046875,
      "step": 2675
    },
    {
      "epoch": 1.4343535708312427,
      "grad_norm": 2.6817616974919565e-06,
      "learning_rate": 6.240613291680795e-07,
      "logits/chosen": 8.827413558959961,
      "logits/rejected": 8.329385757446289,
      "logps/chosen": -44.53593063354492,
      "logps/rejected": -46.4819221496582,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -44.53593063354492,
      "rewards/margins": 1.9459892511367798,
      "rewards/rejected": -46.4819221496582,
      "step": 2680
    },
    {
      "epoch": 1.4370296036126442,
      "grad_norm": 3.7903513812834225e-05,
      "learning_rate": 6.225520292467021e-07,
      "logits/chosen": 9.204416275024414,
      "logits/rejected": 8.248788833618164,
      "logps/chosen": -43.83194351196289,
      "logps/rejected": -46.06584930419922,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.83194351196289,
      "rewards/margins": 2.233898639678955,
      "rewards/rejected": -46.06584930419922,
      "step": 2685
    },
    {
      "epoch": 1.439705636394046,
      "grad_norm": 4.144347034654445e-12,
      "learning_rate": 6.210415403249993e-07,
      "logits/chosen": 8.947945594787598,
      "logits/rejected": 8.148805618286133,
      "logps/chosen": -42.82917404174805,
      "logps/rejected": -45.608917236328125,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -42.82917404174805,
      "rewards/margins": 2.779740333557129,
      "rewards/rejected": -45.608917236328125,
      "step": 2690
    },
    {
      "epoch": 1.4423816691754474,
      "grad_norm": 1.5182155483088923e-07,
      "learning_rate": 6.195298770577415e-07,
      "logits/chosen": 8.511805534362793,
      "logits/rejected": 8.410665512084961,
      "logps/chosen": -44.22201919555664,
      "logps/rejected": -44.78729248046875,
      "loss": 0.0001,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -44.22201919555664,
      "rewards/margins": 0.5652758479118347,
      "rewards/rejected": -44.78729248046875,
      "step": 2695
    },
    {
      "epoch": 1.445057701956849,
      "grad_norm": 1.166783104731775e-07,
      "learning_rate": 6.180170541110923e-07,
      "logits/chosen": 8.863515853881836,
      "logits/rejected": 8.229879379272461,
      "logps/chosen": -43.893157958984375,
      "logps/rejected": -44.5519905090332,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.893157958984375,
      "rewards/margins": 0.658833384513855,
      "rewards/rejected": -44.5519905090332,
      "step": 2700
    },
    {
      "epoch": 1.4477337347382506,
      "grad_norm": 1.3245486902802847e-09,
      "learning_rate": 6.165030861624663e-07,
      "logits/chosen": 9.41827392578125,
      "logits/rejected": 8.334007263183594,
      "logps/chosen": -43.815399169921875,
      "logps/rejected": -44.94819641113281,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.815399169921875,
      "rewards/margins": 1.132794976234436,
      "rewards/rejected": -44.94819641113281,
      "step": 2705
    },
    {
      "epoch": 1.4504097675196521,
      "grad_norm": 1.6810036913258834e-09,
      "learning_rate": 6.149879879003876e-07,
      "logits/chosen": 8.766114234924316,
      "logits/rejected": 8.8961181640625,
      "logps/chosen": -43.911190032958984,
      "logps/rejected": -44.65555953979492,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.911190032958984,
      "rewards/margins": 0.7443699240684509,
      "rewards/rejected": -44.65555953979492,
      "step": 2710
    },
    {
      "epoch": 1.4530858003010536,
      "grad_norm": 1.3000936020840893e-07,
      "learning_rate": 6.13471774024346e-07,
      "logits/chosen": 9.103927612304688,
      "logits/rejected": 8.703275680541992,
      "logps/chosen": -43.20763397216797,
      "logps/rejected": -44.24637985229492,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.20763397216797,
      "rewards/margins": 1.0387444496154785,
      "rewards/rejected": -44.24637985229492,
      "step": 2715
    },
    {
      "epoch": 1.4557618330824553,
      "grad_norm": 1.0194568505416533e-09,
      "learning_rate": 6.119544592446551e-07,
      "logits/chosen": 8.76878547668457,
      "logits/rejected": 8.547409057617188,
      "logps/chosen": -43.992210388183594,
      "logps/rejected": -45.81775665283203,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.992210388183594,
      "rewards/margins": 1.8255468606948853,
      "rewards/rejected": -45.81775665283203,
      "step": 2720
    },
    {
      "epoch": 1.4584378658638568,
      "grad_norm": 1.8581437441859604e-08,
      "learning_rate": 6.104360582823096e-07,
      "logits/chosen": 9.06141185760498,
      "logits/rejected": 8.74266529083252,
      "logps/chosen": -43.121822357177734,
      "logps/rejected": -45.14059829711914,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -43.121822357177734,
      "rewards/margins": 2.0187716484069824,
      "rewards/rejected": -45.14059829711914,
      "step": 2725
    },
    {
      "epoch": 1.4611138986452583,
      "grad_norm": 6.357381367883802e-08,
      "learning_rate": 6.089165858688423e-07,
      "logits/chosen": 8.880109786987305,
      "logits/rejected": 8.219015121459961,
      "logps/chosen": -43.936561584472656,
      "logps/rejected": -44.36549758911133,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.936561584472656,
      "rewards/margins": 0.42893534898757935,
      "rewards/rejected": -44.36549758911133,
      "step": 2730
    },
    {
      "epoch": 1.46378993142666,
      "grad_norm": 6.569229972752515e-08,
      "learning_rate": 6.073960567461811e-07,
      "logits/chosen": 9.197875022888184,
      "logits/rejected": 8.5531587600708,
      "logps/chosen": -43.96082305908203,
      "logps/rejected": -45.73375701904297,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.96082305908203,
      "rewards/margins": 1.7729251384735107,
      "rewards/rejected": -45.73375701904297,
      "step": 2735
    },
    {
      "epoch": 1.4664659642080615,
      "grad_norm": 6.253889865997857e-09,
      "learning_rate": 6.058744856665065e-07,
      "logits/chosen": 8.609704971313477,
      "logits/rejected": 8.199976921081543,
      "logps/chosen": -44.0206184387207,
      "logps/rejected": -45.710941314697266,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.0206184387207,
      "rewards/margins": 1.6903245449066162,
      "rewards/rejected": -45.710941314697266,
      "step": 2740
    },
    {
      "epoch": 1.469141996989463,
      "grad_norm": 1.5465890236316144e-05,
      "learning_rate": 6.043518873921074e-07,
      "logits/chosen": 9.272945404052734,
      "logits/rejected": 8.620854377746582,
      "logps/chosen": -43.11737823486328,
      "logps/rejected": -44.397621154785156,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.11737823486328,
      "rewards/margins": 1.2802444696426392,
      "rewards/rejected": -44.397621154785156,
      "step": 2745
    },
    {
      "epoch": 1.4718180297708647,
      "grad_norm": 6.049031689219591e-09,
      "learning_rate": 6.028282766952393e-07,
      "logits/chosen": 8.60879135131836,
      "logits/rejected": 8.182022094726562,
      "logps/chosen": -44.029380798339844,
      "logps/rejected": -44.99625015258789,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.029380798339844,
      "rewards/margins": 0.966866672039032,
      "rewards/rejected": -44.99625015258789,
      "step": 2750
    },
    {
      "epoch": 1.4744940625522662,
      "grad_norm": 1.1087119127841824e-08,
      "learning_rate": 6.013036683579798e-07,
      "logits/chosen": 8.887524604797363,
      "logits/rejected": 8.562888145446777,
      "logps/chosen": -44.166221618652344,
      "logps/rejected": -45.54884338378906,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.166221618652344,
      "rewards/margins": 1.3826234340667725,
      "rewards/rejected": -45.54884338378906,
      "step": 2755
    },
    {
      "epoch": 1.4771700953336677,
      "grad_norm": 2.4185399312633973e-05,
      "learning_rate": 5.997780771720854e-07,
      "logits/chosen": 8.953954696655273,
      "logits/rejected": 8.117696762084961,
      "logps/chosen": -43.11894226074219,
      "logps/rejected": -44.22502136230469,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.11894226074219,
      "rewards/margins": 1.1060755252838135,
      "rewards/rejected": -44.22502136230469,
      "step": 2760
    },
    {
      "epoch": 1.4798461281150694,
      "grad_norm": 4.136974481000137e-15,
      "learning_rate": 5.982515179388486e-07,
      "logits/chosen": 8.758176803588867,
      "logits/rejected": 8.376714706420898,
      "logps/chosen": -44.05647277832031,
      "logps/rejected": -45.198081970214844,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.05647277832031,
      "rewards/margins": 1.1416116952896118,
      "rewards/rejected": -45.198081970214844,
      "step": 2765
    },
    {
      "epoch": 1.482522160896471,
      "grad_norm": 5.530848719170761e-05,
      "learning_rate": 5.967240054689541e-07,
      "logits/chosen": 9.190610885620117,
      "logits/rejected": 9.088155746459961,
      "logps/chosen": -43.488128662109375,
      "logps/rejected": -44.597965240478516,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -43.488128662109375,
      "rewards/margins": 1.109837293624878,
      "rewards/rejected": -44.597965240478516,
      "step": 2770
    },
    {
      "epoch": 1.4851981936778724,
      "grad_norm": 1.337821956721052e-06,
      "learning_rate": 5.951955545823342e-07,
      "logits/chosen": 9.009981155395508,
      "logits/rejected": 8.726229667663574,
      "logps/chosen": -43.368385314941406,
      "logps/rejected": -45.17705535888672,
      "loss": 0.0,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -43.368385314941406,
      "rewards/margins": 1.8086717128753662,
      "rewards/rejected": -45.17705535888672,
      "step": 2775
    },
    {
      "epoch": 1.4878742264592741,
      "grad_norm": 2.340806593940026e-09,
      "learning_rate": 5.936661801080263e-07,
      "logits/chosen": 8.668088912963867,
      "logits/rejected": 8.333663940429688,
      "logps/chosen": -43.340152740478516,
      "logps/rejected": -44.76644515991211,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.340152740478516,
      "rewards/margins": 1.426295280456543,
      "rewards/rejected": -44.76644515991211,
      "step": 2780
    },
    {
      "epoch": 1.4905502592406756,
      "grad_norm": 3.775314271258639e-07,
      "learning_rate": 5.92135896884028e-07,
      "logits/chosen": 9.418441772460938,
      "logits/rejected": 8.819533348083496,
      "logps/chosen": -45.0187873840332,
      "logps/rejected": -45.530879974365234,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -45.0187873840332,
      "rewards/margins": 0.5120898485183716,
      "rewards/rejected": -45.530879974365234,
      "step": 2785
    },
    {
      "epoch": 1.4932262920220774,
      "grad_norm": 9.063669483752829e-08,
      "learning_rate": 5.906047197571541e-07,
      "logits/chosen": 8.722726821899414,
      "logits/rejected": 8.989908218383789,
      "logps/chosen": -44.75785827636719,
      "logps/rejected": -45.04639434814453,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.75785827636719,
      "rewards/margins": 0.2885281443595886,
      "rewards/rejected": -45.04639434814453,
      "step": 2790
    },
    {
      "epoch": 1.4959023248034788,
      "grad_norm": 1.4904893099178847e-06,
      "learning_rate": 5.890726635828919e-07,
      "logits/chosen": 8.861332893371582,
      "logits/rejected": 8.969846725463867,
      "logps/chosen": -44.98371505737305,
      "logps/rejected": -45.671173095703125,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.98371505737305,
      "rewards/margins": 0.687459409236908,
      "rewards/rejected": -45.671173095703125,
      "step": 2795
    },
    {
      "epoch": 1.4985783575848803,
      "grad_norm": 4.96440599425418e-10,
      "learning_rate": 5.875397432252569e-07,
      "logits/chosen": 8.915472984313965,
      "logits/rejected": 8.888192176818848,
      "logps/chosen": -43.73305892944336,
      "logps/rejected": -44.05973434448242,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -43.73305892944336,
      "rewards/margins": 0.32667845487594604,
      "rewards/rejected": -44.05973434448242,
      "step": 2800
    },
    {
      "epoch": 1.4985783575848803,
      "eval_logits/chosen": 7.463387489318848,
      "eval_logits/rejected": 7.174548149108887,
      "eval_logps/chosen": -43.738250732421875,
      "eval_logps/rejected": -44.85839080810547,
      "eval_loss": 2.48090685772695e-07,
      "eval_rewards/accuracies": 0.5660237669944763,
      "eval_rewards/chosen": -43.738250732421875,
      "eval_rewards/margins": 1.1201441287994385,
      "eval_rewards/rejected": -44.85839080810547,
      "eval_runtime": 40.0468,
      "eval_samples_per_second": 33.586,
      "eval_steps_per_second": 8.415,
      "step": 2800
    },
    {
      "epoch": 1.5012543903662818,
      "grad_norm": 7.100177154191083e-10,
      "learning_rate": 5.860059735566491e-07,
      "logits/chosen": 9.385641098022461,
      "logits/rejected": 8.850804328918457,
      "logps/chosen": -44.666194915771484,
      "logps/rejected": -45.17316818237305,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.666194915771484,
      "rewards/margins": 0.5069742202758789,
      "rewards/rejected": -45.17316818237305,
      "step": 2805
    },
    {
      "epoch": 1.5039304231476835,
      "grad_norm": 4.277165898858123e-12,
      "learning_rate": 5.844713694577087e-07,
      "logits/chosen": 8.89548110961914,
      "logits/rejected": 8.5034761428833,
      "logps/chosen": -43.25994110107422,
      "logps/rejected": -45.0087890625,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.25994110107422,
      "rewards/margins": 1.7488467693328857,
      "rewards/rejected": -45.0087890625,
      "step": 2810
    },
    {
      "epoch": 1.5066064559290853,
      "grad_norm": 5.7415069387955476e-05,
      "learning_rate": 5.829359458171714e-07,
      "logits/chosen": 8.872621536254883,
      "logits/rejected": 8.671123504638672,
      "logps/chosen": -44.34498977661133,
      "logps/rejected": -46.09889221191406,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.34498977661133,
      "rewards/margins": 1.7539066076278687,
      "rewards/rejected": -46.09889221191406,
      "step": 2815
    },
    {
      "epoch": 1.5092824887104868,
      "grad_norm": 9.684451195676176e-06,
      "learning_rate": 5.81399717531724e-07,
      "logits/chosen": 9.255640029907227,
      "logits/rejected": 8.400858879089355,
      "logps/chosen": -44.06698989868164,
      "logps/rejected": -45.42656326293945,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.06698989868164,
      "rewards/margins": 1.3595703840255737,
      "rewards/rejected": -45.42656326293945,
      "step": 2820
    },
    {
      "epoch": 1.5119585214918883,
      "grad_norm": 0.0006130969708790355,
      "learning_rate": 5.798626995058602e-07,
      "logits/chosen": 9.413412094116211,
      "logits/rejected": 8.82717514038086,
      "logps/chosen": -43.92023468017578,
      "logps/rejected": -44.79119873046875,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.92023468017578,
      "rewards/margins": 0.870956301689148,
      "rewards/rejected": -44.79119873046875,
      "step": 2825
    },
    {
      "epoch": 1.51463455427329,
      "grad_norm": 4.790481325114735e-13,
      "learning_rate": 5.783249066517354e-07,
      "logits/chosen": 8.614336013793945,
      "logits/rejected": 8.418572425842285,
      "logps/chosen": -45.310054779052734,
      "logps/rejected": -46.431724548339844,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.310054779052734,
      "rewards/margins": 1.121673822402954,
      "rewards/rejected": -46.431724548339844,
      "step": 2830
    },
    {
      "epoch": 1.5173105870546915,
      "grad_norm": 9.626119213382509e-08,
      "learning_rate": 5.767863538890228e-07,
      "logits/chosen": 9.067800521850586,
      "logits/rejected": 8.664377212524414,
      "logps/chosen": -44.12165451049805,
      "logps/rejected": -45.98723602294922,
      "loss": 0.0006,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.12165451049805,
      "rewards/margins": 1.8655840158462524,
      "rewards/rejected": -45.98723602294922,
      "step": 2835
    },
    {
      "epoch": 1.519986619836093,
      "grad_norm": 6.290323221834649e-10,
      "learning_rate": 5.75247056144768e-07,
      "logits/chosen": 8.993581771850586,
      "logits/rejected": 8.758119583129883,
      "logps/chosen": -44.499183654785156,
      "logps/rejected": -45.44392776489258,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.499183654785156,
      "rewards/margins": 0.9447482228279114,
      "rewards/rejected": -45.44392776489258,
      "step": 2840
    },
    {
      "epoch": 1.5226626526174947,
      "grad_norm": 8.640033443092615e-08,
      "learning_rate": 5.737070283532444e-07,
      "logits/chosen": 9.436294555664062,
      "logits/rejected": 8.941333770751953,
      "logps/chosen": -44.27254867553711,
      "logps/rejected": -45.61143112182617,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.27254867553711,
      "rewards/margins": 1.3388841152191162,
      "rewards/rejected": -45.61143112182617,
      "step": 2845
    },
    {
      "epoch": 1.5253386853988962,
      "grad_norm": 3.4342788792004554e-12,
      "learning_rate": 5.721662854558084e-07,
      "logits/chosen": 9.012344360351562,
      "logits/rejected": 8.6710786819458,
      "logps/chosen": -43.63538360595703,
      "logps/rejected": -44.59769058227539,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.63538360595703,
      "rewards/margins": 0.9623053669929504,
      "rewards/rejected": -44.59769058227539,
      "step": 2850
    },
    {
      "epoch": 1.5280147181802977,
      "grad_norm": 2.7853943910135156e-13,
      "learning_rate": 5.706248424007545e-07,
      "logits/chosen": 8.872308731079102,
      "logits/rejected": 8.421085357666016,
      "logps/chosen": -43.0479621887207,
      "logps/rejected": -44.47206115722656,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.0479621887207,
      "rewards/margins": 1.4240992069244385,
      "rewards/rejected": -44.47206115722656,
      "step": 2855
    },
    {
      "epoch": 1.5306907509616994,
      "grad_norm": 3.219817910824416e-05,
      "learning_rate": 5.690827141431699e-07,
      "logits/chosen": 9.300373077392578,
      "logits/rejected": 8.641509056091309,
      "logps/chosen": -45.172691345214844,
      "logps/rejected": -45.49788284301758,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.172691345214844,
      "rewards/margins": 0.32518890500068665,
      "rewards/rejected": -45.49788284301758,
      "step": 2860
    },
    {
      "epoch": 1.5333667837431009,
      "grad_norm": 7.836597082643575e-11,
      "learning_rate": 5.675399156447897e-07,
      "logits/chosen": 9.264772415161133,
      "logits/rejected": 8.617009162902832,
      "logps/chosen": -43.846649169921875,
      "logps/rejected": -44.46550369262695,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -43.846649169921875,
      "rewards/margins": 0.6188526153564453,
      "rewards/rejected": -44.46550369262695,
      "step": 2865
    },
    {
      "epoch": 1.5360428165245024,
      "grad_norm": 0.00023991522896496084,
      "learning_rate": 5.659964618738515e-07,
      "logits/chosen": 9.301197052001953,
      "logits/rejected": 8.76247787475586,
      "logps/chosen": -43.8774299621582,
      "logps/rejected": -44.85619354248047,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.8774299621582,
      "rewards/margins": 0.9787673950195312,
      "rewards/rejected": -44.85619354248047,
      "step": 2870
    },
    {
      "epoch": 1.538718849305904,
      "grad_norm": 2.0885241535885196e-05,
      "learning_rate": 5.644523678049509e-07,
      "logits/chosen": 9.122513771057129,
      "logits/rejected": 8.773046493530273,
      "logps/chosen": -43.899269104003906,
      "logps/rejected": -44.85614013671875,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.899269104003906,
      "rewards/margins": 0.956866443157196,
      "rewards/rejected": -44.85614013671875,
      "step": 2875
    },
    {
      "epoch": 1.5413948820873056,
      "grad_norm": 1.1518260667685476e-05,
      "learning_rate": 5.629076484188952e-07,
      "logits/chosen": 8.758886337280273,
      "logits/rejected": 8.445196151733398,
      "logps/chosen": -44.264434814453125,
      "logps/rejected": -45.28441619873047,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.264434814453125,
      "rewards/margins": 1.019981026649475,
      "rewards/rejected": -45.28441619873047,
      "step": 2880
    },
    {
      "epoch": 1.544070914868707,
      "grad_norm": 3.91724560115831e-09,
      "learning_rate": 5.613623187025587e-07,
      "logits/chosen": 9.296969413757324,
      "logits/rejected": 8.885337829589844,
      "logps/chosen": -45.53540802001953,
      "logps/rejected": -45.56169891357422,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.53540802001953,
      "rewards/margins": 0.026294361799955368,
      "rewards/rejected": -45.56169891357422,
      "step": 2885
    },
    {
      "epoch": 1.5467469476501088,
      "grad_norm": 1.8308421352758934e-09,
      "learning_rate": 5.598163936487369e-07,
      "logits/chosen": 8.791387557983398,
      "logits/rejected": 8.224920272827148,
      "logps/chosen": -41.973548889160156,
      "logps/rejected": -43.707847595214844,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -41.973548889160156,
      "rewards/margins": 1.7342979907989502,
      "rewards/rejected": -43.707847595214844,
      "step": 2890
    },
    {
      "epoch": 1.5494229804315103,
      "grad_norm": 4.162267470526194e-08,
      "learning_rate": 5.582698882560017e-07,
      "logits/chosen": 9.23347282409668,
      "logits/rejected": 8.481281280517578,
      "logps/chosen": -45.065223693847656,
      "logps/rejected": -45.66847229003906,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.065223693847656,
      "rewards/margins": 0.603253185749054,
      "rewards/rejected": -45.66847229003906,
      "step": 2895
    },
    {
      "epoch": 1.5520990132129118,
      "grad_norm": 8.21961676451442e-09,
      "learning_rate": 5.567228175285549e-07,
      "logits/chosen": 8.954849243164062,
      "logits/rejected": 8.55969524383545,
      "logps/chosen": -44.79142379760742,
      "logps/rejected": -45.13055419921875,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.79142379760742,
      "rewards/margins": 0.3391331136226654,
      "rewards/rejected": -45.13055419921875,
      "step": 2900
    },
    {
      "epoch": 1.5547750459943135,
      "grad_norm": 7.354772894440436e-11,
      "learning_rate": 5.551751964760838e-07,
      "logits/chosen": 8.975966453552246,
      "logits/rejected": 8.975125312805176,
      "logps/chosen": -45.00196838378906,
      "logps/rejected": -45.32769012451172,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.00196838378906,
      "rewards/margins": 0.32572999596595764,
      "rewards/rejected": -45.32769012451172,
      "step": 2905
    },
    {
      "epoch": 1.557451078775715,
      "grad_norm": 1.8991369770752845e-09,
      "learning_rate": 5.536270401136145e-07,
      "logits/chosen": 9.167055130004883,
      "logits/rejected": 8.696390151977539,
      "logps/chosen": -44.45293426513672,
      "logps/rejected": -45.114044189453125,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.45293426513672,
      "rewards/margins": 0.6611086130142212,
      "rewards/rejected": -45.114044189453125,
      "step": 2910
    },
    {
      "epoch": 1.5601271115571165,
      "grad_norm": 1.1789421674013727e-08,
      "learning_rate": 5.520783634613667e-07,
      "logits/chosen": 8.825705528259277,
      "logits/rejected": 8.358338356018066,
      "logps/chosen": -45.0051383972168,
      "logps/rejected": -45.82635498046875,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.0051383972168,
      "rewards/margins": 0.8212140798568726,
      "rewards/rejected": -45.82635498046875,
      "step": 2915
    },
    {
      "epoch": 1.5628031443385182,
      "grad_norm": 2.1720487749145892e-10,
      "learning_rate": 5.505291815446082e-07,
      "logits/chosen": 8.88515567779541,
      "logits/rejected": 8.47895336151123,
      "logps/chosen": -44.1461296081543,
      "logps/rejected": -45.535377502441406,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.1461296081543,
      "rewards/margins": 1.3892475366592407,
      "rewards/rejected": -45.535377502441406,
      "step": 2920
    },
    {
      "epoch": 1.5654791771199197,
      "grad_norm": 4.199863211662947e-09,
      "learning_rate": 5.489795093935089e-07,
      "logits/chosen": 9.114545822143555,
      "logits/rejected": 8.667595863342285,
      "logps/chosen": -44.47600555419922,
      "logps/rejected": -44.89276123046875,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.47600555419922,
      "rewards/margins": 0.4167543351650238,
      "rewards/rejected": -44.89276123046875,
      "step": 2925
    },
    {
      "epoch": 1.5681552099013212,
      "grad_norm": 1.0986896187882187e-10,
      "learning_rate": 5.474293620429946e-07,
      "logits/chosen": 9.153425216674805,
      "logits/rejected": 8.528799057006836,
      "logps/chosen": -43.94269561767578,
      "logps/rejected": -45.45195388793945,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.94269561767578,
      "rewards/margins": 1.509251356124878,
      "rewards/rejected": -45.45195388793945,
      "step": 2930
    },
    {
      "epoch": 1.570831242682723,
      "grad_norm": 2.0497231255196183e-05,
      "learning_rate": 5.458787545326018e-07,
      "logits/chosen": 9.21147632598877,
      "logits/rejected": 8.906046867370605,
      "logps/chosen": -44.090538024902344,
      "logps/rejected": -44.837684631347656,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.090538024902344,
      "rewards/margins": 0.7471483945846558,
      "rewards/rejected": -44.837684631347656,
      "step": 2935
    },
    {
      "epoch": 1.5735072754641244,
      "grad_norm": 3.2758110768174296e-07,
      "learning_rate": 5.443277019063311e-07,
      "logits/chosen": 9.249078750610352,
      "logits/rejected": 8.920819282531738,
      "logps/chosen": -45.23479461669922,
      "logps/rejected": -46.516300201416016,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -45.23479461669922,
      "rewards/margins": 1.2815046310424805,
      "rewards/rejected": -46.516300201416016,
      "step": 2940
    },
    {
      "epoch": 1.5761833082455259,
      "grad_norm": 9.674285285480508e-07,
      "learning_rate": 5.427762192125023e-07,
      "logits/chosen": 8.933552742004395,
      "logits/rejected": 8.40612506866455,
      "logps/chosen": -44.34282684326172,
      "logps/rejected": -44.95331954956055,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.34282684326172,
      "rewards/margins": 0.6104902625083923,
      "rewards/rejected": -44.95331954956055,
      "step": 2945
    },
    {
      "epoch": 1.5788593410269276,
      "grad_norm": 1.894046764857276e-08,
      "learning_rate": 5.41224321503607e-07,
      "logits/chosen": 8.972755432128906,
      "logits/rejected": 8.224937438964844,
      "logps/chosen": -44.24097442626953,
      "logps/rejected": -46.89286804199219,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -44.24097442626953,
      "rewards/margins": 2.6518967151641846,
      "rewards/rejected": -46.89286804199219,
      "step": 2950
    },
    {
      "epoch": 1.5815353738083293,
      "grad_norm": 8.837089488315047e-05,
      "learning_rate": 5.396720238361637e-07,
      "logits/chosen": 8.782453536987305,
      "logits/rejected": 8.35567569732666,
      "logps/chosen": -43.35747528076172,
      "logps/rejected": -45.18345260620117,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.35747528076172,
      "rewards/margins": 1.825979232788086,
      "rewards/rejected": -45.18345260620117,
      "step": 2955
    },
    {
      "epoch": 1.5842114065897306,
      "grad_norm": 2.77752409596975e-06,
      "learning_rate": 5.381193412705711e-07,
      "logits/chosen": 8.93437671661377,
      "logits/rejected": 8.435441970825195,
      "logps/chosen": -43.058895111083984,
      "logps/rejected": -43.84383010864258,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.058895111083984,
      "rewards/margins": 0.7849317789077759,
      "rewards/rejected": -43.84383010864258,
      "step": 2960
    },
    {
      "epoch": 1.5868874393711323,
      "grad_norm": 4.145253533148518e-10,
      "learning_rate": 5.365662888709622e-07,
      "logits/chosen": 9.011127471923828,
      "logits/rejected": 8.565678596496582,
      "logps/chosen": -43.498355865478516,
      "logps/rejected": -44.6573371887207,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.498355865478516,
      "rewards/margins": 1.1589908599853516,
      "rewards/rejected": -44.6573371887207,
      "step": 2965
    },
    {
      "epoch": 1.589563472152534,
      "grad_norm": 1.8312245516354887e-09,
      "learning_rate": 5.350128817050585e-07,
      "logits/chosen": 9.218230247497559,
      "logits/rejected": 8.576069831848145,
      "logps/chosen": -44.43537521362305,
      "logps/rejected": -45.863677978515625,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.43537521362305,
      "rewards/margins": 1.4283016920089722,
      "rewards/rejected": -45.863677978515625,
      "step": 2970
    },
    {
      "epoch": 1.5922395049339353,
      "grad_norm": 5.728340199474309e-10,
      "learning_rate": 5.334591348440229e-07,
      "logits/chosen": 9.00855541229248,
      "logits/rejected": 8.466552734375,
      "logps/chosen": -43.791465759277344,
      "logps/rejected": -44.986698150634766,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -43.791465759277344,
      "rewards/margins": 1.195236325263977,
      "rewards/rejected": -44.986698150634766,
      "step": 2975
    },
    {
      "epoch": 1.594915537715337,
      "grad_norm": 3.174570392602568e-10,
      "learning_rate": 5.319050633623141e-07,
      "logits/chosen": 9.053654670715332,
      "logits/rejected": 8.575790405273438,
      "logps/chosen": -43.92803955078125,
      "logps/rejected": -45.4078483581543,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.92803955078125,
      "rewards/margins": 1.4798088073730469,
      "rewards/rejected": -45.4078483581543,
      "step": 2980
    },
    {
      "epoch": 1.5975915704967387,
      "grad_norm": 7.081511120073261e-10,
      "learning_rate": 5.303506823375409e-07,
      "logits/chosen": 8.856507301330566,
      "logits/rejected": 8.236193656921387,
      "logps/chosen": -43.72004318237305,
      "logps/rejected": -45.751407623291016,
      "loss": 0.0,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -43.72004318237305,
      "rewards/margins": 2.031364679336548,
      "rewards/rejected": -45.751407623291016,
      "step": 2985
    },
    {
      "epoch": 1.60026760327814,
      "grad_norm": 1.1660431933967428e-11,
      "learning_rate": 5.287960068503143e-07,
      "logits/chosen": 9.43662166595459,
      "logits/rejected": 8.824470520019531,
      "logps/chosen": -45.07375717163086,
      "logps/rejected": -46.654319763183594,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.07375717163086,
      "rewards/margins": 1.5805613994598389,
      "rewards/rejected": -46.654319763183594,
      "step": 2990
    },
    {
      "epoch": 1.6029436360595417,
      "grad_norm": 6.59297966882754e-11,
      "learning_rate": 5.272410519841032e-07,
      "logits/chosen": 8.784502029418945,
      "logits/rejected": 8.491594314575195,
      "logps/chosen": -43.351585388183594,
      "logps/rejected": -44.737579345703125,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.351585388183594,
      "rewards/margins": 1.3859916925430298,
      "rewards/rejected": -44.737579345703125,
      "step": 2995
    },
    {
      "epoch": 1.6056196688409434,
      "grad_norm": 0.00012888179765203157,
      "learning_rate": 5.256858328250861e-07,
      "logits/chosen": 9.263972282409668,
      "logits/rejected": 8.59659194946289,
      "logps/chosen": -44.30946350097656,
      "logps/rejected": -45.45436096191406,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.30946350097656,
      "rewards/margins": 1.1448971033096313,
      "rewards/rejected": -45.45436096191406,
      "step": 3000
    },
    {
      "epoch": 1.608295701622345,
      "grad_norm": 2.1602503185659273e-07,
      "learning_rate": 5.241303644620063e-07,
      "logits/chosen": 9.385848999023438,
      "logits/rejected": 8.82504653930664,
      "logps/chosen": -44.934532165527344,
      "logps/rejected": -45.625770568847656,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.934532165527344,
      "rewards/margins": 0.69123774766922,
      "rewards/rejected": -45.625770568847656,
      "step": 3005
    },
    {
      "epoch": 1.6109717344037464,
      "grad_norm": 4.16902103021387e-09,
      "learning_rate": 5.225746619860248e-07,
      "logits/chosen": 9.339444160461426,
      "logits/rejected": 8.85873794555664,
      "logps/chosen": -44.08984375,
      "logps/rejected": -45.21741485595703,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.08984375,
      "rewards/margins": 1.1275746822357178,
      "rewards/rejected": -45.21741485595703,
      "step": 3010
    },
    {
      "epoch": 1.6136477671851481,
      "grad_norm": 2.8089079746387765e-05,
      "learning_rate": 5.210187404905735e-07,
      "logits/chosen": 8.631643295288086,
      "logits/rejected": 8.406475067138672,
      "logps/chosen": -45.26958465576172,
      "logps/rejected": -45.00364685058594,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -45.26958465576172,
      "rewards/margins": -0.26594278216362,
      "rewards/rejected": -45.00364685058594,
      "step": 3015
    },
    {
      "epoch": 1.6163237999665496,
      "grad_norm": 6.129850614519923e-07,
      "learning_rate": 5.194626150712098e-07,
      "logits/chosen": 9.242807388305664,
      "logits/rejected": 8.759637832641602,
      "logps/chosen": -44.524696350097656,
      "logps/rejected": -45.72322463989258,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.524696350097656,
      "rewards/margins": 1.1985244750976562,
      "rewards/rejected": -45.72322463989258,
      "step": 3020
    },
    {
      "epoch": 1.6189998327479511,
      "grad_norm": 7.433345584641093e-06,
      "learning_rate": 5.179063008254695e-07,
      "logits/chosen": 8.925596237182617,
      "logits/rejected": 8.25110149383545,
      "logps/chosen": -43.47417068481445,
      "logps/rejected": -44.26971435546875,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.47417068481445,
      "rewards/margins": 0.795545220375061,
      "rewards/rejected": -44.26971435546875,
      "step": 3025
    },
    {
      "epoch": 1.6216758655293528,
      "grad_norm": 0.20222964614626096,
      "learning_rate": 5.163498128527199e-07,
      "logits/chosen": 8.947025299072266,
      "logits/rejected": 8.450349807739258,
      "logps/chosen": -43.115108489990234,
      "logps/rejected": -45.158409118652344,
      "loss": 0.0003,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.115108489990234,
      "rewards/margins": 2.043299436569214,
      "rewards/rejected": -45.158409118652344,
      "step": 3030
    },
    {
      "epoch": 1.6243518983107543,
      "grad_norm": 2.002902765365557e-09,
      "learning_rate": 5.147931662540144e-07,
      "logits/chosen": 8.742101669311523,
      "logits/rejected": 8.304222106933594,
      "logps/chosen": -43.14806365966797,
      "logps/rejected": -44.483253479003906,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.14806365966797,
      "rewards/margins": 1.3351874351501465,
      "rewards/rejected": -44.483253479003906,
      "step": 3035
    },
    {
      "epoch": 1.6270279310921558,
      "grad_norm": 8.508118856537356e-10,
      "learning_rate": 5.132363761319449e-07,
      "logits/chosen": 8.774137496948242,
      "logits/rejected": 8.631975173950195,
      "logps/chosen": -44.153541564941406,
      "logps/rejected": -45.171112060546875,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.153541564941406,
      "rewards/margins": 1.0175702571868896,
      "rewards/rejected": -45.171112060546875,
      "step": 3040
    },
    {
      "epoch": 1.6297039638735575,
      "grad_norm": 1.2702581829195795e-06,
      "learning_rate": 5.116794575904962e-07,
      "logits/chosen": 8.7122802734375,
      "logits/rejected": 8.468236923217773,
      "logps/chosen": -44.363983154296875,
      "logps/rejected": -45.031578063964844,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.363983154296875,
      "rewards/margins": 0.6675996780395508,
      "rewards/rejected": -45.031578063964844,
      "step": 3045
    },
    {
      "epoch": 1.632379996654959,
      "grad_norm": 4.547147527937683e-09,
      "learning_rate": 5.101224257348987e-07,
      "logits/chosen": 8.997529029846191,
      "logits/rejected": 8.745366096496582,
      "logps/chosen": -44.28626251220703,
      "logps/rejected": -45.018436431884766,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.28626251220703,
      "rewards/margins": 0.7321747541427612,
      "rewards/rejected": -45.018436431884766,
      "step": 3050
    },
    {
      "epoch": 1.6350560294363605,
      "grad_norm": 1.0644492926857492e-07,
      "learning_rate": 5.085652956714823e-07,
      "logits/chosen": 9.522141456604004,
      "logits/rejected": 9.02328872680664,
      "logps/chosen": -44.7415771484375,
      "logps/rejected": -46.29000473022461,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.7415771484375,
      "rewards/margins": 1.5484281778335571,
      "rewards/rejected": -46.29000473022461,
      "step": 3055
    },
    {
      "epoch": 1.6377320622177622,
      "grad_norm": 4.450142375575518e-06,
      "learning_rate": 5.070080825075298e-07,
      "logits/chosen": 9.04645824432373,
      "logits/rejected": 8.45344352722168,
      "logps/chosen": -43.427589416503906,
      "logps/rejected": -44.7748908996582,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.427589416503906,
      "rewards/margins": 1.3473082780838013,
      "rewards/rejected": -44.7748908996582,
      "step": 3060
    },
    {
      "epoch": 1.6404080949991637,
      "grad_norm": 4.620906279769224e-12,
      "learning_rate": 5.0545080135113e-07,
      "logits/chosen": 8.888652801513672,
      "logits/rejected": 8.639145851135254,
      "logps/chosen": -45.13532257080078,
      "logps/rejected": -45.95991134643555,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.13532257080078,
      "rewards/margins": 0.8245919942855835,
      "rewards/rejected": -45.95991134643555,
      "step": 3065
    },
    {
      "epoch": 1.6430841277805652,
      "grad_norm": 2.382274340310994e-07,
      "learning_rate": 5.038934673110316e-07,
      "logits/chosen": 8.792036056518555,
      "logits/rejected": 8.601957321166992,
      "logps/chosen": -43.87910079956055,
      "logps/rejected": -44.90550994873047,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.87910079956055,
      "rewards/margins": 1.026409387588501,
      "rewards/rejected": -44.90550994873047,
      "step": 3070
    },
    {
      "epoch": 1.645760160561967,
      "grad_norm": 1.887154893818732e-08,
      "learning_rate": 5.023360954964963e-07,
      "logits/chosen": 8.988325119018555,
      "logits/rejected": 8.745881080627441,
      "logps/chosen": -44.41101837158203,
      "logps/rejected": -44.872047424316406,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.41101837158203,
      "rewards/margins": 0.46102291345596313,
      "rewards/rejected": -44.872047424316406,
      "step": 3075
    },
    {
      "epoch": 1.6484361933433684,
      "grad_norm": 1.8084563063393043e-07,
      "learning_rate": 5.007787010171524e-07,
      "logits/chosen": 9.057968139648438,
      "logits/rejected": 8.507180213928223,
      "logps/chosen": -44.160926818847656,
      "logps/rejected": -45.26495361328125,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.160926818847656,
      "rewards/margins": 1.104029893875122,
      "rewards/rejected": -45.26495361328125,
      "step": 3080
    },
    {
      "epoch": 1.65111222612477,
      "grad_norm": 3.1143743832198604e-11,
      "learning_rate": 4.992212989828477e-07,
      "logits/chosen": 9.057226181030273,
      "logits/rejected": 8.942174911499023,
      "logps/chosen": -46.557945251464844,
      "logps/rejected": -46.26454544067383,
      "loss": 0.0,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -46.557945251464844,
      "rewards/margins": -0.2934001684188843,
      "rewards/rejected": -46.26454544067383,
      "step": 3085
    },
    {
      "epoch": 1.6537882589061716,
      "grad_norm": 4.348367198888447e-08,
      "learning_rate": 4.976639045035036e-07,
      "logits/chosen": 8.911636352539062,
      "logits/rejected": 8.690635681152344,
      "logps/chosen": -45.5286750793457,
      "logps/rejected": -47.072330474853516,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.5286750793457,
      "rewards/margins": 1.5436522960662842,
      "rewards/rejected": -47.072330474853516,
      "step": 3090
    },
    {
      "epoch": 1.6564642916875731,
      "grad_norm": 0.21954396554611036,
      "learning_rate": 4.961065326889683e-07,
      "logits/chosen": 8.635847091674805,
      "logits/rejected": 8.100770950317383,
      "logps/chosen": -43.7431526184082,
      "logps/rejected": -45.80241394042969,
      "loss": 0.0013,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.7431526184082,
      "rewards/margins": 2.0592589378356934,
      "rewards/rejected": -45.80241394042969,
      "step": 3095
    },
    {
      "epoch": 1.6591403244689746,
      "grad_norm": 1.301002320358724e-12,
      "learning_rate": 4.9454919864887e-07,
      "logits/chosen": 9.048996925354004,
      "logits/rejected": 8.683650970458984,
      "logps/chosen": -44.778358459472656,
      "logps/rejected": -45.48814392089844,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.778358459472656,
      "rewards/margins": 0.7097845077514648,
      "rewards/rejected": -45.48814392089844,
      "step": 3100
    },
    {
      "epoch": 1.6618163572503764,
      "grad_norm": 3.1947790348027506e-11,
      "learning_rate": 4.929919174924701e-07,
      "logits/chosen": 9.414546012878418,
      "logits/rejected": 8.631009101867676,
      "logps/chosen": -44.1229248046875,
      "logps/rejected": -45.368873596191406,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.1229248046875,
      "rewards/margins": 1.245947241783142,
      "rewards/rejected": -45.368873596191406,
      "step": 3105
    },
    {
      "epoch": 1.6644923900317778,
      "grad_norm": 7.612944441853271e-12,
      "learning_rate": 4.914347043285177e-07,
      "logits/chosen": 8.8457612991333,
      "logits/rejected": 8.534770011901855,
      "logps/chosen": -43.92134475708008,
      "logps/rejected": -45.17069625854492,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -43.92134475708008,
      "rewards/margins": 1.2493526935577393,
      "rewards/rejected": -45.17069625854492,
      "step": 3110
    },
    {
      "epoch": 1.6671684228131793,
      "grad_norm": 1.488652709487455e-07,
      "learning_rate": 4.898775742651013e-07,
      "logits/chosen": 8.793787956237793,
      "logits/rejected": 8.45551872253418,
      "logps/chosen": -44.550106048583984,
      "logps/rejected": -45.81063461303711,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.550106048583984,
      "rewards/margins": 1.2605292797088623,
      "rewards/rejected": -45.81063461303711,
      "step": 3115
    },
    {
      "epoch": 1.669844455594581,
      "grad_norm": 1.744035540561211e-07,
      "learning_rate": 4.883205424095037e-07,
      "logits/chosen": 9.141380310058594,
      "logits/rejected": 8.579105377197266,
      "logps/chosen": -43.32773971557617,
      "logps/rejected": -45.79821014404297,
      "loss": 0.0,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -43.32773971557617,
      "rewards/margins": 2.4704694747924805,
      "rewards/rejected": -45.79821014404297,
      "step": 3120
    },
    {
      "epoch": 1.6725204883759828,
      "grad_norm": 0.007013936706540412,
      "learning_rate": 4.86763623868055e-07,
      "logits/chosen": 9.164687156677246,
      "logits/rejected": 8.653786659240723,
      "logps/chosen": -43.58137893676758,
      "logps/rejected": -45.125892639160156,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.58137893676758,
      "rewards/margins": 1.5445135831832886,
      "rewards/rejected": -45.125892639160156,
      "step": 3125
    },
    {
      "epoch": 1.675196521157384,
      "grad_norm": 5.314189117505848e-06,
      "learning_rate": 4.852068337459856e-07,
      "logits/chosen": 8.740901947021484,
      "logits/rejected": 7.896323204040527,
      "logps/chosen": -42.916812896728516,
      "logps/rejected": -44.497398376464844,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -42.916812896728516,
      "rewards/margins": 1.5805844068527222,
      "rewards/rejected": -44.497398376464844,
      "step": 3130
    },
    {
      "epoch": 1.6778725539387858,
      "grad_norm": 8.029539418259507e-13,
      "learning_rate": 4.8365018714728e-07,
      "logits/chosen": 8.81491756439209,
      "logits/rejected": 8.571423530578613,
      "logps/chosen": -42.20915985107422,
      "logps/rejected": -44.726219177246094,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -42.20915985107422,
      "rewards/margins": 2.5170607566833496,
      "rewards/rejected": -44.726219177246094,
      "step": 3135
    },
    {
      "epoch": 1.6805485867201875,
      "grad_norm": 0.012039373631767767,
      "learning_rate": 4.820936991745304e-07,
      "logits/chosen": 9.323692321777344,
      "logits/rejected": 8.982942581176758,
      "logps/chosen": -43.78815460205078,
      "logps/rejected": -45.36547088623047,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.78815460205078,
      "rewards/margins": 1.5773109197616577,
      "rewards/rejected": -45.36547088623047,
      "step": 3140
    },
    {
      "epoch": 1.6832246195015887,
      "grad_norm": 1.507766114191878e-05,
      "learning_rate": 4.8053738492879e-07,
      "logits/chosen": 8.617170333862305,
      "logits/rejected": 8.181614875793457,
      "logps/chosen": -44.20338439941406,
      "logps/rejected": -45.12925720214844,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.20338439941406,
      "rewards/margins": 0.9258725047111511,
      "rewards/rejected": -45.12925720214844,
      "step": 3145
    },
    {
      "epoch": 1.6859006522829905,
      "grad_norm": 0.26293182059909853,
      "learning_rate": 4.789812595094265e-07,
      "logits/chosen": 9.080703735351562,
      "logits/rejected": 8.680227279663086,
      "logps/chosen": -43.022056579589844,
      "logps/rejected": -44.2999267578125,
      "loss": 0.0009,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.022056579589844,
      "rewards/margins": 1.277869701385498,
      "rewards/rejected": -44.2999267578125,
      "step": 3150
    },
    {
      "epoch": 1.6885766850643922,
      "grad_norm": 6.316645509946261e-10,
      "learning_rate": 4.774253380139752e-07,
      "logits/chosen": 9.312246322631836,
      "logits/rejected": 8.868497848510742,
      "logps/chosen": -44.41839599609375,
      "logps/rejected": -45.45479202270508,
      "loss": 0.0006,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.41839599609375,
      "rewards/margins": 1.0363961458206177,
      "rewards/rejected": -45.45479202270508,
      "step": 3155
    },
    {
      "epoch": 1.6912527178457935,
      "grad_norm": 2.8723687313997633e-09,
      "learning_rate": 4.758696355379936e-07,
      "logits/chosen": 8.815282821655273,
      "logits/rejected": 9.04725170135498,
      "logps/chosen": -45.01224899291992,
      "logps/rejected": -45.8141975402832,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -45.01224899291992,
      "rewards/margins": 0.8019444346427917,
      "rewards/rejected": -45.8141975402832,
      "step": 3160
    },
    {
      "epoch": 1.6939287506271952,
      "grad_norm": 6.324086080789563e-08,
      "learning_rate": 4.743141671749138e-07,
      "logits/chosen": 9.385330200195312,
      "logits/rejected": 8.909971237182617,
      "logps/chosen": -43.371559143066406,
      "logps/rejected": -43.863834381103516,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.371559143066406,
      "rewards/margins": 0.4922700822353363,
      "rewards/rejected": -43.863834381103516,
      "step": 3165
    },
    {
      "epoch": 1.6966047834085969,
      "grad_norm": 7.45209385767879e-07,
      "learning_rate": 4.727589480158968e-07,
      "logits/chosen": 9.26764965057373,
      "logits/rejected": 8.89928150177002,
      "logps/chosen": -45.0379753112793,
      "logps/rejected": -46.671791076660156,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.0379753112793,
      "rewards/margins": 1.633811593055725,
      "rewards/rejected": -46.671791076660156,
      "step": 3170
    },
    {
      "epoch": 1.6992808161899984,
      "grad_norm": 6.626771847976568e-07,
      "learning_rate": 4.712039931496855e-07,
      "logits/chosen": 9.002997398376465,
      "logits/rejected": 8.328465461730957,
      "logps/chosen": -44.81950378417969,
      "logps/rejected": -45.96772384643555,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -44.81950378417969,
      "rewards/margins": 1.1482179164886475,
      "rewards/rejected": -45.96772384643555,
      "step": 3175
    },
    {
      "epoch": 1.7019568489713999,
      "grad_norm": 2.319610283324118e-09,
      "learning_rate": 4.6964931766245905e-07,
      "logits/chosen": 8.962507247924805,
      "logits/rejected": 8.796026229858398,
      "logps/chosen": -45.24757385253906,
      "logps/rejected": -45.350589752197266,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -45.24757385253906,
      "rewards/margins": 0.10301361978054047,
      "rewards/rejected": -45.350589752197266,
      "step": 3180
    },
    {
      "epoch": 1.7046328817528016,
      "grad_norm": 6.597709274405706e-07,
      "learning_rate": 4.6809493663768575e-07,
      "logits/chosen": 8.84621810913086,
      "logits/rejected": 8.463728904724121,
      "logps/chosen": -45.923927307128906,
      "logps/rejected": -46.26262664794922,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -45.923927307128906,
      "rewards/margins": 0.3386942744255066,
      "rewards/rejected": -46.26262664794922,
      "step": 3185
    },
    {
      "epoch": 1.707308914534203,
      "grad_norm": 2.8530440139515076e-11,
      "learning_rate": 4.6654086515597716e-07,
      "logits/chosen": 9.278806686401367,
      "logits/rejected": 8.530682563781738,
      "logps/chosen": -43.7806282043457,
      "logps/rejected": -45.83234405517578,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.7806282043457,
      "rewards/margins": 2.051718235015869,
      "rewards/rejected": -45.83234405517578,
      "step": 3190
    },
    {
      "epoch": 1.7099849473156046,
      "grad_norm": 0.13435991648558906,
      "learning_rate": 4.6498711829494154e-07,
      "logits/chosen": 9.012216567993164,
      "logits/rejected": 8.732029914855957,
      "logps/chosen": -44.02513885498047,
      "logps/rejected": -44.78590393066406,
      "loss": 0.0006,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.02513885498047,
      "rewards/margins": 0.7607654333114624,
      "rewards/rejected": -44.78590393066406,
      "step": 3195
    },
    {
      "epoch": 1.7126609800970063,
      "grad_norm": 4.96412939559452e-12,
      "learning_rate": 4.6343371112903777e-07,
      "logits/chosen": 9.082571029663086,
      "logits/rejected": 8.419342994689941,
      "logps/chosen": -45.12734603881836,
      "logps/rejected": -45.620914459228516,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.12734603881836,
      "rewards/margins": 0.493571937084198,
      "rewards/rejected": -45.620914459228516,
      "step": 3200
    },
    {
      "epoch": 1.7126609800970063,
      "eval_logits/chosen": 7.594981670379639,
      "eval_logits/rejected": 7.284818649291992,
      "eval_logps/chosen": -44.49504470825195,
      "eval_logps/rejected": -45.555572509765625,
      "eval_loss": 2.3677523586229654e-07,
      "eval_rewards/accuracies": 0.5637982487678528,
      "eval_rewards/chosen": -44.49504470825195,
      "eval_rewards/margins": 1.0605272054672241,
      "eval_rewards/rejected": -45.555572509765625,
      "eval_runtime": 40.1892,
      "eval_samples_per_second": 33.467,
      "eval_steps_per_second": 8.385,
      "step": 3200
    },
    {
      "epoch": 1.7153370128784078,
      "grad_norm": 1.0694028718160629e-10,
      "learning_rate": 4.618806587294291e-07,
      "logits/chosen": 9.06442928314209,
      "logits/rejected": 8.840324401855469,
      "logps/chosen": -44.341590881347656,
      "logps/rejected": -45.758121490478516,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.341590881347656,
      "rewards/margins": 1.4165369272232056,
      "rewards/rejected": -45.758121490478516,
      "step": 3205
    },
    {
      "epoch": 1.7180130456598093,
      "grad_norm": 0.0002622633126007596,
      "learning_rate": 4.603279761638365e-07,
      "logits/chosen": 8.868464469909668,
      "logits/rejected": 8.483160018920898,
      "logps/chosen": -44.854949951171875,
      "logps/rejected": -45.89757537841797,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.854949951171875,
      "rewards/margins": 1.0426268577575684,
      "rewards/rejected": -45.89757537841797,
      "step": 3210
    },
    {
      "epoch": 1.720689078441211,
      "grad_norm": 5.534403920058748e-09,
      "learning_rate": 4.5877567849639315e-07,
      "logits/chosen": 9.100788116455078,
      "logits/rejected": 8.820245742797852,
      "logps/chosen": -44.247779846191406,
      "logps/rejected": -45.852996826171875,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.247779846191406,
      "rewards/margins": 1.6052172183990479,
      "rewards/rejected": -45.852996826171875,
      "step": 3215
    },
    {
      "epoch": 1.7233651112226125,
      "grad_norm": 1.0887564769962474e-10,
      "learning_rate": 4.572237807874979e-07,
      "logits/chosen": 9.396540641784668,
      "logits/rejected": 8.26921272277832,
      "logps/chosen": -44.187400817871094,
      "logps/rejected": -46.24650192260742,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.187400817871094,
      "rewards/margins": 2.0590977668762207,
      "rewards/rejected": -46.24650192260742,
      "step": 3220
    },
    {
      "epoch": 1.726041144004014,
      "grad_norm": 3.287153742167365e-08,
      "learning_rate": 4.5567229809366895e-07,
      "logits/chosen": 8.91618824005127,
      "logits/rejected": 8.520105361938477,
      "logps/chosen": -44.41497039794922,
      "logps/rejected": -46.025882720947266,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.41497039794922,
      "rewards/margins": 1.6109154224395752,
      "rewards/rejected": -46.025882720947266,
      "step": 3225
    },
    {
      "epoch": 1.7287171767854157,
      "grad_norm": 4.1227083054383196e-07,
      "learning_rate": 4.541212454673984e-07,
      "logits/chosen": 9.229771614074707,
      "logits/rejected": 8.519007682800293,
      "logps/chosen": -44.85152816772461,
      "logps/rejected": -47.07421112060547,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.85152816772461,
      "rewards/margins": 2.2226781845092773,
      "rewards/rejected": -47.07421112060547,
      "step": 3230
    },
    {
      "epoch": 1.7313932095668172,
      "grad_norm": 1.086294120384552e-08,
      "learning_rate": 4.525706379570055e-07,
      "logits/chosen": 9.498385429382324,
      "logits/rejected": 9.169548034667969,
      "logps/chosen": -44.152347564697266,
      "logps/rejected": -45.51213836669922,
      "loss": 0.0002,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.152347564697266,
      "rewards/margins": 1.3597936630249023,
      "rewards/rejected": -45.51213836669922,
      "step": 3235
    },
    {
      "epoch": 1.7340692423482187,
      "grad_norm": 8.505582167202258e-08,
      "learning_rate": 4.510204906064911e-07,
      "logits/chosen": 9.111763954162598,
      "logits/rejected": 8.416470527648926,
      "logps/chosen": -43.987037658691406,
      "logps/rejected": -46.361907958984375,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -43.987037658691406,
      "rewards/margins": 2.374868154525757,
      "rewards/rejected": -46.361907958984375,
      "step": 3240
    },
    {
      "epoch": 1.7367452751296204,
      "grad_norm": 4.593959070816027e-06,
      "learning_rate": 4.4947081845539177e-07,
      "logits/chosen": 9.302237510681152,
      "logits/rejected": 8.826248168945312,
      "logps/chosen": -44.89716720581055,
      "logps/rejected": -46.21928787231445,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.89716720581055,
      "rewards/margins": 1.3221142292022705,
      "rewards/rejected": -46.21928787231445,
      "step": 3245
    },
    {
      "epoch": 1.739421307911022,
      "grad_norm": 5.918138519883763e-07,
      "learning_rate": 4.479216365386333e-07,
      "logits/chosen": 9.084136962890625,
      "logits/rejected": 8.34604549407959,
      "logps/chosen": -43.636314392089844,
      "logps/rejected": -45.5534553527832,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -43.636314392089844,
      "rewards/margins": 1.9171384572982788,
      "rewards/rejected": -45.5534553527832,
      "step": 3250
    },
    {
      "epoch": 1.7420973406924234,
      "grad_norm": 5.5176268027124835e-09,
      "learning_rate": 4.4637295988638555e-07,
      "logits/chosen": 9.118257522583008,
      "logits/rejected": 8.955656051635742,
      "logps/chosen": -43.11465835571289,
      "logps/rejected": -45.639564514160156,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -43.11465835571289,
      "rewards/margins": 2.5249059200286865,
      "rewards/rejected": -45.639564514160156,
      "step": 3255
    },
    {
      "epoch": 1.744773373473825,
      "grad_norm": 1.0946002604136514e-05,
      "learning_rate": 4.4482480352391623e-07,
      "logits/chosen": 8.658828735351562,
      "logits/rejected": 8.588617324829102,
      "logps/chosen": -43.32353973388672,
      "logps/rejected": -44.984195709228516,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -43.32353973388672,
      "rewards/margins": 1.6606624126434326,
      "rewards/rejected": -44.984195709228516,
      "step": 3260
    },
    {
      "epoch": 1.7474494062552266,
      "grad_norm": 2.9172880653176963e-12,
      "learning_rate": 4.4327718247144507e-07,
      "logits/chosen": 8.863751411437988,
      "logits/rejected": 8.424466133117676,
      "logps/chosen": -45.15575408935547,
      "logps/rejected": -45.89250946044922,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.15575408935547,
      "rewards/margins": 0.7367540597915649,
      "rewards/rejected": -45.89250946044922,
      "step": 3265
    },
    {
      "epoch": 1.750125439036628,
      "grad_norm": 6.699298813414161e-06,
      "learning_rate": 4.417301117439984e-07,
      "logits/chosen": 9.300911903381348,
      "logits/rejected": 9.0374174118042,
      "logps/chosen": -43.511348724365234,
      "logps/rejected": -44.943031311035156,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -43.511348724365234,
      "rewards/margins": 1.4316843748092651,
      "rewards/rejected": -44.943031311035156,
      "step": 3270
    },
    {
      "epoch": 1.7528014718180298,
      "grad_norm": 5.340827764518502e-13,
      "learning_rate": 4.401836063512631e-07,
      "logits/chosen": 9.67873764038086,
      "logits/rejected": 8.523398399353027,
      "logps/chosen": -45.80486297607422,
      "logps/rejected": -47.26551818847656,
      "loss": 0.0002,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.80486297607422,
      "rewards/margins": 1.4606564044952393,
      "rewards/rejected": -47.26551818847656,
      "step": 3275
    },
    {
      "epoch": 1.7554775045994313,
      "grad_norm": 1.2883197459860745e-13,
      "learning_rate": 4.386376812974413e-07,
      "logits/chosen": 9.264447212219238,
      "logits/rejected": 9.103300094604492,
      "logps/chosen": -46.562034606933594,
      "logps/rejected": -48.039066314697266,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -46.562034606933594,
      "rewards/margins": 1.4770265817642212,
      "rewards/rejected": -48.039066314697266,
      "step": 3280
    },
    {
      "epoch": 1.7581535373808328,
      "grad_norm": 7.942064988652533e-08,
      "learning_rate": 4.370923515811048e-07,
      "logits/chosen": 9.492040634155273,
      "logits/rejected": 8.796941757202148,
      "logps/chosen": -44.870155334472656,
      "logps/rejected": -45.88899612426758,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.870155334472656,
      "rewards/margins": 1.0188381671905518,
      "rewards/rejected": -45.88899612426758,
      "step": 3285
    },
    {
      "epoch": 1.7608295701622345,
      "grad_norm": 5.549669963100992e-08,
      "learning_rate": 4.35547632195049e-07,
      "logits/chosen": 9.007614135742188,
      "logits/rejected": 8.609139442443848,
      "logps/chosen": -45.14259338378906,
      "logps/rejected": -45.913673400878906,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.14259338378906,
      "rewards/margins": 0.7710763216018677,
      "rewards/rejected": -45.913673400878906,
      "step": 3290
    },
    {
      "epoch": 1.763505602943636,
      "grad_norm": 1.0265411751165003e-10,
      "learning_rate": 4.340035381261484e-07,
      "logits/chosen": 9.28994369506836,
      "logits/rejected": 8.827973365783691,
      "logps/chosen": -45.54672622680664,
      "logps/rejected": -46.20027160644531,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -45.54672622680664,
      "rewards/margins": 0.653546154499054,
      "rewards/rejected": -46.20027160644531,
      "step": 3295
    },
    {
      "epoch": 1.7661816357250375,
      "grad_norm": 5.670679567320231e-09,
      "learning_rate": 4.324600843552104e-07,
      "logits/chosen": 9.292959213256836,
      "logits/rejected": 8.795492172241211,
      "logps/chosen": -45.26659393310547,
      "logps/rejected": -46.33277130126953,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.26659393310547,
      "rewards/margins": 1.0661778450012207,
      "rewards/rejected": -46.33277130126953,
      "step": 3300
    },
    {
      "epoch": 1.7688576685064392,
      "grad_norm": 0.0001614310251418979,
      "learning_rate": 4.309172858568302e-07,
      "logits/chosen": 9.156953811645508,
      "logits/rejected": 8.60557746887207,
      "logps/chosen": -44.87436294555664,
      "logps/rejected": -46.307674407958984,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.87436294555664,
      "rewards/margins": 1.4333099126815796,
      "rewards/rejected": -46.307674407958984,
      "step": 3305
    },
    {
      "epoch": 1.771533701287841,
      "grad_norm": 4.132649629443543e-11,
      "learning_rate": 4.293751575992455e-07,
      "logits/chosen": 8.902022361755371,
      "logits/rejected": 8.878480911254883,
      "logps/chosen": -45.08747863769531,
      "logps/rejected": -46.276634216308594,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -45.08747863769531,
      "rewards/margins": 1.1891568899154663,
      "rewards/rejected": -46.276634216308594,
      "step": 3310
    },
    {
      "epoch": 1.7742097340692422,
      "grad_norm": 1.3228542327544875e-05,
      "learning_rate": 4.278337145441916e-07,
      "logits/chosen": 9.291013717651367,
      "logits/rejected": 8.665119171142578,
      "logps/chosen": -44.231407165527344,
      "logps/rejected": -45.266746520996094,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.231407165527344,
      "rewards/margins": 1.0353434085845947,
      "rewards/rejected": -45.266746520996094,
      "step": 3315
    },
    {
      "epoch": 1.776885766850644,
      "grad_norm": 1.2823468324285221e-08,
      "learning_rate": 4.262929716467556e-07,
      "logits/chosen": 9.337361335754395,
      "logits/rejected": 8.408716201782227,
      "logps/chosen": -45.64391326904297,
      "logps/rejected": -47.033050537109375,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.64391326904297,
      "rewards/margins": 1.38914155960083,
      "rewards/rejected": -47.033050537109375,
      "step": 3320
    },
    {
      "epoch": 1.7795617996320456,
      "grad_norm": 1.9970682138175883e-12,
      "learning_rate": 4.247529438552321e-07,
      "logits/chosen": 9.33411979675293,
      "logits/rejected": 8.666598320007324,
      "logps/chosen": -44.6694221496582,
      "logps/rejected": -45.49016571044922,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -44.6694221496582,
      "rewards/margins": 0.8207443356513977,
      "rewards/rejected": -45.49016571044922,
      "step": 3325
    },
    {
      "epoch": 1.782237832413447,
      "grad_norm": 2.3236450261234825e-07,
      "learning_rate": 4.232136461109773e-07,
      "logits/chosen": 9.214646339416504,
      "logits/rejected": 8.694793701171875,
      "logps/chosen": -45.28386688232422,
      "logps/rejected": -45.74283218383789,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.28386688232422,
      "rewards/margins": 0.45896419882774353,
      "rewards/rejected": -45.74283218383789,
      "step": 3330
    },
    {
      "epoch": 1.7849138651948486,
      "grad_norm": 0.08284321346032689,
      "learning_rate": 4.216750933482646e-07,
      "logits/chosen": 8.820481300354004,
      "logits/rejected": 8.38449764251709,
      "logps/chosen": -44.47235870361328,
      "logps/rejected": -46.04071044921875,
      "loss": 0.0007,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.47235870361328,
      "rewards/margins": 1.5683552026748657,
      "rewards/rejected": -46.04071044921875,
      "step": 3335
    },
    {
      "epoch": 1.7875898979762503,
      "grad_norm": 4.221285838266244e-08,
      "learning_rate": 4.2013730049413986e-07,
      "logits/chosen": 8.990975379943848,
      "logits/rejected": 8.170373916625977,
      "logps/chosen": -43.3891716003418,
      "logps/rejected": -45.585906982421875,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -43.3891716003418,
      "rewards/margins": 2.1967382431030273,
      "rewards/rejected": -45.585906982421875,
      "step": 3340
    },
    {
      "epoch": 1.7902659307576518,
      "grad_norm": 2.575333624317922e-07,
      "learning_rate": 4.1860028246827594e-07,
      "logits/chosen": 9.000231742858887,
      "logits/rejected": 8.150548934936523,
      "logps/chosen": -45.663352966308594,
      "logps/rejected": -46.336387634277344,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.663352966308594,
      "rewards/margins": 0.6730385422706604,
      "rewards/rejected": -46.336387634277344,
      "step": 3345
    },
    {
      "epoch": 1.7929419635390533,
      "grad_norm": 2.170796380468617e-10,
      "learning_rate": 4.170640541828285e-07,
      "logits/chosen": 9.592267036437988,
      "logits/rejected": 9.194437026977539,
      "logps/chosen": -44.966819763183594,
      "logps/rejected": -45.61595916748047,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.966819763183594,
      "rewards/margins": 0.6491453051567078,
      "rewards/rejected": -45.61595916748047,
      "step": 3350
    },
    {
      "epoch": 1.795617996320455,
      "grad_norm": 5.213557002901816e-11,
      "learning_rate": 4.1552863054229116e-07,
      "logits/chosen": 9.181442260742188,
      "logits/rejected": 9.029878616333008,
      "logps/chosen": -44.750633239746094,
      "logps/rejected": -46.416934967041016,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -44.750633239746094,
      "rewards/margins": 1.666303038597107,
      "rewards/rejected": -46.416934967041016,
      "step": 3355
    },
    {
      "epoch": 1.7982940291018565,
      "grad_norm": 2.099684462605139e-06,
      "learning_rate": 4.139940264433508e-07,
      "logits/chosen": 9.143672943115234,
      "logits/rejected": 8.411324501037598,
      "logps/chosen": -45.384033203125,
      "logps/rejected": -46.79613494873047,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.384033203125,
      "rewards/margins": 1.4121065139770508,
      "rewards/rejected": -46.79613494873047,
      "step": 3360
    },
    {
      "epoch": 1.800970061883258,
      "grad_norm": 9.02493308873475e-10,
      "learning_rate": 4.1246025677474303e-07,
      "logits/chosen": 9.587242126464844,
      "logits/rejected": 8.831242561340332,
      "logps/chosen": -45.30094528198242,
      "logps/rejected": -46.592323303222656,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.30094528198242,
      "rewards/margins": 1.2913777828216553,
      "rewards/rejected": -46.592323303222656,
      "step": 3365
    },
    {
      "epoch": 1.8036460946646597,
      "grad_norm": 3.648351691439331e-12,
      "learning_rate": 4.10927336417108e-07,
      "logits/chosen": 9.268774032592773,
      "logits/rejected": 8.551546096801758,
      "logps/chosen": -44.5892448425293,
      "logps/rejected": -45.68513107299805,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.5892448425293,
      "rewards/margins": 1.0958842039108276,
      "rewards/rejected": -45.68513107299805,
      "step": 3370
    },
    {
      "epoch": 1.8063221274460612,
      "grad_norm": 1.70661936825582e-07,
      "learning_rate": 4.093952802428457e-07,
      "logits/chosen": 9.12596607208252,
      "logits/rejected": 9.026418685913086,
      "logps/chosen": -45.5457878112793,
      "logps/rejected": -45.61756134033203,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.5457878112793,
      "rewards/margins": 0.07177647948265076,
      "rewards/rejected": -45.61756134033203,
      "step": 3375
    },
    {
      "epoch": 1.8089981602274627,
      "grad_norm": 1.974305674637144e-12,
      "learning_rate": 4.0786410311597184e-07,
      "logits/chosen": 9.248157501220703,
      "logits/rejected": 8.490348815917969,
      "logps/chosen": -44.32052993774414,
      "logps/rejected": -45.137489318847656,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.32052993774414,
      "rewards/margins": 0.8169604539871216,
      "rewards/rejected": -45.137489318847656,
      "step": 3380
    },
    {
      "epoch": 1.8116741930088645,
      "grad_norm": 2.106705459877558e-08,
      "learning_rate": 4.063338198919737e-07,
      "logits/chosen": 9.356613159179688,
      "logits/rejected": 9.308626174926758,
      "logps/chosen": -44.32353210449219,
      "logps/rejected": -44.884490966796875,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -44.32353210449219,
      "rewards/margins": 0.5609579086303711,
      "rewards/rejected": -44.884490966796875,
      "step": 3385
    },
    {
      "epoch": 1.814350225790266,
      "grad_norm": 5.705093806501091e-14,
      "learning_rate": 4.0480444541766575e-07,
      "logits/chosen": 9.527055740356445,
      "logits/rejected": 8.954641342163086,
      "logps/chosen": -46.183433532714844,
      "logps/rejected": -47.24397277832031,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -46.183433532714844,
      "rewards/margins": 1.0605409145355225,
      "rewards/rejected": -47.24397277832031,
      "step": 3390
    },
    {
      "epoch": 1.8170262585716674,
      "grad_norm": 2.451821660843849e-11,
      "learning_rate": 4.0327599453104606e-07,
      "logits/chosen": 9.311447143554688,
      "logits/rejected": 8.85721206665039,
      "logps/chosen": -43.710479736328125,
      "logps/rejected": -45.643131256103516,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -43.710479736328125,
      "rewards/margins": 1.9326457977294922,
      "rewards/rejected": -45.643131256103516,
      "step": 3395
    },
    {
      "epoch": 1.8197022913530692,
      "grad_norm": 1.9752559937347256e-08,
      "learning_rate": 4.017484820611514e-07,
      "logits/chosen": 8.952548027038574,
      "logits/rejected": 8.433120727539062,
      "logps/chosen": -44.15699005126953,
      "logps/rejected": -45.290706634521484,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.15699005126953,
      "rewards/margins": 1.133719801902771,
      "rewards/rejected": -45.290706634521484,
      "step": 3400
    },
    {
      "epoch": 1.8223783241344707,
      "grad_norm": 2.1809919315401728e-10,
      "learning_rate": 4.002219228279148e-07,
      "logits/chosen": 9.081011772155762,
      "logits/rejected": 8.678295135498047,
      "logps/chosen": -44.32216262817383,
      "logps/rejected": -45.803810119628906,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.32216262817383,
      "rewards/margins": 1.4816484451293945,
      "rewards/rejected": -45.803810119628906,
      "step": 3405
    },
    {
      "epoch": 1.8250543569158721,
      "grad_norm": 1.1241308412841496e-06,
      "learning_rate": 3.9869633164202045e-07,
      "logits/chosen": 9.480367660522461,
      "logits/rejected": 8.383212089538574,
      "logps/chosen": -43.598541259765625,
      "logps/rejected": -44.795326232910156,
      "loss": 0.0001,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.598541259765625,
      "rewards/margins": 1.196785807609558,
      "rewards/rejected": -44.795326232910156,
      "step": 3410
    },
    {
      "epoch": 1.8277303896972739,
      "grad_norm": 1.8371577992314272e-07,
      "learning_rate": 3.9717172330476077e-07,
      "logits/chosen": 9.43813705444336,
      "logits/rejected": 8.96119213104248,
      "logps/chosen": -44.8370246887207,
      "logps/rejected": -45.360557556152344,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.8370246887207,
      "rewards/margins": 0.5235381126403809,
      "rewards/rejected": -45.360557556152344,
      "step": 3415
    },
    {
      "epoch": 1.8304064224786754,
      "grad_norm": 3.116794311011306e-07,
      "learning_rate": 3.956481126078927e-07,
      "logits/chosen": 9.116018295288086,
      "logits/rejected": 8.752016067504883,
      "logps/chosen": -45.700557708740234,
      "logps/rejected": -45.74028778076172,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -45.700557708740234,
      "rewards/margins": 0.03972838073968887,
      "rewards/rejected": -45.74028778076172,
      "step": 3420
    },
    {
      "epoch": 1.8330824552600768,
      "grad_norm": 1.0779931040783106e-10,
      "learning_rate": 3.941255143334937e-07,
      "logits/chosen": 9.044404983520508,
      "logits/rejected": 8.802257537841797,
      "logps/chosen": -45.280479431152344,
      "logps/rejected": -46.155128479003906,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -45.280479431152344,
      "rewards/margins": 0.8746428489685059,
      "rewards/rejected": -46.155128479003906,
      "step": 3425
    },
    {
      "epoch": 1.8357584880414786,
      "grad_norm": 6.432028994600344e-05,
      "learning_rate": 3.9260394325381895e-07,
      "logits/chosen": 8.888750076293945,
      "logits/rejected": 8.433097839355469,
      "logps/chosen": -44.219093322753906,
      "logps/rejected": -45.941043853759766,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.219093322753906,
      "rewards/margins": 1.7219536304473877,
      "rewards/rejected": -45.941043853759766,
      "step": 3430
    },
    {
      "epoch": 1.83843452082288,
      "grad_norm": 1.53327197275372e-09,
      "learning_rate": 3.9108341413115784e-07,
      "logits/chosen": 9.002114295959473,
      "logits/rejected": 8.551782608032227,
      "logps/chosen": -43.9621696472168,
      "logps/rejected": -44.876991271972656,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -43.9621696472168,
      "rewards/margins": 0.914819598197937,
      "rewards/rejected": -44.876991271972656,
      "step": 3435
    },
    {
      "epoch": 1.8411105536042816,
      "grad_norm": 5.580311070428654e-12,
      "learning_rate": 3.895639417176905e-07,
      "logits/chosen": 9.222797393798828,
      "logits/rejected": 8.84571647644043,
      "logps/chosen": -45.45157241821289,
      "logps/rejected": -46.93543243408203,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.45157241821289,
      "rewards/margins": 1.4838628768920898,
      "rewards/rejected": -46.93543243408203,
      "step": 3440
    },
    {
      "epoch": 1.8437865863856833,
      "grad_norm": 4.783211863958131e-07,
      "learning_rate": 3.8804554075534497e-07,
      "logits/chosen": 9.461191177368164,
      "logits/rejected": 8.6389799118042,
      "logps/chosen": -45.1461067199707,
      "logps/rejected": -47.00682830810547,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -45.1461067199707,
      "rewards/margins": 1.8607194423675537,
      "rewards/rejected": -47.00682830810547,
      "step": 3445
    },
    {
      "epoch": 1.8464626191670848,
      "grad_norm": 4.341907515086363e-11,
      "learning_rate": 3.8652822597565403e-07,
      "logits/chosen": 9.17106819152832,
      "logits/rejected": 8.581462860107422,
      "logps/chosen": -44.799530029296875,
      "logps/rejected": -46.195960998535156,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -44.799530029296875,
      "rewards/margins": 1.3964297771453857,
      "rewards/rejected": -46.195960998535156,
      "step": 3450
    },
    {
      "epoch": 1.8491386519484863,
      "grad_norm": 2.6794949181848703e-10,
      "learning_rate": 3.850120120996123e-07,
      "logits/chosen": 9.34289836883545,
      "logits/rejected": 8.584213256835938,
      "logps/chosen": -45.36188507080078,
      "logps/rejected": -46.73574447631836,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -45.36188507080078,
      "rewards/margins": 1.3738635778427124,
      "rewards/rejected": -46.73574447631836,
      "step": 3455
    },
    {
      "epoch": 1.851814684729888,
      "grad_norm": 3.308440880987707e-11,
      "learning_rate": 3.8349691383753356e-07,
      "logits/chosen": 9.056200981140137,
      "logits/rejected": 8.633913040161133,
      "logps/chosen": -46.00835037231445,
      "logps/rejected": -46.129703521728516,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -46.00835037231445,
      "rewards/margins": 0.12135162204504013,
      "rewards/rejected": -46.129703521728516,
      "step": 3460
    },
    {
      "epoch": 1.8544907175112895,
      "grad_norm": 1.8428594327752542e-08,
      "learning_rate": 3.819829458889078e-07,
      "logits/chosen": 9.211648941040039,
      "logits/rejected": 8.840119361877441,
      "logps/chosen": -44.340065002441406,
      "logps/rejected": -45.658870697021484,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.340065002441406,
      "rewards/margins": 1.3188074827194214,
      "rewards/rejected": -45.658870697021484,
      "step": 3465
    },
    {
      "epoch": 1.857166750292691,
      "grad_norm": 2.4358219575739953e-08,
      "learning_rate": 3.804701229422585e-07,
      "logits/chosen": 9.293371200561523,
      "logits/rejected": 9.101160049438477,
      "logps/chosen": -44.973426818847656,
      "logps/rejected": -45.62038803100586,
      "loss": 0.0012,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.973426818847656,
      "rewards/margins": 0.6469655632972717,
      "rewards/rejected": -45.62038803100586,
      "step": 3470
    },
    {
      "epoch": 1.8598427830740927,
      "grad_norm": 1.8702165003935544e-08,
      "learning_rate": 3.789584596750007e-07,
      "logits/chosen": 9.0369234085083,
      "logits/rejected": 8.812845230102539,
      "logps/chosen": -44.45563888549805,
      "logps/rejected": -45.179222106933594,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.45563888549805,
      "rewards/margins": 0.723577082157135,
      "rewards/rejected": -45.179222106933594,
      "step": 3475
    },
    {
      "epoch": 1.8625188158554944,
      "grad_norm": 2.9667040676515464e-09,
      "learning_rate": 3.77447970753298e-07,
      "logits/chosen": 8.777036666870117,
      "logits/rejected": 8.59619426727295,
      "logps/chosen": -45.55571746826172,
      "logps/rejected": -45.620018005371094,
      "loss": 0.0,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -45.55571746826172,
      "rewards/margins": 0.064300037920475,
      "rewards/rejected": -45.620018005371094,
      "step": 3480
    },
    {
      "epoch": 1.8651948486368957,
      "grad_norm": 8.047284193052883e-06,
      "learning_rate": 3.7593867083192057e-07,
      "logits/chosen": 9.050636291503906,
      "logits/rejected": 8.608115196228027,
      "logps/chosen": -45.464298248291016,
      "logps/rejected": -46.49833679199219,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.464298248291016,
      "rewards/margins": 1.034041404724121,
      "rewards/rejected": -46.49833679199219,
      "step": 3485
    },
    {
      "epoch": 1.8678708814182974,
      "grad_norm": 2.205407750311756e-09,
      "learning_rate": 3.7443057455410276e-07,
      "logits/chosen": 9.071859359741211,
      "logits/rejected": 8.676399230957031,
      "logps/chosen": -43.73458480834961,
      "logps/rejected": -45.945255279541016,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.73458480834961,
      "rewards/margins": 2.2106640338897705,
      "rewards/rejected": -45.945255279541016,
      "step": 3490
    },
    {
      "epoch": 1.870546914199699,
      "grad_norm": 9.726591393350278e-13,
      "learning_rate": 3.7292369655140145e-07,
      "logits/chosen": 9.62777328491211,
      "logits/rejected": 9.026641845703125,
      "logps/chosen": -45.54102325439453,
      "logps/rejected": -46.751136779785156,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.54102325439453,
      "rewards/margins": 1.2101147174835205,
      "rewards/rejected": -46.751136779785156,
      "step": 3495
    },
    {
      "epoch": 1.8732229469811004,
      "grad_norm": 7.621280163758726e-06,
      "learning_rate": 3.714180514435534e-07,
      "logits/chosen": 8.73654842376709,
      "logits/rejected": 8.317728996276855,
      "logps/chosen": -44.52265167236328,
      "logps/rejected": -45.2078971862793,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.52265167236328,
      "rewards/margins": 0.6852442622184753,
      "rewards/rejected": -45.2078971862793,
      "step": 3500
    },
    {
      "epoch": 1.875898979762502,
      "grad_norm": 7.98808482875563e-12,
      "learning_rate": 3.6991365383833426e-07,
      "logits/chosen": 9.23396110534668,
      "logits/rejected": 8.694872856140137,
      "logps/chosen": -44.76234817504883,
      "logps/rejected": -46.31821823120117,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.76234817504883,
      "rewards/margins": 1.5558732748031616,
      "rewards/rejected": -46.31821823120117,
      "step": 3505
    },
    {
      "epoch": 1.8785750125439038,
      "grad_norm": 4.293193660131807e-08,
      "learning_rate": 3.684105183314162e-07,
      "logits/chosen": 9.51964282989502,
      "logits/rejected": 8.969693183898926,
      "logps/chosen": -45.994354248046875,
      "logps/rejected": -47.06464767456055,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.994354248046875,
      "rewards/margins": 1.0702959299087524,
      "rewards/rejected": -47.06464767456055,
      "step": 3510
    },
    {
      "epoch": 1.881251045325305,
      "grad_norm": 3.691006918871296e-08,
      "learning_rate": 3.669086595062263e-07,
      "logits/chosen": 9.367341995239258,
      "logits/rejected": 8.668649673461914,
      "logps/chosen": -43.5899543762207,
      "logps/rejected": -45.38178253173828,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -43.5899543762207,
      "rewards/margins": 1.7918258905410767,
      "rewards/rejected": -45.38178253173828,
      "step": 3515
    },
    {
      "epoch": 1.8839270781067068,
      "grad_norm": 1.2159060463539326e-12,
      "learning_rate": 3.654080919338056e-07,
      "logits/chosen": 9.249971389770508,
      "logits/rejected": 8.440373420715332,
      "logps/chosen": -46.17134475708008,
      "logps/rejected": -46.06903839111328,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -46.17134475708008,
      "rewards/margins": -0.10230336338281631,
      "rewards/rejected": -46.06903839111328,
      "step": 3520
    },
    {
      "epoch": 1.8866031108881085,
      "grad_norm": 3.0383750333465535e-09,
      "learning_rate": 3.639088301726673e-07,
      "logits/chosen": 9.00893497467041,
      "logits/rejected": 8.316612243652344,
      "logps/chosen": -44.702972412109375,
      "logps/rejected": -45.52973937988281,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.702972412109375,
      "rewards/margins": 0.8267694711685181,
      "rewards/rejected": -45.52973937988281,
      "step": 3525
    },
    {
      "epoch": 1.88927914366951,
      "grad_norm": 3.8034845199917916e-07,
      "learning_rate": 3.624108887686556e-07,
      "logits/chosen": 9.094563484191895,
      "logits/rejected": 8.910303115844727,
      "logps/chosen": -44.7447395324707,
      "logps/rejected": -45.3892822265625,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.7447395324707,
      "rewards/margins": 0.6445447206497192,
      "rewards/rejected": -45.3892822265625,
      "step": 3530
    },
    {
      "epoch": 1.8919551764509115,
      "grad_norm": 3.2781191080414064e-10,
      "learning_rate": 3.6091428225480433e-07,
      "logits/chosen": 9.246016502380371,
      "logits/rejected": 8.874571800231934,
      "logps/chosen": -44.53443908691406,
      "logps/rejected": -45.921775817871094,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.53443908691406,
      "rewards/margins": 1.3873343467712402,
      "rewards/rejected": -45.921775817871094,
      "step": 3535
    },
    {
      "epoch": 1.8946312092323132,
      "grad_norm": 9.651285649866265e-07,
      "learning_rate": 3.5941902515119674e-07,
      "logits/chosen": 9.404706001281738,
      "logits/rejected": 8.469385147094727,
      "logps/chosen": -45.883079528808594,
      "logps/rejected": -45.9871940612793,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.883079528808594,
      "rewards/margins": 0.1041167601943016,
      "rewards/rejected": -45.9871940612793,
      "step": 3540
    },
    {
      "epoch": 1.8973072420137147,
      "grad_norm": 2.8779766529196333e-08,
      "learning_rate": 3.5792513196482373e-07,
      "logits/chosen": 9.303569793701172,
      "logits/rejected": 8.276330947875977,
      "logps/chosen": -43.323875427246094,
      "logps/rejected": -46.23902130126953,
      "loss": 0.0,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -43.323875427246094,
      "rewards/margins": 2.9151477813720703,
      "rewards/rejected": -46.23902130126953,
      "step": 3545
    },
    {
      "epoch": 1.8999832747951162,
      "grad_norm": 1.1822394086934965e-06,
      "learning_rate": 3.5643261718944346e-07,
      "logits/chosen": 8.586645126342773,
      "logits/rejected": 8.192548751831055,
      "logps/chosen": -43.45540237426758,
      "logps/rejected": -44.325782775878906,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -43.45540237426758,
      "rewards/margins": 0.870381236076355,
      "rewards/rejected": -44.325782775878906,
      "step": 3550
    },
    {
      "epoch": 1.902659307576518,
      "grad_norm": 4.7537397286381586e-12,
      "learning_rate": 3.5494149530544087e-07,
      "logits/chosen": 9.331785202026367,
      "logits/rejected": 8.746190071105957,
      "logps/chosen": -44.875877380371094,
      "logps/rejected": -45.57783126831055,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.875877380371094,
      "rewards/margins": 0.7019538283348083,
      "rewards/rejected": -45.57783126831055,
      "step": 3555
    },
    {
      "epoch": 1.9053353403579194,
      "grad_norm": 1.9502287726844078e-10,
      "learning_rate": 3.534517807796871e-07,
      "logits/chosen": 9.468916893005371,
      "logits/rejected": 8.970396041870117,
      "logps/chosen": -44.85259246826172,
      "logps/rejected": -45.85542297363281,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.85259246826172,
      "rewards/margins": 1.0028345584869385,
      "rewards/rejected": -45.85542297363281,
      "step": 3560
    },
    {
      "epoch": 1.908011373139321,
      "grad_norm": 5.913499599862567e-08,
      "learning_rate": 3.519634880653988e-07,
      "logits/chosen": 9.156373977661133,
      "logits/rejected": 8.890625,
      "logps/chosen": -45.11962890625,
      "logps/rejected": -46.9284782409668,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -45.11962890625,
      "rewards/margins": 1.8088515996932983,
      "rewards/rejected": -46.9284782409668,
      "step": 3565
    },
    {
      "epoch": 1.9106874059207226,
      "grad_norm": 2.900276184295871e-09,
      "learning_rate": 3.504766316019987e-07,
      "logits/chosen": 9.526211738586426,
      "logits/rejected": 8.797994613647461,
      "logps/chosen": -45.73761749267578,
      "logps/rejected": -46.69166564941406,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.73761749267578,
      "rewards/margins": 0.9540489912033081,
      "rewards/rejected": -46.69166564941406,
      "step": 3570
    },
    {
      "epoch": 1.913363438702124,
      "grad_norm": 1.3490910850283383e-11,
      "learning_rate": 3.489912258149745e-07,
      "logits/chosen": 9.244808197021484,
      "logits/rejected": 8.729928970336914,
      "logps/chosen": -45.59259033203125,
      "logps/rejected": -46.43151092529297,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.59259033203125,
      "rewards/margins": 0.8389163017272949,
      "rewards/rejected": -46.43151092529297,
      "step": 3575
    },
    {
      "epoch": 1.9160394714835256,
      "grad_norm": 7.225073665584136e-10,
      "learning_rate": 3.475072851157397e-07,
      "logits/chosen": 9.100488662719727,
      "logits/rejected": 8.985162734985352,
      "logps/chosen": -44.80809020996094,
      "logps/rejected": -46.008811950683594,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.80809020996094,
      "rewards/margins": 1.20072340965271,
      "rewards/rejected": -46.008811950683594,
      "step": 3580
    },
    {
      "epoch": 1.9187155042649273,
      "grad_norm": 4.412552471586721e-09,
      "learning_rate": 3.460248239014936e-07,
      "logits/chosen": 8.756746292114258,
      "logits/rejected": 8.6131010055542,
      "logps/chosen": -44.95341873168945,
      "logps/rejected": -45.993812561035156,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.95341873168945,
      "rewards/margins": 1.0403988361358643,
      "rewards/rejected": -45.993812561035156,
      "step": 3585
    },
    {
      "epoch": 1.9213915370463288,
      "grad_norm": 7.540299028308487e-10,
      "learning_rate": 3.4454385655508134e-07,
      "logits/chosen": 8.940208435058594,
      "logits/rejected": 8.849532127380371,
      "logps/chosen": -44.712223052978516,
      "logps/rejected": -45.487815856933594,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.712223052978516,
      "rewards/margins": 0.7755964994430542,
      "rewards/rejected": -45.487815856933594,
      "step": 3590
    },
    {
      "epoch": 1.9240675698277303,
      "grad_norm": 1.5443581790126504e-11,
      "learning_rate": 3.4306439744485447e-07,
      "logits/chosen": 9.3189697265625,
      "logits/rejected": 8.500635147094727,
      "logps/chosen": -44.964202880859375,
      "logps/rejected": -45.93611526489258,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.964202880859375,
      "rewards/margins": 0.9719089269638062,
      "rewards/rejected": -45.93611526489258,
      "step": 3595
    },
    {
      "epoch": 1.926743602609132,
      "grad_norm": 1.3320226050989407e-06,
      "learning_rate": 3.415864609245322e-07,
      "logits/chosen": 9.067032814025879,
      "logits/rejected": 8.413298606872559,
      "logps/chosen": -45.623313903808594,
      "logps/rejected": -46.80268859863281,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.623313903808594,
      "rewards/margins": 1.179373860359192,
      "rewards/rejected": -46.80268859863281,
      "step": 3600
    },
    {
      "epoch": 1.926743602609132,
      "eval_logits/chosen": 7.594793319702148,
      "eval_logits/rejected": 7.28135347366333,
      "eval_logps/chosen": -44.59578323364258,
      "eval_logps/rejected": -45.656898498535156,
      "eval_loss": 2.138637711368574e-07,
      "eval_rewards/accuracies": 0.5645400881767273,
      "eval_rewards/chosen": -44.59578323364258,
      "eval_rewards/margins": 1.061111330986023,
      "eval_rewards/rejected": -45.656898498535156,
      "eval_runtime": 40.0676,
      "eval_samples_per_second": 33.568,
      "eval_steps_per_second": 8.411,
      "step": 3600
    },
    {
      "epoch": 1.9294196353905335,
      "grad_norm": 1.3526185224858729e-07,
      "learning_rate": 3.401100613330605e-07,
      "logits/chosen": 9.386487007141113,
      "logits/rejected": 9.262727737426758,
      "logps/chosen": -46.18013000488281,
      "logps/rejected": -46.295997619628906,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -46.18013000488281,
      "rewards/margins": 0.11586495488882065,
      "rewards/rejected": -46.295997619628906,
      "step": 3605
    },
    {
      "epoch": 1.932095668171935,
      "grad_norm": 4.325994473718434e-08,
      "learning_rate": 3.3863521299447514e-07,
      "logits/chosen": 9.559442520141602,
      "logits/rejected": 8.802529335021973,
      "logps/chosen": -45.179840087890625,
      "logps/rejected": -46.42104721069336,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.179840087890625,
      "rewards/margins": 1.241209864616394,
      "rewards/rejected": -46.42104721069336,
      "step": 3610
    },
    {
      "epoch": 1.9347717009533367,
      "grad_norm": 3.838669159108073e-07,
      "learning_rate": 3.371619302177609e-07,
      "logits/chosen": 8.959545135498047,
      "logits/rejected": 8.465574264526367,
      "logps/chosen": -43.78969955444336,
      "logps/rejected": -45.14876174926758,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -43.78969955444336,
      "rewards/margins": 1.3590583801269531,
      "rewards/rejected": -45.14876174926758,
      "step": 3615
    },
    {
      "epoch": 1.9374477337347382,
      "grad_norm": 4.0982482261646495e-09,
      "learning_rate": 3.3569022729671393e-07,
      "logits/chosen": 9.089397430419922,
      "logits/rejected": 8.904385566711426,
      "logps/chosen": -43.66358947753906,
      "logps/rejected": -44.96509552001953,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.66358947753906,
      "rewards/margins": 1.3015096187591553,
      "rewards/rejected": -44.96509552001953,
      "step": 3620
    },
    {
      "epoch": 1.9401237665161397,
      "grad_norm": 7.149997307223499e-09,
      "learning_rate": 3.342201185098024e-07,
      "logits/chosen": 8.893362998962402,
      "logits/rejected": 9.245957374572754,
      "logps/chosen": -45.18000030517578,
      "logps/rejected": -45.425907135009766,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.18000030517578,
      "rewards/margins": 0.24590221047401428,
      "rewards/rejected": -45.425907135009766,
      "step": 3625
    },
    {
      "epoch": 1.9427997992975414,
      "grad_norm": 5.8065973577341596e-09,
      "learning_rate": 3.3275161812002807e-07,
      "logits/chosen": 9.162287712097168,
      "logits/rejected": 9.036238670349121,
      "logps/chosen": -45.021793365478516,
      "logps/rejected": -45.946224212646484,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.021793365478516,
      "rewards/margins": 0.9244314432144165,
      "rewards/rejected": -45.946224212646484,
      "step": 3630
    },
    {
      "epoch": 1.945475832078943,
      "grad_norm": 4.588693600826872e-05,
      "learning_rate": 3.312847403747883e-07,
      "logits/chosen": 9.153509140014648,
      "logits/rejected": 8.873466491699219,
      "logps/chosen": -43.759239196777344,
      "logps/rejected": -45.063114166259766,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.759239196777344,
      "rewards/margins": 1.303874135017395,
      "rewards/rejected": -45.063114166259766,
      "step": 3635
    },
    {
      "epoch": 1.9481518648603444,
      "grad_norm": 2.1130044338495463e-11,
      "learning_rate": 3.2981949950573733e-07,
      "logits/chosen": 8.941617965698242,
      "logits/rejected": 9.01390266418457,
      "logps/chosen": -44.7380256652832,
      "logps/rejected": -46.02873611450195,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.7380256652832,
      "rewards/margins": 1.2907090187072754,
      "rewards/rejected": -46.02873611450195,
      "step": 3640
    },
    {
      "epoch": 1.9508278976417461,
      "grad_norm": 3.036408962698141e-11,
      "learning_rate": 3.283559097286486e-07,
      "logits/chosen": 9.053401947021484,
      "logits/rejected": 8.777471542358398,
      "logps/chosen": -45.02251052856445,
      "logps/rejected": -46.47441101074219,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.02251052856445,
      "rewards/margins": 1.451899766921997,
      "rewards/rejected": -46.47441101074219,
      "step": 3645
    },
    {
      "epoch": 1.9535039304231478,
      "grad_norm": 4.45875302093671e-13,
      "learning_rate": 3.268939852432765e-07,
      "logits/chosen": 9.16075325012207,
      "logits/rejected": 8.967235565185547,
      "logps/chosen": -45.10694122314453,
      "logps/rejected": -46.42476272583008,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.10694122314453,
      "rewards/margins": 1.3178173303604126,
      "rewards/rejected": -46.42476272583008,
      "step": 3650
    },
    {
      "epoch": 1.9561799632045491,
      "grad_norm": 0.005874296946359015,
      "learning_rate": 3.254337402332187e-07,
      "logits/chosen": 9.118660926818848,
      "logits/rejected": 8.673898696899414,
      "logps/chosen": -45.06127166748047,
      "logps/rejected": -45.54829788208008,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.06127166748047,
      "rewards/margins": 0.48702558875083923,
      "rewards/rejected": -45.54829788208008,
      "step": 3655
    },
    {
      "epoch": 1.9588559959859508,
      "grad_norm": 2.972323600364537e-10,
      "learning_rate": 3.239751888657788e-07,
      "logits/chosen": 9.04544448852539,
      "logits/rejected": 8.542941093444824,
      "logps/chosen": -43.87335205078125,
      "logps/rejected": -45.54486846923828,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -43.87335205078125,
      "rewards/margins": 1.6715075969696045,
      "rewards/rejected": -45.54486846923828,
      "step": 3660
    },
    {
      "epoch": 1.9615320287673526,
      "grad_norm": 3.876985005970265e-11,
      "learning_rate": 3.2251834529182856e-07,
      "logits/chosen": 9.320975303649902,
      "logits/rejected": 9.008540153503418,
      "logps/chosen": -45.878700256347656,
      "logps/rejected": -47.44437789916992,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.878700256347656,
      "rewards/margins": 1.5656812191009521,
      "rewards/rejected": -47.44437789916992,
      "step": 3665
    },
    {
      "epoch": 1.9642080615487538,
      "grad_norm": 4.364557307526597e-05,
      "learning_rate": 3.2106322364567075e-07,
      "logits/chosen": 9.356664657592773,
      "logits/rejected": 8.841044425964355,
      "logps/chosen": -44.07068634033203,
      "logps/rejected": -45.54160690307617,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -44.07068634033203,
      "rewards/margins": 1.4709235429763794,
      "rewards/rejected": -45.54160690307617,
      "step": 3670
    },
    {
      "epoch": 1.9668840943301555,
      "grad_norm": 8.024482107906481e-11,
      "learning_rate": 3.1960983804490183e-07,
      "logits/chosen": 9.08736515045166,
      "logits/rejected": 8.830459594726562,
      "logps/chosen": -44.478126525878906,
      "logps/rejected": -46.379539489746094,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.478126525878906,
      "rewards/margins": 1.901410698890686,
      "rewards/rejected": -46.379539489746094,
      "step": 3675
    },
    {
      "epoch": 1.9695601271115573,
      "grad_norm": 1.135859941578988e-07,
      "learning_rate": 3.1815820259027537e-07,
      "logits/chosen": 9.239349365234375,
      "logits/rejected": 8.844405174255371,
      "logps/chosen": -46.15561294555664,
      "logps/rejected": -47.18470764160156,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -46.15561294555664,
      "rewards/margins": 1.029096245765686,
      "rewards/rejected": -47.18470764160156,
      "step": 3680
    },
    {
      "epoch": 1.9722361598929585,
      "grad_norm": 7.999953823790667e-10,
      "learning_rate": 3.16708331365565e-07,
      "logits/chosen": 9.081738471984863,
      "logits/rejected": 8.879627227783203,
      "logps/chosen": -44.965293884277344,
      "logps/rejected": -45.97960662841797,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.965293884277344,
      "rewards/margins": 1.014312505722046,
      "rewards/rejected": -45.97960662841797,
      "step": 3685
    },
    {
      "epoch": 1.9749121926743602,
      "grad_norm": 3.9099481262808325e-05,
      "learning_rate": 3.152602384374275e-07,
      "logits/chosen": 9.283867835998535,
      "logits/rejected": 8.637078285217285,
      "logps/chosen": -44.3259162902832,
      "logps/rejected": -45.606971740722656,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.3259162902832,
      "rewards/margins": 1.2810590267181396,
      "rewards/rejected": -45.606971740722656,
      "step": 3690
    },
    {
      "epoch": 1.977588225455762,
      "grad_norm": 8.486385794770616e-14,
      "learning_rate": 3.1381393785526697e-07,
      "logits/chosen": 9.159425735473633,
      "logits/rejected": 8.905111312866211,
      "logps/chosen": -45.864501953125,
      "logps/rejected": -46.577945709228516,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -45.864501953125,
      "rewards/margins": 0.7134426832199097,
      "rewards/rejected": -46.577945709228516,
      "step": 3695
    },
    {
      "epoch": 1.9802642582371635,
      "grad_norm": 5.846864696365749e-05,
      "learning_rate": 3.123694436510979e-07,
      "logits/chosen": 9.246844291687012,
      "logits/rejected": 8.848536491394043,
      "logps/chosen": -45.793861389160156,
      "logps/rejected": -46.526302337646484,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -45.793861389160156,
      "rewards/margins": 0.7324389219284058,
      "rewards/rejected": -46.526302337646484,
      "step": 3700
    },
    {
      "epoch": 1.982940291018565,
      "grad_norm": 5.990250656750355e-14,
      "learning_rate": 3.1092676983940946e-07,
      "logits/chosen": 9.518543243408203,
      "logits/rejected": 9.089153289794922,
      "logps/chosen": -45.391639709472656,
      "logps/rejected": -46.25476837158203,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.391639709472656,
      "rewards/margins": 0.8631294369697571,
      "rewards/rejected": -46.25476837158203,
      "step": 3705
    },
    {
      "epoch": 1.9856163237999667,
      "grad_norm": 8.564751064633309e-07,
      "learning_rate": 3.094859304170293e-07,
      "logits/chosen": 8.792872428894043,
      "logits/rejected": 8.664057731628418,
      "logps/chosen": -43.89430618286133,
      "logps/rejected": -44.99854278564453,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.89430618286133,
      "rewards/margins": 1.1042354106903076,
      "rewards/rejected": -44.99854278564453,
      "step": 3710
    },
    {
      "epoch": 1.9882923565813682,
      "grad_norm": 5.7877938197771896e-11,
      "learning_rate": 3.0804693936298795e-07,
      "logits/chosen": 9.21117877960205,
      "logits/rejected": 9.153080940246582,
      "logps/chosen": -45.481361389160156,
      "logps/rejected": -46.222686767578125,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.481361389160156,
      "rewards/margins": 0.7413271069526672,
      "rewards/rejected": -46.222686767578125,
      "step": 3715
    },
    {
      "epoch": 1.9909683893627697,
      "grad_norm": 1.978064763754491e-10,
      "learning_rate": 3.066098106383826e-07,
      "logits/chosen": 9.162936210632324,
      "logits/rejected": 8.682527542114258,
      "logps/chosen": -44.46044158935547,
      "logps/rejected": -44.681217193603516,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -44.46044158935547,
      "rewards/margins": 0.22077274322509766,
      "rewards/rejected": -44.681217193603516,
      "step": 3720
    },
    {
      "epoch": 1.9936444221441714,
      "grad_norm": 1.0365765390394556e-09,
      "learning_rate": 3.0517455818624263e-07,
      "logits/chosen": 9.314885139465332,
      "logits/rejected": 9.011612892150879,
      "logps/chosen": -45.5214958190918,
      "logps/rejected": -46.24169158935547,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -45.5214958190918,
      "rewards/margins": 0.7201946973800659,
      "rewards/rejected": -46.24169158935547,
      "step": 3725
    },
    {
      "epoch": 1.9963204549255729,
      "grad_norm": 6.038973577588465e-08,
      "learning_rate": 3.037411959313936e-07,
      "logits/chosen": 9.313830375671387,
      "logits/rejected": 8.863197326660156,
      "logps/chosen": -44.33344650268555,
      "logps/rejected": -46.29510498046875,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.33344650268555,
      "rewards/margins": 1.961661696434021,
      "rewards/rejected": -46.29510498046875,
      "step": 3730
    },
    {
      "epoch": 1.9989964877069744,
      "grad_norm": 1.8400692346997477e-08,
      "learning_rate": 3.023097377803224e-07,
      "logits/chosen": 9.077723503112793,
      "logits/rejected": 8.778433799743652,
      "logps/chosen": -45.69194793701172,
      "logps/rejected": -45.78805923461914,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -45.69194793701172,
      "rewards/margins": 0.09610547870397568,
      "rewards/rejected": -45.78805923461914,
      "step": 3735
    },
    {
      "epoch": 2.001672520488376,
      "grad_norm": 4.9116062624471385e-11,
      "learning_rate": 3.008801976210423e-07,
      "logits/chosen": 9.2524995803833,
      "logits/rejected": 8.911211013793945,
      "logps/chosen": -44.5187873840332,
      "logps/rejected": -44.11595916748047,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.5187873840332,
      "rewards/margins": -0.40282851457595825,
      "rewards/rejected": -44.11595916748047,
      "step": 3740
    },
    {
      "epoch": 2.0043485532697773,
      "grad_norm": 3.579080059063778e-11,
      "learning_rate": 2.994525893229581e-07,
      "logits/chosen": 9.15229606628418,
      "logits/rejected": 8.842096328735352,
      "logps/chosen": -45.33643341064453,
      "logps/rejected": -47.261077880859375,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -45.33643341064453,
      "rewards/margins": 1.9246429204940796,
      "rewards/rejected": -47.261077880859375,
      "step": 3745
    },
    {
      "epoch": 2.007024586051179,
      "grad_norm": 2.9526720489246683e-07,
      "learning_rate": 2.98026926736732e-07,
      "logits/chosen": 9.164018630981445,
      "logits/rejected": 8.877763748168945,
      "logps/chosen": -44.869388580322266,
      "logps/rejected": -46.052677154541016,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.869388580322266,
      "rewards/margins": 1.183288335800171,
      "rewards/rejected": -46.052677154541016,
      "step": 3750
    },
    {
      "epoch": 2.0097006188325808,
      "grad_norm": 1.446508420331438e-12,
      "learning_rate": 2.9660322369414846e-07,
      "logits/chosen": 9.142843246459961,
      "logits/rejected": 8.708547592163086,
      "logps/chosen": -44.24568557739258,
      "logps/rejected": -45.95346450805664,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.24568557739258,
      "rewards/margins": 1.7077763080596924,
      "rewards/rejected": -45.95346450805664,
      "step": 3755
    },
    {
      "epoch": 2.0123766516139825,
      "grad_norm": 1.641981608430217e-10,
      "learning_rate": 2.9518149400798063e-07,
      "logits/chosen": 9.358545303344727,
      "logits/rejected": 9.094202995300293,
      "logps/chosen": -43.82788848876953,
      "logps/rejected": -45.10551452636719,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -43.82788848876953,
      "rewards/margins": 1.277624487876892,
      "rewards/rejected": -45.10551452636719,
      "step": 3760
    },
    {
      "epoch": 2.0150526843953838,
      "grad_norm": 4.09862858958351e-11,
      "learning_rate": 2.9376175147185633e-07,
      "logits/chosen": 9.454338073730469,
      "logits/rejected": 8.713216781616211,
      "logps/chosen": -45.2805061340332,
      "logps/rejected": -46.46715545654297,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.2805061340332,
      "rewards/margins": 1.186652421951294,
      "rewards/rejected": -46.46715545654297,
      "step": 3765
    },
    {
      "epoch": 2.0177287171767855,
      "grad_norm": 3.404444354274715e-15,
      "learning_rate": 2.9234400986012376e-07,
      "logits/chosen": 9.280506134033203,
      "logits/rejected": 8.646222114562988,
      "logps/chosen": -45.03093719482422,
      "logps/rejected": -46.94239807128906,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -45.03093719482422,
      "rewards/margins": 1.9114553928375244,
      "rewards/rejected": -46.94239807128906,
      "step": 3770
    },
    {
      "epoch": 2.020404749958187,
      "grad_norm": 6.267616087020187e-10,
      "learning_rate": 2.9092828292771817e-07,
      "logits/chosen": 9.331942558288574,
      "logits/rejected": 9.126152038574219,
      "logps/chosen": -45.44009780883789,
      "logps/rejected": -46.3033561706543,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.44009780883789,
      "rewards/margins": 0.8632590174674988,
      "rewards/rejected": -46.3033561706543,
      "step": 3775
    },
    {
      "epoch": 2.0230807827395885,
      "grad_norm": 1.0686596988840384e-10,
      "learning_rate": 2.8951458441002875e-07,
      "logits/chosen": 8.989980697631836,
      "logits/rejected": 8.762496948242188,
      "logps/chosen": -43.41142654418945,
      "logps/rejected": -43.86376953125,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.41142654418945,
      "rewards/margins": 0.4523501992225647,
      "rewards/rejected": -43.86376953125,
      "step": 3780
    },
    {
      "epoch": 2.02575681552099,
      "grad_norm": 7.81624286480956e-09,
      "learning_rate": 2.881029280227643e-07,
      "logits/chosen": 9.352952003479004,
      "logits/rejected": 8.675131797790527,
      "logps/chosen": -44.77410125732422,
      "logps/rejected": -45.63949966430664,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.77410125732422,
      "rewards/margins": 0.8653966784477234,
      "rewards/rejected": -45.63949966430664,
      "step": 3785
    },
    {
      "epoch": 2.028432848302392,
      "grad_norm": 2.39813811901525e-13,
      "learning_rate": 2.8669332746182177e-07,
      "logits/chosen": 9.49570369720459,
      "logits/rejected": 8.911005973815918,
      "logps/chosen": -45.09392547607422,
      "logps/rejected": -46.648563385009766,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -45.09392547607422,
      "rewards/margins": 1.5546343326568604,
      "rewards/rejected": -46.648563385009766,
      "step": 3790
    },
    {
      "epoch": 2.031108881083793,
      "grad_norm": 3.762638206470958e-06,
      "learning_rate": 2.8528579640315156e-07,
      "logits/chosen": 8.928093910217285,
      "logits/rejected": 8.978708267211914,
      "logps/chosen": -45.453609466552734,
      "logps/rejected": -45.84834671020508,
      "loss": 0.0002,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.453609466552734,
      "rewards/margins": 0.3947386145591736,
      "rewards/rejected": -45.84834671020508,
      "step": 3795
    },
    {
      "epoch": 2.033784913865195,
      "grad_norm": 3.1898197846532945e-07,
      "learning_rate": 2.8388034850262646e-07,
      "logits/chosen": 9.155448913574219,
      "logits/rejected": 8.658851623535156,
      "logps/chosen": -45.79350662231445,
      "logps/rejected": -46.583274841308594,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.79350662231445,
      "rewards/margins": 0.7897666692733765,
      "rewards/rejected": -46.583274841308594,
      "step": 3800
    },
    {
      "epoch": 2.0364609466465966,
      "grad_norm": 2.190370276968802e-09,
      "learning_rate": 2.824769973959079e-07,
      "logits/chosen": 9.182861328125,
      "logits/rejected": 8.834218978881836,
      "logps/chosen": -44.16795349121094,
      "logps/rejected": -46.04777908325195,
      "loss": 0.0,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -44.16795349121094,
      "rewards/margins": 1.8798240423202515,
      "rewards/rejected": -46.04777908325195,
      "step": 3805
    },
    {
      "epoch": 2.039136979427998,
      "grad_norm": 6.556681058984543e-06,
      "learning_rate": 2.81075756698315e-07,
      "logits/chosen": 8.840353012084961,
      "logits/rejected": 8.78870964050293,
      "logps/chosen": -45.95507049560547,
      "logps/rejected": -47.45210647583008,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -45.95507049560547,
      "rewards/margins": 1.4970357418060303,
      "rewards/rejected": -47.45210647583008,
      "step": 3810
    },
    {
      "epoch": 2.0418130122093996,
      "grad_norm": 7.894888815346945e-14,
      "learning_rate": 2.7967664000469035e-07,
      "logits/chosen": 9.368030548095703,
      "logits/rejected": 8.981012344360352,
      "logps/chosen": -45.355674743652344,
      "logps/rejected": -46.373958587646484,
      "loss": 0.0005,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.355674743652344,
      "rewards/margins": 1.0182876586914062,
      "rewards/rejected": -46.373958587646484,
      "step": 3815
    },
    {
      "epoch": 2.0444890449908013,
      "grad_norm": 7.813884240538123e-10,
      "learning_rate": 2.7827966088927095e-07,
      "logits/chosen": 9.441866874694824,
      "logits/rejected": 8.693378448486328,
      "logps/chosen": -45.742454528808594,
      "logps/rejected": -46.97275161743164,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -45.742454528808594,
      "rewards/margins": 1.2302968502044678,
      "rewards/rejected": -46.97275161743164,
      "step": 3820
    },
    {
      "epoch": 2.0471650777722026,
      "grad_norm": 4.978634612577213e-11,
      "learning_rate": 2.768848329055538e-07,
      "logits/chosen": 9.503710746765137,
      "logits/rejected": 9.22178840637207,
      "logps/chosen": -45.83349609375,
      "logps/rejected": -47.25985336303711,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -45.83349609375,
      "rewards/margins": 1.4263614416122437,
      "rewards/rejected": -47.25985336303711,
      "step": 3825
    },
    {
      "epoch": 2.0498411105536043,
      "grad_norm": 3.299485350422559e-10,
      "learning_rate": 2.7549216958616657e-07,
      "logits/chosen": 9.548501968383789,
      "logits/rejected": 8.855688095092773,
      "logps/chosen": -44.516788482666016,
      "logps/rejected": -45.337284088134766,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.516788482666016,
      "rewards/margins": 0.8204957842826843,
      "rewards/rejected": -45.337284088134766,
      "step": 3830
    },
    {
      "epoch": 2.052517143335006,
      "grad_norm": 5.269629934654122e-07,
      "learning_rate": 2.741016844427344e-07,
      "logits/chosen": 9.131202697753906,
      "logits/rejected": 8.484569549560547,
      "logps/chosen": -44.632728576660156,
      "logps/rejected": -45.75881576538086,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.632728576660156,
      "rewards/margins": 1.1260900497436523,
      "rewards/rejected": -45.75881576538086,
      "step": 3835
    },
    {
      "epoch": 2.0551931761164073,
      "grad_norm": 1.554705389564865e-09,
      "learning_rate": 2.7271339096575073e-07,
      "logits/chosen": 9.129579544067383,
      "logits/rejected": 8.696627616882324,
      "logps/chosen": -44.90523147583008,
      "logps/rejected": -46.65637969970703,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -44.90523147583008,
      "rewards/margins": 1.7511470317840576,
      "rewards/rejected": -46.65637969970703,
      "step": 3840
    },
    {
      "epoch": 2.057869208897809,
      "grad_norm": 7.874542750131881e-14,
      "learning_rate": 2.713273026244446e-07,
      "logits/chosen": 9.865469932556152,
      "logits/rejected": 8.985435485839844,
      "logps/chosen": -44.655189514160156,
      "logps/rejected": -46.7956657409668,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -44.655189514160156,
      "rewards/margins": 2.140479564666748,
      "rewards/rejected": -46.7956657409668,
      "step": 3845
    },
    {
      "epoch": 2.0605452416792107,
      "grad_norm": 6.670922737876846e-08,
      "learning_rate": 2.6994343286665156e-07,
      "logits/chosen": 9.544310569763184,
      "logits/rejected": 8.973962783813477,
      "logps/chosen": -43.330848693847656,
      "logps/rejected": -45.52144241333008,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.330848693847656,
      "rewards/margins": 2.1905903816223145,
      "rewards/rejected": -45.52144241333008,
      "step": 3850
    },
    {
      "epoch": 2.063221274460612,
      "grad_norm": 1.7483829393850703e-05,
      "learning_rate": 2.6856179511868156e-07,
      "logits/chosen": 9.428180694580078,
      "logits/rejected": 8.605855941772461,
      "logps/chosen": -43.187374114990234,
      "logps/rejected": -44.761470794677734,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.187374114990234,
      "rewards/margins": 1.5740965604782104,
      "rewards/rejected": -44.761470794677734,
      "step": 3855
    },
    {
      "epoch": 2.0658973072420137,
      "grad_norm": 1.2891861360695786e-08,
      "learning_rate": 2.6718240278519056e-07,
      "logits/chosen": 8.884664535522461,
      "logits/rejected": 8.332504272460938,
      "logps/chosen": -42.93561935424805,
      "logps/rejected": -45.1068229675293,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -42.93561935424805,
      "rewards/margins": 2.171208143234253,
      "rewards/rejected": -45.1068229675293,
      "step": 3860
    },
    {
      "epoch": 2.0685733400234154,
      "grad_norm": 2.1027158710505144e-11,
      "learning_rate": 2.6580526924904866e-07,
      "logits/chosen": 9.492101669311523,
      "logits/rejected": 8.772340774536133,
      "logps/chosen": -45.64497756958008,
      "logps/rejected": -47.19325256347656,
      "loss": 0.001,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -45.64497756958008,
      "rewards/margins": 1.5482736825942993,
      "rewards/rejected": -47.19325256347656,
      "step": 3865
    },
    {
      "epoch": 2.0712493728048167,
      "grad_norm": 1.1732489117881255e-08,
      "learning_rate": 2.6443040787121186e-07,
      "logits/chosen": 9.500373840332031,
      "logits/rejected": 9.093267440795898,
      "logps/chosen": -45.975589752197266,
      "logps/rejected": -46.985389709472656,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.975589752197266,
      "rewards/margins": 1.009800672531128,
      "rewards/rejected": -46.985389709472656,
      "step": 3870
    },
    {
      "epoch": 2.0739254055862184,
      "grad_norm": 1.0198310592262827e-08,
      "learning_rate": 2.6305783199059084e-07,
      "logits/chosen": 9.066644668579102,
      "logits/rejected": 8.836080551147461,
      "logps/chosen": -44.54352951049805,
      "logps/rejected": -46.19464874267578,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.54352951049805,
      "rewards/margins": 1.651116132736206,
      "rewards/rejected": -46.19464874267578,
      "step": 3875
    },
    {
      "epoch": 2.07660143836762,
      "grad_norm": 3.623111354703898e-13,
      "learning_rate": 2.6168755492392324e-07,
      "logits/chosen": 9.532999992370605,
      "logits/rejected": 8.808847427368164,
      "logps/chosen": -44.950660705566406,
      "logps/rejected": -46.17007064819336,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.950660705566406,
      "rewards/margins": 1.219411015510559,
      "rewards/rejected": -46.17007064819336,
      "step": 3880
    },
    {
      "epoch": 2.0792774711490214,
      "grad_norm": 0.0012938588510889796,
      "learning_rate": 2.6031958996564274e-07,
      "logits/chosen": 9.484448432922363,
      "logits/rejected": 9.003042221069336,
      "logps/chosen": -45.464210510253906,
      "logps/rejected": -46.50223922729492,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.464210510253906,
      "rewards/margins": 1.0380312204360962,
      "rewards/rejected": -46.50223922729492,
      "step": 3885
    },
    {
      "epoch": 2.081953503930423,
      "grad_norm": 4.652788733137702e-10,
      "learning_rate": 2.589539503877518e-07,
      "logits/chosen": 8.85987663269043,
      "logits/rejected": 8.54279899597168,
      "logps/chosen": -45.432151794433594,
      "logps/rejected": -46.520538330078125,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.432151794433594,
      "rewards/margins": 1.0883827209472656,
      "rewards/rejected": -46.520538330078125,
      "step": 3890
    },
    {
      "epoch": 2.084629536711825,
      "grad_norm": 9.034388851130708e-08,
      "learning_rate": 2.5759064943969125e-07,
      "logits/chosen": 9.219508171081543,
      "logits/rejected": 8.420819282531738,
      "logps/chosen": -44.55602264404297,
      "logps/rejected": -46.2587776184082,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.55602264404297,
      "rewards/margins": 1.702756643295288,
      "rewards/rejected": -46.2587776184082,
      "step": 3895
    },
    {
      "epoch": 2.087305569493226,
      "grad_norm": 8.516407245082519e-09,
      "learning_rate": 2.562297003482131e-07,
      "logits/chosen": 9.184216499328613,
      "logits/rejected": 9.198433876037598,
      "logps/chosen": -45.359375,
      "logps/rejected": -45.98640823364258,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -45.359375,
      "rewards/margins": 0.6270355582237244,
      "rewards/rejected": -45.98640823364258,
      "step": 3900
    },
    {
      "epoch": 2.089981602274628,
      "grad_norm": 3.341705255759738e-09,
      "learning_rate": 2.548711163172512e-07,
      "logits/chosen": 9.110860824584961,
      "logits/rejected": 8.987802505493164,
      "logps/chosen": -44.51384353637695,
      "logps/rejected": -45.97661590576172,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.51384353637695,
      "rewards/margins": 1.4627739191055298,
      "rewards/rejected": -45.97661590576172,
      "step": 3905
    },
    {
      "epoch": 2.0926576350560295,
      "grad_norm": 3.334280354153858e-10,
      "learning_rate": 2.53514910527794e-07,
      "logits/chosen": 9.172529220581055,
      "logits/rejected": 8.704238891601562,
      "logps/chosen": -45.597076416015625,
      "logps/rejected": -46.3690185546875,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.597076416015625,
      "rewards/margins": 0.7719408273696899,
      "rewards/rejected": -46.3690185546875,
      "step": 3910
    },
    {
      "epoch": 2.095333667837431,
      "grad_norm": 2.156813332494473e-12,
      "learning_rate": 2.5216109613775573e-07,
      "logits/chosen": 9.181436538696289,
      "logits/rejected": 8.83820629119873,
      "logps/chosen": -45.0366096496582,
      "logps/rejected": -46.07899856567383,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.0366096496582,
      "rewards/margins": 1.042391061782837,
      "rewards/rejected": -46.07899856567383,
      "step": 3915
    },
    {
      "epoch": 2.0980097006188325,
      "grad_norm": 3.667407868030345e-07,
      "learning_rate": 2.5080968628184993e-07,
      "logits/chosen": 9.119338989257812,
      "logits/rejected": 8.590522766113281,
      "logps/chosen": -44.952232360839844,
      "logps/rejected": -46.112403869628906,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.952232360839844,
      "rewards/margins": 1.1601797342300415,
      "rewards/rejected": -46.112403869628906,
      "step": 3920
    },
    {
      "epoch": 2.1006857334002342,
      "grad_norm": 3.600540652508295e-11,
      "learning_rate": 2.494606940714605e-07,
      "logits/chosen": 8.969944953918457,
      "logits/rejected": 8.588040351867676,
      "logps/chosen": -44.71038055419922,
      "logps/rejected": -45.47053146362305,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.71038055419922,
      "rewards/margins": 0.7601487040519714,
      "rewards/rejected": -45.47053146362305,
      "step": 3925
    },
    {
      "epoch": 2.103361766181636,
      "grad_norm": 2.6226814644701473e-07,
      "learning_rate": 2.4811413259451625e-07,
      "logits/chosen": 9.265763282775879,
      "logits/rejected": 8.648977279663086,
      "logps/chosen": -43.39482498168945,
      "logps/rejected": -45.24662399291992,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.39482498168945,
      "rewards/margins": 1.8517940044403076,
      "rewards/rejected": -45.24662399291992,
      "step": 3930
    },
    {
      "epoch": 2.106037798963037,
      "grad_norm": 3.3692876924836116e-11,
      "learning_rate": 2.46770014915362e-07,
      "logits/chosen": 9.231999397277832,
      "logits/rejected": 8.959169387817383,
      "logps/chosen": -45.186668395996094,
      "logps/rejected": -46.084815979003906,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.186668395996094,
      "rewards/margins": 0.8981486558914185,
      "rewards/rejected": -46.084815979003906,
      "step": 3935
    },
    {
      "epoch": 2.108713831744439,
      "grad_norm": 9.444594821169201e-12,
      "learning_rate": 2.45428354074634e-07,
      "logits/chosen": 9.113451957702637,
      "logits/rejected": 8.921833038330078,
      "logps/chosen": -45.46223831176758,
      "logps/rejected": -47.02537536621094,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -45.46223831176758,
      "rewards/margins": 1.5631377696990967,
      "rewards/rejected": -47.02537536621094,
      "step": 3940
    },
    {
      "epoch": 2.1113898645258407,
      "grad_norm": 1.0418750658154855e-09,
      "learning_rate": 2.4408916308913105e-07,
      "logits/chosen": 8.970869064331055,
      "logits/rejected": 8.46876335144043,
      "logps/chosen": -43.54448318481445,
      "logps/rejected": -46.08223342895508,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -43.54448318481445,
      "rewards/margins": 2.537745952606201,
      "rewards/rejected": -46.08223342895508,
      "step": 3945
    },
    {
      "epoch": 2.114065897307242,
      "grad_norm": 3.884808305183695e-12,
      "learning_rate": 2.4275245495169025e-07,
      "logits/chosen": 9.135536193847656,
      "logits/rejected": 8.593464851379395,
      "logps/chosen": -44.959922790527344,
      "logps/rejected": -46.51813507080078,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.959922790527344,
      "rewards/margins": 1.5582096576690674,
      "rewards/rejected": -46.51813507080078,
      "step": 3950
    },
    {
      "epoch": 2.1167419300886436,
      "grad_norm": 2.270938727345259e-09,
      "learning_rate": 2.414182426310597e-07,
      "logits/chosen": 9.354995727539062,
      "logits/rejected": 9.300003051757812,
      "logps/chosen": -45.539371490478516,
      "logps/rejected": -46.25625991821289,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -45.539371490478516,
      "rewards/margins": 0.7168853282928467,
      "rewards/rejected": -46.25625991821289,
      "step": 3955
    },
    {
      "epoch": 2.1194179628700454,
      "grad_norm": 3.815352114153528e-08,
      "learning_rate": 2.400865390717734e-07,
      "logits/chosen": 9.278082847595215,
      "logits/rejected": 8.819133758544922,
      "logps/chosen": -45.02953338623047,
      "logps/rejected": -46.43731689453125,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.02953338623047,
      "rewards/margins": 1.4077814817428589,
      "rewards/rejected": -46.43731689453125,
      "step": 3960
    },
    {
      "epoch": 2.1220939956514466,
      "grad_norm": 1.0376528866041012e-06,
      "learning_rate": 2.3875735719402475e-07,
      "logits/chosen": 9.435392379760742,
      "logits/rejected": 9.106009483337402,
      "logps/chosen": -45.4124641418457,
      "logps/rejected": -46.57063674926758,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.4124641418457,
      "rewards/margins": 1.1581709384918213,
      "rewards/rejected": -46.57063674926758,
      "step": 3965
    },
    {
      "epoch": 2.1247700284328483,
      "grad_norm": 2.1505093031771217e-10,
      "learning_rate": 2.3743070989354258e-07,
      "logits/chosen": 9.450298309326172,
      "logits/rejected": 9.165613174438477,
      "logps/chosen": -45.340110778808594,
      "logps/rejected": -47.042686462402344,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.340110778808594,
      "rewards/margins": 1.7025744915008545,
      "rewards/rejected": -47.042686462402344,
      "step": 3970
    },
    {
      "epoch": 2.12744606121425,
      "grad_norm": 1.498928487147093e-09,
      "learning_rate": 2.3610661004146454e-07,
      "logits/chosen": 9.185544967651367,
      "logits/rejected": 8.768010139465332,
      "logps/chosen": -44.21394348144531,
      "logps/rejected": -45.38459014892578,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.21394348144531,
      "rewards/margins": 1.1706488132476807,
      "rewards/rejected": -45.38459014892578,
      "step": 3975
    },
    {
      "epoch": 2.1301220939956513,
      "grad_norm": 2.358709539714924e-05,
      "learning_rate": 2.3478507048421314e-07,
      "logits/chosen": 9.06254768371582,
      "logits/rejected": 8.614270210266113,
      "logps/chosen": -44.136199951171875,
      "logps/rejected": -45.13253402709961,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.136199951171875,
      "rewards/margins": 0.9963349103927612,
      "rewards/rejected": -45.13253402709961,
      "step": 3980
    },
    {
      "epoch": 2.132798126777053,
      "grad_norm": 3.9538394344270254e-13,
      "learning_rate": 2.334661040433713e-07,
      "logits/chosen": 9.60548210144043,
      "logits/rejected": 9.145254135131836,
      "logps/chosen": -46.31865310668945,
      "logps/rejected": -47.91405487060547,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -46.31865310668945,
      "rewards/margins": 1.5954011678695679,
      "rewards/rejected": -47.91405487060547,
      "step": 3985
    },
    {
      "epoch": 2.1354741595584548,
      "grad_norm": 9.651153398306607e-11,
      "learning_rate": 2.321497235155568e-07,
      "logits/chosen": 9.451119422912598,
      "logits/rejected": 8.900880813598633,
      "logps/chosen": -44.477203369140625,
      "logps/rejected": -45.651214599609375,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.477203369140625,
      "rewards/margins": 1.174009919166565,
      "rewards/rejected": -45.651214599609375,
      "step": 3990
    },
    {
      "epoch": 2.138150192339856,
      "grad_norm": 2.1726651716548215e-11,
      "learning_rate": 2.3083594167229965e-07,
      "logits/chosen": 9.45460319519043,
      "logits/rejected": 8.565498352050781,
      "logps/chosen": -44.546714782714844,
      "logps/rejected": -45.86309051513672,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -44.546714782714844,
      "rewards/margins": 1.3163725137710571,
      "rewards/rejected": -45.86309051513672,
      "step": 3995
    },
    {
      "epoch": 2.1408262251212578,
      "grad_norm": 3.6103402725677112e-09,
      "learning_rate": 2.295247712599167e-07,
      "logits/chosen": 9.18407154083252,
      "logits/rejected": 8.986404418945312,
      "logps/chosen": -44.876014709472656,
      "logps/rejected": -45.80879211425781,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.876014709472656,
      "rewards/margins": 0.932785153388977,
      "rewards/rejected": -45.80879211425781,
      "step": 4000
    },
    {
      "epoch": 2.1408262251212578,
      "eval_logits/chosen": 7.743565559387207,
      "eval_logits/rejected": 7.423532009124756,
      "eval_logps/chosen": -44.82708740234375,
      "eval_logps/rejected": -45.84105682373047,
      "eval_loss": 1.8925777567346813e-07,
      "eval_rewards/accuracies": 0.5667656064033508,
      "eval_rewards/chosen": -44.82708740234375,
      "eval_rewards/margins": 1.0139695405960083,
      "eval_rewards/rejected": -45.84105682373047,
      "eval_runtime": 40.0254,
      "eval_samples_per_second": 33.604,
      "eval_steps_per_second": 8.42,
      "step": 4000
    },
    {
      "epoch": 2.1435022579026595,
      "grad_norm": 2.575094921310585e-06,
      "learning_rate": 2.2821622499938948e-07,
      "logits/chosen": 9.21353530883789,
      "logits/rejected": 8.408825874328613,
      "logps/chosen": -44.120628356933594,
      "logps/rejected": -45.213050842285156,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.120628356933594,
      "rewards/margins": 1.092421531677246,
      "rewards/rejected": -45.213050842285156,
      "step": 4005
    },
    {
      "epoch": 2.1461782906840607,
      "grad_norm": 2.604295987021044e-12,
      "learning_rate": 2.269103155862391e-07,
      "logits/chosen": 9.495182991027832,
      "logits/rejected": 8.887582778930664,
      "logps/chosen": -45.84088134765625,
      "logps/rejected": -46.23398971557617,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -45.84088134765625,
      "rewards/margins": 0.393104612827301,
      "rewards/rejected": -46.23398971557617,
      "step": 4010
    },
    {
      "epoch": 2.1488543234654625,
      "grad_norm": 9.97463566022257e-08,
      "learning_rate": 2.2560705569040483e-07,
      "logits/chosen": 9.217655181884766,
      "logits/rejected": 8.236234664916992,
      "logps/chosen": -44.97712707519531,
      "logps/rejected": -45.84660720825195,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.97712707519531,
      "rewards/margins": 0.869475245475769,
      "rewards/rejected": -45.84660720825195,
      "step": 4015
    },
    {
      "epoch": 2.151530356246864,
      "grad_norm": 6.911903917782392e-08,
      "learning_rate": 2.2430645795611963e-07,
      "logits/chosen": 9.325763702392578,
      "logits/rejected": 8.859232902526855,
      "logps/chosen": -44.20629119873047,
      "logps/rejected": -45.07987594604492,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.20629119873047,
      "rewards/margins": 0.8735834956169128,
      "rewards/rejected": -45.07987594604492,
      "step": 4020
    },
    {
      "epoch": 2.1542063890282654,
      "grad_norm": 2.427651519136142e-07,
      "learning_rate": 2.230085350017884e-07,
      "logits/chosen": 9.240030288696289,
      "logits/rejected": 8.765321731567383,
      "logps/chosen": -44.92817687988281,
      "logps/rejected": -46.06142044067383,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -44.92817687988281,
      "rewards/margins": 1.1332423686981201,
      "rewards/rejected": -46.06142044067383,
      "step": 4025
    },
    {
      "epoch": 2.156882421809667,
      "grad_norm": 2.1989765230612784e-15,
      "learning_rate": 2.2171329941986554e-07,
      "logits/chosen": 9.407346725463867,
      "logits/rejected": 9.212428092956543,
      "logps/chosen": -45.88409423828125,
      "logps/rejected": -47.289405822753906,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.88409423828125,
      "rewards/margins": 1.405310034751892,
      "rewards/rejected": -47.289405822753906,
      "step": 4030
    },
    {
      "epoch": 2.159558454591069,
      "grad_norm": 1.7702818435996948e-11,
      "learning_rate": 2.2042076377673202e-07,
      "logits/chosen": 9.01850700378418,
      "logits/rejected": 9.015161514282227,
      "logps/chosen": -44.85331344604492,
      "logps/rejected": -45.72483825683594,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.85331344604492,
      "rewards/margins": 0.8715214729309082,
      "rewards/rejected": -45.72483825683594,
      "step": 4035
    },
    {
      "epoch": 2.16223448737247,
      "grad_norm": 2.3290275355281818e-09,
      "learning_rate": 2.1913094061257476e-07,
      "logits/chosen": 9.142797470092773,
      "logits/rejected": 9.041338920593262,
      "logps/chosen": -45.284690856933594,
      "logps/rejected": -46.05985641479492,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -45.284690856933594,
      "rewards/margins": 0.7751662135124207,
      "rewards/rejected": -46.05985641479492,
      "step": 4040
    },
    {
      "epoch": 2.164910520153872,
      "grad_norm": 1.0552966832848103e-10,
      "learning_rate": 2.178438424412633e-07,
      "logits/chosen": 9.181341171264648,
      "logits/rejected": 8.764431953430176,
      "logps/chosen": -45.255245208740234,
      "logps/rejected": -46.364707946777344,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.255245208740234,
      "rewards/margins": 1.1094658374786377,
      "rewards/rejected": -46.364707946777344,
      "step": 4045
    },
    {
      "epoch": 2.1675865529352736,
      "grad_norm": 1.3956487210970813e-09,
      "learning_rate": 2.165594817502302e-07,
      "logits/chosen": 9.58380126953125,
      "logits/rejected": 8.988260269165039,
      "logps/chosen": -44.96337127685547,
      "logps/rejected": -46.00679397583008,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.96337127685547,
      "rewards/margins": 1.0434272289276123,
      "rewards/rejected": -46.00679397583008,
      "step": 4050
    },
    {
      "epoch": 2.170262585716675,
      "grad_norm": 1.3727426767258255e-07,
      "learning_rate": 2.1527787100034806e-07,
      "logits/chosen": 9.012346267700195,
      "logits/rejected": 8.913850784301758,
      "logps/chosen": -44.37690734863281,
      "logps/rejected": -45.588130950927734,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.37690734863281,
      "rewards/margins": 1.2112195491790771,
      "rewards/rejected": -45.588130950927734,
      "step": 4055
    },
    {
      "epoch": 2.1729386184980766,
      "grad_norm": 3.339350580442168e-09,
      "learning_rate": 2.1399902262581037e-07,
      "logits/chosen": 9.36424446105957,
      "logits/rejected": 8.728506088256836,
      "logps/chosen": -44.87565994262695,
      "logps/rejected": -46.128761291503906,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.87565994262695,
      "rewards/margins": 1.253103256225586,
      "rewards/rejected": -46.128761291503906,
      "step": 4060
    },
    {
      "epoch": 2.1756146512794783,
      "grad_norm": 1.6117771042831145e-10,
      "learning_rate": 2.127229490340094e-07,
      "logits/chosen": 9.26677131652832,
      "logits/rejected": 8.867010116577148,
      "logps/chosen": -44.972373962402344,
      "logps/rejected": -46.0651741027832,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -44.972373962402344,
      "rewards/margins": 1.0927984714508057,
      "rewards/rejected": -46.0651741027832,
      "step": 4065
    },
    {
      "epoch": 2.1782906840608796,
      "grad_norm": 6.129177216163308e-09,
      "learning_rate": 2.1144966260541698e-07,
      "logits/chosen": 9.329187393188477,
      "logits/rejected": 8.583409309387207,
      "logps/chosen": -44.58380126953125,
      "logps/rejected": -45.702613830566406,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.58380126953125,
      "rewards/margins": 1.1188137531280518,
      "rewards/rejected": -45.702613830566406,
      "step": 4070
    },
    {
      "epoch": 2.1809667168422813,
      "grad_norm": 7.980799630564147e-05,
      "learning_rate": 2.1017917569346332e-07,
      "logits/chosen": 9.277470588684082,
      "logits/rejected": 8.55964469909668,
      "logps/chosen": -44.04616928100586,
      "logps/rejected": -45.53752899169922,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.04616928100586,
      "rewards/margins": 1.491363286972046,
      "rewards/rejected": -45.53752899169922,
      "step": 4075
    },
    {
      "epoch": 2.183642749623683,
      "grad_norm": 2.5600713390836112e-05,
      "learning_rate": 2.0891150062441837e-07,
      "logits/chosen": 9.310317993164062,
      "logits/rejected": 8.673260688781738,
      "logps/chosen": -44.63154220581055,
      "logps/rejected": -45.97618865966797,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.63154220581055,
      "rewards/margins": 1.344651460647583,
      "rewards/rejected": -45.97618865966797,
      "step": 4080
    },
    {
      "epoch": 2.1863187824050843,
      "grad_norm": 8.033503748981239e-13,
      "learning_rate": 2.0764664969727086e-07,
      "logits/chosen": 8.919893264770508,
      "logits/rejected": 8.994022369384766,
      "logps/chosen": -43.82129669189453,
      "logps/rejected": -44.625526428222656,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -43.82129669189453,
      "rewards/margins": 0.8042244911193848,
      "rewards/rejected": -44.625526428222656,
      "step": 4085
    },
    {
      "epoch": 2.188994815186486,
      "grad_norm": 4.604717080931137e-10,
      "learning_rate": 2.0638463518361033e-07,
      "logits/chosen": 9.738365173339844,
      "logits/rejected": 8.983895301818848,
      "logps/chosen": -44.650657653808594,
      "logps/rejected": -45.93735122680664,
      "loss": 0.0001,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.650657653808594,
      "rewards/margins": 1.2866928577423096,
      "rewards/rejected": -45.93735122680664,
      "step": 4090
    },
    {
      "epoch": 2.1916708479678877,
      "grad_norm": 1.7231417652249943e-12,
      "learning_rate": 2.0512546932750702e-07,
      "logits/chosen": 9.511911392211914,
      "logits/rejected": 9.241863250732422,
      "logps/chosen": -44.86798858642578,
      "logps/rejected": -45.76915740966797,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.86798858642578,
      "rewards/margins": 0.9011713862419128,
      "rewards/rejected": -45.76915740966797,
      "step": 4095
    },
    {
      "epoch": 2.194346880749289,
      "grad_norm": 2.733946065685473e-07,
      "learning_rate": 2.0386916434539343e-07,
      "logits/chosen": 9.160719871520996,
      "logits/rejected": 8.472604751586914,
      "logps/chosen": -46.77503204345703,
      "logps/rejected": -47.63618087768555,
      "loss": 0.0005,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -46.77503204345703,
      "rewards/margins": 0.861148476600647,
      "rewards/rejected": -47.63618087768555,
      "step": 4100
    },
    {
      "epoch": 2.1970229135306907,
      "grad_norm": 9.646196504964925e-11,
      "learning_rate": 2.0261573242594627e-07,
      "logits/chosen": 9.295928001403809,
      "logits/rejected": 8.694162368774414,
      "logps/chosen": -45.074073791503906,
      "logps/rejected": -45.78377151489258,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.074073791503906,
      "rewards/margins": 0.7096951603889465,
      "rewards/rejected": -45.78377151489258,
      "step": 4105
    },
    {
      "epoch": 2.1996989463120924,
      "grad_norm": 1.0327416988813133e-11,
      "learning_rate": 2.0136518572996724e-07,
      "logits/chosen": 9.27946662902832,
      "logits/rejected": 8.708243370056152,
      "logps/chosen": -45.14055252075195,
      "logps/rejected": -46.46326446533203,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -45.14055252075195,
      "rewards/margins": 1.3227152824401855,
      "rewards/rejected": -46.46326446533203,
      "step": 4110
    },
    {
      "epoch": 2.202374979093494,
      "grad_norm": 1.080286743951806e-09,
      "learning_rate": 2.0011753639026617e-07,
      "logits/chosen": 9.049102783203125,
      "logits/rejected": 8.623174667358398,
      "logps/chosen": -45.614173889160156,
      "logps/rejected": -46.34154510498047,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.614173889160156,
      "rewards/margins": 0.7273696660995483,
      "rewards/rejected": -46.34154510498047,
      "step": 4115
    },
    {
      "epoch": 2.2050510118748954,
      "grad_norm": 3.7796021022353616e-10,
      "learning_rate": 1.988727965115421e-07,
      "logits/chosen": 9.531824111938477,
      "logits/rejected": 9.05683708190918,
      "logps/chosen": -45.27906036376953,
      "logps/rejected": -46.1784553527832,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.27906036376953,
      "rewards/margins": 0.8993936777114868,
      "rewards/rejected": -46.1784553527832,
      "step": 4120
    },
    {
      "epoch": 2.207727044656297,
      "grad_norm": 2.3843721613217753e-09,
      "learning_rate": 1.9763097817026713e-07,
      "logits/chosen": 9.593295097351074,
      "logits/rejected": 8.75585651397705,
      "logps/chosen": -45.413814544677734,
      "logps/rejected": -46.98648452758789,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -45.413814544677734,
      "rewards/margins": 1.5726661682128906,
      "rewards/rejected": -46.98648452758789,
      "step": 4125
    },
    {
      "epoch": 2.210403077437699,
      "grad_norm": 6.961385160378615e-13,
      "learning_rate": 1.9639209341456796e-07,
      "logits/chosen": 8.99570369720459,
      "logits/rejected": 8.69746208190918,
      "logps/chosen": -45.26372528076172,
      "logps/rejected": -45.88838195800781,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -45.26372528076172,
      "rewards/margins": 0.6246573328971863,
      "rewards/rejected": -45.88838195800781,
      "step": 4130
    },
    {
      "epoch": 2.2130791102191,
      "grad_norm": 1.0001253762452942e-05,
      "learning_rate": 1.951561542641102e-07,
      "logits/chosen": 8.708776473999023,
      "logits/rejected": 8.892117500305176,
      "logps/chosen": -45.701873779296875,
      "logps/rejected": -45.65106964111328,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -45.701873779296875,
      "rewards/margins": -0.05080683156847954,
      "rewards/rejected": -45.65106964111328,
      "step": 4135
    },
    {
      "epoch": 2.215755143000502,
      "grad_norm": 9.918551272718425e-11,
      "learning_rate": 1.939231727099806e-07,
      "logits/chosen": 9.794648170471191,
      "logits/rejected": 9.639742851257324,
      "logps/chosen": -44.663673400878906,
      "logps/rejected": -45.982357025146484,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.663673400878906,
      "rewards/margins": 1.3186814785003662,
      "rewards/rejected": -45.982357025146484,
      "step": 4140
    },
    {
      "epoch": 2.2184311757819035,
      "grad_norm": 4.035932251174062e-08,
      "learning_rate": 1.926931607145719e-07,
      "logits/chosen": 8.842588424682617,
      "logits/rejected": 8.291573524475098,
      "logps/chosen": -44.751625061035156,
      "logps/rejected": -45.99313735961914,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -44.751625061035156,
      "rewards/margins": 1.2415127754211426,
      "rewards/rejected": -45.99313735961914,
      "step": 4145
    },
    {
      "epoch": 2.221107208563305,
      "grad_norm": 3.911718985187506e-12,
      "learning_rate": 1.9146613021146564e-07,
      "logits/chosen": 9.318704605102539,
      "logits/rejected": 8.82827377319336,
      "logps/chosen": -45.755897521972656,
      "logps/rejected": -47.46888732910156,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.755897521972656,
      "rewards/margins": 1.7129875421524048,
      "rewards/rejected": -47.46888732910156,
      "step": 4150
    },
    {
      "epoch": 2.2237832413447065,
      "grad_norm": 1.0652510359044194e-09,
      "learning_rate": 1.9024209310531736e-07,
      "logits/chosen": 9.457426071166992,
      "logits/rejected": 9.38670539855957,
      "logps/chosen": -46.131744384765625,
      "logps/rejected": -46.005130767822266,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -46.131744384765625,
      "rewards/margins": -0.12661492824554443,
      "rewards/rejected": -46.005130767822266,
      "step": 4155
    },
    {
      "epoch": 2.2264592741261082,
      "grad_norm": 1.4924644704860259e-05,
      "learning_rate": 1.890210612717401e-07,
      "logits/chosen": 9.204139709472656,
      "logits/rejected": 9.008432388305664,
      "logps/chosen": -44.67188262939453,
      "logps/rejected": -46.08615493774414,
      "loss": 0.0005,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.67188262939453,
      "rewards/margins": 1.4142718315124512,
      "rewards/rejected": -46.08615493774414,
      "step": 4160
    },
    {
      "epoch": 2.2291353069075095,
      "grad_norm": 1.633406465780551e-11,
      "learning_rate": 1.8780304655719054e-07,
      "logits/chosen": 9.546049118041992,
      "logits/rejected": 9.035723686218262,
      "logps/chosen": -45.32472610473633,
      "logps/rejected": -46.75824737548828,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -45.32472610473633,
      "rewards/margins": 1.4335224628448486,
      "rewards/rejected": -46.75824737548828,
      "step": 4165
    },
    {
      "epoch": 2.231811339688911,
      "grad_norm": 7.89677764765043e-13,
      "learning_rate": 1.865880607788523e-07,
      "logits/chosen": 8.880742073059082,
      "logits/rejected": 8.672422409057617,
      "logps/chosen": -45.667503356933594,
      "logps/rejected": -46.04290008544922,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -45.667503356933594,
      "rewards/margins": 0.37539854645729065,
      "rewards/rejected": -46.04290008544922,
      "step": 4170
    },
    {
      "epoch": 2.234487372470313,
      "grad_norm": 4.881623930514966e-10,
      "learning_rate": 1.8537611572452316e-07,
      "logits/chosen": 9.35511302947998,
      "logits/rejected": 8.961983680725098,
      "logps/chosen": -44.94432830810547,
      "logps/rejected": -45.75162887573242,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.94432830810547,
      "rewards/margins": 0.8073033094406128,
      "rewards/rejected": -45.75162887573242,
      "step": 4175
    },
    {
      "epoch": 2.237163405251714,
      "grad_norm": 1.5120567542599632e-08,
      "learning_rate": 1.84167223152499e-07,
      "logits/chosen": 9.398344039916992,
      "logits/rejected": 8.564884185791016,
      "logps/chosen": -43.814964294433594,
      "logps/rejected": -45.29149627685547,
      "loss": 0.0011,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -43.814964294433594,
      "rewards/margins": 1.4765417575836182,
      "rewards/rejected": -45.29149627685547,
      "step": 4180
    },
    {
      "epoch": 2.239839438033116,
      "grad_norm": 1.763460570648625e-10,
      "learning_rate": 1.8296139479146112e-07,
      "logits/chosen": 9.090723037719727,
      "logits/rejected": 9.141990661621094,
      "logps/chosen": -45.409095764160156,
      "logps/rejected": -45.64834213256836,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -45.409095764160156,
      "rewards/margins": 0.23925387859344482,
      "rewards/rejected": -45.64834213256836,
      "step": 4185
    },
    {
      "epoch": 2.2425154708145176,
      "grad_norm": 5.570686374141531e-13,
      "learning_rate": 1.8175864234036132e-07,
      "logits/chosen": 8.940000534057617,
      "logits/rejected": 8.688929557800293,
      "logps/chosen": -45.17580795288086,
      "logps/rejected": -46.13033676147461,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.17580795288086,
      "rewards/margins": 0.9545255899429321,
      "rewards/rejected": -46.13033676147461,
      "step": 4190
    },
    {
      "epoch": 2.245191503595919,
      "grad_norm": 5.24875672194833e-12,
      "learning_rate": 1.805589774683094e-07,
      "logits/chosen": 9.52067756652832,
      "logits/rejected": 9.070625305175781,
      "logps/chosen": -45.96797561645508,
      "logps/rejected": -46.76105880737305,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.96797561645508,
      "rewards/margins": 0.7930809259414673,
      "rewards/rejected": -46.76105880737305,
      "step": 4195
    },
    {
      "epoch": 2.2478675363773206,
      "grad_norm": 5.684646384467382e-10,
      "learning_rate": 1.79362411814459e-07,
      "logits/chosen": 8.911266326904297,
      "logits/rejected": 8.833349227905273,
      "logps/chosen": -45.118751525878906,
      "logps/rejected": -45.6392822265625,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -45.118751525878906,
      "rewards/margins": 0.5205384492874146,
      "rewards/rejected": -45.6392822265625,
      "step": 4200
    },
    {
      "epoch": 2.2505435691587223,
      "grad_norm": 1.208601285810954e-12,
      "learning_rate": 1.7816895698789552e-07,
      "logits/chosen": 9.220732688903809,
      "logits/rejected": 9.135797500610352,
      "logps/chosen": -44.417327880859375,
      "logps/rejected": -46.14222717285156,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.417327880859375,
      "rewards/margins": 1.7248995304107666,
      "rewards/rejected": -46.14222717285156,
      "step": 4205
    },
    {
      "epoch": 2.2532196019401236,
      "grad_norm": 6.515298439391098e-12,
      "learning_rate": 1.7697862456752271e-07,
      "logits/chosen": 9.287664413452148,
      "logits/rejected": 8.877995491027832,
      "logps/chosen": -44.734798431396484,
      "logps/rejected": -45.94930648803711,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -44.734798431396484,
      "rewards/margins": 1.2145054340362549,
      "rewards/rejected": -45.94930648803711,
      "step": 4210
    },
    {
      "epoch": 2.2558956347215253,
      "grad_norm": 1.3664984291552614e-06,
      "learning_rate": 1.7579142610195124e-07,
      "logits/chosen": 9.049276351928711,
      "logits/rejected": 8.553240776062012,
      "logps/chosen": -45.8105354309082,
      "logps/rejected": -46.29276657104492,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.8105354309082,
      "rewards/margins": 0.48222923278808594,
      "rewards/rejected": -46.29276657104492,
      "step": 4215
    },
    {
      "epoch": 2.258571667502927,
      "grad_norm": 1.1067677487420494e-10,
      "learning_rate": 1.7460737310938568e-07,
      "logits/chosen": 9.711051940917969,
      "logits/rejected": 8.900574684143066,
      "logps/chosen": -45.82500076293945,
      "logps/rejected": -46.460479736328125,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.82500076293945,
      "rewards/margins": 0.6354772448539734,
      "rewards/rejected": -46.460479736328125,
      "step": 4220
    },
    {
      "epoch": 2.2612477002843283,
      "grad_norm": 1.6742511096402045e-10,
      "learning_rate": 1.734264770775133e-07,
      "logits/chosen": 9.123899459838867,
      "logits/rejected": 8.245651245117188,
      "logps/chosen": -44.41241455078125,
      "logps/rejected": -46.23500061035156,
      "loss": 0.0002,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.41241455078125,
      "rewards/margins": 1.8225882053375244,
      "rewards/rejected": -46.23500061035156,
      "step": 4225
    },
    {
      "epoch": 2.26392373306573,
      "grad_norm": 1.6411043957297404e-10,
      "learning_rate": 1.7224874946339241e-07,
      "logits/chosen": 9.433492660522461,
      "logits/rejected": 8.985888481140137,
      "logps/chosen": -45.03028869628906,
      "logps/rejected": -46.2167854309082,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -45.03028869628906,
      "rewards/margins": 1.1864937543869019,
      "rewards/rejected": -46.2167854309082,
      "step": 4230
    },
    {
      "epoch": 2.2665997658471317,
      "grad_norm": 7.030148954393853e-11,
      "learning_rate": 1.7107420169334186e-07,
      "logits/chosen": 9.29533576965332,
      "logits/rejected": 9.031789779663086,
      "logps/chosen": -46.05066680908203,
      "logps/rejected": -46.200172424316406,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -46.05066680908203,
      "rewards/margins": 0.14950624108314514,
      "rewards/rejected": -46.200172424316406,
      "step": 4235
    },
    {
      "epoch": 2.269275798628533,
      "grad_norm": 1.3018252940323773e-08,
      "learning_rate": 1.6990284516282893e-07,
      "logits/chosen": 9.241679191589355,
      "logits/rejected": 8.87092399597168,
      "logps/chosen": -46.68424987792969,
      "logps/rejected": -48.03684997558594,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -46.68424987792969,
      "rewards/margins": 1.3526033163070679,
      "rewards/rejected": -48.03684997558594,
      "step": 4240
    },
    {
      "epoch": 2.2719518314099347,
      "grad_norm": 5.699462309866782e-07,
      "learning_rate": 1.687346912363602e-07,
      "logits/chosen": 9.422359466552734,
      "logits/rejected": 8.819128036499023,
      "logps/chosen": -45.358642578125,
      "logps/rejected": -46.635498046875,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -45.358642578125,
      "rewards/margins": 1.276852011680603,
      "rewards/rejected": -46.635498046875,
      "step": 4245
    },
    {
      "epoch": 2.2746278641913364,
      "grad_norm": 1.3921456961566741e-06,
      "learning_rate": 1.675697512473697e-07,
      "logits/chosen": 9.070808410644531,
      "logits/rejected": 8.285497665405273,
      "logps/chosen": -44.79621887207031,
      "logps/rejected": -45.449378967285156,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.79621887207031,
      "rewards/margins": 0.6531566977500916,
      "rewards/rejected": -45.449378967285156,
      "step": 4250
    },
    {
      "epoch": 2.2773038969727377,
      "grad_norm": 9.225351688846631e-06,
      "learning_rate": 1.6640803649811087e-07,
      "logits/chosen": 9.049978256225586,
      "logits/rejected": 8.00257682800293,
      "logps/chosen": -44.055931091308594,
      "logps/rejected": -46.00001525878906,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.055931091308594,
      "rewards/margins": 1.9440851211547852,
      "rewards/rejected": -46.00001525878906,
      "step": 4255
    },
    {
      "epoch": 2.2799799297541394,
      "grad_norm": 2.6226592864341017e-11,
      "learning_rate": 1.6524955825954472e-07,
      "logits/chosen": 9.163142204284668,
      "logits/rejected": 8.996142387390137,
      "logps/chosen": -45.974876403808594,
      "logps/rejected": -46.11400604248047,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -45.974876403808594,
      "rewards/margins": 0.13912814855575562,
      "rewards/rejected": -46.11400604248047,
      "step": 4260
    },
    {
      "epoch": 2.282655962535541,
      "grad_norm": 1.6641137853379333e-11,
      "learning_rate": 1.6409432777123277e-07,
      "logits/chosen": 9.670583724975586,
      "logits/rejected": 8.970295906066895,
      "logps/chosen": -44.53575897216797,
      "logps/rejected": -46.32887649536133,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.53575897216797,
      "rewards/margins": 1.793117880821228,
      "rewards/rejected": -46.32887649536133,
      "step": 4265
    },
    {
      "epoch": 2.285331995316943,
      "grad_norm": 1.0196521982884407e-06,
      "learning_rate": 1.6294235624122577e-07,
      "logits/chosen": 9.029888153076172,
      "logits/rejected": 8.152617454528809,
      "logps/chosen": -43.733238220214844,
      "logps/rejected": -45.61360549926758,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -43.733238220214844,
      "rewards/margins": 1.8803634643554688,
      "rewards/rejected": -45.61360549926758,
      "step": 4270
    },
    {
      "epoch": 2.288008028098344,
      "grad_norm": 8.002693343220748e-06,
      "learning_rate": 1.6179365484595697e-07,
      "logits/chosen": 9.204007148742676,
      "logits/rejected": 8.759544372558594,
      "logps/chosen": -45.035400390625,
      "logps/rejected": -46.14873504638672,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.035400390625,
      "rewards/margins": 1.1133339405059814,
      "rewards/rejected": -46.14873504638672,
      "step": 4275
    },
    {
      "epoch": 2.290684060879746,
      "grad_norm": 9.86993720669966e-08,
      "learning_rate": 1.60648234730132e-07,
      "logits/chosen": 9.386974334716797,
      "logits/rejected": 8.950389862060547,
      "logps/chosen": -44.32321548461914,
      "logps/rejected": -45.66289520263672,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.32321548461914,
      "rewards/margins": 1.3396800756454468,
      "rewards/rejected": -45.66289520263672,
      "step": 4280
    },
    {
      "epoch": 2.293360093661147,
      "grad_norm": 5.159426828272243e-07,
      "learning_rate": 1.595061070066222e-07,
      "logits/chosen": 9.203662872314453,
      "logits/rejected": 9.324064254760742,
      "logps/chosen": -46.127281188964844,
      "logps/rejected": -46.32810974121094,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -46.127281188964844,
      "rewards/margins": 0.20082557201385498,
      "rewards/rejected": -46.32810974121094,
      "step": 4285
    },
    {
      "epoch": 2.296036126442549,
      "grad_norm": 1.1584664217674032e-08,
      "learning_rate": 1.5836728275635542e-07,
      "logits/chosen": 9.55512809753418,
      "logits/rejected": 8.806375503540039,
      "logps/chosen": -46.00165557861328,
      "logps/rejected": -47.35778045654297,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -46.00165557861328,
      "rewards/margins": 1.3561303615570068,
      "rewards/rejected": -47.35778045654297,
      "step": 4290
    },
    {
      "epoch": 2.2987121592239506,
      "grad_norm": 2.9416250266813807e-13,
      "learning_rate": 1.5723177302820984e-07,
      "logits/chosen": 9.338932991027832,
      "logits/rejected": 9.083527565002441,
      "logps/chosen": -45.26051330566406,
      "logps/rejected": -46.9761848449707,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.26051330566406,
      "rewards/margins": 1.71566903591156,
      "rewards/rejected": -46.9761848449707,
      "step": 4295
    },
    {
      "epoch": 2.3013881920053523,
      "grad_norm": 5.687839243583107e-11,
      "learning_rate": 1.5609958883890544e-07,
      "logits/chosen": 8.963685035705566,
      "logits/rejected": 8.548189163208008,
      "logps/chosen": -45.0977783203125,
      "logps/rejected": -45.46226501464844,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.0977783203125,
      "rewards/margins": 0.3644913136959076,
      "rewards/rejected": -45.46226501464844,
      "step": 4300
    },
    {
      "epoch": 2.3040642247867535,
      "grad_norm": 6.951516551357573e-15,
      "learning_rate": 1.5497074117289865e-07,
      "logits/chosen": 9.411312103271484,
      "logits/rejected": 8.932104110717773,
      "logps/chosen": -46.19283676147461,
      "logps/rejected": -46.69925308227539,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -46.19283676147461,
      "rewards/margins": 0.5064181089401245,
      "rewards/rejected": -46.69925308227539,
      "step": 4305
    },
    {
      "epoch": 2.3067402575681553,
      "grad_norm": 9.243508994256584e-09,
      "learning_rate": 1.5384524098227402e-07,
      "logits/chosen": 9.599885940551758,
      "logits/rejected": 8.822911262512207,
      "logps/chosen": -45.800811767578125,
      "logps/rejected": -47.12258529663086,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -45.800811767578125,
      "rewards/margins": 1.3217742443084717,
      "rewards/rejected": -47.12258529663086,
      "step": 4310
    },
    {
      "epoch": 2.3094162903495565,
      "grad_norm": 4.2301775051796314e-10,
      "learning_rate": 1.5272309918663974e-07,
      "logits/chosen": 9.307069778442383,
      "logits/rejected": 8.760488510131836,
      "logps/chosen": -45.543357849121094,
      "logps/rejected": -46.798744201660156,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.543357849121094,
      "rewards/margins": 1.2553884983062744,
      "rewards/rejected": -46.798744201660156,
      "step": 4315
    },
    {
      "epoch": 2.3120923231309582,
      "grad_norm": 3.3819624038596377e-09,
      "learning_rate": 1.516043266730201e-07,
      "logits/chosen": 9.427566528320312,
      "logits/rejected": 8.85187816619873,
      "logps/chosen": -45.796932220458984,
      "logps/rejected": -46.85143280029297,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -45.796932220458984,
      "rewards/margins": 1.0545002222061157,
      "rewards/rejected": -46.85143280029297,
      "step": 4320
    },
    {
      "epoch": 2.31476835591236,
      "grad_norm": 1.0769471810406897e-08,
      "learning_rate": 1.504889342957512e-07,
      "logits/chosen": 9.37326717376709,
      "logits/rejected": 8.801453590393066,
      "logps/chosen": -45.548072814941406,
      "logps/rejected": -46.69108581542969,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.548072814941406,
      "rewards/margins": 1.1430118083953857,
      "rewards/rejected": -46.69108581542969,
      "step": 4325
    },
    {
      "epoch": 2.3174443886937617,
      "grad_norm": 5.2673984838149584e-08,
      "learning_rate": 1.4937693287637453e-07,
      "logits/chosen": 9.347002983093262,
      "logits/rejected": 8.92799186706543,
      "logps/chosen": -45.899471282958984,
      "logps/rejected": -46.31129455566406,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.899471282958984,
      "rewards/margins": 0.41181984543800354,
      "rewards/rejected": -46.31129455566406,
      "step": 4330
    },
    {
      "epoch": 2.320120421475163,
      "grad_norm": 5.078356344748222e-11,
      "learning_rate": 1.4826833320353305e-07,
      "logits/chosen": 9.294052124023438,
      "logits/rejected": 8.87785530090332,
      "logps/chosen": -45.23823547363281,
      "logps/rejected": -45.81166458129883,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -45.23823547363281,
      "rewards/margins": 0.5734270215034485,
      "rewards/rejected": -45.81166458129883,
      "step": 4335
    },
    {
      "epoch": 2.3227964542565647,
      "grad_norm": 5.0152870867273174e-09,
      "learning_rate": 1.4716314603286528e-07,
      "logits/chosen": 9.406847953796387,
      "logits/rejected": 8.600555419921875,
      "logps/chosen": -45.07960510253906,
      "logps/rejected": -46.85123825073242,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -45.07960510253906,
      "rewards/margins": 1.7716360092163086,
      "rewards/rejected": -46.85123825073242,
      "step": 4340
    },
    {
      "epoch": 2.3254724870379664,
      "grad_norm": 2.2549829241739764e-08,
      "learning_rate": 1.4606138208690233e-07,
      "logits/chosen": 8.978201866149902,
      "logits/rejected": 8.895700454711914,
      "logps/chosen": -45.859066009521484,
      "logps/rejected": -46.40438461303711,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.859066009521484,
      "rewards/margins": 0.5453125834465027,
      "rewards/rejected": -46.40438461303711,
      "step": 4345
    },
    {
      "epoch": 2.3281485198193677,
      "grad_norm": 1.536675255596879e-12,
      "learning_rate": 1.4496305205496251e-07,
      "logits/chosen": 9.103460311889648,
      "logits/rejected": 8.869914054870605,
      "logps/chosen": -46.76239013671875,
      "logps/rejected": -46.868370056152344,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -46.76239013671875,
      "rewards/margins": 0.10597886890172958,
      "rewards/rejected": -46.868370056152344,
      "step": 4350
    },
    {
      "epoch": 2.3308245526007694,
      "grad_norm": 1.1495155803179659e-08,
      "learning_rate": 1.4386816659304895e-07,
      "logits/chosen": 9.627204895019531,
      "logits/rejected": 8.978019714355469,
      "logps/chosen": -46.2531623840332,
      "logps/rejected": -46.31879425048828,
      "loss": 0.001,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -46.2531623840332,
      "rewards/margins": 0.06563319265842438,
      "rewards/rejected": -46.31879425048828,
      "step": 4355
    },
    {
      "epoch": 2.333500585382171,
      "grad_norm": 2.687909505317397e-05,
      "learning_rate": 1.4277673632374492e-07,
      "logits/chosen": 9.195300102233887,
      "logits/rejected": 8.230979919433594,
      "logps/chosen": -44.62238311767578,
      "logps/rejected": -46.647987365722656,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.62238311767578,
      "rewards/margins": 2.0256028175354004,
      "rewards/rejected": -46.647987365722656,
      "step": 4360
    },
    {
      "epoch": 2.3361766181635724,
      "grad_norm": 1.1063188784833087e-05,
      "learning_rate": 1.416887718361119e-07,
      "logits/chosen": 9.244245529174805,
      "logits/rejected": 9.243672370910645,
      "logps/chosen": -46.36935806274414,
      "logps/rejected": -47.31113052368164,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -46.36935806274414,
      "rewards/margins": 0.9417756199836731,
      "rewards/rejected": -47.31113052368164,
      "step": 4365
    },
    {
      "epoch": 2.338852650944974,
      "grad_norm": 7.022896570590801e-12,
      "learning_rate": 1.406042836855859e-07,
      "logits/chosen": 9.601293563842773,
      "logits/rejected": 9.134517669677734,
      "logps/chosen": -46.14667510986328,
      "logps/rejected": -46.56935501098633,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -46.14667510986328,
      "rewards/margins": 0.422677606344223,
      "rewards/rejected": -46.56935501098633,
      "step": 4370
    },
    {
      "epoch": 2.341528683726376,
      "grad_norm": 1.4720556989439093e-07,
      "learning_rate": 1.3952328239387595e-07,
      "logits/chosen": 9.258251190185547,
      "logits/rejected": 8.387868881225586,
      "logps/chosen": -44.60669708251953,
      "logps/rejected": -45.86088943481445,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.60669708251953,
      "rewards/margins": 1.254192590713501,
      "rewards/rejected": -45.86088943481445,
      "step": 4375
    },
    {
      "epoch": 2.344204716507777,
      "grad_norm": 2.8409351428302185e-09,
      "learning_rate": 1.3844577844886109e-07,
      "logits/chosen": 9.402308464050293,
      "logits/rejected": 8.589438438415527,
      "logps/chosen": -43.92802810668945,
      "logps/rejected": -45.45700454711914,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -43.92802810668945,
      "rewards/margins": 1.5289745330810547,
      "rewards/rejected": -45.45700454711914,
      "step": 4380
    },
    {
      "epoch": 2.346880749289179,
      "grad_norm": 1.3624963243637793e-13,
      "learning_rate": 1.3737178230448955e-07,
      "logits/chosen": 9.153677940368652,
      "logits/rejected": 8.826898574829102,
      "logps/chosen": -45.93329620361328,
      "logps/rejected": -46.705078125,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -45.93329620361328,
      "rewards/margins": 0.7717849612236023,
      "rewards/rejected": -46.705078125,
      "step": 4385
    },
    {
      "epoch": 2.3495567820705805,
      "grad_norm": 4.949251575235949e-14,
      "learning_rate": 1.363013043806764e-07,
      "logits/chosen": 9.47992992401123,
      "logits/rejected": 9.112804412841797,
      "logps/chosen": -45.50286102294922,
      "logps/rejected": -46.640342712402344,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.50286102294922,
      "rewards/margins": 1.1374781131744385,
      "rewards/rejected": -46.640342712402344,
      "step": 4390
    },
    {
      "epoch": 2.3522328148519818,
      "grad_norm": 1.136500027629646e-09,
      "learning_rate": 1.352343550632034e-07,
      "logits/chosen": 9.524073600769043,
      "logits/rejected": 8.909246444702148,
      "logps/chosen": -45.00959014892578,
      "logps/rejected": -47.09957504272461,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -45.00959014892578,
      "rewards/margins": 2.0899882316589355,
      "rewards/rejected": -47.09957504272461,
      "step": 4395
    },
    {
      "epoch": 2.3549088476333835,
      "grad_norm": 3.991235279244279e-11,
      "learning_rate": 1.3417094470361722e-07,
      "logits/chosen": 9.218130111694336,
      "logits/rejected": 8.74543571472168,
      "logps/chosen": -45.18308639526367,
      "logps/rejected": -46.49921798706055,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.18308639526367,
      "rewards/margins": 1.3161300420761108,
      "rewards/rejected": -46.49921798706055,
      "step": 4400
    },
    {
      "epoch": 2.3549088476333835,
      "eval_logits/chosen": 7.683070659637451,
      "eval_logits/rejected": 7.35262393951416,
      "eval_logps/chosen": -45.134376525878906,
      "eval_logps/rejected": -46.13737869262695,
      "eval_loss": 1.7626304327222897e-07,
      "eval_rewards/accuracies": 0.5652819275856018,
      "eval_rewards/chosen": -45.134376525878906,
      "eval_rewards/margins": 1.0030003786087036,
      "eval_rewards/rejected": -46.13737869262695,
      "eval_runtime": 39.9946,
      "eval_samples_per_second": 33.63,
      "eval_steps_per_second": 8.426,
      "step": 4400
    },
    {
      "epoch": 2.357584880414785,
      "grad_norm": 7.24606566183061e-07,
      "learning_rate": 1.3311108361913015e-07,
      "logits/chosen": 9.239240646362305,
      "logits/rejected": 9.264859199523926,
      "logps/chosen": -46.154388427734375,
      "logps/rejected": -46.573265075683594,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -46.154388427734375,
      "rewards/margins": 0.4188779890537262,
      "rewards/rejected": -46.573265075683594,
      "step": 4405
    },
    {
      "epoch": 2.3602609131961865,
      "grad_norm": 4.631412221361335e-07,
      "learning_rate": 1.3205478209251874e-07,
      "logits/chosen": 9.432363510131836,
      "logits/rejected": 9.208551406860352,
      "logps/chosen": -45.91524124145508,
      "logps/rejected": -46.409584045410156,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.91524124145508,
      "rewards/margins": 0.4943412244319916,
      "rewards/rejected": -46.409584045410156,
      "step": 4410
    },
    {
      "epoch": 2.362936945977588,
      "grad_norm": 1.1204142178340912e-08,
      "learning_rate": 1.310020503720254e-07,
      "logits/chosen": 9.017038345336914,
      "logits/rejected": 8.273764610290527,
      "logps/chosen": -44.94533157348633,
      "logps/rejected": -46.261451721191406,
      "loss": 0.0002,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.94533157348633,
      "rewards/margins": 1.3161191940307617,
      "rewards/rejected": -46.261451721191406,
      "step": 4415
    },
    {
      "epoch": 2.36561297875899,
      "grad_norm": 3.5695191901198563e-13,
      "learning_rate": 1.2995289867125752e-07,
      "logits/chosen": 9.239496231079102,
      "logits/rejected": 8.94532585144043,
      "logps/chosen": -46.1478271484375,
      "logps/rejected": -47.16299819946289,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -46.1478271484375,
      "rewards/margins": 1.0151664018630981,
      "rewards/rejected": -47.16299819946289,
      "step": 4420
    },
    {
      "epoch": 2.368289011540391,
      "grad_norm": 3.94843649897232e-09,
      "learning_rate": 1.2890733716908986e-07,
      "logits/chosen": 9.131144523620605,
      "logits/rejected": 9.097776412963867,
      "logps/chosen": -44.41114044189453,
      "logps/rejected": -45.267616271972656,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.41114044189453,
      "rewards/margins": 0.8564763069152832,
      "rewards/rejected": -45.267616271972656,
      "step": 4425
    },
    {
      "epoch": 2.370965044321793,
      "grad_norm": 1.1257750213026578e-10,
      "learning_rate": 1.2786537600956454e-07,
      "logits/chosen": 9.508583068847656,
      "logits/rejected": 8.710375785827637,
      "logps/chosen": -44.781517028808594,
      "logps/rejected": -46.413063049316406,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -44.781517028808594,
      "rewards/margins": 1.6315491199493408,
      "rewards/rejected": -46.413063049316406,
      "step": 4430
    },
    {
      "epoch": 2.3736410771031946,
      "grad_norm": 3.8515860687231255e-08,
      "learning_rate": 1.268270253017933e-07,
      "logits/chosen": 9.412229537963867,
      "logits/rejected": 8.651571273803711,
      "logps/chosen": -45.56482696533203,
      "logps/rejected": -46.47025680541992,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.56482696533203,
      "rewards/margins": 0.9054354429244995,
      "rewards/rejected": -46.47025680541992,
      "step": 4435
    },
    {
      "epoch": 2.376317109884596,
      "grad_norm": 1.7666303876799908e-09,
      "learning_rate": 1.257922951198591e-07,
      "logits/chosen": 9.349123001098633,
      "logits/rejected": 8.273117065429688,
      "logps/chosen": -44.951072692871094,
      "logps/rejected": -45.71271514892578,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.951072692871094,
      "rewards/margins": 0.7616437673568726,
      "rewards/rejected": -45.71271514892578,
      "step": 4440
    },
    {
      "epoch": 2.3789931426659976,
      "grad_norm": 1.5327142779836557e-09,
      "learning_rate": 1.24761195502719e-07,
      "logits/chosen": 9.236307144165039,
      "logits/rejected": 8.378153800964355,
      "logps/chosen": -45.35531234741211,
      "logps/rejected": -46.683067321777344,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -45.35531234741211,
      "rewards/margins": 1.327751874923706,
      "rewards/rejected": -46.683067321777344,
      "step": 4445
    },
    {
      "epoch": 2.3816691754473993,
      "grad_norm": 0.0001557290511347757,
      "learning_rate": 1.2373373645410573e-07,
      "logits/chosen": 9.191245079040527,
      "logits/rejected": 8.729669570922852,
      "logps/chosen": -46.34674835205078,
      "logps/rejected": -46.783287048339844,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -46.34674835205078,
      "rewards/margins": 0.43653780221939087,
      "rewards/rejected": -46.783287048339844,
      "step": 4450
    },
    {
      "epoch": 2.384345208228801,
      "grad_norm": 9.924119677675345e-11,
      "learning_rate": 1.2270992794243175e-07,
      "logits/chosen": 9.469690322875977,
      "logits/rejected": 8.919861793518066,
      "logps/chosen": -46.28891372680664,
      "logps/rejected": -47.067176818847656,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -46.28891372680664,
      "rewards/margins": 0.7782581448554993,
      "rewards/rejected": -47.067176818847656,
      "step": 4455
    },
    {
      "epoch": 2.3870212410102023,
      "grad_norm": 9.790016500948385e-10,
      "learning_rate": 1.2168977990069147e-07,
      "logits/chosen": 9.747946739196777,
      "logits/rejected": 8.933435440063477,
      "logps/chosen": -46.40081024169922,
      "logps/rejected": -47.27013397216797,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -46.40081024169922,
      "rewards/margins": 0.8693264722824097,
      "rewards/rejected": -47.27013397216797,
      "step": 4460
    },
    {
      "epoch": 2.389697273791604,
      "grad_norm": 1.447470691979032e-10,
      "learning_rate": 1.206733022263659e-07,
      "logits/chosen": 9.325894355773926,
      "logits/rejected": 8.80048656463623,
      "logps/chosen": -44.30507278442383,
      "logps/rejected": -45.68259811401367,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.30507278442383,
      "rewards/margins": 1.3775246143341064,
      "rewards/rejected": -45.68259811401367,
      "step": 4465
    },
    {
      "epoch": 2.3923733065730053,
      "grad_norm": 7.348283309899994e-10,
      "learning_rate": 1.1966050478132572e-07,
      "logits/chosen": 9.299378395080566,
      "logits/rejected": 8.914216995239258,
      "logps/chosen": -46.35230255126953,
      "logps/rejected": -46.77234649658203,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -46.35230255126953,
      "rewards/margins": 0.4200368821620941,
      "rewards/rejected": -46.77234649658203,
      "step": 4470
    },
    {
      "epoch": 2.395049339354407,
      "grad_norm": 8.29060641631744e-09,
      "learning_rate": 1.1865139739173635e-07,
      "logits/chosen": 9.377392768859863,
      "logits/rejected": 8.560579299926758,
      "logps/chosen": -47.37395095825195,
      "logps/rejected": -47.59191131591797,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -47.37395095825195,
      "rewards/margins": 0.21795757114887238,
      "rewards/rejected": -47.59191131591797,
      "step": 4475
    },
    {
      "epoch": 2.3977253721358087,
      "grad_norm": 3.926671285348592e-08,
      "learning_rate": 1.1764598984796187e-07,
      "logits/chosen": 9.74540901184082,
      "logits/rejected": 9.097370147705078,
      "logps/chosen": -45.65620040893555,
      "logps/rejected": -46.510093688964844,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.65620040893555,
      "rewards/margins": 0.8538895845413208,
      "rewards/rejected": -46.510093688964844,
      "step": 4480
    },
    {
      "epoch": 2.4004014049172104,
      "grad_norm": 8.507423999212602e-11,
      "learning_rate": 1.1664429190447095e-07,
      "logits/chosen": 8.89933967590332,
      "logits/rejected": 8.700769424438477,
      "logps/chosen": -44.45652770996094,
      "logps/rejected": -46.14180374145508,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.45652770996094,
      "rewards/margins": 1.6852710247039795,
      "rewards/rejected": -46.14180374145508,
      "step": 4485
    },
    {
      "epoch": 2.4030774376986117,
      "grad_norm": 6.139442033288743e-09,
      "learning_rate": 1.1564631327974122e-07,
      "logits/chosen": 9.319207191467285,
      "logits/rejected": 8.695261001586914,
      "logps/chosen": -45.07764434814453,
      "logps/rejected": -46.976680755615234,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -45.07764434814453,
      "rewards/margins": 1.8990408182144165,
      "rewards/rejected": -46.976680755615234,
      "step": 4490
    },
    {
      "epoch": 2.4057534704800134,
      "grad_norm": 5.735900730615429e-10,
      "learning_rate": 1.1465206365616587e-07,
      "logits/chosen": 9.48769474029541,
      "logits/rejected": 8.742941856384277,
      "logps/chosen": -45.64276885986328,
      "logps/rejected": -47.07363510131836,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -45.64276885986328,
      "rewards/margins": 1.430870771408081,
      "rewards/rejected": -47.07363510131836,
      "step": 4495
    },
    {
      "epoch": 2.408429503261415,
      "grad_norm": 2.93358601467871e-13,
      "learning_rate": 1.1366155267995887e-07,
      "logits/chosen": 9.417936325073242,
      "logits/rejected": 9.324278831481934,
      "logps/chosen": -46.866615295410156,
      "logps/rejected": -47.489280700683594,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -46.866615295410156,
      "rewards/margins": 0.6226621866226196,
      "rewards/rejected": -47.489280700683594,
      "step": 4500
    },
    {
      "epoch": 2.4111055360428164,
      "grad_norm": 1.7033020208334332e-06,
      "learning_rate": 1.1267478996106228e-07,
      "logits/chosen": 9.425454139709473,
      "logits/rejected": 8.606499671936035,
      "logps/chosen": -44.77401351928711,
      "logps/rejected": -46.28554916381836,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.77401351928711,
      "rewards/margins": 1.5115344524383545,
      "rewards/rejected": -46.28554916381836,
      "step": 4505
    },
    {
      "epoch": 2.413781568824218,
      "grad_norm": 9.149577410786956e-09,
      "learning_rate": 1.116917850730521e-07,
      "logits/chosen": 9.437515258789062,
      "logits/rejected": 8.742735862731934,
      "logps/chosen": -46.2396354675293,
      "logps/rejected": -46.017433166503906,
      "loss": 0.0,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -46.2396354675293,
      "rewards/margins": -0.2221999615430832,
      "rewards/rejected": -46.017433166503906,
      "step": 4510
    },
    {
      "epoch": 2.41645760160562,
      "grad_norm": 5.698531082970119e-09,
      "learning_rate": 1.1071254755304637e-07,
      "logits/chosen": 9.066153526306152,
      "logits/rejected": 8.694692611694336,
      "logps/chosen": -45.34435272216797,
      "logps/rejected": -45.92352294921875,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.34435272216797,
      "rewards/margins": 0.5791710019111633,
      "rewards/rejected": -45.92352294921875,
      "step": 4515
    },
    {
      "epoch": 2.419133634387021,
      "grad_norm": 9.304096003270801e-11,
      "learning_rate": 1.0973708690161143e-07,
      "logits/chosen": 9.382495880126953,
      "logits/rejected": 8.875944137573242,
      "logps/chosen": -44.22921371459961,
      "logps/rejected": -46.228904724121094,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -44.22921371459961,
      "rewards/margins": 1.9996871948242188,
      "rewards/rejected": -46.228904724121094,
      "step": 4520
    },
    {
      "epoch": 2.421809667168423,
      "grad_norm": 3.110317215988051e-11,
      "learning_rate": 1.0876541258267119e-07,
      "logits/chosen": 9.274861335754395,
      "logits/rejected": 8.571756362915039,
      "logps/chosen": -44.45564651489258,
      "logps/rejected": -45.201568603515625,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.45564651489258,
      "rewards/margins": 0.7459216117858887,
      "rewards/rejected": -45.201568603515625,
      "step": 4525
    },
    {
      "epoch": 2.4244856999498245,
      "grad_norm": 3.0318135619720165e-07,
      "learning_rate": 1.0779753402341379e-07,
      "logits/chosen": 9.577934265136719,
      "logits/rejected": 9.126605033874512,
      "logps/chosen": -44.972469329833984,
      "logps/rejected": -46.0164680480957,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -44.972469329833984,
      "rewards/margins": 1.043993592262268,
      "rewards/rejected": -46.0164680480957,
      "step": 4530
    },
    {
      "epoch": 2.427161732731226,
      "grad_norm": 2.5505909153490574e-09,
      "learning_rate": 1.0683346061420157e-07,
      "logits/chosen": 9.555131912231445,
      "logits/rejected": 9.077537536621094,
      "logps/chosen": -45.720611572265625,
      "logps/rejected": -47.41533660888672,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -45.720611572265625,
      "rewards/margins": 1.6947218179702759,
      "rewards/rejected": -47.41533660888672,
      "step": 4535
    },
    {
      "epoch": 2.4298377655126275,
      "grad_norm": 5.27733242231109e-08,
      "learning_rate": 1.0587320170847874e-07,
      "logits/chosen": 9.469239234924316,
      "logits/rejected": 8.999317169189453,
      "logps/chosen": -47.020782470703125,
      "logps/rejected": -47.056549072265625,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -47.020782470703125,
      "rewards/margins": 0.03576944023370743,
      "rewards/rejected": -47.056549072265625,
      "step": 4540
    },
    {
      "epoch": 2.4325137982940293,
      "grad_norm": 3.798754829927539e-08,
      "learning_rate": 1.0491676662268156e-07,
      "logits/chosen": 9.277535438537598,
      "logits/rejected": 8.860095024108887,
      "logps/chosen": -46.3672981262207,
      "logps/rejected": -47.542755126953125,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -46.3672981262207,
      "rewards/margins": 1.1754497289657593,
      "rewards/rejected": -47.542755126953125,
      "step": 4545
    },
    {
      "epoch": 2.4351898310754305,
      "grad_norm": 3.6315119125631423e-07,
      "learning_rate": 1.0396416463614732e-07,
      "logits/chosen": 9.187960624694824,
      "logits/rejected": 9.037254333496094,
      "logps/chosen": -44.55503845214844,
      "logps/rejected": -45.899452209472656,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.55503845214844,
      "rewards/margins": 1.3444080352783203,
      "rewards/rejected": -45.899452209472656,
      "step": 4550
    },
    {
      "epoch": 2.4378658638568322,
      "grad_norm": 1.778875126239864e-11,
      "learning_rate": 1.0301540499102479e-07,
      "logits/chosen": 9.221136093139648,
      "logits/rejected": 9.0938081741333,
      "logps/chosen": -45.838050842285156,
      "logps/rejected": -46.866294860839844,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -45.838050842285156,
      "rewards/margins": 1.028247594833374,
      "rewards/rejected": -46.866294860839844,
      "step": 4555
    },
    {
      "epoch": 2.440541896638234,
      "grad_norm": 4.960396175846367e-08,
      "learning_rate": 1.0207049689218405e-07,
      "logits/chosen": 9.349725723266602,
      "logits/rejected": 8.359013557434082,
      "logps/chosen": -44.666236877441406,
      "logps/rejected": -46.120059967041016,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.666236877441406,
      "rewards/margins": 1.4538233280181885,
      "rewards/rejected": -46.120059967041016,
      "step": 4560
    },
    {
      "epoch": 2.4432179294196352,
      "grad_norm": 0.16436842523927525,
      "learning_rate": 1.0112944950712782e-07,
      "logits/chosen": 8.953794479370117,
      "logits/rejected": 8.54833698272705,
      "logps/chosen": -44.6590576171875,
      "logps/rejected": -45.62831115722656,
      "loss": 0.0005,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.6590576171875,
      "rewards/margins": 0.9692533612251282,
      "rewards/rejected": -45.62831115722656,
      "step": 4565
    },
    {
      "epoch": 2.445893962201037,
      "grad_norm": 6.274198190798086e-08,
      "learning_rate": 1.0019227196590174e-07,
      "logits/chosen": 9.414281845092773,
      "logits/rejected": 8.802621841430664,
      "logps/chosen": -45.33159637451172,
      "logps/rejected": -47.08720779418945,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -45.33159637451172,
      "rewards/margins": 1.7556155920028687,
      "rewards/rejected": -47.08720779418945,
      "step": 4570
    },
    {
      "epoch": 2.4485699949824387,
      "grad_norm": 1.2481204070808861e-11,
      "learning_rate": 9.925897336100664e-08,
      "logits/chosen": 8.701372146606445,
      "logits/rejected": 8.632661819458008,
      "logps/chosen": -46.3024787902832,
      "logps/rejected": -46.93836212158203,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -46.3024787902832,
      "rewards/margins": 0.6358833312988281,
      "rewards/rejected": -46.93836212158203,
      "step": 4575
    },
    {
      "epoch": 2.45124602776384,
      "grad_norm": 3.6093327078131443e-10,
      "learning_rate": 9.832956274730946e-08,
      "logits/chosen": 8.811094284057617,
      "logits/rejected": 8.597399711608887,
      "logps/chosen": -44.7505989074707,
      "logps/rejected": -45.92190170288086,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.7505989074707,
      "rewards/margins": 1.1713066101074219,
      "rewards/rejected": -45.92190170288086,
      "step": 4580
    },
    {
      "epoch": 2.4539220605452416,
      "grad_norm": 2.857435071134369e-07,
      "learning_rate": 9.740404914195633e-08,
      "logits/chosen": 9.29952621459961,
      "logits/rejected": 8.922680854797363,
      "logps/chosen": -45.47762680053711,
      "logps/rejected": -46.04093551635742,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.47762680053711,
      "rewards/margins": 0.5633118748664856,
      "rewards/rejected": -46.04093551635742,
      "step": 4585
    },
    {
      "epoch": 2.4565980933266434,
      "grad_norm": 1.7215797841753174e-09,
      "learning_rate": 9.648244152428392e-08,
      "logits/chosen": 9.362550735473633,
      "logits/rejected": 9.023419380187988,
      "logps/chosen": -45.469871520996094,
      "logps/rejected": -47.061222076416016,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -45.469871520996094,
      "rewards/margins": 1.591349720954895,
      "rewards/rejected": -47.061222076416016,
      "step": 4590
    },
    {
      "epoch": 2.4592741261080446,
      "grad_norm": 2.5432610267504006e-06,
      "learning_rate": 9.556474883573379e-08,
      "logits/chosen": 9.353912353515625,
      "logits/rejected": 8.983583450317383,
      "logps/chosen": -45.581565856933594,
      "logps/rejected": -47.159000396728516,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -45.581565856933594,
      "rewards/margins": 1.5774341821670532,
      "rewards/rejected": -47.159000396728516,
      "step": 4595
    },
    {
      "epoch": 2.4619501588894463,
      "grad_norm": 1.0522490338372249e-14,
      "learning_rate": 9.465097997976412e-08,
      "logits/chosen": 9.360940933227539,
      "logits/rejected": 8.549973487854004,
      "logps/chosen": -44.684513092041016,
      "logps/rejected": -46.31632614135742,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.684513092041016,
      "rewards/margins": 1.6318145990371704,
      "rewards/rejected": -46.31632614135742,
      "step": 4600
    },
    {
      "epoch": 2.464626191670848,
      "grad_norm": 1.919910469591367e-09,
      "learning_rate": 9.374114382176457e-08,
      "logits/chosen": 9.609962463378906,
      "logits/rejected": 8.819144248962402,
      "logps/chosen": -45.517860412597656,
      "logps/rejected": -45.98442840576172,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -45.517860412597656,
      "rewards/margins": 0.4665655195713043,
      "rewards/rejected": -45.98442840576172,
      "step": 4605
    },
    {
      "epoch": 2.46730222445225,
      "grad_norm": 0.0002873129507900905,
      "learning_rate": 9.283524918896945e-08,
      "logits/chosen": 9.610032081604004,
      "logits/rejected": 9.007634162902832,
      "logps/chosen": -46.157405853271484,
      "logps/rejected": -47.55231475830078,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -46.157405853271484,
      "rewards/margins": 1.3949075937271118,
      "rewards/rejected": -47.55231475830078,
      "step": 4610
    },
    {
      "epoch": 2.469978257233651,
      "grad_norm": 5.626665096009374e-14,
      "learning_rate": 9.193330487037232e-08,
      "logits/chosen": 9.33863639831543,
      "logits/rejected": 8.881044387817383,
      "logps/chosen": -45.47136688232422,
      "logps/rejected": -47.027915954589844,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -45.47136688232422,
      "rewards/margins": 1.55655038356781,
      "rewards/rejected": -47.027915954589844,
      "step": 4615
    },
    {
      "epoch": 2.4726542900150528,
      "grad_norm": 2.784722405085902e-14,
      "learning_rate": 9.103531961664118e-08,
      "logits/chosen": 9.091102600097656,
      "logits/rejected": 8.526533126831055,
      "logps/chosen": -46.0606689453125,
      "logps/rejected": -47.78614044189453,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -46.0606689453125,
      "rewards/margins": 1.7254736423492432,
      "rewards/rejected": -47.78614044189453,
      "step": 4620
    },
    {
      "epoch": 2.475330322796454,
      "grad_norm": 7.646662074779495e-11,
      "learning_rate": 9.014130214003269e-08,
      "logits/chosen": 9.005826950073242,
      "logits/rejected": 9.009138107299805,
      "logps/chosen": -44.47298049926758,
      "logps/rejected": -45.722564697265625,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.47298049926758,
      "rewards/margins": 1.2495874166488647,
      "rewards/rejected": -45.722564697265625,
      "step": 4625
    },
    {
      "epoch": 2.4780063555778558,
      "grad_norm": 3.6155206269495154e-08,
      "learning_rate": 8.925126111430848e-08,
      "logits/chosen": 8.5174560546875,
      "logits/rejected": 8.313937187194824,
      "logps/chosen": -44.49368667602539,
      "logps/rejected": -45.462669372558594,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -44.49368667602539,
      "rewards/margins": 0.9689850807189941,
      "rewards/rejected": -45.462669372558594,
      "step": 4630
    },
    {
      "epoch": 2.4806823883592575,
      "grad_norm": 6.034274801515113e-09,
      "learning_rate": 8.83652051746504e-08,
      "logits/chosen": 9.42755126953125,
      "logits/rejected": 8.577604293823242,
      "logps/chosen": -45.6042366027832,
      "logps/rejected": -45.777984619140625,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -45.6042366027832,
      "rewards/margins": 0.17375202476978302,
      "rewards/rejected": -45.777984619140625,
      "step": 4635
    },
    {
      "epoch": 2.483358421140659,
      "grad_norm": 8.410236931568268e-12,
      "learning_rate": 8.748314291757696e-08,
      "logits/chosen": 8.98786735534668,
      "logits/rejected": 8.812253952026367,
      "logps/chosen": -44.325767517089844,
      "logps/rejected": -46.640419006347656,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.325767517089844,
      "rewards/margins": 2.3146510124206543,
      "rewards/rejected": -46.640419006347656,
      "step": 4640
    },
    {
      "epoch": 2.4860344539220605,
      "grad_norm": 0.00022929275780233266,
      "learning_rate": 8.660508290086032e-08,
      "logits/chosen": 9.35904598236084,
      "logits/rejected": 8.609199523925781,
      "logps/chosen": -44.49699020385742,
      "logps/rejected": -45.26287078857422,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.49699020385742,
      "rewards/margins": 0.7658793330192566,
      "rewards/rejected": -45.26287078857422,
      "step": 4645
    },
    {
      "epoch": 2.488710486703462,
      "grad_norm": 2.1788396257575245e-10,
      "learning_rate": 8.573103364344231e-08,
      "logits/chosen": 9.593141555786133,
      "logits/rejected": 8.84683609008789,
      "logps/chosen": -44.61894607543945,
      "logps/rejected": -47.17866516113281,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -44.61894607543945,
      "rewards/margins": 2.5597167015075684,
      "rewards/rejected": -47.17866516113281,
      "step": 4650
    },
    {
      "epoch": 2.4913865194848634,
      "grad_norm": 4.96122854968039e-11,
      "learning_rate": 8.486100362535292e-08,
      "logits/chosen": 9.201153755187988,
      "logits/rejected": 8.769639015197754,
      "logps/chosen": -44.683860778808594,
      "logps/rejected": -45.771324157714844,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.683860778808594,
      "rewards/margins": 1.0874602794647217,
      "rewards/rejected": -45.771324157714844,
      "step": 4655
    },
    {
      "epoch": 2.494062552266265,
      "grad_norm": 4.463398118261067e-06,
      "learning_rate": 8.399500128762693e-08,
      "logits/chosen": 8.940571784973145,
      "logits/rejected": 8.48813247680664,
      "logps/chosen": -45.75236511230469,
      "logps/rejected": -46.339393615722656,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -45.75236511230469,
      "rewards/margins": 0.5870294570922852,
      "rewards/rejected": -46.339393615722656,
      "step": 4660
    },
    {
      "epoch": 2.496738585047667,
      "grad_norm": 8.822704159807692e-09,
      "learning_rate": 8.313303503222313e-08,
      "logits/chosen": 9.366979598999023,
      "logits/rejected": 8.9149751663208,
      "logps/chosen": -45.231842041015625,
      "logps/rejected": -44.39390182495117,
      "loss": 0.0,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -45.231842041015625,
      "rewards/margins": -0.837939441204071,
      "rewards/rejected": -44.39390182495117,
      "step": 4665
    },
    {
      "epoch": 2.4994146178290686,
      "grad_norm": 2.3537000462418865e-08,
      "learning_rate": 8.227511322194164e-08,
      "logits/chosen": 9.373997688293457,
      "logits/rejected": 8.870220184326172,
      "logps/chosen": -44.252403259277344,
      "logps/rejected": -45.56930160522461,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.252403259277344,
      "rewards/margins": 1.3168978691101074,
      "rewards/rejected": -45.56930160522461,
      "step": 4670
    },
    {
      "epoch": 2.50209065061047,
      "grad_norm": 4.1760686522277836e-10,
      "learning_rate": 8.142124418034385e-08,
      "logits/chosen": 9.223678588867188,
      "logits/rejected": 8.315411567687988,
      "logps/chosen": -45.74673843383789,
      "logps/rejected": -46.79120635986328,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.74673843383789,
      "rewards/margins": 1.0444693565368652,
      "rewards/rejected": -46.79120635986328,
      "step": 4675
    },
    {
      "epoch": 2.5047666833918716,
      "grad_norm": 4.035979974833743e-10,
      "learning_rate": 8.057143619167073e-08,
      "logits/chosen": 8.787276268005371,
      "logits/rejected": 8.599661827087402,
      "logps/chosen": -45.0388298034668,
      "logps/rejected": -46.18868637084961,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -45.0388298034668,
      "rewards/margins": 1.1498557329177856,
      "rewards/rejected": -46.18868637084961,
      "step": 4680
    },
    {
      "epoch": 2.507442716173273,
      "grad_norm": 2.369139555014432e-11,
      "learning_rate": 7.97256975007633e-08,
      "logits/chosen": 9.203855514526367,
      "logits/rejected": 8.421099662780762,
      "logps/chosen": -44.24137496948242,
      "logps/rejected": -45.653568267822266,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -44.24137496948242,
      "rewards/margins": 1.4121909141540527,
      "rewards/rejected": -45.653568267822266,
      "step": 4685
    },
    {
      "epoch": 2.5101187489546746,
      "grad_norm": 0.08629806542730972,
      "learning_rate": 7.888403631298186e-08,
      "logits/chosen": 8.74711799621582,
      "logits/rejected": 8.82441520690918,
      "logps/chosen": -44.985877990722656,
      "logps/rejected": -46.11347198486328,
      "loss": 0.0001,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -44.985877990722656,
      "rewards/margins": 1.127590298652649,
      "rewards/rejected": -46.11347198486328,
      "step": 4690
    },
    {
      "epoch": 2.5127947817360763,
      "grad_norm": 1.1983961976269943e-09,
      "learning_rate": 7.804646079412719e-08,
      "logits/chosen": 8.926512718200684,
      "logits/rejected": 8.411669731140137,
      "logps/chosen": -44.288780212402344,
      "logps/rejected": -45.819637298583984,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.288780212402344,
      "rewards/margins": 1.5308585166931152,
      "rewards/rejected": -45.819637298583984,
      "step": 4695
    },
    {
      "epoch": 2.515470814517478,
      "grad_norm": 2.3654903468343515e-10,
      "learning_rate": 7.72129790703604e-08,
      "logits/chosen": 9.372162818908691,
      "logits/rejected": 8.877138137817383,
      "logps/chosen": -44.409523010253906,
      "logps/rejected": -45.571022033691406,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.409523010253906,
      "rewards/margins": 1.1614948511123657,
      "rewards/rejected": -45.571022033691406,
      "step": 4700
    },
    {
      "epoch": 2.5181468472988793,
      "grad_norm": 4.356253305242245e-05,
      "learning_rate": 7.638359922812504e-08,
      "logits/chosen": 9.020631790161133,
      "logits/rejected": 8.69045639038086,
      "logps/chosen": -45.277976989746094,
      "logps/rejected": -45.649879455566406,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.277976989746094,
      "rewards/margins": 0.3719016909599304,
      "rewards/rejected": -45.649879455566406,
      "step": 4705
    },
    {
      "epoch": 2.520822880080281,
      "grad_norm": 4.9689550402185556e-08,
      "learning_rate": 7.555832931406774e-08,
      "logits/chosen": 9.383692741394043,
      "logits/rejected": 8.596406936645508,
      "logps/chosen": -44.811134338378906,
      "logps/rejected": -46.374969482421875,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -44.811134338378906,
      "rewards/margins": 1.5638368129730225,
      "rewards/rejected": -46.374969482421875,
      "step": 4710
    },
    {
      "epoch": 2.5234989128616827,
      "grad_norm": 1.1020249633319474e-13,
      "learning_rate": 7.47371773349611e-08,
      "logits/chosen": 9.111364364624023,
      "logits/rejected": 9.082585334777832,
      "logps/chosen": -45.052207946777344,
      "logps/rejected": -46.28043746948242,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -45.052207946777344,
      "rewards/margins": 1.22823166847229,
      "rewards/rejected": -46.28043746948242,
      "step": 4715
    },
    {
      "epoch": 2.526174945643084,
      "grad_norm": 8.562396458008921e-06,
      "learning_rate": 7.392015125762496e-08,
      "logits/chosen": 9.160844802856445,
      "logits/rejected": 8.723586082458496,
      "logps/chosen": -45.42751693725586,
      "logps/rejected": -46.31050491333008,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -45.42751693725586,
      "rewards/margins": 0.8829911947250366,
      "rewards/rejected": -46.31050491333008,
      "step": 4720
    },
    {
      "epoch": 2.5288509784244857,
      "grad_norm": 1.0929138127294639e-07,
      "learning_rate": 7.310725900885018e-08,
      "logits/chosen": 9.284099578857422,
      "logits/rejected": 9.149104118347168,
      "logps/chosen": -45.62548065185547,
      "logps/rejected": -45.44000244140625,
      "loss": 0.0,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -45.62548065185547,
      "rewards/margins": -0.1854783147573471,
      "rewards/rejected": -45.44000244140625,
      "step": 4725
    },
    {
      "epoch": 2.5315270112058874,
      "grad_norm": 3.4945987257728354e-10,
      "learning_rate": 7.229850847532076e-08,
      "logits/chosen": 8.984869003295898,
      "logits/rejected": 8.462522506713867,
      "logps/chosen": -45.275177001953125,
      "logps/rejected": -46.592857360839844,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.275177001953125,
      "rewards/margins": 1.3176777362823486,
      "rewards/rejected": -46.592857360839844,
      "step": 4730
    },
    {
      "epoch": 2.5342030439872887,
      "grad_norm": 2.997115039571154e-09,
      "learning_rate": 7.149390750353779e-08,
      "logits/chosen": 8.56629753112793,
      "logits/rejected": 8.94212532043457,
      "logps/chosen": -44.24767303466797,
      "logps/rejected": -44.74664306640625,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.24767303466797,
      "rewards/margins": 0.49897074699401855,
      "rewards/rejected": -44.74664306640625,
      "step": 4735
    },
    {
      "epoch": 2.5368790767686904,
      "grad_norm": 5.046984975993097e-11,
      "learning_rate": 7.069346389974374e-08,
      "logits/chosen": 9.294145584106445,
      "logits/rejected": 8.642403602600098,
      "logps/chosen": -44.31879425048828,
      "logps/rejected": -45.810272216796875,
      "loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -44.31879425048828,
      "rewards/margins": 1.4914792776107788,
      "rewards/rejected": -45.810272216796875,
      "step": 4740
    },
    {
      "epoch": 2.539555109550092,
      "grad_norm": 1.0604687831645627e-08,
      "learning_rate": 6.989718542984563e-08,
      "logits/chosen": 9.119512557983398,
      "logits/rejected": 9.08793830871582,
      "logps/chosen": -45.18333435058594,
      "logps/rejected": -45.79257583618164,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -45.18333435058594,
      "rewards/margins": 0.6092401742935181,
      "rewards/rejected": -45.79257583618164,
      "step": 4745
    },
    {
      "epoch": 2.5422311423314934,
      "grad_norm": 7.272757017627416e-12,
      "learning_rate": 6.9105079819341e-08,
      "logits/chosen": 9.195124626159668,
      "logits/rejected": 7.980881690979004,
      "logps/chosen": -44.97095489501953,
      "logps/rejected": -46.54754638671875,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -44.97095489501953,
      "rewards/margins": 1.576589822769165,
      "rewards/rejected": -46.54754638671875,
      "step": 4750
    },
    {
      "epoch": 2.544907175112895,
      "grad_norm": 1.4940094569002793e-10,
      "learning_rate": 6.831715475324163e-08,
      "logits/chosen": 9.517915725708008,
      "logits/rejected": 8.83462142944336,
      "logps/chosen": -45.40341567993164,
      "logps/rejected": -46.67591094970703,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.40341567993164,
      "rewards/margins": 1.2724964618682861,
      "rewards/rejected": -46.67591094970703,
      "step": 4755
    },
    {
      "epoch": 2.547583207894297,
      "grad_norm": 8.162091286711141e-08,
      "learning_rate": 6.753341787600026e-08,
      "logits/chosen": 9.697515487670898,
      "logits/rejected": 9.264740943908691,
      "logps/chosen": -45.728389739990234,
      "logps/rejected": -46.27559280395508,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.728389739990234,
      "rewards/margins": 0.5472058057785034,
      "rewards/rejected": -46.27559280395508,
      "step": 4760
    },
    {
      "epoch": 2.5502592406756985,
      "grad_norm": 3.0021203671761005e-10,
      "learning_rate": 6.67538767914353e-08,
      "logits/chosen": 9.38265323638916,
      "logits/rejected": 8.777417182922363,
      "logps/chosen": -45.686248779296875,
      "logps/rejected": -46.63222885131836,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.686248779296875,
      "rewards/margins": 0.9459837079048157,
      "rewards/rejected": -46.63222885131836,
      "step": 4765
    },
    {
      "epoch": 2.5529352734571,
      "grad_norm": 1.0035318183347984e-05,
      "learning_rate": 6.597853906265793e-08,
      "logits/chosen": 8.941511154174805,
      "logits/rejected": 8.485795021057129,
      "logps/chosen": -44.67109298706055,
      "logps/rejected": -46.38446807861328,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.67109298706055,
      "rewards/margins": 1.7133716344833374,
      "rewards/rejected": -46.38446807861328,
      "step": 4770
    },
    {
      "epoch": 2.5556113062385015,
      "grad_norm": 8.42635325733941e-12,
      "learning_rate": 6.5207412211998e-08,
      "logits/chosen": 9.072559356689453,
      "logits/rejected": 8.78048324584961,
      "logps/chosen": -45.05380630493164,
      "logps/rejected": -45.90123748779297,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -45.05380630493164,
      "rewards/margins": 0.8474318385124207,
      "rewards/rejected": -45.90123748779297,
      "step": 4775
    },
    {
      "epoch": 2.558287339019903,
      "grad_norm": 3.3124994529049685e-07,
      "learning_rate": 6.444050372093186e-08,
      "logits/chosen": 9.298035621643066,
      "logits/rejected": 9.047917366027832,
      "logps/chosen": -45.26776885986328,
      "logps/rejected": -45.835784912109375,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -45.26776885986328,
      "rewards/margins": 0.5680132508277893,
      "rewards/rejected": -45.835784912109375,
      "step": 4780
    },
    {
      "epoch": 2.5609633718013045,
      "grad_norm": 6.303632951392163e-10,
      "learning_rate": 6.367782103000873e-08,
      "logits/chosen": 9.396265029907227,
      "logits/rejected": 8.952186584472656,
      "logps/chosen": -45.63679885864258,
      "logps/rejected": -46.1491813659668,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -45.63679885864258,
      "rewards/margins": 0.5123839378356934,
      "rewards/rejected": -46.1491813659668,
      "step": 4785
    },
    {
      "epoch": 2.5636394045827062,
      "grad_norm": 1.2025937520918248e-11,
      "learning_rate": 6.29193715387798e-08,
      "logits/chosen": 9.129429817199707,
      "logits/rejected": 8.66586685180664,
      "logps/chosen": -44.47456359863281,
      "logps/rejected": -45.37948989868164,
      "loss": 0.0005,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.47456359863281,
      "rewards/margins": 0.9049239158630371,
      "rewards/rejected": -45.37948989868164,
      "step": 4790
    },
    {
      "epoch": 2.566315437364108,
      "grad_norm": 1.1171569650958958e-11,
      "learning_rate": 6.216516260572502e-08,
      "logits/chosen": 9.242927551269531,
      "logits/rejected": 8.784564971923828,
      "logps/chosen": -45.46044158935547,
      "logps/rejected": -46.668663024902344,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.46044158935547,
      "rewards/margins": 1.2082191705703735,
      "rewards/rejected": -46.668663024902344,
      "step": 4795
    },
    {
      "epoch": 2.568991470145509,
      "grad_norm": 1.157158189837623e-08,
      "learning_rate": 6.141520154818297e-08,
      "logits/chosen": 9.50001049041748,
      "logits/rejected": 9.037002563476562,
      "logps/chosen": -45.338504791259766,
      "logps/rejected": -46.17444610595703,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.338504791259766,
      "rewards/margins": 0.8359421491622925,
      "rewards/rejected": -46.17444610595703,
      "step": 4800
    },
    {
      "epoch": 2.568991470145509,
      "eval_logits/chosen": 7.710291385650635,
      "eval_logits/rejected": 7.384316444396973,
      "eval_logps/chosen": -45.02006530761719,
      "eval_logps/rejected": -46.050052642822266,
      "eval_loss": 1.989791087453341e-07,
      "eval_rewards/accuracies": 0.5652819275856018,
      "eval_rewards/chosen": -45.02006530761719,
      "eval_rewards/margins": 1.0299879312515259,
      "eval_rewards/rejected": -46.050052642822266,
      "eval_runtime": 40.0333,
      "eval_samples_per_second": 33.597,
      "eval_steps_per_second": 8.418,
      "step": 4800
    },
    {
      "epoch": 2.571667502926911,
      "grad_norm": 6.499214345871628e-06,
      "learning_rate": 6.066949564227897e-08,
      "logits/chosen": 9.507647514343262,
      "logits/rejected": 9.029126167297363,
      "logps/chosen": -44.45769119262695,
      "logps/rejected": -45.58419418334961,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.45769119262695,
      "rewards/margins": 1.126503825187683,
      "rewards/rejected": -45.58419418334961,
      "step": 4805
    },
    {
      "epoch": 2.574343535708312,
      "grad_norm": 1.4421281941197793e-07,
      "learning_rate": 5.992805212285523e-08,
      "logits/chosen": 9.183524131774902,
      "logits/rejected": 8.913390159606934,
      "logps/chosen": -44.8923225402832,
      "logps/rejected": -46.285552978515625,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.8923225402832,
      "rewards/margins": 1.3932315111160278,
      "rewards/rejected": -46.285552978515625,
      "step": 4810
    },
    {
      "epoch": 2.577019568489714,
      "grad_norm": 4.722180713675045e-09,
      "learning_rate": 5.9190878183399684e-08,
      "logits/chosen": 9.28604793548584,
      "logits/rejected": 8.589982032775879,
      "logps/chosen": -45.522682189941406,
      "logps/rejected": -47.07054138183594,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.522682189941406,
      "rewards/margins": 1.5478566884994507,
      "rewards/rejected": -47.07054138183594,
      "step": 4815
    },
    {
      "epoch": 2.5796956012711156,
      "grad_norm": 1.4947791319226982e-07,
      "learning_rate": 5.845798097597748e-08,
      "logits/chosen": 8.920289039611816,
      "logits/rejected": 8.7565336227417,
      "logps/chosen": -45.001441955566406,
      "logps/rejected": -45.923892974853516,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.001441955566406,
      "rewards/margins": 0.9224470257759094,
      "rewards/rejected": -45.923892974853516,
      "step": 4820
    },
    {
      "epoch": 2.5823716340525174,
      "grad_norm": 2.4532938854060864e-08,
      "learning_rate": 5.772936761116026e-08,
      "logits/chosen": 9.255111694335938,
      "logits/rejected": 8.704314231872559,
      "logps/chosen": -45.045448303222656,
      "logps/rejected": -45.898258209228516,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.045448303222656,
      "rewards/margins": 0.8528038263320923,
      "rewards/rejected": -45.898258209228516,
      "step": 4825
    },
    {
      "epoch": 2.5850476668339186,
      "grad_norm": 3.2503159442402865e-13,
      "learning_rate": 5.700504515795829e-08,
      "logits/chosen": 9.270207405090332,
      "logits/rejected": 8.756840705871582,
      "logps/chosen": -44.40122604370117,
      "logps/rejected": -46.42242431640625,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.40122604370117,
      "rewards/margins": 2.02119517326355,
      "rewards/rejected": -46.42242431640625,
      "step": 4830
    },
    {
      "epoch": 2.5877236996153203,
      "grad_norm": 6.754692122295384e-13,
      "learning_rate": 5.628502064375101e-08,
      "logits/chosen": 9.396100997924805,
      "logits/rejected": 8.850330352783203,
      "logps/chosen": -45.30399703979492,
      "logps/rejected": -47.65460968017578,
      "loss": 0.0,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -45.30399703979492,
      "rewards/margins": 2.350609540939331,
      "rewards/rejected": -47.65460968017578,
      "step": 4835
    },
    {
      "epoch": 2.5903997323967216,
      "grad_norm": 1.1909399844058021e-09,
      "learning_rate": 5.55693010542197e-08,
      "logits/chosen": 9.542749404907227,
      "logits/rejected": 8.576563835144043,
      "logps/chosen": -45.77398681640625,
      "logps/rejected": -46.796165466308594,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.77398681640625,
      "rewards/margins": 1.0221812725067139,
      "rewards/rejected": -46.796165466308594,
      "step": 4840
    },
    {
      "epoch": 2.5930757651781233,
      "grad_norm": 9.533766066417347e-15,
      "learning_rate": 5.485789333327856e-08,
      "logits/chosen": 9.174176216125488,
      "logits/rejected": 8.996587753295898,
      "logps/chosen": -45.76953125,
      "logps/rejected": -47.08142852783203,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -45.76953125,
      "rewards/margins": 1.3118951320648193,
      "rewards/rejected": -47.08142852783203,
      "step": 4845
    },
    {
      "epoch": 2.595751797959525,
      "grad_norm": 8.564673224342437e-09,
      "learning_rate": 5.4150804383008675e-08,
      "logits/chosen": 9.380346298217773,
      "logits/rejected": 8.829231262207031,
      "logps/chosen": -44.38100051879883,
      "logps/rejected": -46.20612716674805,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -44.38100051879883,
      "rewards/margins": 1.8251266479492188,
      "rewards/rejected": -46.20612716674805,
      "step": 4850
    },
    {
      "epoch": 2.5984278307409268,
      "grad_norm": 1.320415489823604e-06,
      "learning_rate": 5.344804106359002e-08,
      "logits/chosen": 9.127296447753906,
      "logits/rejected": 8.540373802185059,
      "logps/chosen": -44.92371368408203,
      "logps/rejected": -46.46620559692383,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -44.92371368408203,
      "rewards/margins": 1.5424927473068237,
      "rewards/rejected": -46.46620559692383,
      "step": 4855
    },
    {
      "epoch": 2.601103863522328,
      "grad_norm": 2.961571037301974e-05,
      "learning_rate": 5.274961019323559e-08,
      "logits/chosen": 9.083234786987305,
      "logits/rejected": 8.71281909942627,
      "logps/chosen": -44.62846374511719,
      "logps/rejected": -46.3775749206543,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -44.62846374511719,
      "rewards/margins": 1.7491109371185303,
      "rewards/rejected": -46.3775749206543,
      "step": 4860
    },
    {
      "epoch": 2.6037798963037297,
      "grad_norm": 2.2087904674864526e-05,
      "learning_rate": 5.205551854812451e-08,
      "logits/chosen": 9.683609962463379,
      "logits/rejected": 9.186744689941406,
      "logps/chosen": -44.877342224121094,
      "logps/rejected": -44.92319107055664,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.877342224121094,
      "rewards/margins": 0.04584855958819389,
      "rewards/rejected": -44.92319107055664,
      "step": 4865
    },
    {
      "epoch": 2.606455929085131,
      "grad_norm": 5.4131213651298994e-12,
      "learning_rate": 5.1365772862337177e-08,
      "logits/chosen": 9.131199836730957,
      "logits/rejected": 8.844860076904297,
      "logps/chosen": -45.55270767211914,
      "logps/rejected": -47.496360778808594,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -45.55270767211914,
      "rewards/margins": 1.9436581134796143,
      "rewards/rejected": -47.496360778808594,
      "step": 4870
    },
    {
      "epoch": 2.6091319618665327,
      "grad_norm": 7.395400343027402e-06,
      "learning_rate": 5.068037982778905e-08,
      "logits/chosen": 9.156936645507812,
      "logits/rejected": 8.87824535369873,
      "logps/chosen": -45.61544418334961,
      "logps/rejected": -46.70695877075195,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.61544418334961,
      "rewards/margins": 1.091510534286499,
      "rewards/rejected": -46.70695877075195,
      "step": 4875
    },
    {
      "epoch": 2.6118079946479344,
      "grad_norm": 9.81709626915334e-12,
      "learning_rate": 4.999934609416656e-08,
      "logits/chosen": 9.462814331054688,
      "logits/rejected": 8.919404029846191,
      "logps/chosen": -45.523868560791016,
      "logps/rejected": -47.12161636352539,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.523868560791016,
      "rewards/margins": 1.5977489948272705,
      "rewards/rejected": -47.12161636352539,
      "step": 4880
    },
    {
      "epoch": 2.614484027429336,
      "grad_norm": 1.2635584094542929e-06,
      "learning_rate": 4.932267826886183e-08,
      "logits/chosen": 9.188573837280273,
      "logits/rejected": 9.109978675842285,
      "logps/chosen": -45.007667541503906,
      "logps/rejected": -45.94084930419922,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.007667541503906,
      "rewards/margins": 0.9331871867179871,
      "rewards/rejected": -45.94084930419922,
      "step": 4885
    },
    {
      "epoch": 2.6171600602107374,
      "grad_norm": 3.1849432091594705e-12,
      "learning_rate": 4.8650382916909206e-08,
      "logits/chosen": 9.601472854614258,
      "logits/rejected": 8.907073020935059,
      "logps/chosen": -45.633819580078125,
      "logps/rejected": -46.83782196044922,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.633819580078125,
      "rewards/margins": 1.2039997577667236,
      "rewards/rejected": -46.83782196044922,
      "step": 4890
    },
    {
      "epoch": 2.619836092992139,
      "grad_norm": 0.10605037394882505,
      "learning_rate": 4.7982466560920976e-08,
      "logits/chosen": 9.186003684997559,
      "logits/rejected": 8.89367389678955,
      "logps/chosen": -45.44621658325195,
      "logps/rejected": -44.77003479003906,
      "loss": 0.0008,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -45.44621658325195,
      "rewards/margins": -0.6761819124221802,
      "rewards/rejected": -44.77003479003906,
      "step": 4895
    },
    {
      "epoch": 2.622512125773541,
      "grad_norm": 1.4580485237284184e-05,
      "learning_rate": 4.7318935681024685e-08,
      "logits/chosen": 9.082255363464355,
      "logits/rejected": 8.590585708618164,
      "logps/chosen": -45.023277282714844,
      "logps/rejected": -45.838409423828125,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.023277282714844,
      "rewards/margins": 0.8151308298110962,
      "rewards/rejected": -45.838409423828125,
      "step": 4900
    },
    {
      "epoch": 2.625188158554942,
      "grad_norm": 9.91795759185946e-07,
      "learning_rate": 4.6659796714799745e-08,
      "logits/chosen": 8.826153755187988,
      "logits/rejected": 8.30756950378418,
      "logps/chosen": -44.310428619384766,
      "logps/rejected": -45.227027893066406,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.310428619384766,
      "rewards/margins": 0.916598916053772,
      "rewards/rejected": -45.227027893066406,
      "step": 4905
    },
    {
      "epoch": 2.627864191336344,
      "grad_norm": 1.9415197095929207e-09,
      "learning_rate": 4.60050560572155e-08,
      "logits/chosen": 9.118809700012207,
      "logits/rejected": 9.228595733642578,
      "logps/chosen": -46.4311408996582,
      "logps/rejected": -46.904972076416016,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -46.4311408996582,
      "rewards/margins": 0.47382792830467224,
      "rewards/rejected": -46.904972076416016,
      "step": 4910
    },
    {
      "epoch": 2.6305402241177456,
      "grad_norm": 7.816571800878812e-11,
      "learning_rate": 4.535472006056834e-08,
      "logits/chosen": 9.280956268310547,
      "logits/rejected": 8.73022747039795,
      "logps/chosen": -45.65574645996094,
      "logps/rejected": -46.302879333496094,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.65574645996094,
      "rewards/margins": 0.6471337676048279,
      "rewards/rejected": -46.302879333496094,
      "step": 4915
    },
    {
      "epoch": 2.6332162568991473,
      "grad_norm": 1.0681511440772666e-13,
      "learning_rate": 4.470879503442132e-08,
      "logits/chosen": 9.052552223205566,
      "logits/rejected": 8.837255477905273,
      "logps/chosen": -45.657676696777344,
      "logps/rejected": -47.061119079589844,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.657676696777344,
      "rewards/margins": 1.4034405946731567,
      "rewards/rejected": -47.061119079589844,
      "step": 4920
    },
    {
      "epoch": 2.6358922896805486,
      "grad_norm": 1.6849100782655674e-13,
      "learning_rate": 4.406728724554154e-08,
      "logits/chosen": 9.313118934631348,
      "logits/rejected": 8.454846382141113,
      "logps/chosen": -44.2215576171875,
      "logps/rejected": -46.773765563964844,
      "loss": 0.0,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -44.2215576171875,
      "rewards/margins": 2.5522091388702393,
      "rewards/rejected": -46.773765563964844,
      "step": 4925
    },
    {
      "epoch": 2.6385683224619503,
      "grad_norm": 2.1591515824794977e-05,
      "learning_rate": 4.3430202917840664e-08,
      "logits/chosen": 9.579002380371094,
      "logits/rejected": 8.895158767700195,
      "logps/chosen": -46.80788803100586,
      "logps/rejected": -48.37885284423828,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -46.80788803100586,
      "rewards/margins": 1.5709649324417114,
      "rewards/rejected": -48.37885284423828,
      "step": 4930
    },
    {
      "epoch": 2.6412443552433515,
      "grad_norm": 9.05555407126786e-08,
      "learning_rate": 4.279754823231346e-08,
      "logits/chosen": 9.354592323303223,
      "logits/rejected": 8.649497985839844,
      "logps/chosen": -44.20353317260742,
      "logps/rejected": -45.705047607421875,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.20353317260742,
      "rewards/margins": 1.5015130043029785,
      "rewards/rejected": -45.705047607421875,
      "step": 4935
    },
    {
      "epoch": 2.6439203880247533,
      "grad_norm": 4.926092428209623e-12,
      "learning_rate": 4.216932932697859e-08,
      "logits/chosen": 9.376448631286621,
      "logits/rejected": 9.002198219299316,
      "logps/chosen": -45.52629852294922,
      "logps/rejected": -46.99915313720703,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.52629852294922,
      "rewards/margins": 1.4728606939315796,
      "rewards/rejected": -46.99915313720703,
      "step": 4940
    },
    {
      "epoch": 2.646596420806155,
      "grad_norm": 3.047843924350103e-08,
      "learning_rate": 4.154555229681844e-08,
      "logits/chosen": 9.16755485534668,
      "logits/rejected": 8.610645294189453,
      "logps/chosen": -45.5439338684082,
      "logps/rejected": -46.50552749633789,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.5439338684082,
      "rewards/margins": 0.9615978002548218,
      "rewards/rejected": -46.50552749633789,
      "step": 4945
    },
    {
      "epoch": 2.6492724535875567,
      "grad_norm": 9.344081139350164e-10,
      "learning_rate": 4.092622319372069e-08,
      "logits/chosen": 9.295737266540527,
      "logits/rejected": 8.617795944213867,
      "logps/chosen": -45.558250427246094,
      "logps/rejected": -46.47163391113281,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -45.558250427246094,
      "rewards/margins": 0.9133817553520203,
      "rewards/rejected": -46.47163391113281,
      "step": 4950
    },
    {
      "epoch": 2.651948486368958,
      "grad_norm": 1.3824635521284955e-08,
      "learning_rate": 4.031134802641889e-08,
      "logits/chosen": 9.21855354309082,
      "logits/rejected": 9.146310806274414,
      "logps/chosen": -45.980079650878906,
      "logps/rejected": -46.13520812988281,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.980079650878906,
      "rewards/margins": 0.1551315188407898,
      "rewards/rejected": -46.13520812988281,
      "step": 4955
    },
    {
      "epoch": 2.6546245191503597,
      "grad_norm": 9.875485289732448e-12,
      "learning_rate": 3.970093276043468e-08,
      "logits/chosen": 9.156174659729004,
      "logits/rejected": 8.933629035949707,
      "logps/chosen": -45.51990509033203,
      "logps/rejected": -46.665130615234375,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.51990509033203,
      "rewards/margins": 1.1452271938323975,
      "rewards/rejected": -46.665130615234375,
      "step": 4960
    },
    {
      "epoch": 2.657300551931761,
      "grad_norm": 2.0745797901265324e-09,
      "learning_rate": 3.9094983318019584e-08,
      "logits/chosen": 9.447578430175781,
      "logits/rejected": 8.80170726776123,
      "logps/chosen": -45.238975524902344,
      "logps/rejected": -46.198219299316406,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.238975524902344,
      "rewards/margins": 0.9592388272285461,
      "rewards/rejected": -46.198219299316406,
      "step": 4965
    },
    {
      "epoch": 2.6599765847131627,
      "grad_norm": 7.054178266188731e-10,
      "learning_rate": 3.849350557809789e-08,
      "logits/chosen": 9.304219245910645,
      "logits/rejected": 9.122222900390625,
      "logps/chosen": -45.77711868286133,
      "logps/rejected": -46.44526672363281,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.77711868286133,
      "rewards/margins": 0.668146014213562,
      "rewards/rejected": -46.44526672363281,
      "step": 4970
    },
    {
      "epoch": 2.6626526174945644,
      "grad_norm": 9.045265403895608e-14,
      "learning_rate": 3.789650537620903e-08,
      "logits/chosen": 9.211756706237793,
      "logits/rejected": 8.97603988647461,
      "logps/chosen": -46.72168731689453,
      "logps/rejected": -47.54497528076172,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -46.72168731689453,
      "rewards/margins": 0.8232835531234741,
      "rewards/rejected": -47.54497528076172,
      "step": 4975
    },
    {
      "epoch": 2.665328650275966,
      "grad_norm": 7.49628341313709e-08,
      "learning_rate": 3.730398850445182e-08,
      "logits/chosen": 9.036661148071289,
      "logits/rejected": 8.906078338623047,
      "logps/chosen": -45.29460144042969,
      "logps/rejected": -45.756813049316406,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.29460144042969,
      "rewards/margins": 0.46220913529396057,
      "rewards/rejected": -45.756813049316406,
      "step": 4980
    },
    {
      "epoch": 2.6680046830573674,
      "grad_norm": 6.978082709671817e-12,
      "learning_rate": 3.671596071142735e-08,
      "logits/chosen": 9.275309562683105,
      "logits/rejected": 8.708935737609863,
      "logps/chosen": -45.900875091552734,
      "logps/rejected": -47.92578887939453,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -45.900875091552734,
      "rewards/margins": 2.0249156951904297,
      "rewards/rejected": -47.92578887939453,
      "step": 4985
    },
    {
      "epoch": 2.670680715838769,
      "grad_norm": 2.0080878125339568e-09,
      "learning_rate": 3.6132427702183996e-08,
      "logits/chosen": 9.540891647338867,
      "logits/rejected": 8.97695541381836,
      "logps/chosen": -45.53911209106445,
      "logps/rejected": -46.811180114746094,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.53911209106445,
      "rewards/margins": 1.272071123123169,
      "rewards/rejected": -46.811180114746094,
      "step": 4990
    },
    {
      "epoch": 2.6733567486201704,
      "grad_norm": 4.3990498776643074e-13,
      "learning_rate": 3.555339513816147e-08,
      "logits/chosen": 9.489121437072754,
      "logits/rejected": 9.421063423156738,
      "logps/chosen": -47.22208786010742,
      "logps/rejected": -46.29930114746094,
      "loss": 0.0,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -47.22208786010742,
      "rewards/margins": -0.9227871894836426,
      "rewards/rejected": -46.29930114746094,
      "step": 4995
    },
    {
      "epoch": 2.676032781401572,
      "grad_norm": 5.171870798097416e-09,
      "learning_rate": 3.497886863713639e-08,
      "logits/chosen": 9.506311416625977,
      "logits/rejected": 9.286051750183105,
      "logps/chosen": -47.10966491699219,
      "logps/rejected": -46.7911262512207,
      "loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -47.10966491699219,
      "rewards/margins": -0.3185359835624695,
      "rewards/rejected": -46.7911262512207,
      "step": 5000
    },
    {
      "epoch": 2.678708814182974,
      "grad_norm": 1.9938813831662465e-09,
      "learning_rate": 3.440885377316721e-08,
      "logits/chosen": 9.017499923706055,
      "logits/rejected": 8.768988609313965,
      "logps/chosen": -45.000633239746094,
      "logps/rejected": -45.58954620361328,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.000633239746094,
      "rewards/margins": 0.5889142155647278,
      "rewards/rejected": -45.58954620361328,
      "step": 5005
    },
    {
      "epoch": 2.6813848469643755,
      "grad_norm": 8.178477953149043e-10,
      "learning_rate": 3.384335607654082e-08,
      "logits/chosen": 8.908024787902832,
      "logits/rejected": 8.513559341430664,
      "logps/chosen": -45.63285446166992,
      "logps/rejected": -46.390174865722656,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -45.63285446166992,
      "rewards/margins": 0.7573186159133911,
      "rewards/rejected": -46.390174865722656,
      "step": 5010
    },
    {
      "epoch": 2.684060879745777,
      "grad_norm": 0.00010346335837991947,
      "learning_rate": 3.328238103371811e-08,
      "logits/chosen": 9.248283386230469,
      "logits/rejected": 9.00572681427002,
      "logps/chosen": -46.236793518066406,
      "logps/rejected": -46.884788513183594,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -46.236793518066406,
      "rewards/margins": 0.6480017900466919,
      "rewards/rejected": -46.884788513183594,
      "step": 5015
    },
    {
      "epoch": 2.6867369125271785,
      "grad_norm": 1.9101256012146997e-12,
      "learning_rate": 3.272593408728169e-08,
      "logits/chosen": 9.2693510055542,
      "logits/rejected": 8.378210067749023,
      "logps/chosen": -45.769142150878906,
      "logps/rejected": -47.50373077392578,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.769142150878906,
      "rewards/margins": 1.734585165977478,
      "rewards/rejected": -47.50373077392578,
      "step": 5020
    },
    {
      "epoch": 2.6894129453085798,
      "grad_norm": 6.908359975296131e-12,
      "learning_rate": 3.217402063588204e-08,
      "logits/chosen": 9.387495040893555,
      "logits/rejected": 8.876842498779297,
      "logps/chosen": -45.69883728027344,
      "logps/rejected": -46.85731887817383,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.69883728027344,
      "rewards/margins": 1.1584804058074951,
      "rewards/rejected": -46.85731887817383,
      "step": 5025
    },
    {
      "epoch": 2.6920889780899815,
      "grad_norm": 5.498973808362937e-11,
      "learning_rate": 3.162664603418608e-08,
      "logits/chosen": 9.505193710327148,
      "logits/rejected": 9.121057510375977,
      "logps/chosen": -46.05498123168945,
      "logps/rejected": -47.25499725341797,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -46.05498123168945,
      "rewards/margins": 1.2000218629837036,
      "rewards/rejected": -47.25499725341797,
      "step": 5030
    },
    {
      "epoch": 2.694765010871383,
      "grad_norm": 1.1609313466705298e-10,
      "learning_rate": 3.1083815592824416e-08,
      "logits/chosen": 9.13976001739502,
      "logits/rejected": 8.92640495300293,
      "logps/chosen": -45.334659576416016,
      "logps/rejected": -46.82782745361328,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -45.334659576416016,
      "rewards/margins": 1.4931678771972656,
      "rewards/rejected": -46.82782745361328,
      "step": 5035
    },
    {
      "epoch": 2.697441043652785,
      "grad_norm": 6.403743639509696e-09,
      "learning_rate": 3.054553457834053e-08,
      "logits/chosen": 8.904398918151855,
      "logits/rejected": 9.061676979064941,
      "logps/chosen": -44.84414291381836,
      "logps/rejected": -45.527713775634766,
      "loss": 0.0,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -44.84414291381836,
      "rewards/margins": 0.6835751533508301,
      "rewards/rejected": -45.527713775634766,
      "step": 5040
    },
    {
      "epoch": 2.700117076434186,
      "grad_norm": 3.7626728885651544e-08,
      "learning_rate": 3.0011808213139036e-08,
      "logits/chosen": 8.967598915100098,
      "logits/rejected": 9.13463020324707,
      "logps/chosen": -45.867469787597656,
      "logps/rejected": -46.632999420166016,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.867469787597656,
      "rewards/margins": 0.7655293941497803,
      "rewards/rejected": -46.632999420166016,
      "step": 5045
    },
    {
      "epoch": 2.702793109215588,
      "grad_norm": 9.045006046459982e-09,
      "learning_rate": 2.948264167543568e-08,
      "logits/chosen": 9.107101440429688,
      "logits/rejected": 8.803953170776367,
      "logps/chosen": -46.371883392333984,
      "logps/rejected": -46.62939453125,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -46.371883392333984,
      "rewards/margins": 0.25751060247421265,
      "rewards/rejected": -46.62939453125,
      "step": 5050
    },
    {
      "epoch": 2.7054691419969896,
      "grad_norm": 9.95089990328451e-11,
      "learning_rate": 2.8958040099206216e-08,
      "logits/chosen": 9.511360168457031,
      "logits/rejected": 9.174965858459473,
      "logps/chosen": -46.9873046875,
      "logps/rejected": -47.3683967590332,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -46.9873046875,
      "rewards/margins": 0.3810928761959076,
      "rewards/rejected": -47.3683967590332,
      "step": 5055
    },
    {
      "epoch": 2.708145174778391,
      "grad_norm": 1.4668913432934932e-11,
      "learning_rate": 2.843800857413775e-08,
      "logits/chosen": 8.837248802185059,
      "logits/rejected": 8.727644920349121,
      "logps/chosen": -44.951210021972656,
      "logps/rejected": -45.855194091796875,
      "loss": 0.0,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -44.951210021972656,
      "rewards/margins": 0.9039837121963501,
      "rewards/rejected": -45.855194091796875,
      "step": 5060
    },
    {
      "epoch": 2.7108212075597926,
      "grad_norm": 1.329590770707339e-11,
      "learning_rate": 2.7922552145578203e-08,
      "logits/chosen": 9.241259574890137,
      "logits/rejected": 8.34737491607666,
      "logps/chosen": -45.47701644897461,
      "logps/rejected": -46.37939453125,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.47701644897461,
      "rewards/margins": 0.9023796319961548,
      "rewards/rejected": -46.37939453125,
      "step": 5065
    },
    {
      "epoch": 2.7134972403411943,
      "grad_norm": 3.6797210446523834e-11,
      "learning_rate": 2.7411675814488277e-08,
      "logits/chosen": 9.067750930786133,
      "logits/rejected": 8.636190414428711,
      "logps/chosen": -45.23744583129883,
      "logps/rejected": -45.43699645996094,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -45.23744583129883,
      "rewards/margins": 0.1995445191860199,
      "rewards/rejected": -45.43699645996094,
      "step": 5070
    },
    {
      "epoch": 2.7161732731225956,
      "grad_norm": 2.127061882210916e-06,
      "learning_rate": 2.690538453739216e-08,
      "logits/chosen": 9.242752075195312,
      "logits/rejected": 8.9752836227417,
      "logps/chosen": -44.51603317260742,
      "logps/rejected": -45.54743576049805,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -44.51603317260742,
      "rewards/margins": 1.0313999652862549,
      "rewards/rejected": -45.54743576049805,
      "step": 5075
    },
    {
      "epoch": 2.7188493059039973,
      "grad_norm": 3.5517196854701234e-10,
      "learning_rate": 2.6403683226330298e-08,
      "logits/chosen": 9.47146987915039,
      "logits/rejected": 8.898198127746582,
      "logps/chosen": -45.674041748046875,
      "logps/rejected": -47.228580474853516,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -45.674041748046875,
      "rewards/margins": 1.5545380115509033,
      "rewards/rejected": -47.228580474853516,
      "step": 5080
    },
    {
      "epoch": 2.721525338685399,
      "grad_norm": 1.2511023136291464e-11,
      "learning_rate": 2.5906576748810804e-08,
      "logits/chosen": 9.433286666870117,
      "logits/rejected": 8.915594100952148,
      "logps/chosen": -45.973018646240234,
      "logps/rejected": -47.39091491699219,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.973018646240234,
      "rewards/margins": 1.417893409729004,
      "rewards/rejected": -47.39091491699219,
      "step": 5085
    },
    {
      "epoch": 2.7242013714668003,
      "grad_norm": 4.9165179090934564e-06,
      "learning_rate": 2.5414069927763016e-08,
      "logits/chosen": 9.71883487701416,
      "logits/rejected": 8.982449531555176,
      "logps/chosen": -45.15052032470703,
      "logps/rejected": -46.02933120727539,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.15052032470703,
      "rewards/margins": 0.8788101077079773,
      "rewards/rejected": -46.02933120727539,
      "step": 5090
    },
    {
      "epoch": 2.726877404248202,
      "grad_norm": 3.4514428621715805e-09,
      "learning_rate": 2.4926167541490185e-08,
      "logits/chosen": 9.462326049804688,
      "logits/rejected": 8.5897798538208,
      "logps/chosen": -44.90679931640625,
      "logps/rejected": -46.62962341308594,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -44.90679931640625,
      "rewards/margins": 1.7228221893310547,
      "rewards/rejected": -46.62962341308594,
      "step": 5095
    },
    {
      "epoch": 2.7295534370296037,
      "grad_norm": 2.513215730608072e-10,
      "learning_rate": 2.4442874323623574e-08,
      "logits/chosen": 9.198725700378418,
      "logits/rejected": 8.60053539276123,
      "logps/chosen": -44.8150634765625,
      "logps/rejected": -46.2679557800293,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -44.8150634765625,
      "rewards/margins": 1.452892541885376,
      "rewards/rejected": -46.2679557800293,
      "step": 5100
    },
    {
      "epoch": 2.7322294698110055,
      "grad_norm": 4.010001517814102e-08,
      "learning_rate": 2.396419496307589e-08,
      "logits/chosen": 9.045036315917969,
      "logits/rejected": 8.533232688903809,
      "logps/chosen": -44.816314697265625,
      "logps/rejected": -46.530521392822266,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -44.816314697265625,
      "rewards/margins": 1.7142038345336914,
      "rewards/rejected": -46.530521392822266,
      "step": 5105
    },
    {
      "epoch": 2.7349055025924067,
      "grad_norm": 2.064175313278354e-15,
      "learning_rate": 2.349013410399653e-08,
      "logits/chosen": 9.023319244384766,
      "logits/rejected": 8.801127433776855,
      "logps/chosen": -45.231300354003906,
      "logps/rejected": -46.802330017089844,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -45.231300354003906,
      "rewards/margins": 1.5710254907608032,
      "rewards/rejected": -46.802330017089844,
      "step": 5110
    },
    {
      "epoch": 2.7375815353738084,
      "grad_norm": 5.726384894675597e-12,
      "learning_rate": 2.3020696345725954e-08,
      "logits/chosen": 9.486570358276367,
      "logits/rejected": 8.559287071228027,
      "logps/chosen": -44.43461227416992,
      "logps/rejected": -45.779090881347656,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.43461227416992,
      "rewards/margins": 1.3444724082946777,
      "rewards/rejected": -45.779090881347656,
      "step": 5115
    },
    {
      "epoch": 2.7402575681552097,
      "grad_norm": 1.8552036058255396e-06,
      "learning_rate": 2.2555886242751398e-08,
      "logits/chosen": 9.53734302520752,
      "logits/rejected": 9.222439765930176,
      "logps/chosen": -45.25102996826172,
      "logps/rejected": -46.01683807373047,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -45.25102996826172,
      "rewards/margins": 0.7658067941665649,
      "rewards/rejected": -46.01683807373047,
      "step": 5120
    },
    {
      "epoch": 2.7429336009366114,
      "grad_norm": 4.125917231733744e-10,
      "learning_rate": 2.2095708304662453e-08,
      "logits/chosen": 9.679773330688477,
      "logits/rejected": 8.958166122436523,
      "logps/chosen": -46.162208557128906,
      "logps/rejected": -46.909725189208984,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -46.162208557128906,
      "rewards/margins": 0.7475177645683289,
      "rewards/rejected": -46.909725189208984,
      "step": 5125
    },
    {
      "epoch": 2.745609633718013,
      "grad_norm": 9.498947137305709e-08,
      "learning_rate": 2.16401669961076e-08,
      "logits/chosen": 9.602405548095703,
      "logits/rejected": 8.846227645874023,
      "logps/chosen": -44.744991302490234,
      "logps/rejected": -46.215084075927734,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -44.744991302490234,
      "rewards/margins": 1.4700872898101807,
      "rewards/rejected": -46.215084075927734,
      "step": 5130
    },
    {
      "epoch": 2.748285666499415,
      "grad_norm": 1.7194083931363393e-09,
      "learning_rate": 2.1189266736750532e-08,
      "logits/chosen": 9.120603561401367,
      "logits/rejected": 8.862998962402344,
      "logps/chosen": -46.00870895385742,
      "logps/rejected": -46.72013854980469,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -46.00870895385742,
      "rewards/margins": 0.7114282846450806,
      "rewards/rejected": -46.72013854980469,
      "step": 5135
    },
    {
      "epoch": 2.750961699280816,
      "grad_norm": 5.23018255369677e-07,
      "learning_rate": 2.0743011901227623e-08,
      "logits/chosen": 9.266596794128418,
      "logits/rejected": 8.745198249816895,
      "logps/chosen": -43.74271011352539,
      "logps/rejected": -44.4289665222168,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -43.74271011352539,
      "rewards/margins": 0.6862567663192749,
      "rewards/rejected": -44.4289665222168,
      "step": 5140
    },
    {
      "epoch": 2.753637732062218,
      "grad_norm": 8.238195545236475e-07,
      "learning_rate": 2.030140681910508e-08,
      "logits/chosen": 9.078023910522461,
      "logits/rejected": 8.59467887878418,
      "logps/chosen": -45.07929992675781,
      "logps/rejected": -46.49383544921875,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -45.07929992675781,
      "rewards/margins": 1.414538025856018,
      "rewards/rejected": -46.49383544921875,
      "step": 5145
    },
    {
      "epoch": 2.756313764843619,
      "grad_norm": 3.632666509241067e-11,
      "learning_rate": 1.986445577483753e-08,
      "logits/chosen": 9.499336242675781,
      "logits/rejected": 8.920842170715332,
      "logps/chosen": -45.081443786621094,
      "logps/rejected": -45.711849212646484,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -45.081443786621094,
      "rewards/margins": 0.630403995513916,
      "rewards/rejected": -45.711849212646484,
      "step": 5150
    },
    {
      "epoch": 2.758989797625021,
      "grad_norm": 8.624187391622431e-13,
      "learning_rate": 1.9432163007725765e-08,
      "logits/chosen": 9.421464920043945,
      "logits/rejected": 9.20478630065918,
      "logps/chosen": -45.046199798583984,
      "logps/rejected": -46.70404815673828,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -45.046199798583984,
      "rewards/margins": 1.6578500270843506,
      "rewards/rejected": -46.70404815673828,
      "step": 5155
    },
    {
      "epoch": 2.7616658304064226,
      "grad_norm": 2.304018096380105e-06,
      "learning_rate": 1.9004532711876297e-08,
      "logits/chosen": 9.018564224243164,
      "logits/rejected": 9.155081748962402,
      "logps/chosen": -44.680206298828125,
      "logps/rejected": -45.13641357421875,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -44.680206298828125,
      "rewards/margins": 0.4562087059020996,
      "rewards/rejected": -45.13641357421875,
      "step": 5160
    },
    {
      "epoch": 2.7643418631878243,
      "grad_norm": 4.149626071851763e-07,
      "learning_rate": 1.8581569036159928e-08,
      "logits/chosen": 9.165428161621094,
      "logits/rejected": 8.500818252563477,
      "logps/chosen": -44.28517532348633,
      "logps/rejected": -46.5477294921875,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -44.28517532348633,
      "rewards/margins": 2.2625536918640137,
      "rewards/rejected": -46.5477294921875,
      "step": 5165
    },
    {
      "epoch": 2.7670178959692255,
      "grad_norm": 4.228184952721746e-08,
      "learning_rate": 1.8163276084172285e-08,
      "logits/chosen": 9.442312240600586,
      "logits/rejected": 9.16694450378418,
      "logps/chosen": -44.13327407836914,
      "logps/rejected": -45.20075225830078,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -44.13327407836914,
      "rewards/margins": 1.0674808025360107,
      "rewards/rejected": -45.20075225830078,
      "step": 5170
    },
    {
      "epoch": 2.7696939287506273,
      "grad_norm": 8.656219636396584e-07,
      "learning_rate": 1.7749657914193194e-08,
      "logits/chosen": 9.039213180541992,
      "logits/rejected": 8.899674415588379,
      "logps/chosen": -45.28321075439453,
      "logps/rejected": -46.11141586303711,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.28321075439453,
      "rewards/margins": 0.8282037973403931,
      "rewards/rejected": -46.11141586303711,
      "step": 5175
    },
    {
      "epoch": 2.7723699615320285,
      "grad_norm": 1.6876599341144144e-09,
      "learning_rate": 1.7340718539148203e-08,
      "logits/chosen": 9.122260093688965,
      "logits/rejected": 8.889673233032227,
      "logps/chosen": -45.152854919433594,
      "logps/rejected": -46.281497955322266,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.152854919433594,
      "rewards/margins": 1.1286396980285645,
      "rewards/rejected": -46.281497955322266,
      "step": 5180
    },
    {
      "epoch": 2.7750459943134302,
      "grad_norm": 6.913985784745342e-12,
      "learning_rate": 1.6936461926568724e-08,
      "logits/chosen": 9.346076965332031,
      "logits/rejected": 8.638724327087402,
      "logps/chosen": -45.47941207885742,
      "logps/rejected": -47.605812072753906,
      "loss": 0.0011,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.47941207885742,
      "rewards/margins": 2.126397132873535,
      "rewards/rejected": -47.605812072753906,
      "step": 5185
    },
    {
      "epoch": 2.777722027094832,
      "grad_norm": 1.3752126114130894e-12,
      "learning_rate": 1.6536891998554346e-08,
      "logits/chosen": 9.24070930480957,
      "logits/rejected": 8.748628616333008,
      "logps/chosen": -44.88689041137695,
      "logps/rejected": -46.4335823059082,
      "loss": 0.0001,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.88689041137695,
      "rewards/margins": 1.5466945171356201,
      "rewards/rejected": -46.4335823059082,
      "step": 5190
    },
    {
      "epoch": 2.7803980598762337,
      "grad_norm": 2.254231474464554e-06,
      "learning_rate": 1.6142012631734093e-08,
      "logits/chosen": 9.12055778503418,
      "logits/rejected": 8.69666576385498,
      "logps/chosen": -45.850746154785156,
      "logps/rejected": -46.16967010498047,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -45.850746154785156,
      "rewards/margins": 0.31892484426498413,
      "rewards/rejected": -46.16967010498047,
      "step": 5195
    },
    {
      "epoch": 2.783074092657635,
      "grad_norm": 9.589278440270112e-12,
      "learning_rate": 1.575182765722949e-08,
      "logits/chosen": 9.317351341247559,
      "logits/rejected": 8.72008228302002,
      "logps/chosen": -45.6714973449707,
      "logps/rejected": -47.0482063293457,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.6714973449707,
      "rewards/margins": 1.3767114877700806,
      "rewards/rejected": -47.0482063293457,
      "step": 5200
    },
    {
      "epoch": 2.783074092657635,
      "eval_logits/chosen": 7.7830023765563965,
      "eval_logits/rejected": 7.449924945831299,
      "eval_logps/chosen": -45.34323501586914,
      "eval_logps/rejected": -46.339378356933594,
      "eval_loss": 2.017032159074006e-07,
      "eval_rewards/accuracies": 0.5652819275856018,
      "eval_rewards/chosen": -45.34323501586914,
      "eval_rewards/margins": 0.9961463809013367,
      "eval_rewards/rejected": -46.339378356933594,
      "eval_runtime": 40.0738,
      "eval_samples_per_second": 33.563,
      "eval_steps_per_second": 8.409,
      "step": 5200
    },
    {
      "epoch": 2.7857501254390367,
      "grad_norm": 7.816740176199239e-08,
      "learning_rate": 1.536634086061672e-08,
      "logits/chosen": 9.10354232788086,
      "logits/rejected": 8.885189056396484,
      "logps/chosen": -44.331356048583984,
      "logps/rejected": -45.00513458251953,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -44.331356048583984,
      "rewards/margins": 0.6737779378890991,
      "rewards/rejected": -45.00513458251953,
      "step": 5205
    },
    {
      "epoch": 2.788426158220438,
      "grad_norm": 1.2487996289028488e-08,
      "learning_rate": 1.4985555981890495e-08,
      "logits/chosen": 9.244466781616211,
      "logits/rejected": 9.004443168640137,
      "logps/chosen": -45.558616638183594,
      "logps/rejected": -47.24901580810547,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -45.558616638183594,
      "rewards/margins": 1.6903969049453735,
      "rewards/rejected": -47.24901580810547,
      "step": 5210
    },
    {
      "epoch": 2.7911021910018396,
      "grad_norm": 9.226104024774014e-12,
      "learning_rate": 1.4609476715427226e-08,
      "logits/chosen": 9.444185256958008,
      "logits/rejected": 9.038237571716309,
      "logps/chosen": -43.968360900878906,
      "logps/rejected": -44.89006805419922,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -43.968360900878906,
      "rewards/margins": 0.9217087626457214,
      "rewards/rejected": -44.89006805419922,
      "step": 5215
    },
    {
      "epoch": 2.7937782237832414,
      "grad_norm": 9.021047754274223e-08,
      "learning_rate": 1.4238106709949792e-08,
      "logits/chosen": 9.282184600830078,
      "logits/rejected": 8.902387619018555,
      "logps/chosen": -45.662330627441406,
      "logps/rejected": -45.58094024658203,
      "loss": 0.0,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -45.662330627441406,
      "rewards/margins": -0.08139518648386002,
      "rewards/rejected": -45.58094024658203,
      "step": 5220
    },
    {
      "epoch": 2.796454256564643,
      "grad_norm": 6.034615818294384e-08,
      "learning_rate": 1.3871449568491511e-08,
      "logits/chosen": 9.006844520568848,
      "logits/rejected": 8.497916221618652,
      "logps/chosen": -46.34284973144531,
      "logps/rejected": -47.13788604736328,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -46.34284973144531,
      "rewards/margins": 0.7950329184532166,
      "rewards/rejected": -47.13788604736328,
      "step": 5225
    },
    {
      "epoch": 2.7991302893460444,
      "grad_norm": 4.689134215211333e-09,
      "learning_rate": 1.3509508848361606e-08,
      "logits/chosen": 9.407037734985352,
      "logits/rejected": 8.859153747558594,
      "logps/chosen": -44.994224548339844,
      "logps/rejected": -46.497501373291016,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -44.994224548339844,
      "rewards/margins": 1.5032765865325928,
      "rewards/rejected": -46.497501373291016,
      "step": 5230
    },
    {
      "epoch": 2.801806322127446,
      "grad_norm": 2.2475561945601365e-06,
      "learning_rate": 1.3152288061110517e-08,
      "logits/chosen": 9.219392776489258,
      "logits/rejected": 8.73949146270752,
      "logps/chosen": -44.67304611206055,
      "logps/rejected": -45.740394592285156,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.67304611206055,
      "rewards/margins": 1.067342758178711,
      "rewards/rejected": -45.740394592285156,
      "step": 5235
    },
    {
      "epoch": 2.804482354908848,
      "grad_norm": 1.4758056355489937e-10,
      "learning_rate": 1.2799790672495814e-08,
      "logits/chosen": 9.460224151611328,
      "logits/rejected": 8.719059944152832,
      "logps/chosen": -45.48836898803711,
      "logps/rejected": -47.160301208496094,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.48836898803711,
      "rewards/margins": 1.6719379425048828,
      "rewards/rejected": -47.160301208496094,
      "step": 5240
    },
    {
      "epoch": 2.807158387690249,
      "grad_norm": 2.915325064731587e-13,
      "learning_rate": 1.2452020102448835e-08,
      "logits/chosen": 9.069089889526367,
      "logits/rejected": 8.860189437866211,
      "logps/chosen": -44.70355987548828,
      "logps/rejected": -45.467857360839844,
      "loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -44.70355987548828,
      "rewards/margins": 0.7642976641654968,
      "rewards/rejected": -45.467857360839844,
      "step": 5245
    },
    {
      "epoch": 2.8098344204716508,
      "grad_norm": 3.5323922845195516e-10,
      "learning_rate": 1.2108979725041103e-08,
      "logits/chosen": 9.311979293823242,
      "logits/rejected": 8.985702514648438,
      "logps/chosen": -44.827545166015625,
      "logps/rejected": -45.708839416503906,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.827545166015625,
      "rewards/margins": 0.8812910914421082,
      "rewards/rejected": -45.708839416503906,
      "step": 5250
    },
    {
      "epoch": 2.8125104532530525,
      "grad_norm": 1.7011555812427142e-08,
      "learning_rate": 1.1770672868451958e-08,
      "logits/chosen": 9.217060089111328,
      "logits/rejected": 8.561985969543457,
      "logps/chosen": -43.92146682739258,
      "logps/rejected": -45.702178955078125,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -43.92146682739258,
      "rewards/margins": 1.7807128429412842,
      "rewards/rejected": -45.702178955078125,
      "step": 5255
    },
    {
      "epoch": 2.8151864860344538,
      "grad_norm": 2.1579077354302896e-07,
      "learning_rate": 1.1437102814935872e-08,
      "logits/chosen": 8.787565231323242,
      "logits/rejected": 8.522884368896484,
      "logps/chosen": -45.5779914855957,
      "logps/rejected": -45.379493713378906,
      "loss": 0.0002,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.5779914855957,
      "rewards/margins": -0.19850048422813416,
      "rewards/rejected": -45.379493713378906,
      "step": 5260
    },
    {
      "epoch": 2.8178625188158555,
      "grad_norm": 2.57246551712737e-08,
      "learning_rate": 1.1108272800791018e-08,
      "logits/chosen": 9.877340316772461,
      "logits/rejected": 8.877042770385742,
      "logps/chosen": -44.234466552734375,
      "logps/rejected": -46.3248291015625,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.234466552734375,
      "rewards/margins": 2.0903666019439697,
      "rewards/rejected": -46.3248291015625,
      "step": 5265
    },
    {
      "epoch": 2.820538551597257,
      "grad_norm": 2.24112402995779e-09,
      "learning_rate": 1.078418601632769e-08,
      "logits/chosen": 9.219143867492676,
      "logits/rejected": 8.762103080749512,
      "logps/chosen": -44.727333068847656,
      "logps/rejected": -45.62724304199219,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.727333068847656,
      "rewards/margins": 0.8999112844467163,
      "rewards/rejected": -45.62724304199219,
      "step": 5270
    },
    {
      "epoch": 2.8232145843786585,
      "grad_norm": 1.0600951813872122e-10,
      "learning_rate": 1.0464845605837159e-08,
      "logits/chosen": 9.17064094543457,
      "logits/rejected": 8.770169258117676,
      "logps/chosen": -45.47101593017578,
      "logps/rejected": -46.60496139526367,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -45.47101593017578,
      "rewards/margins": 1.1339523792266846,
      "rewards/rejected": -46.60496139526367,
      "step": 5275
    },
    {
      "epoch": 2.82589061716006,
      "grad_norm": 3.9635631312028664e-05,
      "learning_rate": 1.0150254667561642e-08,
      "logits/chosen": 9.25227165222168,
      "logits/rejected": 8.457999229431152,
      "logps/chosen": -45.351966857910156,
      "logps/rejected": -46.87631607055664,
      "loss": 0.0017,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -45.351966857910156,
      "rewards/margins": 1.524350881576538,
      "rewards/rejected": -46.87631607055664,
      "step": 5280
    },
    {
      "epoch": 2.828566649941462,
      "grad_norm": 3.966087847304576e-10,
      "learning_rate": 9.840416253663719e-09,
      "logits/chosen": 9.477801322937012,
      "logits/rejected": 9.114805221557617,
      "logps/chosen": -46.06488800048828,
      "logps/rejected": -47.42600631713867,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -46.06488800048828,
      "rewards/margins": 1.3611249923706055,
      "rewards/rejected": -47.42600631713867,
      "step": 5285
    },
    {
      "epoch": 2.8312426827228636,
      "grad_norm": 3.5715790020175666e-14,
      "learning_rate": 9.535333370197074e-09,
      "logits/chosen": 9.245917320251465,
      "logits/rejected": 8.656416893005371,
      "logps/chosen": -45.50297164916992,
      "logps/rejected": -46.674781799316406,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.50297164916992,
      "rewards/margins": 1.1718120574951172,
      "rewards/rejected": -46.674781799316406,
      "step": 5290
    },
    {
      "epoch": 2.833918715504265,
      "grad_norm": 3.542054199915874e-07,
      "learning_rate": 9.23500897707713e-09,
      "logits/chosen": 9.498045921325684,
      "logits/rejected": 8.81795883178711,
      "logps/chosen": -45.44274139404297,
      "logps/rejected": -46.454368591308594,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.44274139404297,
      "rewards/margins": 1.0116297006607056,
      "rewards/rejected": -46.454368591308594,
      "step": 5295
    },
    {
      "epoch": 2.8365947482856666,
      "grad_norm": 7.420593015548149e-12,
      "learning_rate": 8.939445988052574e-09,
      "logits/chosen": 9.181685447692871,
      "logits/rejected": 8.880643844604492,
      "logps/chosen": -45.54419708251953,
      "logps/rejected": -45.80699920654297,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -45.54419708251953,
      "rewards/margins": 0.26280778646469116,
      "rewards/rejected": -45.80699920654297,
      "step": 5300
    },
    {
      "epoch": 2.839270781067068,
      "grad_norm": 1.3569035486556837e-09,
      "learning_rate": 8.648647270676656e-09,
      "logits/chosen": 8.99543571472168,
      "logits/rejected": 8.507521629333496,
      "logps/chosen": -44.479496002197266,
      "logps/rejected": -46.040218353271484,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.479496002197266,
      "rewards/margins": 1.56071937084198,
      "rewards/rejected": -46.040218353271484,
      "step": 5305
    },
    {
      "epoch": 2.8419468138484696,
      "grad_norm": 1.1028957578871877e-14,
      "learning_rate": 8.362615646279991e-09,
      "logits/chosen": 9.993020057678223,
      "logits/rejected": 8.945398330688477,
      "logps/chosen": -46.47921371459961,
      "logps/rejected": -47.681915283203125,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -46.47921371459961,
      "rewards/margins": 1.2026946544647217,
      "rewards/rejected": -47.681915283203125,
      "step": 5310
    },
    {
      "epoch": 2.8446228466298713,
      "grad_norm": 1.6061629732451726e-10,
      "learning_rate": 8.081353889942466e-09,
      "logits/chosen": 9.332159042358398,
      "logits/rejected": 8.767170906066895,
      "logps/chosen": -45.59265899658203,
      "logps/rejected": -46.954933166503906,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.59265899658203,
      "rewards/margins": 1.3622767925262451,
      "rewards/rejected": -46.954933166503906,
      "step": 5315
    },
    {
      "epoch": 2.847298879411273,
      "grad_norm": 3.0339073626390855e-10,
      "learning_rate": 7.804864730467042e-09,
      "logits/chosen": 9.176513671875,
      "logits/rejected": 9.093063354492188,
      "logps/chosen": -45.16083908081055,
      "logps/rejected": -46.383872985839844,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.16083908081055,
      "rewards/margins": 1.2230373620986938,
      "rewards/rejected": -46.383872985839844,
      "step": 5320
    },
    {
      "epoch": 2.8499749121926743,
      "grad_norm": 2.9036699408029805e-10,
      "learning_rate": 7.533150850352665e-09,
      "logits/chosen": 9.224966049194336,
      "logits/rejected": 8.548715591430664,
      "logps/chosen": -46.75611114501953,
      "logps/rejected": -46.84228515625,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -46.75611114501953,
      "rewards/margins": 0.08617454767227173,
      "rewards/rejected": -46.84228515625,
      "step": 5325
    },
    {
      "epoch": 2.852650944974076,
      "grad_norm": 1.0653354866986469e-08,
      "learning_rate": 7.2662148857686175e-09,
      "logits/chosen": 8.694648742675781,
      "logits/rejected": 8.436681747436523,
      "logps/chosen": -45.037750244140625,
      "logps/rejected": -45.7553825378418,
      "loss": 0.0004,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.037750244140625,
      "rewards/margins": 0.717631459236145,
      "rewards/rejected": -45.7553825378418,
      "step": 5330
    },
    {
      "epoch": 2.8553269777554773,
      "grad_norm": 2.6182167552303917e-10,
      "learning_rate": 7.0040594265287635e-09,
      "logits/chosen": 8.752364158630371,
      "logits/rejected": 8.914209365844727,
      "logps/chosen": -45.395790100097656,
      "logps/rejected": -46.17589569091797,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -45.395790100097656,
      "rewards/margins": 0.7801089286804199,
      "rewards/rejected": -46.17589569091797,
      "step": 5335
    },
    {
      "epoch": 2.858003010536879,
      "grad_norm": 3.2068251965503893e-13,
      "learning_rate": 6.746687016066566e-09,
      "logits/chosen": 9.286540031433105,
      "logits/rejected": 9.058765411376953,
      "logps/chosen": -45.74053192138672,
      "logps/rejected": -46.500648498535156,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.74053192138672,
      "rewards/margins": 0.7601187825202942,
      "rewards/rejected": -46.500648498535156,
      "step": 5340
    },
    {
      "epoch": 2.8606790433182807,
      "grad_norm": 7.359458194381693e-09,
      "learning_rate": 6.494100151410276e-09,
      "logits/chosen": 9.41588306427002,
      "logits/rejected": 8.668153762817383,
      "logps/chosen": -44.76633071899414,
      "logps/rejected": -46.34720993041992,
      "loss": 0.0,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -44.76633071899414,
      "rewards/margins": 1.5808814764022827,
      "rewards/rejected": -46.34720993041992,
      "step": 5345
    },
    {
      "epoch": 2.8633550760996824,
      "grad_norm": 4.804962516992645e-09,
      "learning_rate": 6.246301283158728e-09,
      "logits/chosen": 9.029601097106934,
      "logits/rejected": 8.94938850402832,
      "logps/chosen": -45.265342712402344,
      "logps/rejected": -45.678558349609375,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -45.265342712402344,
      "rewards/margins": 0.4132116734981537,
      "rewards/rejected": -45.678558349609375,
      "step": 5350
    },
    {
      "epoch": 2.8660311088810837,
      "grad_norm": 2.0324028652120222e-10,
      "learning_rate": 6.0032928154576944e-09,
      "logits/chosen": 9.365392684936523,
      "logits/rejected": 9.195672035217285,
      "logps/chosen": -44.81023406982422,
      "logps/rejected": -46.05611801147461,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -44.81023406982422,
      "rewards/margins": 1.2458800077438354,
      "rewards/rejected": -46.05611801147461,
      "step": 5355
    },
    {
      "epoch": 2.8687071416624854,
      "grad_norm": 1.2188510532329644e-09,
      "learning_rate": 5.76507710597629e-09,
      "logits/chosen": 9.305898666381836,
      "logits/rejected": 8.563643455505371,
      "logps/chosen": -44.4002685546875,
      "logps/rejected": -46.452125549316406,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.4002685546875,
      "rewards/margins": 2.051856279373169,
      "rewards/rejected": -46.452125549316406,
      "step": 5360
    },
    {
      "epoch": 2.8713831744438867,
      "grad_norm": 8.289851665405591e-13,
      "learning_rate": 5.531656465884438e-09,
      "logits/chosen": 9.142070770263672,
      "logits/rejected": 8.563552856445312,
      "logps/chosen": -45.080833435058594,
      "logps/rejected": -46.668304443359375,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -45.080833435058594,
      "rewards/margins": 1.5874735116958618,
      "rewards/rejected": -46.668304443359375,
      "step": 5365
    },
    {
      "epoch": 2.8740592072252884,
      "grad_norm": 1.5826421621316423e-12,
      "learning_rate": 5.303033159830217e-09,
      "logits/chosen": 9.022242546081543,
      "logits/rejected": 8.992324829101562,
      "logps/chosen": -45.019309997558594,
      "logps/rejected": -46.437828063964844,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.019309997558594,
      "rewards/margins": 1.4185160398483276,
      "rewards/rejected": -46.437828063964844,
      "step": 5370
    },
    {
      "epoch": 2.87673524000669,
      "grad_norm": 1.1874671979051657e-13,
      "learning_rate": 5.079209405917939e-09,
      "logits/chosen": 8.996851921081543,
      "logits/rejected": 8.904409408569336,
      "logps/chosen": -45.653846740722656,
      "logps/rejected": -47.350955963134766,
      "loss": 0.0,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -45.653846740722656,
      "rewards/margins": 1.6971060037612915,
      "rewards/rejected": -47.350955963134766,
      "step": 5375
    },
    {
      "epoch": 2.879411272788092,
      "grad_norm": 1.9170652014312904e-10,
      "learning_rate": 4.860187375686664e-09,
      "logits/chosen": 9.73114013671875,
      "logits/rejected": 8.676279067993164,
      "logps/chosen": -44.765777587890625,
      "logps/rejected": -46.24297332763672,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -44.765777587890625,
      "rewards/margins": 1.47720205783844,
      "rewards/rejected": -46.24297332763672,
      "step": 5380
    },
    {
      "epoch": 2.882087305569493,
      "grad_norm": 1.2402195112146414e-12,
      "learning_rate": 4.64596919408905e-09,
      "logits/chosen": 9.051892280578613,
      "logits/rejected": 9.013568878173828,
      "logps/chosen": -44.13518524169922,
      "logps/rejected": -45.68724822998047,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -44.13518524169922,
      "rewards/margins": 1.5520589351654053,
      "rewards/rejected": -45.68724822998047,
      "step": 5385
    },
    {
      "epoch": 2.884763338350895,
      "grad_norm": 1.8799493641565875e-06,
      "learning_rate": 4.436556939470814e-09,
      "logits/chosen": 9.390496253967285,
      "logits/rejected": 8.861213684082031,
      "logps/chosen": -46.65007400512695,
      "logps/rejected": -47.16893768310547,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -46.65007400512695,
      "rewards/margins": 0.5188626050949097,
      "rewards/rejected": -47.16893768310547,
      "step": 5390
    },
    {
      "epoch": 2.887439371132296,
      "grad_norm": 2.650463698018818e-07,
      "learning_rate": 4.23195264355064e-09,
      "logits/chosen": 9.392367362976074,
      "logits/rejected": 8.64184856414795,
      "logps/chosen": -45.28086853027344,
      "logps/rejected": -46.37200164794922,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.28086853027344,
      "rewards/margins": 1.0911357402801514,
      "rewards/rejected": -46.37200164794922,
      "step": 5395
    },
    {
      "epoch": 2.890115403913698,
      "grad_norm": 7.853834194822404e-07,
      "learning_rate": 4.032158291400245e-09,
      "logits/chosen": 9.48828411102295,
      "logits/rejected": 8.497381210327148,
      "logps/chosen": -44.835670471191406,
      "logps/rejected": -46.30108642578125,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.835670471191406,
      "rewards/margins": 1.465419054031372,
      "rewards/rejected": -46.30108642578125,
      "step": 5400
    },
    {
      "epoch": 2.8927914366950995,
      "grad_norm": 8.635637158994779e-05,
      "learning_rate": 3.837175821425398e-09,
      "logits/chosen": 9.104378700256348,
      "logits/rejected": 8.779329299926758,
      "logps/chosen": -45.33580017089844,
      "logps/rejected": -45.78975296020508,
      "loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -45.33580017089844,
      "rewards/margins": 0.4539506435394287,
      "rewards/rejected": -45.78975296020508,
      "step": 5405
    },
    {
      "epoch": 2.8954674694765012,
      "grad_norm": 2.71609408822428e-07,
      "learning_rate": 3.6470071253467683e-09,
      "logits/chosen": 9.407661437988281,
      "logits/rejected": 8.914694786071777,
      "logps/chosen": -45.22163009643555,
      "logps/rejected": -46.47017288208008,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.22163009643555,
      "rewards/margins": 1.248543620109558,
      "rewards/rejected": -46.47017288208008,
      "step": 5410
    },
    {
      "epoch": 2.8981435022579025,
      "grad_norm": 5.3050851173949576e-11,
      "learning_rate": 3.461654048181939e-09,
      "logits/chosen": 9.624258995056152,
      "logits/rejected": 8.95285701751709,
      "logps/chosen": -45.516815185546875,
      "logps/rejected": -46.962730407714844,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.516815185546875,
      "rewards/margins": 1.4459118843078613,
      "rewards/rejected": -46.962730407714844,
      "step": 5415
    },
    {
      "epoch": 2.9008195350393042,
      "grad_norm": 4.041242689645956e-13,
      "learning_rate": 3.281118388227255e-09,
      "logits/chosen": 9.329256057739258,
      "logits/rejected": 9.042010307312012,
      "logps/chosen": -45.84162139892578,
      "logps/rejected": -46.053077697753906,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.84162139892578,
      "rewards/margins": 0.21145661175251007,
      "rewards/rejected": -46.053077697753906,
      "step": 5420
    },
    {
      "epoch": 2.903495567820706,
      "grad_norm": 1.273978876382391e-10,
      "learning_rate": 3.1054018970405048e-09,
      "logits/chosen": 9.346078872680664,
      "logits/rejected": 8.737161636352539,
      "logps/chosen": -45.39017868041992,
      "logps/rejected": -46.47900390625,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -45.39017868041992,
      "rewards/margins": 1.08882737159729,
      "rewards/rejected": -46.47900390625,
      "step": 5425
    },
    {
      "epoch": 2.906171600602107,
      "grad_norm": 9.099359880226555e-07,
      "learning_rate": 2.9345062794238207e-09,
      "logits/chosen": 9.591211318969727,
      "logits/rejected": 8.986166000366211,
      "logps/chosen": -45.41316223144531,
      "logps/rejected": -46.36024475097656,
      "loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -45.41316223144531,
      "rewards/margins": 0.9470812678337097,
      "rewards/rejected": -46.36024475097656,
      "step": 5430
    },
    {
      "epoch": 2.908847633383509,
      "grad_norm": 4.0064969826156453e-14,
      "learning_rate": 2.7684331934072492e-09,
      "logits/chosen": 9.369043350219727,
      "logits/rejected": 9.077577590942383,
      "logps/chosen": -45.00037384033203,
      "logps/rejected": -46.1748161315918,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.00037384033203,
      "rewards/margins": 1.1744441986083984,
      "rewards/rejected": -46.1748161315918,
      "step": 5435
    },
    {
      "epoch": 2.9115236661649107,
      "grad_norm": 1.2751168126151237e-08,
      "learning_rate": 2.6071842502326526e-09,
      "logits/chosen": 9.392855644226074,
      "logits/rejected": 8.840311050415039,
      "logps/chosen": -44.9502067565918,
      "logps/rejected": -45.607967376708984,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -44.9502067565918,
      "rewards/margins": 0.6577606201171875,
      "rewards/rejected": -45.607967376708984,
      "step": 5440
    },
    {
      "epoch": 2.9141996989463124,
      "grad_norm": 3.7907033093303556e-06,
      "learning_rate": 2.450761014337888e-09,
      "logits/chosen": 8.839734077453613,
      "logits/rejected": 8.636930465698242,
      "logps/chosen": -45.072608947753906,
      "logps/rejected": -45.858760833740234,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -45.072608947753906,
      "rewards/margins": 0.7861508131027222,
      "rewards/rejected": -45.858760833740234,
      "step": 5445
    },
    {
      "epoch": 2.9168757317277136,
      "grad_norm": 2.578104706307563e-07,
      "learning_rate": 2.299165003341985e-09,
      "logits/chosen": 8.770320892333984,
      "logits/rejected": 8.441905975341797,
      "logps/chosen": -44.49614334106445,
      "logps/rejected": -45.10933303833008,
      "loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -44.49614334106445,
      "rewards/margins": 0.6131892800331116,
      "rewards/rejected": -45.10933303833008,
      "step": 5450
    },
    {
      "epoch": 2.9195517645091154,
      "grad_norm": 2.4492008268516684e-06,
      "learning_rate": 2.1523976880299945e-09,
      "logits/chosen": 9.372171401977539,
      "logits/rejected": 8.936022758483887,
      "logps/chosen": -45.319339752197266,
      "logps/rejected": -46.52952194213867,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -45.319339752197266,
      "rewards/margins": 1.210187554359436,
      "rewards/rejected": -46.52952194213867,
      "step": 5455
    },
    {
      "epoch": 2.9222277972905166,
      "grad_norm": 7.83577334249431e-15,
      "learning_rate": 2.010460492339161e-09,
      "logits/chosen": 9.247393608093262,
      "logits/rejected": 8.673940658569336,
      "logps/chosen": -45.221656799316406,
      "logps/rejected": -46.91360855102539,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -45.221656799316406,
      "rewards/margins": 1.6919498443603516,
      "rewards/rejected": -46.91360855102539,
      "step": 5460
    },
    {
      "epoch": 2.9249038300719183,
      "grad_norm": 0.0006585802376002892,
      "learning_rate": 1.8733547933446614e-09,
      "logits/chosen": 9.516321182250977,
      "logits/rejected": 8.72892951965332,
      "logps/chosen": -45.13756561279297,
      "logps/rejected": -45.5636100769043,
      "loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -45.13756561279297,
      "rewards/margins": 0.4260477125644684,
      "rewards/rejected": -45.5636100769043,
      "step": 5465
    },
    {
      "epoch": 2.92757986285332,
      "grad_norm": 1.1911102157756876e-09,
      "learning_rate": 1.7410819212467231e-09,
      "logits/chosen": 9.350858688354492,
      "logits/rejected": 9.068004608154297,
      "logps/chosen": -47.18770217895508,
      "logps/rejected": -47.46491622924805,
      "loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -47.18770217895508,
      "rewards/margins": 0.27721983194351196,
      "rewards/rejected": -47.46491622924805,
      "step": 5470
    },
    {
      "epoch": 2.9302558956347218,
      "grad_norm": 5.178060400408307e-12,
      "learning_rate": 1.613643159357192e-09,
      "logits/chosen": 9.355071067810059,
      "logits/rejected": 9.311732292175293,
      "logps/chosen": -46.11331558227539,
      "logps/rejected": -46.16367721557617,
      "loss": 0.0,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -46.11331558227539,
      "rewards/margins": 0.05036578327417374,
      "rewards/rejected": -46.16367721557617,
      "step": 5475
    },
    {
      "epoch": 2.932931928416123,
      "grad_norm": 1.4159944028613333e-09,
      "learning_rate": 1.4910397440875967e-09,
      "logits/chosen": 9.341100692749023,
      "logits/rejected": 8.961751937866211,
      "logps/chosen": -45.71449661254883,
      "logps/rejected": -46.8352165222168,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.71449661254883,
      "rewards/margins": 1.1207211017608643,
      "rewards/rejected": -46.8352165222168,
      "step": 5480
    },
    {
      "epoch": 2.9356079611975248,
      "grad_norm": 2.654807486469358e-10,
      "learning_rate": 1.3732728649368253e-09,
      "logits/chosen": 9.172346115112305,
      "logits/rejected": 8.382214546203613,
      "logps/chosen": -45.7579231262207,
      "logps/rejected": -46.55530548095703,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -45.7579231262207,
      "rewards/margins": 0.7973812818527222,
      "rewards/rejected": -46.55530548095703,
      "step": 5485
    },
    {
      "epoch": 2.938283993978926,
      "grad_norm": 8.171140764261645e-13,
      "learning_rate": 1.260343664479524e-09,
      "logits/chosen": 8.913924217224121,
      "logits/rejected": 8.826963424682617,
      "logps/chosen": -45.91178512573242,
      "logps/rejected": -46.22658920288086,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -45.91178512573242,
      "rewards/margins": 0.31480032205581665,
      "rewards/rejected": -46.22658920288086,
      "step": 5490
    },
    {
      "epoch": 2.9409600267603278,
      "grad_norm": 2.569638301922221e-06,
      "learning_rate": 1.1522532383554384e-09,
      "logits/chosen": 9.545568466186523,
      "logits/rejected": 8.770270347595215,
      "logps/chosen": -45.8795280456543,
      "logps/rejected": -46.69158172607422,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.8795280456543,
      "rewards/margins": 0.8120635747909546,
      "rewards/rejected": -46.69158172607422,
      "step": 5495
    },
    {
      "epoch": 2.9436360595417295,
      "grad_norm": 1.0704362526178386e-10,
      "learning_rate": 1.049002635258256e-09,
      "logits/chosen": 9.066648483276367,
      "logits/rejected": 8.688053131103516,
      "logps/chosen": -45.68961715698242,
      "logps/rejected": -46.245445251464844,
      "loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -45.68961715698242,
      "rewards/margins": 0.5558298826217651,
      "rewards/rejected": -46.245445251464844,
      "step": 5500
    },
    {
      "epoch": 2.946312092323131,
      "grad_norm": 3.3702573529026385e-05,
      "learning_rate": 9.505928569258358e-10,
      "logits/chosen": 9.145792961120605,
      "logits/rejected": 9.183855056762695,
      "logps/chosen": -45.642555236816406,
      "logps/rejected": -46.52573776245117,
      "loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -45.642555236816406,
      "rewards/margins": 0.8831820487976074,
      "rewards/rejected": -46.52573776245117,
      "step": 5505
    },
    {
      "epoch": 2.9489881251045325,
      "grad_norm": 1.7818564844080353e-12,
      "learning_rate": 8.57024858130273e-10,
      "logits/chosen": 9.309082984924316,
      "logits/rejected": 8.756964683532715,
      "logps/chosen": -46.07358932495117,
      "logps/rejected": -46.978416442871094,
      "loss": 0.0,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -46.07358932495117,
      "rewards/margins": 0.9048234820365906,
      "rewards/rejected": -46.978416442871094,
      "step": 5510
    },
    {
      "epoch": 2.951664157885934,
      "grad_norm": 1.1929994163049427e-12,
      "learning_rate": 7.682995466686826e-10,
      "logits/chosen": 9.70967960357666,
      "logits/rejected": 9.164298057556152,
      "logps/chosen": -45.57379913330078,
      "logps/rejected": -46.675018310546875,
      "loss": 0.0,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -45.57379913330078,
      "rewards/margins": 1.1012215614318848,
      "rewards/rejected": -46.675018310546875,
      "step": 5515
    },
    {
      "epoch": 2.9543401906673354,
      "grad_norm": 2.7526309928573536e-11,
      "learning_rate": 6.844177833543741e-10,
      "logits/chosen": 9.421499252319336,
      "logits/rejected": 9.199018478393555,
      "logps/chosen": -45.561363220214844,
      "logps/rejected": -47.19806671142578,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.561363220214844,
      "rewards/margins": 1.6366981267929077,
      "rewards/rejected": -47.19806671142578,
      "step": 5520
    },
    {
      "epoch": 2.957016223448737,
      "grad_norm": 7.67522819439805e-14,
      "learning_rate": 6.053803820087467e-10,
      "logits/chosen": 9.304147720336914,
      "logits/rejected": 8.788652420043945,
      "logps/chosen": -46.47734069824219,
      "logps/rejected": -47.225914001464844,
      "loss": 0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -46.47734069824219,
      "rewards/margins": 0.7485787272453308,
      "rewards/rejected": -47.225914001464844,
      "step": 5525
    },
    {
      "epoch": 2.959692256230139,
      "grad_norm": 7.099943834794598e-09,
      "learning_rate": 5.311881094528514e-10,
      "logits/chosen": 9.374969482421875,
      "logits/rejected": 8.625144958496094,
      "logps/chosen": -45.38831329345703,
      "logps/rejected": -46.69354248046875,
      "loss": 0.0,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -45.38831329345703,
      "rewards/margins": 1.3052278757095337,
      "rewards/rejected": -46.69354248046875,
      "step": 5530
    },
    {
      "epoch": 2.9623682890115406,
      "grad_norm": 2.881600760535787e-05,
      "learning_rate": 4.6184168550050806e-10,
      "logits/chosen": 8.875592231750488,
      "logits/rejected": 8.807694435119629,
      "logps/chosen": -45.00106430053711,
      "logps/rejected": -45.43553161621094,
      "loss": 0.0,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -45.00106430053711,
      "rewards/margins": 0.43446406722068787,
      "rewards/rejected": -45.43553161621094,
      "step": 5535
    },
    {
      "epoch": 2.965044321792942,
      "grad_norm": 1.3352744908639852e-10,
      "learning_rate": 3.973417829510328e-10,
      "logits/chosen": 9.539215087890625,
      "logits/rejected": 9.029739379882812,
      "logps/chosen": -44.717552185058594,
      "logps/rejected": -46.43828201293945,
      "loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -44.717552185058594,
      "rewards/margins": 1.7207313776016235,
      "rewards/rejected": -46.43828201293945,
      "step": 5540
    },
    {
      "epoch": 2.9677203545743436,
      "grad_norm": 4.653986466432795e-12,
      "learning_rate": 3.3768902758274377e-10,
      "logits/chosen": 9.230944633483887,
      "logits/rejected": 8.877386093139648,
      "logps/chosen": -44.17393112182617,
      "logps/rejected": -45.24363327026367,
      "loss": 0.0,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -44.17393112182617,
      "rewards/margins": 1.0697017908096313,
      "rewards/rejected": -45.24363327026367,
      "step": 5545
    },
    {
      "epoch": 2.970396387355745,
      "grad_norm": 1.3098552433508314e-07,
      "learning_rate": 2.8288399814691e-10,
      "logits/chosen": 9.120819091796875,
      "logits/rejected": 9.103508949279785,
      "logps/chosen": -45.6077766418457,
      "logps/rejected": -46.726951599121094,
      "loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -45.6077766418457,
      "rewards/margins": 1.1191757917404175,
      "rewards/rejected": -46.726951599121094,
      "step": 5550
    },
    {
      "epoch": 2.9730724201371466,
      "grad_norm": 7.928332538132965e-11,
      "learning_rate": 2.3292722636220066e-10,
      "logits/chosen": 9.421287536621094,
      "logits/rejected": 8.7330322265625,
      "logps/chosen": -44.95536804199219,
      "logps/rejected": -47.126197814941406,
      "loss": 0.0,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -44.95536804199219,
      "rewards/margins": 2.17083477973938,
      "rewards/rejected": -47.126197814941406,
      "step": 5555
    },
    {
      "epoch": 2.9757484529185483,
      "grad_norm": 1.189054194031107e-11,
      "learning_rate": 1.8781919690946668e-10,
      "logits/chosen": 8.81137466430664,
      "logits/rejected": 8.718222618103027,
      "logps/chosen": -46.6141242980957,
      "logps/rejected": -46.707923889160156,
      "loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -46.6141242980957,
      "rewards/margins": 0.09379738569259644,
      "rewards/rejected": -46.707923889160156,
      "step": 5560
    },
    {
      "epoch": 2.97842448569995,
      "grad_norm": 2.2027557720890748e-08,
      "learning_rate": 1.4756034742696711e-10,
      "logits/chosen": 9.41294002532959,
      "logits/rejected": 9.040725708007812,
      "logps/chosen": -45.043243408203125,
      "logps/rejected": -45.9727668762207,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.043243408203125,
      "rewards/margins": 0.9295236468315125,
      "rewards/rejected": -45.9727668762207,
      "step": 5565
    },
    {
      "epoch": 2.9811005184813513,
      "grad_norm": 2.5254663495301706e-09,
      "learning_rate": 1.12151068506261e-10,
      "logits/chosen": 9.255581855773926,
      "logits/rejected": 8.883371353149414,
      "logps/chosen": -45.58115768432617,
      "logps/rejected": -47.051422119140625,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.58115768432617,
      "rewards/margins": 1.4702672958374023,
      "rewards/rejected": -47.051422119140625,
      "step": 5570
    },
    {
      "epoch": 2.983776551262753,
      "grad_norm": 2.9596700578146245e-10,
      "learning_rate": 8.159170368826629e-11,
      "logits/chosen": 9.244758605957031,
      "logits/rejected": 8.61176586151123,
      "logps/chosen": -44.7607536315918,
      "logps/rejected": -46.781761169433594,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -44.7607536315918,
      "rewards/margins": 2.021002769470215,
      "rewards/rejected": -46.781761169433594,
      "step": 5575
    },
    {
      "epoch": 2.9864525840441547,
      "grad_norm": 4.347859196681055e-10,
      "learning_rate": 5.588254946015114e-11,
      "logits/chosen": 9.469440460205078,
      "logits/rejected": 8.532564163208008,
      "logps/chosen": -44.822479248046875,
      "logps/rejected": -46.4453125,
      "loss": 0.0,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -44.822479248046875,
      "rewards/margins": 1.6228355169296265,
      "rewards/rejected": -46.4453125,
      "step": 5580
    },
    {
      "epoch": 2.989128616825556,
      "grad_norm": 2.7871857291810235e-13,
      "learning_rate": 3.502385525216978e-11,
      "logits/chosen": 9.081156730651855,
      "logits/rejected": 8.444694519042969,
      "logps/chosen": -45.75415802001953,
      "logps/rejected": -46.766876220703125,
      "loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -45.75415802001953,
      "rewards/margins": 1.012715458869934,
      "rewards/rejected": -46.766876220703125,
      "step": 5585
    },
    {
      "epoch": 2.9918046496069577,
      "grad_norm": 2.251584556282719e-09,
      "learning_rate": 1.901582343555308e-11,
      "logits/chosen": 8.882328033447266,
      "logits/rejected": 8.731029510498047,
      "logps/chosen": -44.618873596191406,
      "logps/rejected": -45.2707633972168,
      "loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -44.618873596191406,
      "rewards/margins": 0.6518921256065369,
      "rewards/rejected": -45.2707633972168,
      "step": 5590
    },
    {
      "epoch": 2.9944806823883594,
      "grad_norm": 2.01164221885118e-09,
      "learning_rate": 7.858609320232634e-12,
      "logits/chosen": 8.997359275817871,
      "logits/rejected": 8.305047988891602,
      "logps/chosen": -45.42284393310547,
      "logps/rejected": -47.415523529052734,
      "loss": 0.0,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -45.42284393310547,
      "rewards/margins": 1.9926798343658447,
      "rewards/rejected": -47.415523529052734,
      "step": 5595
    },
    {
      "epoch": 2.9971567151697607,
      "grad_norm": 1.718387865118827e-14,
      "learning_rate": 1.5523211535639624e-12,
      "logits/chosen": 9.273758888244629,
      "logits/rejected": 8.704572677612305,
      "logps/chosen": -45.442649841308594,
      "logps/rejected": -46.81134796142578,
      "loss": 0.0,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -45.442649841308594,
      "rewards/margins": 1.3686975240707397,
      "rewards/rejected": -46.81134796142578,
      "step": 5600
    },
    {
      "epoch": 2.9971567151697607,
      "eval_logits/chosen": 7.7498650550842285,
      "eval_logits/rejected": 7.4245452880859375,
      "eval_logps/chosen": -45.089073181152344,
      "eval_logps/rejected": -46.10942077636719,
      "eval_loss": 1.940507559083926e-07,
      "eval_rewards/accuracies": 0.5682492852210999,
      "eval_rewards/chosen": -45.089073181152344,
      "eval_rewards/margins": 1.0203527212142944,
      "eval_rewards/rejected": -46.10942077636719,
      "eval_runtime": 40.0049,
      "eval_samples_per_second": 33.621,
      "eval_steps_per_second": 8.424,
      "step": 5600
    },
    {
      "epoch": 2.999297541394882,
      "step": 5604,
      "total_flos": 0.0,
      "train_loss": 0.00884905898078017,
      "train_runtime": 29905.5175,
      "train_samples_per_second": 5.998,
      "train_steps_per_second": 0.187
    }
  ],
  "logging_steps": 5,
  "max_steps": 5604,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}