Model save

README.md

@@ -3,16 +3,10 @@ library_name: transformers
 license: apache-2.0
 base_model: alignment-handbook/zephyr-7b-sft-full
 tags:
-- alignment-handbook
-- trl
-- dpo
-- generated_from_trainer
 - trl
 - dpo
 - alignment-handbook
 - generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 model-index:
 - name: zephyr-7b-dpo-full
   results: []
@@ -23,17 +17,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-full
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.2473
-- Rewards/chosen: -4.6815
-- Rewards/rejected: -10.5131
-- Rewards/accuracies: 0.8525
-- Rewards/margins: 5.8316
-- Logps/rejected: -1354.8135
-- Logps/chosen: -759.6055
-- Logits/rejected: -1.2709
-- Logits/chosen: -1.7157
+- Loss: 0.2490
+- Rewards/chosen: -4.6026
+- Rewards/rejected: -9.7319
+- Rewards/accuracies: 0.8463
+- Rewards/margins: 5.1294
+- Logps/rejected: -1276.6984
+- Logps/chosen: -751.7161
+- Logits/rejected: -1.1260
+- Logits/chosen: -1.5108
 
 ## Model description
 
@@ -68,11 +62,11 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
-|:-------------:|:------:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 0.3043        | 0.2559 | 100  | -2.9645       | -2.9613         | -589.3575    | -935.3794      | 0.3080          | 0.8245             | -2.9790        | 3.3398          | -6.3188          |
-| 0.2557        | 0.5118 | 200  | -2.2748       | -2.0707         | -709.4976    | -1222.8809     | 0.2607          | 0.8470             | -4.1804        | 5.0134          | -9.1938          |
-| 0.2515        | 0.7678 | 300  | -1.8309       | -1.4322         | -726.7409    | -1277.9103     | 0.2493          | 0.8509             | -4.3528        | 5.3912          | -9.7441          |
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.3131        | 0.2559 | 100  | 0.3047          | -2.7883        | -6.0602          | 0.8292             | 3.2718          | -909.5203      | -570.2890    | -2.9402         | -2.9498       |
+| 0.2541        | 0.5118 | 200  | 0.2608          | -5.2681        | -10.7685         | 0.8463             | 5.5004          | -1380.3505     | -818.2686    | -1.4771         | -1.8230       |
+| 0.2604        | 0.7678 | 300  | 0.2490          | -4.6026        | -9.7319          | 0.8463             | 5.1294          | -1276.6984     | -751.7161    | -1.1260         | -1.5108       |
 
 
 ### Framework versions

all_results.json

@@ -14,9 +14,9 @@
     "eval_samples_per_second": 11.099,
     "eval_steps_per_second": 0.349,
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 0.0175,
+    "train_loss": 0.32177775089557353,
+    "train_runtime": 13656.3788,
     "train_samples": 50000,
-    "train_samples_per_second": 2864609.543,
-    "train_steps_per_second": 22343.954
+    "train_samples_per_second": 3.661,
+    "train_steps_per_second": 0.029
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.9980806142034548,
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 0.0175,
+    "train_loss": 0.32177775089557353,
+    "train_runtime": 13656.3788,
     "train_samples": 50000,
-    "train_samples_per_second": 2864609.543,
-    "train_steps_per_second": 22343.954
+    "train_samples_per_second": 3.661,
+    "train_steps_per_second": 0.029
 }

trainer_state.json

@@ -10,12 +10,12 @@
   "log_history": [
     {
       "epoch": 0.0025591810620601407,
-      "grad_norm": 8.372040796681393,
+      "grad_norm": 9.300729627928584,
       "learning_rate": 1.282051282051282e-08,
-      "logits/chosen": -2.9558680057525635,
-      "logits/rejected": -2.9835896492004395,
-      "logps/chosen": -287.1746520996094,
-      "logps/rejected": -318.6817626953125,
+      "logits/chosen": -3.076528787612915,
+      "logits/rejected": -3.05928111076355,
+      "logps/chosen": -298.5160217285156,
+      "logps/rejected": -280.6475524902344,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,645 +25,645 @@
     },
     {
       "epoch": 0.025591810620601407,
-      "grad_norm": 9.212524406222368,
+      "grad_norm": 8.91999986583984,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -3.0168228149414062,
-      "logits/rejected": -3.0099453926086426,
-      "logps/chosen": -286.0946044921875,
-      "logps/rejected": -304.9287414550781,
-      "loss": 0.693,
+      "logits/chosen": -3.0055878162384033,
+      "logits/rejected": -3.006002187728882,
+      "logps/chosen": -295.4976806640625,
+      "logps/rejected": -299.0694885253906,
+      "loss": 0.6932,
       "rewards/accuracies": 0.4097222089767456,
-      "rewards/chosen": -0.00034835602855309844,
-      "rewards/margins": -5.7743654906516895e-05,
-      "rewards/rejected": -0.00029061237000860274,
+      "rewards/chosen": -0.00027675420278683305,
+      "rewards/margins": -0.00019084251835010946,
+      "rewards/rejected": -8.591161895310506e-05,
       "step": 10
     },
     {
       "epoch": 0.05118362124120281,
-      "grad_norm": 8.914663082765845,
+      "grad_norm": 8.795980264234538,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -3.006526470184326,
-      "logits/rejected": -2.9971041679382324,
-      "logps/chosen": -283.1783447265625,
-      "logps/rejected": -298.82427978515625,
-      "loss": 0.69,
-      "rewards/accuracies": 0.6468750238418579,
-      "rewards/chosen": 0.0020886282436549664,
-      "rewards/margins": 0.00550027284771204,
-      "rewards/rejected": -0.0034116446040570736,
+      "logits/chosen": -3.013876438140869,
+      "logits/rejected": -2.9846599102020264,
+      "logps/chosen": -278.1435546875,
+      "logps/rejected": -280.6507568359375,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": 0.004018495324999094,
+      "rewards/margins": 0.008613762445747852,
+      "rewards/rejected": -0.004595267120748758,
       "step": 20
     },
     {
       "epoch": 0.07677543186180422,
-      "grad_norm": 9.211645338164717,
+      "grad_norm": 8.739321551260307,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -3.0022165775299072,
-      "logits/rejected": -2.997166156768799,
-      "logps/chosen": -280.0216369628906,
-      "logps/rejected": -295.76959228515625,
-      "loss": 0.6714,
-      "rewards/accuracies": 0.7593749761581421,
-      "rewards/chosen": 0.009928617626428604,
-      "rewards/margins": 0.04372577741742134,
-      "rewards/rejected": -0.03379715979099274,
+      "logits/chosen": -2.9977359771728516,
+      "logits/rejected": -2.9877142906188965,
+      "logps/chosen": -286.0644836425781,
+      "logps/rejected": -299.9364318847656,
+      "loss": 0.6715,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": 0.01204680372029543,
+      "rewards/margins": 0.042642779648303986,
+      "rewards/rejected": -0.03059597872197628,
       "step": 30
     },
     {
       "epoch": 0.10236724248240563,
-      "grad_norm": 10.081033679769522,
+      "grad_norm": 10.25775431193367,
       "learning_rate": 4.99989986344963e-07,
-      "logits/chosen": -3.0331904888153076,
-      "logits/rejected": -3.0275347232818604,
-      "logps/chosen": -285.2835998535156,
-      "logps/rejected": -319.5372314453125,
-      "loss": 0.615,
-      "rewards/accuracies": 0.828125,
-      "rewards/chosen": -0.042057085782289505,
-      "rewards/margins": 0.17162299156188965,
-      "rewards/rejected": -0.21368007361888885,
+      "logits/chosen": -3.0262513160705566,
+      "logits/rejected": -3.018004894256592,
+      "logps/chosen": -292.0692138671875,
+      "logps/rejected": -319.06201171875,
+      "loss": 0.6172,
+      "rewards/accuracies": 0.7906249761581421,
+      "rewards/chosen": -0.040852729231119156,
+      "rewards/margins": 0.15366603434085846,
+      "rewards/rejected": -0.19451875984668732,
       "step": 40
     },
     {
       "epoch": 0.12795905310300704,
-      "grad_norm": 13.895203665337698,
+      "grad_norm": 13.197584682554714,
       "learning_rate": 4.987893180827479e-07,
-      "logits/chosen": -3.0868072509765625,
-      "logits/rejected": -3.0783658027648926,
-      "logps/chosen": -368.4632263183594,
-      "logps/rejected": -446.92608642578125,
-      "loss": 0.5101,
-      "rewards/accuracies": 0.784375011920929,
-      "rewards/chosen": -0.7513679265975952,
-      "rewards/margins": 0.5743271112442017,
-      "rewards/rejected": -1.3256951570510864,
+      "logits/chosen": -3.042942762374878,
+      "logits/rejected": -3.0136711597442627,
+      "logps/chosen": -354.50311279296875,
+      "logps/rejected": -408.25933837890625,
+      "loss": 0.5182,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.7526591420173645,
+      "rewards/margins": 0.5003793239593506,
+      "rewards/rejected": -1.2530385255813599,
       "step": 50
     },
     {
       "epoch": 0.15355086372360843,
-      "grad_norm": 17.530401994512683,
+      "grad_norm": 17.586826775035103,
       "learning_rate": 4.955969343539162e-07,
-      "logits/chosen": -3.0945253372192383,
-      "logits/rejected": -3.077105760574341,
-      "logps/chosen": -525.1121215820312,
-      "logps/rejected": -675.7432861328125,
-      "loss": 0.4203,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.4605610370635986,
-      "rewards/margins": 1.4077235460281372,
-      "rewards/rejected": -3.8682847023010254,
+      "logits/chosen": -3.068596363067627,
+      "logits/rejected": -3.05415415763855,
+      "logps/chosen": -529.9071044921875,
+      "logps/rejected": -668.5487670898438,
+      "loss": 0.4414,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -2.2428505420684814,
+      "rewards/margins": 1.1905735731124878,
+      "rewards/rejected": -3.4334239959716797,
       "step": 60
     },
     {
       "epoch": 0.17914267434420986,
-      "grad_norm": 20.655420138174726,
+      "grad_norm": 29.094385760333942,
       "learning_rate": 4.90438392204474e-07,
-      "logits/chosen": -3.1209347248077393,
-      "logits/rejected": -3.1095337867736816,
-      "logps/chosen": -557.0096435546875,
-      "logps/rejected": -769.7672119140625,
-      "loss": 0.3553,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -2.73195219039917,
-      "rewards/margins": 1.957765817642212,
-      "rewards/rejected": -4.6897172927856445,
+      "logits/chosen": -3.1505279541015625,
+      "logits/rejected": -3.1332974433898926,
+      "logps/chosen": -545.3878173828125,
+      "logps/rejected": -748.8613891601562,
+      "loss": 0.3467,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.668983221054077,
+      "rewards/margins": 1.957241415977478,
+      "rewards/rejected": -4.626224994659424,
       "step": 70
     },
     {
       "epoch": 0.20473448496481125,
-      "grad_norm": 34.20074706717272,
+      "grad_norm": 19.23827111496606,
       "learning_rate": 4.83354989019146e-07,
-      "logits/chosen": -3.0491955280303955,
-      "logits/rejected": -3.05132794380188,
-      "logps/chosen": -558.9640502929688,
-      "logps/rejected": -801.9044799804688,
-      "loss": 0.3429,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -2.753610134124756,
-      "rewards/margins": 2.3364205360412598,
-      "rewards/rejected": -5.090030193328857,
+      "logits/chosen": -3.1376254558563232,
+      "logits/rejected": -3.1242246627807617,
+      "logps/chosen": -567.9331665039062,
+      "logps/rejected": -824.6702880859375,
+      "loss": 0.3436,
+      "rewards/accuracies": 0.8218749761581421,
+      "rewards/chosen": -2.782257556915283,
+      "rewards/margins": 2.4667036533355713,
+      "rewards/rejected": -5.248961448669434,
       "step": 80
     },
     {
       "epoch": 0.23032629558541268,
-      "grad_norm": 25.187798132399784,
+      "grad_norm": 19.580017636565856,
       "learning_rate": 4.7440343190975353e-07,
-      "logits/chosen": -3.0432305335998535,
-      "logits/rejected": -3.041344165802002,
-      "logps/chosen": -570.7512817382812,
-      "logps/rejected": -843.0035400390625,
-      "loss": 0.3284,
-      "rewards/accuracies": 0.8343750238418579,
-      "rewards/chosen": -2.7428793907165527,
-      "rewards/margins": 2.6616787910461426,
-      "rewards/rejected": -5.404558181762695,
+      "logits/chosen": -3.0505166053771973,
+      "logits/rejected": -3.042973279953003,
+      "logps/chosen": -611.1217041015625,
+      "logps/rejected": -919.2169799804688,
+      "loss": 0.3345,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -3.080747604370117,
+      "rewards/margins": 3.0196213722229004,
+      "rewards/rejected": -6.100369453430176,
       "step": 90
     },
     {
       "epoch": 0.2559181062060141,
-      "grad_norm": 39.301413976140616,
+      "grad_norm": 25.054112621298355,
       "learning_rate": 4.6365538373900506e-07,
-      "logits/chosen": -3.000190019607544,
-      "logits/rejected": -3.000822067260742,
-      "logps/chosen": -588.419677734375,
-      "logps/rejected": -898.4981689453125,
-      "loss": 0.3043,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.9578394889831543,
-      "rewards/margins": 3.0813093185424805,
-      "rewards/rejected": -6.039149284362793,
+      "logits/chosen": -2.9996213912963867,
+      "logits/rejected": -2.990790843963623,
+      "logps/chosen": -554.5274658203125,
+      "logps/rejected": -862.6036987304688,
+      "loss": 0.3131,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -2.494730234146118,
+      "rewards/margins": 2.9665369987487793,
+      "rewards/rejected": -5.461266994476318,
       "step": 100
     },
     {
       "epoch": 0.2559181062060141,
-      "eval_logits/chosen": -2.96449875831604,
-      "eval_logits/rejected": -2.961296796798706,
-      "eval_logps/chosen": -589.3575439453125,
-      "eval_logps/rejected": -935.37939453125,
-      "eval_loss": 0.3079955577850342,
-      "eval_rewards/accuracies": 0.8245341777801514,
-      "eval_rewards/chosen": -2.9790048599243164,
-      "eval_rewards/margins": 3.339751958847046,
-      "eval_rewards/rejected": -6.318756580352783,
-      "eval_runtime": 475.0898,
-      "eval_samples_per_second": 10.79,
-      "eval_steps_per_second": 0.339,
+      "eval_logits/chosen": -2.949781656265259,
+      "eval_logits/rejected": -2.940216541290283,
+      "eval_logps/chosen": -570.2890014648438,
+      "eval_logps/rejected": -909.520263671875,
+      "eval_loss": 0.3047349452972412,
+      "eval_rewards/accuracies": 0.8291925191879272,
+      "eval_rewards/chosen": -2.7883195877075195,
+      "eval_rewards/margins": 3.2718467712402344,
+      "eval_rewards/rejected": -6.060166358947754,
+      "eval_runtime": 463.0554,
+      "eval_samples_per_second": 11.07,
+      "eval_steps_per_second": 0.348,
       "step": 100
     },
     {
       "epoch": 0.28150991682661547,
-      "grad_norm": 23.2464532886725,
+      "grad_norm": 28.064453271068697,
       "learning_rate": 4.5119688941406386e-07,
-      "logits/chosen": -2.9700212478637695,
-      "logits/rejected": -2.974587917327881,
-      "logps/chosen": -593.6639404296875,
-      "logps/rejected": -905.212890625,
-      "loss": 0.3259,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.0440046787261963,
-      "rewards/margins": 2.9606070518493652,
-      "rewards/rejected": -6.004611492156982,
+      "logits/chosen": -2.917959451675415,
+      "logits/rejected": -2.907701253890991,
+      "logps/chosen": -648.3985595703125,
+      "logps/rejected": -1051.9510498046875,
+      "loss": 0.313,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -3.600653886795044,
+      "rewards/margins": 3.9409751892089844,
+      "rewards/rejected": -7.541630744934082,
       "step": 110
     },
     {
       "epoch": 0.30710172744721687,
-      "grad_norm": 20.92968428786604,
+      "grad_norm": 24.128631827809503,
       "learning_rate": 4.3712768704277524e-07,
-      "logits/chosen": -2.954521417617798,
-      "logits/rejected": -2.959869861602783,
-      "logps/chosen": -536.2525634765625,
-      "logps/rejected": -830.0895385742188,
-      "loss": 0.3015,
+      "logits/chosen": -2.886887550354004,
+      "logits/rejected": -2.8801417350769043,
+      "logps/chosen": -532.00048828125,
+      "logps/rejected": -861.47802734375,
+      "loss": 0.2992,
       "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.4020705223083496,
-      "rewards/margins": 2.877993106842041,
-      "rewards/rejected": -5.280063629150391,
+      "rewards/chosen": -2.393364429473877,
+      "rewards/margins": 3.2483468055725098,
+      "rewards/rejected": -5.641711235046387,
       "step": 120
     },
     {
       "epoch": 0.3326935380678183,
-      "grad_norm": 28.437529952019855,
+      "grad_norm": 41.03517320506039,
       "learning_rate": 4.2156040946718343e-07,
-      "logits/chosen": -2.862247943878174,
-      "logits/rejected": -2.8785834312438965,
-      "logps/chosen": -636.6513061523438,
-      "logps/rejected": -1044.999755859375,
-      "loss": 0.2841,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -3.4623122215270996,
-      "rewards/margins": 3.8583245277404785,
-      "rewards/rejected": -7.320636749267578,
+      "logits/chosen": -2.7591373920440674,
+      "logits/rejected": -2.7252399921417236,
+      "logps/chosen": -654.140380859375,
+      "logps/rejected": -1031.537109375,
+      "loss": 0.2747,
+      "rewards/accuracies": 0.846875011920929,
+      "rewards/chosen": -3.605952501296997,
+      "rewards/margins": 3.7166149616241455,
+      "rewards/rejected": -7.322567939758301,
       "step": 130
     },
     {
       "epoch": 0.3582853486884197,
-      "grad_norm": 19.315717522096396,
+      "grad_norm": 29.778477903537254,
       "learning_rate": 4.046196825665637e-07,
-      "logits/chosen": -2.832946538925171,
-      "logits/rejected": -2.8259646892547607,
-      "logps/chosen": -593.8186645507812,
-      "logps/rejected": -979.2440185546875,
-      "loss": 0.2622,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -2.943942070007324,
-      "rewards/margins": 3.7794156074523926,
-      "rewards/rejected": -6.723358154296875,
+      "logits/chosen": -2.559828519821167,
+      "logits/rejected": -2.479792594909668,
+      "logps/chosen": -711.0362548828125,
+      "logps/rejected": -1145.183837890625,
+      "loss": 0.2739,
+      "rewards/accuracies": 0.846875011920929,
+      "rewards/chosen": -4.220881938934326,
+      "rewards/margins": 4.242983341217041,
+      "rewards/rejected": -8.463865280151367,
       "step": 140
     },
     {
       "epoch": 0.3838771593090211,
-      "grad_norm": 22.976543958848772,
+      "grad_norm": 26.95404487166528,
       "learning_rate": 3.864411275486261e-07,
-      "logits/chosen": -2.760894536972046,
-      "logits/rejected": -2.7563464641571045,
-      "logps/chosen": -634.1366577148438,
-      "logps/rejected": -1078.597412109375,
-      "loss": 0.2794,
-      "rewards/accuracies": 0.8218749761581421,
-      "rewards/chosen": -3.5211944580078125,
-      "rewards/margins": 4.23276424407959,
-      "rewards/rejected": -7.753958702087402,
+      "logits/chosen": -2.3361523151397705,
+      "logits/rejected": -2.17087721824646,
+      "logps/chosen": -759.3307495117188,
+      "logps/rejected": -1217.0687255859375,
+      "loss": 0.2708,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": -4.6963276863098145,
+      "rewards/margins": 4.567781925201416,
+      "rewards/rejected": -9.264108657836914,
       "step": 150
     },
     {
       "epoch": 0.4094689699296225,
-      "grad_norm": 21.849223996398678,
+      "grad_norm": 36.77841932269942,
       "learning_rate": 3.671702752161759e-07,
-      "logits/chosen": -2.6357340812683105,
-      "logits/rejected": -2.5901741981506348,
-      "logps/chosen": -729.1080322265625,
-      "logps/rejected": -1168.1109619140625,
-      "loss": 0.2781,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -4.447979927062988,
-      "rewards/margins": 4.25943660736084,
-      "rewards/rejected": -8.707415580749512,
+      "logits/chosen": -2.2813222408294678,
+      "logits/rejected": -2.133829116821289,
+      "logps/chosen": -743.0094604492188,
+      "logps/rejected": -1178.709716796875,
+      "loss": 0.2646,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -4.425657272338867,
+      "rewards/margins": 4.283566951751709,
+      "rewards/rejected": -8.709224700927734,
       "step": 160
     },
     {
       "epoch": 0.4350607805502239,
-      "grad_norm": 34.268492856409395,
+      "grad_norm": 31.269061340589527,
       "learning_rate": 3.4696140090121375e-07,
-      "logits/chosen": -2.5432353019714355,
-      "logits/rejected": -2.4383697509765625,
-      "logps/chosen": -768.44775390625,
-      "logps/rejected": -1241.2236328125,
-      "loss": 0.2592,
-      "rewards/accuracies": 0.8531249761581421,
-      "rewards/chosen": -4.733465194702148,
-      "rewards/margins": 4.677088737487793,
-      "rewards/rejected": -9.410554885864258,
+      "logits/chosen": -2.316734790802002,
+      "logits/rejected": -2.122131824493408,
+      "logps/chosen": -777.4652099609375,
+      "logps/rejected": -1272.863525390625,
+      "loss": 0.2719,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": -4.7845916748046875,
+      "rewards/margins": 4.937060356140137,
+      "rewards/rejected": -9.721651077270508,
       "step": 170
     },
     {
       "epoch": 0.46065259117082535,
-      "grad_norm": 24.22372375688885,
+      "grad_norm": 20.7412696930549,
       "learning_rate": 3.259762893935617e-07,
-      "logits/chosen": -2.5379650592803955,
-      "logits/rejected": -2.4227848052978516,
-      "logps/chosen": -643.3690185546875,
-      "logps/rejected": -1086.7647705078125,
-      "loss": 0.2982,
-      "rewards/accuracies": 0.8218749761581421,
-      "rewards/chosen": -3.5777480602264404,
-      "rewards/margins": 4.262465476989746,
-      "rewards/rejected": -7.840213775634766,
+      "logits/chosen": -2.1940865516662598,
+      "logits/rejected": -1.979815125465393,
+      "logps/chosen": -725.8430786132812,
+      "logps/rejected": -1174.8927001953125,
+      "loss": 0.2459,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": -4.354685306549072,
+      "rewards/margins": 4.337596893310547,
+      "rewards/rejected": -8.692281723022461,
       "step": 180
     },
     {
       "epoch": 0.48624440179142675,
-      "grad_norm": 22.276548976639525,
+      "grad_norm": 22.840820454830215,
       "learning_rate": 3.0438293975154184e-07,
-      "logits/chosen": -2.4349989891052246,
-      "logits/rejected": -2.2799932956695557,
-      "logps/chosen": -682.0303955078125,
-      "logps/rejected": -1134.5205078125,
-      "loss": 0.2551,
+      "logits/chosen": -2.1304101943969727,
+      "logits/rejected": -1.8553383350372314,
+      "logps/chosen": -694.48193359375,
+      "logps/rejected": -1045.50927734375,
+      "loss": 0.2737,
       "rewards/accuracies": 0.8531249761581421,
-      "rewards/chosen": -3.8549671173095703,
-      "rewards/margins": 4.42364501953125,
-      "rewards/rejected": -8.27861213684082,
+      "rewards/chosen": -4.092527866363525,
+      "rewards/margins": 3.4944236278533936,
+      "rewards/rejected": -7.58695125579834,
       "step": 190
     },
     {
       "epoch": 0.5118362124120281,
-      "grad_norm": 23.143027388197456,
+      "grad_norm": 48.315922100487725,
       "learning_rate": 2.823542203635138e-07,
-      "logits/chosen": -2.3481929302215576,
-      "logits/rejected": -2.147021770477295,
-      "logps/chosen": -698.4183959960938,
-      "logps/rejected": -1117.36962890625,
-      "loss": 0.2557,
-      "rewards/accuracies": 0.8031250238418579,
-      "rewards/chosen": -3.897473096847534,
-      "rewards/margins": 4.184942722320557,
-      "rewards/rejected": -8.082415580749512,
+      "logits/chosen": -1.982703447341919,
+      "logits/rejected": -1.6738717555999756,
+      "logps/chosen": -708.0183715820312,
+      "logps/rejected": -1162.70556640625,
+      "loss": 0.2541,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": -4.193836688995361,
+      "rewards/margins": 4.424074649810791,
+      "rewards/rejected": -8.617910385131836,
       "step": 200
     },
     {
       "epoch": 0.5118362124120281,
-      "eval_logits/chosen": -2.2748405933380127,
-      "eval_logits/rejected": -2.0707473754882812,
-      "eval_logps/chosen": -709.49755859375,
-      "eval_logps/rejected": -1222.880859375,
-      "eval_loss": 0.2607395350933075,
-      "eval_rewards/accuracies": 0.8470497131347656,
-      "eval_rewards/chosen": -4.1804046630859375,
-      "eval_rewards/margins": 5.013367652893066,
-      "eval_rewards/rejected": -9.193772315979004,
-      "eval_runtime": 467.0944,
-      "eval_samples_per_second": 10.974,
-      "eval_steps_per_second": 0.345,
+      "eval_logits/chosen": -1.8229694366455078,
+      "eval_logits/rejected": -1.4771463871002197,
+      "eval_logps/chosen": -818.2685546875,
+      "eval_logps/rejected": -1380.3504638671875,
+      "eval_loss": 0.2607860863208771,
+      "eval_rewards/accuracies": 0.8462733030319214,
+      "eval_rewards/chosen": -5.268115043640137,
+      "eval_rewards/margins": 5.500354290008545,
+      "eval_rewards/rejected": -10.768467903137207,
+      "eval_runtime": 462.7794,
+      "eval_samples_per_second": 11.077,
+      "eval_steps_per_second": 0.348,
       "step": 200
     },
     {
       "epoch": 0.5374280230326296,
-      "grad_norm": 29.714729651434116,
+      "grad_norm": 31.413529562290883,
       "learning_rate": 2.600664850273538e-07,
-      "logits/chosen": -2.220996379852295,
-      "logits/rejected": -2.0096168518066406,
-      "logps/chosen": -736.2384033203125,
-      "logps/rejected": -1215.465576171875,
-      "loss": 0.265,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -4.440661907196045,
-      "rewards/margins": 4.793159484863281,
-      "rewards/rejected": -9.2338228225708,
+      "logits/chosen": -1.7659847736358643,
+      "logits/rejected": -1.4307024478912354,
+      "logps/chosen": -810.8310546875,
+      "logps/rejected": -1331.2171630859375,
+      "loss": 0.25,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": -5.195899486541748,
+      "rewards/margins": 5.097092151641846,
+      "rewards/rejected": -10.292991638183594,
       "step": 210
     },
     {
       "epoch": 0.5630198336532309,
-      "grad_norm": 30.279152935247957,
+      "grad_norm": 27.081174599726534,
       "learning_rate": 2.3769816112703045e-07,
-      "logits/chosen": -2.0213561058044434,
-      "logits/rejected": -1.708433747291565,
-      "logps/chosen": -803.655029296875,
-      "logps/rejected": -1369.3001708984375,
-      "loss": 0.253,
-      "rewards/accuracies": 0.828125,
-      "rewards/chosen": -5.206329345703125,
-      "rewards/margins": 5.589818000793457,
-      "rewards/rejected": -10.796146392822266,
+      "logits/chosen": -1.6221271753311157,
+      "logits/rejected": -1.2494385242462158,
+      "logps/chosen": -812.7271728515625,
+      "logps/rejected": -1295.7265625,
+      "loss": 0.2522,
+      "rewards/accuracies": 0.846875011920929,
+      "rewards/chosen": -5.059515953063965,
+      "rewards/margins": 4.7599945068359375,
+      "rewards/rejected": -9.819511413574219,
       "step": 220
     },
     {
       "epoch": 0.5886116442738324,
-      "grad_norm": 25.614713397243474,
+      "grad_norm": 17.333742936417842,
       "learning_rate": 2.1542832120881677e-07,
-      "logits/chosen": -1.8582950830459595,
-      "logits/rejected": -1.4825233221054077,
-      "logps/chosen": -796.53857421875,
-      "logps/rejected": -1305.88818359375,
-      "loss": 0.2601,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -5.0762786865234375,
-      "rewards/margins": 5.091577053070068,
-      "rewards/rejected": -10.167856216430664,
+      "logits/chosen": -1.681460976600647,
+      "logits/rejected": -1.3104435205459595,
+      "logps/chosen": -759.7596435546875,
+      "logps/rejected": -1202.4127197265625,
+      "loss": 0.2744,
+      "rewards/accuracies": 0.809374988079071,
+      "rewards/chosen": -4.643033027648926,
+      "rewards/margins": 4.396432876586914,
+      "rewards/rejected": -9.039464950561523,
       "step": 230
     },
     {
       "epoch": 0.6142034548944337,
-      "grad_norm": 24.31506504955288,
+      "grad_norm": 26.442974160839057,
       "learning_rate": 1.934352493925695e-07,
-      "logits/chosen": -1.9886703491210938,
-      "logits/rejected": -1.6142040491104126,
-      "logps/chosen": -769.7489013671875,
-      "logps/rejected": -1311.3548583984375,
-      "loss": 0.2748,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -4.7199506759643555,
-      "rewards/margins": 5.420409202575684,
-      "rewards/rejected": -10.140359878540039,
+      "logits/chosen": -1.5410155057907104,
+      "logits/rejected": -1.2406994104385376,
+      "logps/chosen": -764.8502197265625,
+      "logps/rejected": -1239.8050537109375,
+      "loss": 0.3065,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -4.678874492645264,
+      "rewards/margins": 4.619391441345215,
+      "rewards/rejected": -9.29826545715332,
       "step": 240
     },
     {
       "epoch": 0.6397952655150352,
-      "grad_norm": 31.574156427087846,
+      "grad_norm": 25.27878081759285,
       "learning_rate": 1.7189501409486059e-07,
-      "logits/chosen": -2.0121378898620605,
-      "logits/rejected": -1.6347030401229858,
-      "logps/chosen": -716.2213134765625,
-      "logps/rejected": -1249.190185546875,
-      "loss": 0.2809,
-      "rewards/accuracies": 0.840624988079071,
-      "rewards/chosen": -4.3439483642578125,
-      "rewards/margins": 5.274473667144775,
-      "rewards/rejected": -9.61842155456543,
+      "logits/chosen": -1.6293185949325562,
+      "logits/rejected": -1.3014212846755981,
+      "logps/chosen": -790.1380004882812,
+      "logps/rejected": -1304.321044921875,
+      "loss": 0.243,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": -4.863690376281738,
+      "rewards/margins": 5.004323482513428,
+      "rewards/rejected": -9.868013381958008,
       "step": 250
     },
     {
       "epoch": 0.6653870761356366,
-      "grad_norm": 19.51529401796244,
+      "grad_norm": 33.57879613298401,
       "learning_rate": 1.5098005849021078e-07,
-      "logits/chosen": -2.051848888397217,
-      "logits/rejected": -1.7610851526260376,
-      "logps/chosen": -730.6099853515625,
-      "logps/rejected": -1209.3929443359375,
-      "loss": 0.2457,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -4.3370680809021,
-      "rewards/margins": 4.67025089263916,
-      "rewards/rejected": -9.007319450378418,
+      "logits/chosen": -1.5503108501434326,
+      "logits/rejected": -1.2434519529342651,
+      "logps/chosen": -808.1937255859375,
+      "logps/rejected": -1313.421875,
+      "loss": 0.2634,
+      "rewards/accuracies": 0.809374988079071,
+      "rewards/chosen": -5.169132709503174,
+      "rewards/margins": 4.917426109313965,
+      "rewards/rejected": -10.086559295654297,
       "step": 260
     },
     {
       "epoch": 0.690978886756238,
-      "grad_norm": 34.445303465962446,
+      "grad_norm": 29.480034521734677,
       "learning_rate": 1.30857819994673e-07,
-      "logits/chosen": -1.923056960105896,
-      "logits/rejected": -1.6418602466583252,
-      "logps/chosen": -726.4392700195312,
-      "logps/rejected": -1264.95458984375,
-      "loss": 0.256,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -4.332821369171143,
-      "rewards/margins": 5.251183032989502,
-      "rewards/rejected": -9.584003448486328,
+      "logits/chosen": -1.4918172359466553,
+      "logits/rejected": -1.1054009199142456,
+      "logps/chosen": -813.1369018554688,
+      "logps/rejected": -1310.81640625,
+      "loss": 0.2593,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -5.231433868408203,
+      "rewards/margins": 4.96165132522583,
+      "rewards/rejected": -10.193084716796875,
       "step": 270
     },
     {
       "epoch": 0.7165706973768394,
-      "grad_norm": 36.88942757740681,
+      "grad_norm": 31.86429892286265,
       "learning_rate": 1.116893898236716e-07,
-      "logits/chosen": -1.9537960290908813,
-      "logits/rejected": -1.6011472940444946,
-      "logps/chosen": -746.1478271484375,
-      "logps/rejected": -1309.915283203125,
-      "loss": 0.2386,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -4.4233832359313965,
-      "rewards/margins": 5.588069438934326,
-      "rewards/rejected": -10.011453628540039,
+      "logits/chosen": -1.7091087102890015,
+      "logits/rejected": -1.4138612747192383,
+      "logps/chosen": -746.4378051757812,
+      "logps/rejected": -1237.638427734375,
+      "loss": 0.2394,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -4.613304615020752,
+      "rewards/margins": 4.798872947692871,
+      "rewards/rejected": -9.412178039550781,
       "step": 280
     },
     {
       "epoch": 0.7421625079974408,
-      "grad_norm": 49.4114473741805,
+      "grad_norm": 17.438412834057193,
       "learning_rate": 9.362822335518062e-08,
-      "logits/chosen": -1.8809627294540405,
-      "logits/rejected": -1.427119493484497,
-      "logps/chosen": -769.44140625,
-      "logps/rejected": -1329.4346923828125,
-      "loss": 0.2622,
-      "rewards/accuracies": 0.8843749761581421,
-      "rewards/chosen": -4.582036972045898,
-      "rewards/margins": 5.65748929977417,
-      "rewards/rejected": -10.239526748657227,
+      "logits/chosen": -1.6236486434936523,
+      "logits/rejected": -1.2768752574920654,
+      "logps/chosen": -760.7342529296875,
+      "logps/rejected": -1256.605712890625,
+      "loss": 0.2524,
+      "rewards/accuracies": 0.840624988079071,
+      "rewards/chosen": -4.530442237854004,
+      "rewards/margins": 4.826045036315918,
+      "rewards/rejected": -9.356486320495605,
       "step": 290
     },
     {
       "epoch": 0.7677543186180422,
-      "grad_norm": 19.047743120052225,
+      "grad_norm": 16.423055035755162,
       "learning_rate": 7.681891162260015e-08,
-      "logits/chosen": -1.828704833984375,
-      "logits/rejected": -1.5141593217849731,
-      "logps/chosen": -717.1990966796875,
-      "logps/rejected": -1231.6229248046875,
-      "loss": 0.2515,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -4.390562057495117,
-      "rewards/margins": 4.981083869934082,
-      "rewards/rejected": -9.3716459274292,
+      "logits/chosen": -1.4279536008834839,
+      "logits/rejected": -1.0810502767562866,
+      "logps/chosen": -718.3739013671875,
+      "logps/rejected": -1134.891357421875,
+      "loss": 0.2604,
+      "rewards/accuracies": 0.815625011920929,
+      "rewards/chosen": -4.416540622711182,
+      "rewards/margins": 4.05845832824707,
+      "rewards/rejected": -8.47499942779541,
       "step": 300
     },
     {
       "epoch": 0.7677543186180422,
-      "eval_logits/chosen": -1.830853819847107,
-      "eval_logits/rejected": -1.4321902990341187,
-      "eval_logps/chosen": -726.7409057617188,
-      "eval_logps/rejected": -1277.9102783203125,
-      "eval_loss": 0.24932526051998138,
-      "eval_rewards/accuracies": 0.850931704044342,
-      "eval_rewards/chosen": -4.352837562561035,
-      "eval_rewards/margins": 5.391228675842285,
-      "eval_rewards/rejected": -9.74406623840332,
-      "eval_runtime": 468.6767,
-      "eval_samples_per_second": 10.937,
-      "eval_steps_per_second": 0.344,
+      "eval_logits/chosen": -1.5107975006103516,
+      "eval_logits/rejected": -1.1260188817977905,
+      "eval_logps/chosen": -751.716064453125,
+      "eval_logps/rejected": -1276.6983642578125,
+      "eval_loss": 0.24895673990249634,
+      "eval_rewards/accuracies": 0.8462733030319214,
+      "eval_rewards/chosen": -4.602590084075928,
+      "eval_rewards/margins": 5.129357814788818,
+      "eval_rewards/rejected": -9.73194694519043,
+      "eval_runtime": 461.9008,
+      "eval_samples_per_second": 11.098,
+      "eval_steps_per_second": 0.349,
       "step": 300
     },
     {
       "epoch": 0.7933461292386437,
-      "grad_norm": 27.67097540824916,
+      "grad_norm": 24.363324466471035,
       "learning_rate": 6.139602377230247e-08,
-      "logits/chosen": -1.7593371868133545,
-      "logits/rejected": -1.3604390621185303,
-      "logps/chosen": -744.55078125,
-      "logps/rejected": -1289.401611328125,
-      "loss": 0.2523,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": -4.4812517166137695,
-      "rewards/margins": 5.370087623596191,
-      "rewards/rejected": -9.851339340209961,
+      "logits/chosen": -1.552236557006836,
+      "logits/rejected": -1.2730642557144165,
+      "logps/chosen": -777.7206420898438,
+      "logps/rejected": -1248.6849365234375,
+      "loss": 0.2592,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -4.85605001449585,
+      "rewards/margins": 4.622461318969727,
+      "rewards/rejected": -9.478510856628418,
       "step": 310
     },
     {
       "epoch": 0.818937939859245,
-      "grad_norm": 27.140457734231973,
+      "grad_norm": 27.20082426255916,
       "learning_rate": 4.748302975270837e-08,
-      "logits/chosen": -1.7739003896713257,
-      "logits/rejected": -1.296608805656433,
-      "logps/chosen": -752.3242797851562,
-      "logps/rejected": -1264.2474365234375,
-      "loss": 0.2396,
-      "rewards/accuracies": 0.8343750238418579,
-      "rewards/chosen": -4.523016452789307,
-      "rewards/margins": 5.08230447769165,
-      "rewards/rejected": -9.605320930480957,
+      "logits/chosen": -1.5342400074005127,
+      "logits/rejected": -1.1317594051361084,
+      "logps/chosen": -757.9732666015625,
+      "logps/rejected": -1286.5670166015625,
+      "loss": 0.2424,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -4.64093542098999,
+      "rewards/margins": 5.158129692077637,
+      "rewards/rejected": -9.799064636230469,
       "step": 320
     },
     {
       "epoch": 0.8445297504798465,
-      "grad_norm": 19.26284094768001,
+      "grad_norm": 25.05529887885906,
       "learning_rate": 3.5191311859445795e-08,
-      "logits/chosen": -1.762459397315979,
-      "logits/rejected": -1.3729654550552368,
-      "logps/chosen": -762.8904418945312,
-      "logps/rejected": -1324.27197265625,
-      "loss": 0.2321,
-      "rewards/accuracies": 0.828125,
-      "rewards/chosen": -4.7744550704956055,
-      "rewards/margins": 5.492222785949707,
-      "rewards/rejected": -10.266677856445312,
+      "logits/chosen": -1.6275312900543213,
+      "logits/rejected": -1.1424721479415894,
+      "logps/chosen": -807.8544311523438,
+      "logps/rejected": -1404.4608154296875,
+      "loss": 0.2424,
+      "rewards/accuracies": 0.8656250238418579,
+      "rewards/chosen": -5.022496223449707,
+      "rewards/margins": 5.926790237426758,
+      "rewards/rejected": -10.949285507202148,
       "step": 330
     },
     {
       "epoch": 0.8701215611004478,
-      "grad_norm": 24.386284614263385,
+      "grad_norm": 26.4529069102665,
       "learning_rate": 2.4619273049795996e-08,
-      "logits/chosen": -1.7161592245101929,
-      "logits/rejected": -1.374194860458374,
-      "logps/chosen": -761.0625,
-      "logps/rejected": -1358.326416015625,
-      "loss": 0.2605,
-      "rewards/accuracies": 0.8656250238418579,
-      "rewards/chosen": -4.776429653167725,
-      "rewards/margins": 5.739912986755371,
-      "rewards/rejected": -10.516342163085938,
+      "logits/chosen": -1.4998157024383545,
+      "logits/rejected": -1.1587274074554443,
+      "logps/chosen": -786.7511596679688,
+      "logps/rejected": -1342.9837646484375,
+      "loss": 0.2452,
+      "rewards/accuracies": 0.859375,
+      "rewards/chosen": -5.053293704986572,
+      "rewards/margins": 5.45106840133667,
+      "rewards/rejected": -10.504362106323242,
       "step": 340
     },
     {
       "epoch": 0.8957133717210493,
-      "grad_norm": 25.575727288966945,
+      "grad_norm": 29.52957594878875,
       "learning_rate": 1.5851549164932115e-08,
-      "logits/chosen": -1.698293924331665,
-      "logits/rejected": -1.249987006187439,
-      "logps/chosen": -781.341552734375,
-      "logps/rejected": -1365.3270263671875,
-      "loss": 0.2447,
-      "rewards/accuracies": 0.8218749761581421,
-      "rewards/chosen": -4.80244255065918,
-      "rewards/margins": 5.769272804260254,
-      "rewards/rejected": -10.571714401245117,
+      "logits/chosen": -1.5455517768859863,
+      "logits/rejected": -1.0819389820098877,
+      "logps/chosen": -793.215087890625,
+      "logps/rejected": -1346.713623046875,
+      "loss": 0.2349,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -5.030755519866943,
+      "rewards/margins": 5.420226573944092,
+      "rewards/rejected": -10.450983047485352,
       "step": 350
     },
     {
       "epoch": 0.9213051823416507,
-      "grad_norm": 28.833379096024903,
+      "grad_norm": 25.548406142288144,
       "learning_rate": 8.958331366609423e-09,
-      "logits/chosen": -1.625765085220337,
-      "logits/rejected": -1.215453863143921,
-      "logps/chosen": -729.2706298828125,
-      "logps/rejected": -1337.571044921875,
-      "loss": 0.2386,
+      "logits/chosen": -1.5059678554534912,
+      "logits/rejected": -1.0360510349273682,
+      "logps/chosen": -829.7648315429688,
+      "logps/rejected": -1369.6993408203125,
+      "loss": 0.2633,
       "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.483765602111816,
-      "rewards/margins": 5.966723442077637,
-      "rewards/rejected": -10.450489044189453,
+      "rewards/chosen": -5.2703986167907715,
+      "rewards/margins": 5.340662479400635,
+      "rewards/rejected": -10.611061096191406,
       "step": 360
     },
     {
       "epoch": 0.946896992962252,
-      "grad_norm": 32.57067435027107,
+      "grad_norm": 21.11557499289859,
       "learning_rate": 3.994804212627461e-09,
-      "logits/chosen": -1.7305755615234375,
-      "logits/rejected": -1.2781140804290771,
-      "logps/chosen": -772.3482666015625,
-      "logps/rejected": -1352.6353759765625,
-      "loss": 0.2442,
+      "logits/chosen": -1.5344655513763428,
+      "logits/rejected": -1.0639396905899048,
+      "logps/chosen": -785.5733642578125,
+      "logps/rejected": -1357.8092041015625,
+      "loss": 0.2495,
       "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": -4.808593273162842,
-      "rewards/margins": 5.681990623474121,
-      "rewards/rejected": -10.490584373474121,
+      "rewards/chosen": -4.919853210449219,
+      "rewards/margins": 5.675114154815674,
+      "rewards/rejected": -10.594966888427734,
       "step": 370
     },
     {
       "epoch": 0.9724888035828535,
-      "grad_norm": 52.329250984527555,
+      "grad_norm": 36.07951763974912,
       "learning_rate": 1.0007038696262516e-09,
-      "logits/chosen": -1.7279059886932373,
-      "logits/rejected": -1.3428099155426025,
-      "logps/chosen": -765.5159301757812,
-      "logps/rejected": -1355.879150390625,
-      "loss": 0.2353,
-      "rewards/accuracies": 0.84375,
-      "rewards/chosen": -4.687448024749756,
-      "rewards/margins": 5.837033271789551,
-      "rewards/rejected": -10.524479866027832,
+      "logits/chosen": -1.5291458368301392,
+      "logits/rejected": -0.9540739059448242,
+      "logps/chosen": -803.43017578125,
+      "logps/rejected": -1367.823486328125,
+      "loss": 0.2557,
+      "rewards/accuracies": 0.871874988079071,
+      "rewards/chosen": -5.007359504699707,
+      "rewards/margins": 5.686868667602539,
+      "rewards/rejected": -10.69422721862793,
       "step": 380
     },
     {
       "epoch": 0.9980806142034548,
-      "grad_norm": 22.836707059503702,
+      "grad_norm": 36.10595960803006,
       "learning_rate": 0.0,
-      "logits/chosen": -1.6817991733551025,
-      "logits/rejected": -1.2504949569702148,
-      "logps/chosen": -785.3814086914062,
-      "logps/rejected": -1332.812744140625,
-      "loss": 0.2478,
-      "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": -4.990485191345215,
-      "rewards/margins": 5.404683589935303,
-      "rewards/rejected": -10.395169258117676,
+      "logits/chosen": -1.6231262683868408,
+      "logits/rejected": -1.1367409229278564,
+      "logps/chosen": -790.3123779296875,
+      "logps/rejected": -1342.495849609375,
+      "loss": 0.2454,
+      "rewards/accuracies": 0.840624988079071,
+      "rewards/chosen": -4.837635040283203,
+      "rewards/margins": 5.484267234802246,
+      "rewards/rejected": -10.32190227508545,
       "step": 390
     },
     {
       "epoch": 0.9980806142034548,
       "step": 390,
       "total_flos": 0.0,
-      "train_loss": 0.0,
-      "train_runtime": 0.0175,
-      "train_samples_per_second": 2864609.543,
-      "train_steps_per_second": 22343.954
+      "train_loss": 0.32177775089557353,
+      "train_runtime": 13656.3788,
+      "train_samples_per_second": 3.661,
+      "train_steps_per_second": 0.029
     }
   ],
   "logging_steps": 10,