Training in progress, step 390

README.md

@@ -7,29 +7,33 @@ tags:
 - trl
 - dpo
 - generated_from_trainer
+- trl
+- dpo
+- alignment-handbook
+- generated_from_trainer
 datasets:
 - HuggingFaceH4/ultrafeedback_binarized
 model-index:
-- name: mixed_zephyr-7b-dpo-full
+- name: zephyr-7b-dpo-full
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# mixed_zephyr-7b-dpo-full
+# zephyr-7b-dpo-full
 
 This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
 It achieves the following results on the evaluation set:
-- Logits/chosen: -0.3096
-- Logits/rejected: 0.6049
-- Logps/chosen: -755.9323
-- Logps/rejected: -1192.5621
-- Loss: 0.3152
-- Rewards/accuracies: 0.8184
-- Rewards/chosen: -4.6496
-- Rewards/margins: 4.3751
-- Rewards/rejected: -9.0247
+- Loss: 0.2473
+- Rewards/chosen: -4.6815
+- Rewards/rejected: -10.5131
+- Rewards/accuracies: 0.8525
+- Rewards/margins: 5.8316
+- Logps/rejected: -1354.8135
+- Logps/chosen: -759.6055
+- Logits/rejected: -1.2709
+- Logits/chosen: -1.7157
 
 ## Model description
 
@@ -66,14 +70,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
 |:-------------:|:------:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 0.5385        | 0.1152 | 100  | -2.9012       | -2.8749         | -433.4271    | -527.4997      | 0.4593          | 0.7539             | -1.4246        | 0.9495          | -2.3741          |
-| 0.4369        | 0.2303 | 200  | -1.5078       | -1.1798         | -594.2914    | -823.1062      | 0.3590          | 0.7915             | -3.0332        | 2.2969          | -5.3301          |
-| 0.4119        | 0.3455 | 300  | -0.6166       | -0.1140         | -677.2002    | -996.9340      | 0.3369          | 0.8156             | -3.8623        | 3.2061          | -7.0684          |
-| 0.3964        | 0.4607 | 400  | -0.6209       | 0.2313          | -753.4187    | -1128.0946     | 0.3311          | 0.8178             | -4.6245        | 3.7555          | -8.3800          |
-| 0.3858        | 0.5759 | 500  | -0.7776       | 0.1893          | -694.4181    | -1049.8429     | 0.3247          | 0.8167             | -4.0345        | 3.5630          | -7.5975          |
-| 0.4031        | 0.6910 | 600  | -0.2605       | 0.6163          | -748.3096    | -1143.1573     | 0.3191          | 0.8201             | -4.5734        | 3.9572          | -8.5306          |
-| 0.4007        | 0.8062 | 700  | -0.4982       | 0.4411          | -753.0112    | -1189.4250     | 0.3171          | 0.8178             | -4.6204        | 4.3729          | -8.9933          |
-| 0.3644        | 0.9214 | 800  | -0.3096       | 0.6049          | -755.9323    | -1192.5621     | 0.3152          | 0.8184             | -4.6496        | 4.3751          | -9.0247          |
+| 0.3043        | 0.2559 | 100  | -2.9645       | -2.9613         | -589.3575    | -935.3794      | 0.3080          | 0.8245             | -2.9790        | 3.3398          | -6.3188          |
+| 0.2557        | 0.5118 | 200  | -2.2748       | -2.0707         | -709.4976    | -1222.8809     | 0.2607          | 0.8470             | -4.1804        | 5.0134          | -9.1938          |
+| 0.2515        | 0.7678 | 300  | -1.8309       | -1.4322         | -726.7409    | -1277.9103     | 0.2493          | 0.8509             | -4.3528        | 5.3912          | -9.7441          |
 
 
 ### Framework versions

all_results.json

@@ -1,22 +1,22 @@
 {
-    "epoch": 0.9997120644975526,
-    "eval_logits/chosen": -0.29694831371307373,
-    "eval_logits/rejected": 0.6135479807853699,
-    "eval_logps/chosen": -754.9666748046875,
-    "eval_logps/rejected": -1189.8031005859375,
-    "eval_loss": 0.3148016333580017,
-    "eval_rewards/accuracies": 0.8178251385688782,
-    "eval_rewards/chosen": -4.639986038208008,
-    "eval_rewards/margins": 4.357123374938965,
-    "eval_rewards/rejected": -8.997109413146973,
-    "eval_runtime": 645.0187,
-    "eval_samples": 7126,
-    "eval_samples_per_second": 11.048,
-    "eval_steps_per_second": 0.346,
+    "epoch": 0.9980806142034548,
+    "eval_logits/chosen": -1.7157304286956787,
+    "eval_logits/rejected": -1.2709392309188843,
+    "eval_logps/chosen": -759.6055297851562,
+    "eval_logps/rejected": -1354.8134765625,
+    "eval_loss": 0.24732786417007446,
+    "eval_rewards/accuracies": 0.8524844646453857,
+    "eval_rewards/chosen": -4.681485176086426,
+    "eval_rewards/margins": 5.831614017486572,
+    "eval_rewards/rejected": -10.513099670410156,
+    "eval_runtime": 461.8444,
+    "eval_samples": 5126,
+    "eval_samples_per_second": 11.099,
+    "eval_steps_per_second": 0.349,
     "total_flos": 0.0,
     "train_loss": 0.0,
-    "train_runtime": 0.0211,
-    "train_samples": 111134,
-    "train_samples_per_second": 5273498.215,
-    "train_steps_per_second": 41188.083
+    "train_runtime": 0.0175,
+    "train_samples": 50000,
+    "train_samples_per_second": 2864609.543,
+    "train_steps_per_second": 22343.954
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 0.9997120644975526,
-    "eval_logits/chosen": -0.29694831371307373,
-    "eval_logits/rejected": 0.6135479807853699,
-    "eval_logps/chosen": -754.9666748046875,
-    "eval_logps/rejected": -1189.8031005859375,
-    "eval_loss": 0.3148016333580017,
-    "eval_rewards/accuracies": 0.8178251385688782,
-    "eval_rewards/chosen": -4.639986038208008,
-    "eval_rewards/margins": 4.357123374938965,
-    "eval_rewards/rejected": -8.997109413146973,
-    "eval_runtime": 645.0187,
-    "eval_samples": 7126,
-    "eval_samples_per_second": 11.048,
-    "eval_steps_per_second": 0.346
+    "epoch": 0.9980806142034548,
+    "eval_logits/chosen": -1.7157304286956787,
+    "eval_logits/rejected": -1.2709392309188843,
+    "eval_logps/chosen": -759.6055297851562,
+    "eval_logps/rejected": -1354.8134765625,
+    "eval_loss": 0.24732786417007446,
+    "eval_rewards/accuracies": 0.8524844646453857,
+    "eval_rewards/chosen": -4.681485176086426,
+    "eval_rewards/margins": 5.831614017486572,
+    "eval_rewards/rejected": -10.513099670410156,
+    "eval_runtime": 461.8444,
+    "eval_samples": 5126,
+    "eval_samples_per_second": 11.099,
+    "eval_steps_per_second": 0.349
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2823922804834020966f253a901cab560b3eec3de5bbf71aefec047b40da9bc
+oid sha256:ff4b5a5be76e37572a78f77a91dcee0c5204506ab23eb4ee88dcab323a866a5e
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8f86a008d69991a14daf14453cdf46b66a4990dad6bee154e27b71df98c46d5
+oid sha256:50d5ae8f96013aa6864002408c60efade71e83b8ca8570c55938b71db1aae965
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52931932e9e3ec9cae3635aa4f549d705e718b16ccf550e0c51676afc63d3629
+oid sha256:b45bebc5b40929e9d3c0fd0dcbc06b83d0bf868469735d62f6ab994cb44442b7
 size 4540516344

runs/Aug25_02-05-26_ip-10-0-9-154.ec2.internal/events.out.tfevents.1724552305.ip-10-0-9-154.ec2.internal.56487.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf78747043d32ce75cbab5d2c7810ff04c16dd815569f0a5a6e77bb38ede02e4
-size 29537
+oid sha256:2a3e6262f79c979ad0d185d03abeb223d260c90b920c5b4c29cdb2fed5b8b88e
+size 36083

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.9997120644975526,
+    "epoch": 0.9980806142034548,
     "total_flos": 0.0,
     "train_loss": 0.0,
-    "train_runtime": 0.0211,
-    "train_samples": 111134,
-    "train_samples_per_second": 5273498.215,
-    "train_steps_per_second": 41188.083
+    "train_runtime": 0.0175,
+    "train_samples": 50000,
+    "train_samples_per_second": 2864609.543,
+    "train_steps_per_second": 22343.954
 }

trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9997120644975526,
+  "epoch": 0.9980806142034548,
   "eval_steps": 100,
-  "global_step": 868,
+  "global_step": 390,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.0,
-      "grad_norm": 13.015832712288159,
-      "learning_rate": 5e-07,
-      "logits/chosen": -2.605381965637207,
-      "logits/rejected": -2.5362534523010254,
-      "logps/chosen": -197.4033660888672,
-      "logps/rejected": -176.15130615234375,
+      "epoch": 0.0025591810620601407,
+      "grad_norm": 8.372040796681393,
+      "learning_rate": 1.282051282051282e-08,
+      "logits/chosen": -2.9558680057525635,
+      "logits/rejected": -2.9835896492004395,
+      "logps/chosen": -287.1746520996094,
+      "logps/rejected": -318.6817626953125,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -24,1435 +24,650 @@
       "step": 1
     },
     {
-      "epoch": 0.01151742009789807,
-      "grad_norm": 8.230031374538095,
-      "learning_rate": 5.747126436781609e-08,
-      "logits/chosen": -2.797184705734253,
-      "logits/rejected": -2.768812417984009,
-      "logps/chosen": -266.24053955078125,
-      "logps/rejected": -265.971923828125,
-      "loss": 0.6928,
-      "rewards/accuracies": 0.4305555522441864,
-      "rewards/chosen": -0.00021778659720439464,
-      "rewards/margins": -0.00010571091843303293,
-      "rewards/rejected": -0.00011207569332327694,
+      "epoch": 0.025591810620601407,
+      "grad_norm": 9.212524406222368,
+      "learning_rate": 1.2820512820512818e-07,
+      "logits/chosen": -3.0168228149414062,
+      "logits/rejected": -3.0099453926086426,
+      "logps/chosen": -286.0946044921875,
+      "logps/rejected": -304.9287414550781,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4097222089767456,
+      "rewards/chosen": -0.00034835602855309844,
+      "rewards/margins": -5.7743654906516895e-05,
+      "rewards/rejected": -0.00029061237000860274,
       "step": 10
     },
     {
-      "epoch": 0.02303484019579614,
-      "grad_norm": 8.31009452460146,
-      "learning_rate": 1.1494252873563217e-07,
-      "logits/chosen": -2.802431583404541,
-      "logits/rejected": -2.773219347000122,
-      "logps/chosen": -287.32781982421875,
-      "logps/rejected": -273.28900146484375,
-      "loss": 0.6923,
-      "rewards/accuracies": 0.5406249761581421,
-      "rewards/chosen": 0.0006046505295671523,
-      "rewards/margins": 0.0014849099097773433,
-      "rewards/rejected": -0.0008802594384178519,
+      "epoch": 0.05118362124120281,
+      "grad_norm": 8.914663082765845,
+      "learning_rate": 2.5641025641025636e-07,
+      "logits/chosen": -3.006526470184326,
+      "logits/rejected": -2.9971041679382324,
+      "logps/chosen": -283.1783447265625,
+      "logps/rejected": -298.82427978515625,
+      "loss": 0.69,
+      "rewards/accuracies": 0.6468750238418579,
+      "rewards/chosen": 0.0020886282436549664,
+      "rewards/margins": 0.00550027284771204,
+      "rewards/rejected": -0.0034116446040570736,
       "step": 20
     },
     {
-      "epoch": 0.03455226029369421,
-      "grad_norm": 8.066889291282722,
-      "learning_rate": 1.7241379310344828e-07,
-      "logits/chosen": -2.804356098175049,
-      "logits/rejected": -2.7821590900421143,
-      "logps/chosen": -278.156494140625,
-      "logps/rejected": -270.8301086425781,
-      "loss": 0.6905,
-      "rewards/accuracies": 0.684374988079071,
-      "rewards/chosen": 0.0025812473613768816,
-      "rewards/margins": 0.005251543130725622,
-      "rewards/rejected": -0.0026702960021793842,
+      "epoch": 0.07677543186180422,
+      "grad_norm": 9.211645338164717,
+      "learning_rate": 3.8461538461538463e-07,
+      "logits/chosen": -3.0022165775299072,
+      "logits/rejected": -2.997166156768799,
+      "logps/chosen": -280.0216369628906,
+      "logps/rejected": -295.76959228515625,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.7593749761581421,
+      "rewards/chosen": 0.009928617626428604,
+      "rewards/margins": 0.04372577741742134,
+      "rewards/rejected": -0.03379715979099274,
       "step": 30
     },
     {
-      "epoch": 0.04606968039159228,
-      "grad_norm": 8.939044393747595,
-      "learning_rate": 2.2988505747126435e-07,
-      "logits/chosen": -2.8080034255981445,
-      "logits/rejected": -2.7811412811279297,
-      "logps/chosen": -272.1091003417969,
-      "logps/rejected": -268.6837158203125,
-      "loss": 0.6858,
-      "rewards/accuracies": 0.703125,
-      "rewards/chosen": 0.007119017653167248,
-      "rewards/margins": 0.016155635938048363,
-      "rewards/rejected": -0.009036618284881115,
+      "epoch": 0.10236724248240563,
+      "grad_norm": 10.081033679769522,
+      "learning_rate": 4.99989986344963e-07,
+      "logits/chosen": -3.0331904888153076,
+      "logits/rejected": -3.0275347232818604,
+      "logps/chosen": -285.2835998535156,
+      "logps/rejected": -319.5372314453125,
+      "loss": 0.615,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -0.042057085782289505,
+      "rewards/margins": 0.17162299156188965,
+      "rewards/rejected": -0.21368007361888885,
       "step": 40
     },
     {
-      "epoch": 0.05758710048949035,
-      "grad_norm": 9.805284456793881,
-      "learning_rate": 2.873563218390804e-07,
-      "logits/chosen": -2.834063768386841,
-      "logits/rejected": -2.7892394065856934,
-      "logps/chosen": -284.08453369140625,
-      "logps/rejected": -282.91802978515625,
-      "loss": 0.6762,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.014029329642653465,
-      "rewards/margins": 0.03542623296380043,
-      "rewards/rejected": -0.021396907046437263,
+      "epoch": 0.12795905310300704,
+      "grad_norm": 13.895203665337698,
+      "learning_rate": 4.987893180827479e-07,
+      "logits/chosen": -3.0868072509765625,
+      "logits/rejected": -3.0783658027648926,
+      "logps/chosen": -368.4632263183594,
+      "logps/rejected": -446.92608642578125,
+      "loss": 0.5101,
+      "rewards/accuracies": 0.784375011920929,
+      "rewards/chosen": -0.7513679265975952,
+      "rewards/margins": 0.5743271112442017,
+      "rewards/rejected": -1.3256951570510864,
       "step": 50
     },
     {
-      "epoch": 0.06910452058738842,
-      "grad_norm": 8.352607046334498,
-      "learning_rate": 3.4482758620689656e-07,
-      "logits/chosen": -2.805022716522217,
-      "logits/rejected": -2.796321392059326,
-      "logps/chosen": -292.1920166015625,
-      "logps/rejected": -302.4415588378906,
-      "loss": 0.6624,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": 0.006681998260319233,
-      "rewards/margins": 0.06977846473455429,
-      "rewards/rejected": -0.06309647113084793,
+      "epoch": 0.15355086372360843,
+      "grad_norm": 17.530401994512683,
+      "learning_rate": 4.955969343539162e-07,
+      "logits/chosen": -3.0945253372192383,
+      "logits/rejected": -3.077105760574341,
+      "logps/chosen": -525.1121215820312,
+      "logps/rejected": -675.7432861328125,
+      "loss": 0.4203,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -2.4605610370635986,
+      "rewards/margins": 1.4077235460281372,
+      "rewards/rejected": -3.8682847023010254,
       "step": 60
     },
     {
-      "epoch": 0.0806219406852865,
-      "grad_norm": 9.823702522936284,
-      "learning_rate": 4.0229885057471266e-07,
-      "logits/chosen": -2.75339674949646,
-      "logits/rejected": -2.751986026763916,
-      "logps/chosen": -281.77618408203125,
-      "logps/rejected": -300.4095153808594,
-      "loss": 0.6322,
-      "rewards/accuracies": 0.734375,
-      "rewards/chosen": -0.07559685409069061,
-      "rewards/margins": 0.150864839553833,
-      "rewards/rejected": -0.22646169364452362,
+      "epoch": 0.17914267434420986,
+      "grad_norm": 20.655420138174726,
+      "learning_rate": 4.90438392204474e-07,
+      "logits/chosen": -3.1209347248077393,
+      "logits/rejected": -3.1095337867736816,
+      "logps/chosen": -557.0096435546875,
+      "logps/rejected": -769.7672119140625,
+      "loss": 0.3553,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.73195219039917,
+      "rewards/margins": 1.957765817642212,
+      "rewards/rejected": -4.6897172927856445,
       "step": 70
     },
     {
-      "epoch": 0.09213936078318456,
-      "grad_norm": 11.550756640744595,
-      "learning_rate": 4.597701149425287e-07,
-      "logits/chosen": -2.8751022815704346,
-      "logits/rejected": -2.8525900840759277,
-      "logps/chosen": -316.79888916015625,
-      "logps/rejected": -340.1561584472656,
-      "loss": 0.5913,
-      "rewards/accuracies": 0.765625,
-      "rewards/chosen": -0.24550755321979523,
-      "rewards/margins": 0.29761967062950134,
-      "rewards/rejected": -0.5431272387504578,
+      "epoch": 0.20473448496481125,
+      "grad_norm": 34.20074706717272,
+      "learning_rate": 4.83354989019146e-07,
+      "logits/chosen": -3.0491955280303955,
+      "logits/rejected": -3.05132794380188,
+      "logps/chosen": -558.9640502929688,
+      "logps/rejected": -801.9044799804688,
+      "loss": 0.3429,
+      "rewards/accuracies": 0.778124988079071,
+      "rewards/chosen": -2.753610134124756,
+      "rewards/margins": 2.3364205360412598,
+      "rewards/rejected": -5.090030193328857,
       "step": 80
     },
     {
-      "epoch": 0.10365678088108264,
-      "grad_norm": 18.102002209139584,
-      "learning_rate": 4.999817969178237e-07,
-      "logits/chosen": -2.8152594566345215,
-      "logits/rejected": -2.7724924087524414,
-      "logps/chosen": -363.1444396972656,
-      "logps/rejected": -401.7603759765625,
-      "loss": 0.5547,
-      "rewards/accuracies": 0.746874988079071,
-      "rewards/chosen": -0.6734243631362915,
-      "rewards/margins": 0.5091755986213684,
-      "rewards/rejected": -1.1825997829437256,
+      "epoch": 0.23032629558541268,
+      "grad_norm": 25.187798132399784,
+      "learning_rate": 4.7440343190975353e-07,
+      "logits/chosen": -3.0432305335998535,
+      "logits/rejected": -3.041344165802002,
+      "logps/chosen": -570.7512817382812,
+      "logps/rejected": -843.0035400390625,
+      "loss": 0.3284,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": -2.7428793907165527,
+      "rewards/margins": 2.6616787910461426,
+      "rewards/rejected": -5.404558181762695,
       "step": 90
     },
     {
-      "epoch": 0.1151742009789807,
-      "grad_norm": 15.592173368744417,
-      "learning_rate": 4.996582603056428e-07,
-      "logits/chosen": -2.7807068824768066,
-      "logits/rejected": -2.75152325630188,
-      "logps/chosen": -403.1298828125,
-      "logps/rejected": -451.24072265625,
-      "loss": 0.5385,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": -1.0208370685577393,
-      "rewards/margins": 0.6278557181358337,
-      "rewards/rejected": -1.6486928462982178,
+      "epoch": 0.2559181062060141,
+      "grad_norm": 39.301413976140616,
+      "learning_rate": 4.6365538373900506e-07,
+      "logits/chosen": -3.000190019607544,
+      "logits/rejected": -3.000822067260742,
+      "logps/chosen": -588.419677734375,
+      "logps/rejected": -898.4981689453125,
+      "loss": 0.3043,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.9578394889831543,
+      "rewards/margins": 3.0813093185424805,
+      "rewards/rejected": -6.039149284362793,
       "step": 100
     },
     {
-      "epoch": 0.1151742009789807,
-      "eval_logits/chosen": -2.9011571407318115,
-      "eval_logits/rejected": -2.874889373779297,
-      "eval_logps/chosen": -433.42706298828125,
-      "eval_logps/rejected": -527.4996948242188,
-      "eval_loss": 0.45933064818382263,
-      "eval_rewards/accuracies": 0.753923773765564,
-      "eval_rewards/chosen": -1.424589991569519,
-      "eval_rewards/margins": 0.9494837522506714,
-      "eval_rewards/rejected": -2.3740737438201904,
-      "eval_runtime": 651.6627,
-      "eval_samples_per_second": 10.935,
-      "eval_steps_per_second": 0.342,
+      "epoch": 0.2559181062060141,
+      "eval_logits/chosen": -2.96449875831604,
+      "eval_logits/rejected": -2.961296796798706,
+      "eval_logps/chosen": -589.3575439453125,
+      "eval_logps/rejected": -935.37939453125,
+      "eval_loss": 0.3079955577850342,
+      "eval_rewards/accuracies": 0.8245341777801514,
+      "eval_rewards/chosen": -2.9790048599243164,
+      "eval_rewards/margins": 3.339751958847046,
+      "eval_rewards/rejected": -6.318756580352783,
+      "eval_runtime": 475.0898,
+      "eval_samples_per_second": 10.79,
+      "eval_steps_per_second": 0.339,
       "step": 100
     },
     {
-      "epoch": 0.12669162107687879,
-      "grad_norm": 17.802035855151065,
-      "learning_rate": 4.989308132738126e-07,
-      "logits/chosen": -2.731767416000366,
-      "logits/rejected": -2.702854633331299,
-      "logps/chosen": -390.03009033203125,
-      "logps/rejected": -461.499755859375,
-      "loss": 0.4959,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -1.160954236984253,
-      "rewards/margins": 0.7525253295898438,
-      "rewards/rejected": -1.9134795665740967,
+      "epoch": 0.28150991682661547,
+      "grad_norm": 23.2464532886725,
+      "learning_rate": 4.5119688941406386e-07,
+      "logits/chosen": -2.9700212478637695,
+      "logits/rejected": -2.974587917327881,
+      "logps/chosen": -593.6639404296875,
+      "logps/rejected": -905.212890625,
+      "loss": 0.3259,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -3.0440046787261963,
+      "rewards/margins": 2.9606070518493652,
+      "rewards/rejected": -6.004611492156982,
       "step": 110
     },
     {
-      "epoch": 0.13820904117477684,
-      "grad_norm": 26.847609346017396,
-      "learning_rate": 4.978006327248536e-07,
-      "logits/chosen": -2.6494832038879395,
-      "logits/rejected": -2.6402511596679688,
-      "logps/chosen": -438.6656799316406,
-      "logps/rejected": -550.1033325195312,
-      "loss": 0.475,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.6023308038711548,
-      "rewards/margins": 1.078300952911377,
-      "rewards/rejected": -2.680631637573242,
+      "epoch": 0.30710172744721687,
+      "grad_norm": 20.92968428786604,
+      "learning_rate": 4.3712768704277524e-07,
+      "logits/chosen": -2.954521417617798,
+      "logits/rejected": -2.959869861602783,
+      "logps/chosen": -536.2525634765625,
+      "logps/rejected": -830.0895385742188,
+      "loss": 0.3015,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -2.4020705223083496,
+      "rewards/margins": 2.877993106842041,
+      "rewards/rejected": -5.280063629150391,
       "step": 120
     },
     {
-      "epoch": 0.14972646127267492,
-      "grad_norm": 25.087856993190254,
-      "learning_rate": 4.962695471250032e-07,
-      "logits/chosen": -2.4692533016204834,
-      "logits/rejected": -2.435044050216675,
-      "logps/chosen": -499.8922424316406,
-      "logps/rejected": -645.5679931640625,
-      "loss": 0.468,
-      "rewards/accuracies": 0.746874988079071,
-      "rewards/chosen": -2.0733580589294434,
-      "rewards/margins": 1.5583977699279785,
-      "rewards/rejected": -3.631755828857422,
+      "epoch": 0.3326935380678183,
+      "grad_norm": 28.437529952019855,
+      "learning_rate": 4.2156040946718343e-07,
+      "logits/chosen": -2.862247943878174,
+      "logits/rejected": -2.8785834312438965,
+      "logps/chosen": -636.6513061523438,
+      "logps/rejected": -1044.999755859375,
+      "loss": 0.2841,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -3.4623122215270996,
+      "rewards/margins": 3.8583245277404785,
+      "rewards/rejected": -7.320636749267578,
       "step": 130
     },
     {
-      "epoch": 0.161243881370573,
-      "grad_norm": 26.974432330966298,
-      "learning_rate": 4.94340033546025e-07,
-      "logits/chosen": -1.697016716003418,
-      "logits/rejected": -1.593400239944458,
-      "logps/chosen": -511.65814208984375,
-      "logps/rejected": -659.9658813476562,
-      "loss": 0.4654,
-      "rewards/accuracies": 0.746874988079071,
-      "rewards/chosen": -2.305452823638916,
-      "rewards/margins": 1.5949369668960571,
-      "rewards/rejected": -3.9003894329071045,
+      "epoch": 0.3582853486884197,
+      "grad_norm": 19.315717522096396,
+      "learning_rate": 4.046196825665637e-07,
+      "logits/chosen": -2.832946538925171,
+      "logits/rejected": -2.8259646892547607,
+      "logps/chosen": -593.8186645507812,
+      "logps/rejected": -979.2440185546875,
+      "loss": 0.2622,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -2.943942070007324,
+      "rewards/margins": 3.7794156074523926,
+      "rewards/rejected": -6.723358154296875,
       "step": 140
     },
     {
-      "epoch": 0.17276130146847107,
-      "grad_norm": 21.115401587052915,
-      "learning_rate": 4.920152136576705e-07,
-      "logits/chosen": -1.4327126741409302,
-      "logits/rejected": -1.2659103870391846,
-      "logps/chosen": -538.796630859375,
-      "logps/rejected": -664.4251098632812,
-      "loss": 0.4789,
-      "rewards/accuracies": 0.753125011920929,
-      "rewards/chosen": -2.33674955368042,
-      "rewards/margins": 1.4603914022445679,
-      "rewards/rejected": -3.7971413135528564,
+      "epoch": 0.3838771593090211,
+      "grad_norm": 22.976543958848772,
+      "learning_rate": 3.864411275486261e-07,
+      "logits/chosen": -2.760894536972046,
+      "logits/rejected": -2.7563464641571045,
+      "logps/chosen": -634.1366577148438,
+      "logps/rejected": -1078.597412109375,
+      "loss": 0.2794,
+      "rewards/accuracies": 0.8218749761581421,
+      "rewards/chosen": -3.5211944580078125,
+      "rewards/margins": 4.23276424407959,
+      "rewards/rejected": -7.753958702087402,
       "step": 150
     },
     {
-      "epoch": 0.18427872156636912,
-      "grad_norm": 24.637364700318916,
-      "learning_rate": 4.892988486772756e-07,
-      "logits/chosen": -1.4591898918151855,
-      "logits/rejected": -1.3274848461151123,
-      "logps/chosen": -468.7333068847656,
-      "logps/rejected": -612.8162841796875,
-      "loss": 0.4462,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.950823187828064,
-      "rewards/margins": 1.431302785873413,
-      "rewards/rejected": -3.3821263313293457,
+      "epoch": 0.4094689699296225,
+      "grad_norm": 21.849223996398678,
+      "learning_rate": 3.671702752161759e-07,
+      "logits/chosen": -2.6357340812683105,
+      "logits/rejected": -2.5901741981506348,
+      "logps/chosen": -729.1080322265625,
+      "logps/rejected": -1168.1109619140625,
+      "loss": 0.2781,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -4.447979927062988,
+      "rewards/margins": 4.25943660736084,
+      "rewards/rejected": -8.707415580749512,
       "step": 160
     },
     {
-      "epoch": 0.1957961416642672,
-      "grad_norm": 27.13923752480491,
-      "learning_rate": 4.861953332846629e-07,
-      "logits/chosen": -1.2759544849395752,
-      "logits/rejected": -1.0808634757995605,
-      "logps/chosen": -469.6282653808594,
-      "logps/rejected": -628.2378540039062,
-      "loss": 0.444,
-      "rewards/accuracies": 0.7593749761581421,
-      "rewards/chosen": -2.0090174674987793,
-      "rewards/margins": 1.5872033834457397,
-      "rewards/rejected": -3.5962207317352295,
+      "epoch": 0.4350607805502239,
+      "grad_norm": 34.268492856409395,
+      "learning_rate": 3.4696140090121375e-07,
+      "logits/chosen": -2.5432353019714355,
+      "logits/rejected": -2.4383697509765625,
+      "logps/chosen": -768.44775390625,
+      "logps/rejected": -1241.2236328125,
+      "loss": 0.2592,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": -4.733465194702148,
+      "rewards/margins": 4.677088737487793,
+      "rewards/rejected": -9.410554885864258,
       "step": 170
     },
     {
-      "epoch": 0.20731356176216528,
-      "grad_norm": 22.29941288426432,
-      "learning_rate": 4.827096885121953e-07,
-      "logits/chosen": -0.8839688301086426,
-      "logits/rejected": -0.664128839969635,
-      "logps/chosen": -591.6177978515625,
-      "logps/rejected": -778.9203491210938,
-      "loss": 0.4486,
-      "rewards/accuracies": 0.7406250238418579,
-      "rewards/chosen": -3.2479281425476074,
-      "rewards/margins": 1.83078134059906,
-      "rewards/rejected": -5.078709125518799,
+      "epoch": 0.46065259117082535,
+      "grad_norm": 24.22372375688885,
+      "learning_rate": 3.259762893935617e-07,
+      "logits/chosen": -2.5379650592803955,
+      "logits/rejected": -2.4227848052978516,
+      "logps/chosen": -643.3690185546875,
+      "logps/rejected": -1086.7647705078125,
+      "loss": 0.2982,
+      "rewards/accuracies": 0.8218749761581421,
+      "rewards/chosen": -3.5777480602264404,
+      "rewards/margins": 4.262465476989746,
+      "rewards/rejected": -7.840213775634766,
       "step": 180
     },
     {
-      "epoch": 0.21883098186006333,
-      "grad_norm": 20.150152801800882,
-      "learning_rate": 4.788475536214821e-07,
-      "logits/chosen": -1.1295298337936401,
-      "logits/rejected": -0.8731690645217896,
-      "logps/chosen": -518.4920654296875,
-      "logps/rejected": -677.3343505859375,
-      "loss": 0.4248,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -2.0726380348205566,
-      "rewards/margins": 1.7125848531723022,
-      "rewards/rejected": -3.7852234840393066,
+      "epoch": 0.48624440179142675,
+      "grad_norm": 22.276548976639525,
+      "learning_rate": 3.0438293975154184e-07,
+      "logits/chosen": -2.4349989891052246,
+      "logits/rejected": -2.2799932956695557,
+      "logps/chosen": -682.0303955078125,
+      "logps/rejected": -1134.5205078125,
+      "loss": 0.2551,
+      "rewards/accuracies": 0.8531249761581421,
+      "rewards/chosen": -3.8549671173095703,
+      "rewards/margins": 4.42364501953125,
+      "rewards/rejected": -8.27861213684082,
       "step": 190
     },
     {
-      "epoch": 0.2303484019579614,
-      "grad_norm": 24.4341951464939,
-      "learning_rate": 4.746151769798818e-07,
-      "logits/chosen": -0.9307588338851929,
-      "logits/rejected": -0.6262258291244507,
-      "logps/chosen": -524.0397338867188,
-      "logps/rejected": -701.8967895507812,
-      "loss": 0.4369,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -2.360715389251709,
-      "rewards/margins": 1.8277909755706787,
-      "rewards/rejected": -4.188506603240967,
+      "epoch": 0.5118362124120281,
+      "grad_norm": 23.143027388197456,
+      "learning_rate": 2.823542203635138e-07,
+      "logits/chosen": -2.3481929302215576,
+      "logits/rejected": -2.147021770477295,
+      "logps/chosen": -698.4183959960938,
+      "logps/rejected": -1117.36962890625,
+      "loss": 0.2557,
+      "rewards/accuracies": 0.8031250238418579,
+      "rewards/chosen": -3.897473096847534,
+      "rewards/margins": 4.184942722320557,
+      "rewards/rejected": -8.082415580749512,
       "step": 200
     },
     {
-      "epoch": 0.2303484019579614,
-      "eval_logits/chosen": -1.5077687501907349,
-      "eval_logits/rejected": -1.1797598600387573,
-      "eval_logps/chosen": -594.2913818359375,
-      "eval_logps/rejected": -823.106201171875,
-      "eval_loss": 0.3589639961719513,
-      "eval_rewards/accuracies": 0.7914798259735107,
-      "eval_rewards/chosen": -3.033234119415283,
-      "eval_rewards/margins": 2.2969048023223877,
-      "eval_rewards/rejected": -5.330138683319092,
-      "eval_runtime": 650.6064,
-      "eval_samples_per_second": 10.953,
-      "eval_steps_per_second": 0.343,
+      "epoch": 0.5118362124120281,
+      "eval_logits/chosen": -2.2748405933380127,
+      "eval_logits/rejected": -2.0707473754882812,
+      "eval_logps/chosen": -709.49755859375,
+      "eval_logps/rejected": -1222.880859375,
+      "eval_loss": 0.2607395350933075,
+      "eval_rewards/accuracies": 0.8470497131347656,
+      "eval_rewards/chosen": -4.1804046630859375,
+      "eval_rewards/margins": 5.013367652893066,
+      "eval_rewards/rejected": -9.193772315979004,
+      "eval_runtime": 467.0944,
+      "eval_samples_per_second": 10.974,
+      "eval_steps_per_second": 0.345,
       "step": 200
     },
     {
-      "epoch": 0.2418658220558595,
-      "grad_norm": 27.352856519591263,
-      "learning_rate": 4.7001940595156055e-07,
-      "logits/chosen": -0.7815187573432922,
-      "logits/rejected": -0.46700936555862427,
-      "logps/chosen": -518.8436279296875,
-      "logps/rejected": -683.1966552734375,
-      "loss": 0.4274,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -2.385855197906494,
-      "rewards/margins": 1.621694564819336,
-      "rewards/rejected": -4.00754976272583,
+      "epoch": 0.5374280230326296,
+      "grad_norm": 29.714729651434116,
+      "learning_rate": 2.600664850273538e-07,
+      "logits/chosen": -2.220996379852295,
+      "logits/rejected": -2.0096168518066406,
+      "logps/chosen": -736.2384033203125,
+      "logps/rejected": -1215.465576171875,
+      "loss": 0.265,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -4.440661907196045,
+      "rewards/margins": 4.793159484863281,
+      "rewards/rejected": -9.2338228225708,
       "step": 210
     },
     {
-      "epoch": 0.25338324215375757,
-      "grad_norm": 29.897947419384028,
-      "learning_rate": 4.650676758194623e-07,
-      "logits/chosen": -0.5421683192253113,
-      "logits/rejected": -0.02623056247830391,
-      "logps/chosen": -606.1685791015625,
-      "logps/rejected": -831.0916137695312,
-      "loss": 0.4012,
-      "rewards/accuracies": 0.765625,
-      "rewards/chosen": -3.0587515830993652,
-      "rewards/margins": 2.499514102935791,
-      "rewards/rejected": -5.558266639709473,
+      "epoch": 0.5630198336532309,
+      "grad_norm": 30.279152935247957,
+      "learning_rate": 2.3769816112703045e-07,
+      "logits/chosen": -2.0213561058044434,
+      "logits/rejected": -1.708433747291565,
+      "logps/chosen": -803.655029296875,
+      "logps/rejected": -1369.3001708984375,
+      "loss": 0.253,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -5.206329345703125,
+      "rewards/margins": 5.589818000793457,
+      "rewards/rejected": -10.796146392822266,
       "step": 220
     },
     {
-      "epoch": 0.26490066225165565,
-      "grad_norm": 28.31850344555953,
-      "learning_rate": 4.5976799775611215e-07,
-      "logits/chosen": -0.28304657340049744,
-      "logits/rejected": 0.2166980504989624,
-      "logps/chosen": -565.9539794921875,
-      "logps/rejected": -766.6756591796875,
-      "loss": 0.4392,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": -2.7024905681610107,
-      "rewards/margins": 2.067142963409424,
-      "rewards/rejected": -4.7696332931518555,
+      "epoch": 0.5886116442738324,
+      "grad_norm": 25.614713397243474,
+      "learning_rate": 2.1542832120881677e-07,
+      "logits/chosen": -1.8582950830459595,
+      "logits/rejected": -1.4825233221054077,
+      "logps/chosen": -796.53857421875,
+      "logps/rejected": -1305.88818359375,
+      "loss": 0.2601,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -5.0762786865234375,
+      "rewards/margins": 5.091577053070068,
+      "rewards/rejected": -10.167856216430664,
       "step": 230
     },
     {
-      "epoch": 0.2764180823495537,
-      "grad_norm": 25.790552553148434,
-      "learning_rate": 4.5412894586271543e-07,
-      "logits/chosen": -0.3281463384628296,
-      "logits/rejected": 0.12199939787387848,
-      "logps/chosen": -534.4832763671875,
-      "logps/rejected": -700.3882446289062,
-      "loss": 0.4403,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -2.3464341163635254,
-      "rewards/margins": 1.9047329425811768,
-      "rewards/rejected": -4.251167297363281,
+      "epoch": 0.6142034548944337,
+      "grad_norm": 24.31506504955288,
+      "learning_rate": 1.934352493925695e-07,
+      "logits/chosen": -1.9886703491210938,
+      "logits/rejected": -1.6142040491104126,
+      "logps/chosen": -769.7489013671875,
+      "logps/rejected": -1311.3548583984375,
+      "loss": 0.2748,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -4.7199506759643555,
+      "rewards/margins": 5.420409202575684,
+      "rewards/rejected": -10.140359878540039,
       "step": 240
     },
     {
-      "epoch": 0.28793550244745175,
-      "grad_norm": 29.425669097369397,
-      "learning_rate": 4.481596432975201e-07,
-      "logits/chosen": -0.6021678447723389,
-      "logits/rejected": -0.20536144077777863,
-      "logps/chosen": -615.7349853515625,
-      "logps/rejected": -839.0997924804688,
-      "loss": 0.4298,
-      "rewards/accuracies": 0.7593749761581421,
-      "rewards/chosen": -3.1481640338897705,
-      "rewards/margins": 2.2502574920654297,
-      "rewards/rejected": -5.398421764373779,
+      "epoch": 0.6397952655150352,
+      "grad_norm": 31.574156427087846,
+      "learning_rate": 1.7189501409486059e-07,
+      "logits/chosen": -2.0121378898620605,
+      "logits/rejected": -1.6347030401229858,
+      "logps/chosen": -716.2213134765625,
+      "logps/rejected": -1249.190185546875,
+      "loss": 0.2809,
+      "rewards/accuracies": 0.840624988079071,
+      "rewards/chosen": -4.3439483642578125,
+      "rewards/margins": 5.274473667144775,
+      "rewards/rejected": -9.61842155456543,
       "step": 250
     },
     {
-      "epoch": 0.29945292254534983,
-      "grad_norm": 23.62933629230091,
-      "learning_rate": 4.41869747515886e-07,
-      "logits/chosen": -0.2845512330532074,
-      "logits/rejected": 0.14756298065185547,
-      "logps/chosen": -572.5442504882812,
-      "logps/rejected": -812.703125,
-      "loss": 0.3968,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -2.856945514678955,
-      "rewards/margins": 2.3578898906707764,
-      "rewards/rejected": -5.214835166931152,
+      "epoch": 0.6653870761356366,
+      "grad_norm": 19.51529401796244,
+      "learning_rate": 1.5098005849021078e-07,
+      "logits/chosen": -2.051848888397217,
+      "logits/rejected": -1.7610851526260376,
+      "logps/chosen": -730.6099853515625,
+      "logps/rejected": -1209.3929443359375,
+      "loss": 0.2457,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -4.3370680809021,
+      "rewards/margins": 4.67025089263916,
+      "rewards/rejected": -9.007319450378418,
       "step": 260
     },
     {
-      "epoch": 0.3109703426432479,
-      "grad_norm": 36.01630964835951,
-      "learning_rate": 4.352694346459396e-07,
-      "logits/chosen": -0.057602040469646454,
-      "logits/rejected": 0.40555334091186523,
-      "logps/chosen": -587.2971801757812,
-      "logps/rejected": -866.1613159179688,
-      "loss": 0.4006,
-      "rewards/accuracies": 0.784375011920929,
-      "rewards/chosen": -3.123883008956909,
-      "rewards/margins": 2.7192797660827637,
-      "rewards/rejected": -5.84316349029541,
+      "epoch": 0.690978886756238,
+      "grad_norm": 34.445303465962446,
+      "learning_rate": 1.30857819994673e-07,
+      "logits/chosen": -1.923056960105896,
+      "logits/rejected": -1.6418602466583252,
+      "logps/chosen": -726.4392700195312,
+      "logps/rejected": -1264.95458984375,
+      "loss": 0.256,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -4.332821369171143,
+      "rewards/margins": 5.251183032989502,
+      "rewards/rejected": -9.584003448486328,
       "step": 270
     },
     {
-      "epoch": 0.322487762741146,
-      "grad_norm": 26.73415377993604,
-      "learning_rate": 4.2836938302509256e-07,
-      "logits/chosen": -0.25706934928894043,
-      "logits/rejected": 0.16837282478809357,
-      "logps/chosen": -575.8345947265625,
-      "logps/rejected": -808.24267578125,
-      "loss": 0.4075,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -2.9973578453063965,
-      "rewards/margins": 2.355498790740967,
-      "rewards/rejected": -5.352856636047363,
+      "epoch": 0.7165706973768394,
+      "grad_norm": 36.88942757740681,
+      "learning_rate": 1.116893898236716e-07,
+      "logits/chosen": -1.9537960290908813,
+      "logits/rejected": -1.6011472940444946,
+      "logps/chosen": -746.1478271484375,
+      "logps/rejected": -1309.915283203125,
+      "loss": 0.2386,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -4.4233832359313965,
+      "rewards/margins": 5.588069438934326,
+      "rewards/rejected": -10.011453628540039,
       "step": 280
     },
     {
-      "epoch": 0.33400518283904407,
-      "grad_norm": 29.332592595497015,
-      "learning_rate": 4.2118075592405874e-07,
-      "logits/chosen": -0.3039420247077942,
-      "logits/rejected": 0.07993211597204208,
-      "logps/chosen": -582.0941162109375,
-      "logps/rejected": -830.5714111328125,
-      "loss": 0.3976,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -2.9472875595092773,
-      "rewards/margins": 2.5015506744384766,
-      "rewards/rejected": -5.448838233947754,
+      "epoch": 0.7421625079974408,
+      "grad_norm": 49.4114473741805,
+      "learning_rate": 9.362822335518062e-08,
+      "logits/chosen": -1.8809627294540405,
+      "logits/rejected": -1.427119493484497,
+      "logps/chosen": -769.44140625,
+      "logps/rejected": -1329.4346923828125,
+      "loss": 0.2622,
+      "rewards/accuracies": 0.8843749761581421,
+      "rewards/chosen": -4.582036972045898,
+      "rewards/margins": 5.65748929977417,
+      "rewards/rejected": -10.239526748657227,
       "step": 290
     },
     {
-      "epoch": 0.34552260293694215,
-      "grad_norm": 30.91612291215343,
-      "learning_rate": 4.137151834863213e-07,
-      "logits/chosen": -0.10641048848628998,
-      "logits/rejected": 0.6166712641716003,
-      "logps/chosen": -632.7642822265625,
-      "logps/rejected": -849.4898681640625,
-      "loss": 0.4119,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -3.399906873703003,
-      "rewards/margins": 2.507375478744507,
-      "rewards/rejected": -5.90728235244751,
+      "epoch": 0.7677543186180422,
+      "grad_norm": 19.047743120052225,
+      "learning_rate": 7.681891162260015e-08,
+      "logits/chosen": -1.828704833984375,
+      "logits/rejected": -1.5141593217849731,
+      "logps/chosen": -717.1990966796875,
+      "logps/rejected": -1231.6229248046875,
+      "loss": 0.2515,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -4.390562057495117,
+      "rewards/margins": 4.981083869934082,
+      "rewards/rejected": -9.3716459274292,
       "step": 300
     },
     {
-      "epoch": 0.34552260293694215,
-      "eval_logits/chosen": -0.6165890693664551,
-      "eval_logits/rejected": -0.11399216204881668,
-      "eval_logps/chosen": -677.2001953125,
-      "eval_logps/rejected": -996.9340209960938,
-      "eval_loss": 0.336904913187027,
-      "eval_rewards/accuracies": 0.8155829310417175,
-      "eval_rewards/chosen": -3.862321615219116,
-      "eval_rewards/margins": 3.206094741821289,
-      "eval_rewards/rejected": -7.068417072296143,
-      "eval_runtime": 656.6921,
-      "eval_samples_per_second": 10.851,
-      "eval_steps_per_second": 0.34,
+      "epoch": 0.7677543186180422,
+      "eval_logits/chosen": -1.830853819847107,
+      "eval_logits/rejected": -1.4321902990341187,
+      "eval_logps/chosen": -726.7409057617188,
+      "eval_logps/rejected": -1277.9102783203125,
+      "eval_loss": 0.24932526051998138,
+      "eval_rewards/accuracies": 0.850931704044342,
+      "eval_rewards/chosen": -4.352837562561035,
+      "eval_rewards/margins": 5.391228675842285,
+      "eval_rewards/rejected": -9.74406623840332,
+      "eval_runtime": 468.6767,
+      "eval_samples_per_second": 10.937,
+      "eval_steps_per_second": 0.344,
       "step": 300
     },
     {
-      "epoch": 0.35704002303484017,
-      "grad_norm": 22.38837991601497,
-      "learning_rate": 4.059847439122671e-07,
-      "logits/chosen": -0.46659454703330994,
-      "logits/rejected": 0.0826030969619751,
-      "logps/chosen": -515.8815307617188,
-      "logps/rejected": -717.310302734375,
-      "loss": 0.4112,
-      "rewards/accuracies": 0.765625,
-      "rewards/chosen": -2.256371021270752,
-      "rewards/margins": 2.008225679397583,
-      "rewards/rejected": -4.264596462249756,
+      "epoch": 0.7933461292386437,
+      "grad_norm": 27.67097540824916,
+      "learning_rate": 6.139602377230247e-08,
+      "logits/chosen": -1.7593371868133545,
+      "logits/rejected": -1.3604390621185303,
+      "logps/chosen": -744.55078125,
+      "logps/rejected": -1289.401611328125,
+      "loss": 0.2523,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -4.4812517166137695,
+      "rewards/margins": 5.370087623596191,
+      "rewards/rejected": -9.851339340209961,
       "step": 310
     },
     {
-      "epoch": 0.36855744313273825,
-      "grad_norm": 21.515754430109986,
-      "learning_rate": 3.98001943918432e-07,
-      "logits/chosen": -0.8846302032470703,
-      "logits/rejected": -0.03813103586435318,
-      "logps/chosen": -544.3895263671875,
-      "logps/rejected": -746.3841552734375,
-      "loss": 0.3939,
-      "rewards/accuracies": 0.809374988079071,
-      "rewards/chosen": -2.379772663116455,
-      "rewards/margins": 2.3421151638031006,
-      "rewards/rejected": -4.721888542175293,
+      "epoch": 0.818937939859245,
+      "grad_norm": 27.140457734231973,
+      "learning_rate": 4.748302975270837e-08,
+      "logits/chosen": -1.7739003896713257,
+      "logits/rejected": -1.296608805656433,
+      "logps/chosen": -752.3242797851562,
+      "logps/rejected": -1264.2474365234375,
+      "loss": 0.2396,
+      "rewards/accuracies": 0.8343750238418579,
+      "rewards/chosen": -4.523016452789307,
+      "rewards/margins": 5.08230447769165,
+      "rewards/rejected": -9.605320930480957,
       "step": 320
     },
     {
-      "epoch": 0.38007486323063633,
-      "grad_norm": 33.71230207361674,
-      "learning_rate": 3.8977969850346866e-07,
-      "logits/chosen": 0.13661722838878632,
-      "logits/rejected": 0.7041386365890503,
-      "logps/chosen": -666.94482421875,
-      "logps/rejected": -926.0341796875,
-      "loss": 0.3873,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -3.78490948677063,
-      "rewards/margins": 2.646435022354126,
-      "rewards/rejected": -6.431344509124756,
+      "epoch": 0.8445297504798465,
+      "grad_norm": 19.26284094768001,
+      "learning_rate": 3.5191311859445795e-08,
+      "logits/chosen": -1.762459397315979,
+      "logits/rejected": -1.3729654550552368,
+      "logps/chosen": -762.8904418945312,
+      "logps/rejected": -1324.27197265625,
+      "loss": 0.2321,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -4.7744550704956055,
+      "rewards/margins": 5.492222785949707,
+      "rewards/rejected": -10.266677856445312,
       "step": 330
     },
     {
-      "epoch": 0.3915922833285344,
-      "grad_norm": 28.524858622055092,
-      "learning_rate": 3.8133131005357465e-07,
-      "logits/chosen": -0.015070567838847637,
-      "logits/rejected": 0.6914359927177429,
-      "logps/chosen": -646.4139404296875,
-      "logps/rejected": -965.0103759765625,
-      "loss": 0.3971,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -3.5984835624694824,
-      "rewards/margins": 3.210897922515869,
-      "rewards/rejected": -6.809381008148193,
+      "epoch": 0.8701215611004478,
+      "grad_norm": 24.386284614263385,
+      "learning_rate": 2.4619273049795996e-08,
+      "logits/chosen": -1.7161592245101929,
+      "logits/rejected": -1.374194860458374,
+      "logps/chosen": -761.0625,
+      "logps/rejected": -1358.326416015625,
+      "loss": 0.2605,
+      "rewards/accuracies": 0.8656250238418579,
+      "rewards/chosen": -4.776429653167725,
+      "rewards/margins": 5.739912986755371,
+      "rewards/rejected": -10.516342163085938,
       "step": 340
     },
     {
-      "epoch": 0.4031097034264325,
-      "grad_norm": 32.078697347416266,
-      "learning_rate": 3.7267044682118435e-07,
-      "logits/chosen": -0.002132108900696039,
-      "logits/rejected": 0.7953078150749207,
-      "logps/chosen": -604.9791259765625,
-      "logps/rejected": -838.1949462890625,
-      "loss": 0.4191,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -3.1062846183776855,
-      "rewards/margins": 2.339332342147827,
-      "rewards/rejected": -5.445616722106934,
+      "epoch": 0.8957133717210493,
+      "grad_norm": 25.575727288966945,
+      "learning_rate": 1.5851549164932115e-08,
+      "logits/chosen": -1.698293924331665,
+      "logits/rejected": -1.249987006187439,
+      "logps/chosen": -781.341552734375,
+      "logps/rejected": -1365.3270263671875,
+      "loss": 0.2447,
+      "rewards/accuracies": 0.8218749761581421,
+      "rewards/chosen": -4.80244255065918,
+      "rewards/margins": 5.769272804260254,
+      "rewards/rejected": -10.571714401245117,
       "step": 350
     },
     {
-      "epoch": 0.41462712352433057,
-      "grad_norm": 28.020517011807925,
-      "learning_rate": 3.638111208117425e-07,
-      "logits/chosen": -0.1473531574010849,
-      "logits/rejected": 0.490295946598053,
-      "logps/chosen": -583.7153930664062,
-      "logps/rejected": -761.9363403320312,
-      "loss": 0.4035,
-      "rewards/accuracies": 0.765625,
-      "rewards/chosen": -3.0424270629882812,
-      "rewards/margins": 1.7358585596084595,
-      "rewards/rejected": -4.778285026550293,
+      "epoch": 0.9213051823416507,
+      "grad_norm": 28.833379096024903,
+      "learning_rate": 8.958331366609423e-09,
+      "logits/chosen": -1.625765085220337,
+      "logits/rejected": -1.215453863143921,
+      "logps/chosen": -729.2706298828125,
+      "logps/rejected": -1337.571044921875,
+      "loss": 0.2386,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -4.483765602111816,
+      "rewards/margins": 5.966723442077637,
+      "rewards/rejected": -10.450489044189453,
       "step": 360
     },
     {
-      "epoch": 0.42614454362222864,
-      "grad_norm": 25.853288738352997,
-      "learning_rate": 3.5476766511433605e-07,
-      "logits/chosen": -0.25570568442344666,
-      "logits/rejected": 0.6842668652534485,
-      "logps/chosen": -590.0350341796875,
-      "logps/rejected": -811.4537353515625,
-      "loss": 0.3968,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -3.002671480178833,
-      "rewards/margins": 2.369654655456543,
-      "rewards/rejected": -5.372325897216797,
+      "epoch": 0.946896992962252,
+      "grad_norm": 32.57067435027107,
+      "learning_rate": 3.994804212627461e-09,
+      "logits/chosen": -1.7305755615234375,
+      "logits/rejected": -1.2781140804290771,
+      "logps/chosen": -772.3482666015625,
+      "logps/rejected": -1352.6353759765625,
+      "loss": 0.2442,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -4.808593273162842,
+      "rewards/margins": 5.681990623474121,
+      "rewards/rejected": -10.490584373474121,
       "step": 370
     },
     {
-      "epoch": 0.43766196372012667,
-      "grad_norm": 21.591809702398923,
-      "learning_rate": 3.455547107128602e-07,
-      "logits/chosen": -0.12841393053531647,
-      "logits/rejected": 0.6481091380119324,
-      "logps/chosen": -580.2199096679688,
-      "logps/rejected": -826.1383666992188,
-      "loss": 0.3958,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -3.195159435272217,
-      "rewards/margins": 2.441926956176758,
-      "rewards/rejected": -5.637085914611816,
+      "epoch": 0.9724888035828535,
+      "grad_norm": 52.329250984527555,
+      "learning_rate": 1.0007038696262516e-09,
+      "logits/chosen": -1.7279059886932373,
+      "logits/rejected": -1.3428099155426025,
+      "logps/chosen": -765.5159301757812,
+      "logps/rejected": -1355.879150390625,
+      "loss": 0.2353,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -4.687448024749756,
+      "rewards/margins": 5.837033271789551,
+      "rewards/rejected": -10.524479866027832,
       "step": 380
     },
     {
-      "epoch": 0.44917938381802475,
-      "grad_norm": 39.83795352564531,
-      "learning_rate": 3.361871628152338e-07,
-      "logits/chosen": -0.23047828674316406,
-      "logits/rejected": 0.7577739953994751,
-      "logps/chosen": -605.4849853515625,
-      "logps/rejected": -883.64501953125,
-      "loss": 0.4085,
-      "rewards/accuracies": 0.809374988079071,
-      "rewards/chosen": -3.1104187965393066,
-      "rewards/margins": 3.0135536193847656,
-      "rewards/rejected": -6.123971939086914,
+      "epoch": 0.9980806142034548,
+      "grad_norm": 22.836707059503702,
+      "learning_rate": 0.0,
+      "logits/chosen": -1.6817991733551025,
+      "logits/rejected": -1.2504949569702148,
+      "logps/chosen": -785.3814086914062,
+      "logps/rejected": -1332.812744140625,
+      "loss": 0.2478,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -4.990485191345215,
+      "rewards/margins": 5.404683589935303,
+      "rewards/rejected": -10.395169258117676,
       "step": 390
     },
     {
-      "epoch": 0.4606968039159228,
-      "grad_norm": 22.463302227367222,
-      "learning_rate": 3.2668017673896077e-07,
-      "logits/chosen": -0.22118684649467468,
-      "logits/rejected": 0.6193957924842834,
-      "logps/chosen": -640.8189697265625,
-      "logps/rejected": -955.4924926757812,
-      "loss": 0.3964,
-      "rewards/accuracies": 0.809374988079071,
-      "rewards/chosen": -3.495349884033203,
-      "rewards/margins": 3.084470748901367,
-      "rewards/rejected": -6.579820156097412,
-      "step": 400
-    },
-    {
-      "epoch": 0.4606968039159228,
-      "eval_logits/chosen": -0.6209221482276917,
-      "eval_logits/rejected": 0.23131267726421356,
-      "eval_logps/chosen": -753.418701171875,
-      "eval_logps/rejected": -1128.0946044921875,
-      "eval_loss": 0.33106523752212524,
-      "eval_rewards/accuracies": 0.8178251385688782,
-      "eval_rewards/chosen": -4.624506950378418,
-      "eval_rewards/margins": 3.7555172443389893,
-      "eval_rewards/rejected": -8.380023956298828,
-      "eval_runtime": 655.865,
-      "eval_samples_per_second": 10.865,
-      "eval_steps_per_second": 0.34,
-      "step": 400
-    },
-    {
-      "epoch": 0.4722142240138209,
-      "grad_norm": 27.33004967085911,
-      "learning_rate": 3.1704913339205103e-07,
-      "logits/chosen": 0.38320040702819824,
-      "logits/rejected": 1.2441421747207642,
-      "logps/chosen": -592.7208862304688,
-      "logps/rejected": -816.4508666992188,
-      "loss": 0.407,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -3.1608848571777344,
-      "rewards/margins": 2.369687080383301,
-      "rewards/rejected": -5.530571937561035,
-      "step": 410
-    },
-    {
-      "epoch": 0.483731644111719,
-      "grad_norm": 29.22769569320565,
-      "learning_rate": 3.0730961438896885e-07,
-      "logits/chosen": -0.32711368799209595,
-      "logits/rejected": 0.6167188882827759,
-      "logps/chosen": -647.5065307617188,
-      "logps/rejected": -920.5850830078125,
-      "loss": 0.3864,
-      "rewards/accuracies": 0.815625011920929,
-      "rewards/chosen": -3.577653408050537,
-      "rewards/margins": 2.697723865509033,
-      "rewards/rejected": -6.27537727355957,
-      "step": 420
-    },
-    {
-      "epoch": 0.49524906420961706,
-      "grad_norm": 29.1628367265211,
-      "learning_rate": 2.9747737684186795e-07,
-      "logits/chosen": -0.8004047274589539,
-      "logits/rejected": 0.0654061958193779,
-      "logps/chosen": -586.2633056640625,
-      "logps/rejected": -828.8479614257812,
-      "loss": 0.4008,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -3.018512487411499,
-      "rewards/margins": 2.515615701675415,
-      "rewards/rejected": -5.534128189086914,
-      "step": 430
-    },
-    {
-      "epoch": 0.5067664843075151,
-      "grad_norm": 43.05788588925481,
-      "learning_rate": 2.8756832786789663e-07,
-      "logits/chosen": -0.7165388464927673,
-      "logits/rejected": 0.3907933533191681,
-      "logps/chosen": -558.6912231445312,
-      "logps/rejected": -839.3739013671875,
-      "loss": 0.3988,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": -2.8957457542419434,
-      "rewards/margins": 2.8470349311828613,
-      "rewards/rejected": -5.742780685424805,
-      "step": 440
-    },
-    {
-      "epoch": 0.5182839044054132,
-      "grad_norm": 26.95003512302597,
-      "learning_rate": 2.7759849885381747e-07,
-      "logits/chosen": -0.43579286336898804,
-      "logits/rejected": 0.7088162302970886,
-      "logps/chosen": -564.5299072265625,
-      "logps/rejected": -807.0545043945312,
-      "loss": 0.3965,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -2.7139556407928467,
-      "rewards/margins": 2.583310127258301,
-      "rewards/rejected": -5.297266483306885,
-      "step": 450
-    },
-    {
-      "epoch": 0.5298013245033113,
-      "grad_norm": 37.40829093424466,
-      "learning_rate": 2.675840195195762e-07,
-      "logits/chosen": -0.4753951132297516,
-      "logits/rejected": 0.5207837224006653,
-      "logps/chosen": -559.075927734375,
-      "logps/rejected": -858.9351806640625,
-      "loss": 0.3858,
-      "rewards/accuracies": 0.809374988079071,
-      "rewards/chosen": -2.8625900745391846,
-      "rewards/margins": 2.9560627937316895,
-      "rewards/rejected": -5.818652153015137,
-      "step": 460
-    },
-    {
-      "epoch": 0.5413187446012093,
-      "grad_norm": 28.860389068235733,
-      "learning_rate": 2.575410918227829e-07,
-      "logits/chosen": -0.4289991855621338,
-      "logits/rejected": 0.41408976912498474,
-      "logps/chosen": -583.07763671875,
-      "logps/rejected": -848.7003784179688,
-      "loss": 0.3851,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -2.932926654815674,
-      "rewards/margins": 2.7647881507873535,
-      "rewards/rejected": -5.697714805603027,
-      "step": 470
-    },
-    {
-      "epoch": 0.5528361646991073,
-      "grad_norm": 25.478968182398468,
-      "learning_rate": 2.474859637463226e-07,
-      "logits/chosen": 0.019112158566713333,
-      "logits/rejected": 0.9573495984077454,
-      "logps/chosen": -578.31005859375,
-      "logps/rejected": -817.9622192382812,
-      "loss": 0.4001,
-      "rewards/accuracies": 0.753125011920929,
-      "rewards/chosen": -3.071147918701172,
-      "rewards/margins": 2.528298854827881,
-      "rewards/rejected": -5.599446773529053,
-      "step": 480
-    },
-    {
-      "epoch": 0.5643535847970055,
-      "grad_norm": 22.69267875960799,
-      "learning_rate": 2.3743490301150355e-07,
-      "logits/chosen": 0.03456907719373703,
-      "logits/rejected": 0.9821624755859375,
-      "logps/chosen": -616.0007934570312,
-      "logps/rejected": -855.6959228515625,
-      "loss": 0.395,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -3.1994495391845703,
-      "rewards/margins": 2.4747273921966553,
-      "rewards/rejected": -5.6741766929626465,
-      "step": 490
-    },
-    {
-      "epoch": 0.5758710048949035,
-      "grad_norm": 26.70832967985792,
-      "learning_rate": 2.274041707592724e-07,
-      "logits/chosen": -0.4122609496116638,
-      "logits/rejected": 0.6060948371887207,
-      "logps/chosen": -594.303466796875,
-      "logps/rejected": -892.1234130859375,
-      "loss": 0.3858,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.213183879852295,
-      "rewards/margins": 2.860560894012451,
-      "rewards/rejected": -6.0737457275390625,
-      "step": 500
-    },
-    {
-      "epoch": 0.5758710048949035,
-      "eval_logits/chosen": -0.7776147127151489,
-      "eval_logits/rejected": 0.18928049504756927,
-      "eval_logps/chosen": -694.4180908203125,
-      "eval_logps/rejected": -1049.8428955078125,
-      "eval_loss": 0.3246955871582031,
-      "eval_rewards/accuracies": 0.8167040348052979,
-      "eval_rewards/chosen": -4.034500598907471,
-      "eval_rewards/margins": 3.563004732131958,
-      "eval_rewards/rejected": -7.59750509262085,
-      "eval_runtime": 874.6942,
-      "eval_samples_per_second": 8.147,
-      "eval_steps_per_second": 0.255,
-      "step": 500
-    },
-    {
-      "epoch": 0.5873884249928016,
-      "grad_norm": 21.857166040982808,
-      "learning_rate": 2.17409995242075e-07,
-      "logits/chosen": -0.3013337552547455,
-      "logits/rejected": 0.687148928642273,
-      "logps/chosen": -590.9053955078125,
-      "logps/rejected": -848.279296875,
-      "loss": 0.3623,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": -3.1760306358337402,
-      "rewards/margins": 2.6910133361816406,
-      "rewards/rejected": -5.867043972015381,
-      "step": 510
-    },
-    {
-      "epoch": 0.5989058450906997,
-      "grad_norm": 32.93018464240502,
-      "learning_rate": 2.0746854556892544e-07,
-      "logits/chosen": -0.28416475653648376,
-      "logits/rejected": 0.760982871055603,
-      "logps/chosen": -584.7510986328125,
-      "logps/rejected": -825.0016479492188,
-      "loss": 0.3654,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -3.0093677043914795,
-      "rewards/margins": 2.5775859355926514,
-      "rewards/rejected": -5.586953639984131,
-      "step": 520
-    },
-    {
-      "epoch": 0.6104232651885978,
-      "grad_norm": 31.84439684571111,
-      "learning_rate": 1.9759590554616173e-07,
-      "logits/chosen": -0.21416716277599335,
-      "logits/rejected": 0.8462156057357788,
-      "logps/chosen": -591.3154296875,
-      "logps/rejected": -826.24853515625,
-      "loss": 0.39,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -3.1136324405670166,
-      "rewards/margins": 2.453207015991211,
-      "rewards/rejected": -5.56683874130249,
-      "step": 530
-    },
-    {
-      "epoch": 0.6219406852864958,
-      "grad_norm": 28.506645648712848,
-      "learning_rate": 1.8780804765620746e-07,
-      "logits/chosen": -0.06838655471801758,
-      "logits/rejected": 1.0294172763824463,
-      "logps/chosen": -577.9981689453125,
-      "logps/rejected": -835.3642578125,
-      "loss": 0.3793,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -2.834435224533081,
-      "rewards/margins": 2.704789876937866,
-      "rewards/rejected": -5.539225101470947,
-      "step": 540
-    },
-    {
-      "epoch": 0.6334581053843938,
-      "grad_norm": 30.179970032375,
-      "learning_rate": 1.7812080721643973e-07,
-      "logits/chosen": -0.30736392736434937,
-      "logits/rejected": 0.8852709531784058,
-      "logps/chosen": -576.0755615234375,
-      "logps/rejected": -836.8414306640625,
-      "loss": 0.381,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -2.8606371879577637,
-      "rewards/margins": 2.6888041496276855,
-      "rewards/rejected": -5.549441337585449,
-      "step": 550
-    },
-    {
-      "epoch": 0.644975525482292,
-      "grad_norm": 26.709457513505647,
-      "learning_rate": 1.6854985675997063e-07,
-      "logits/chosen": -0.26044386625289917,
-      "logits/rejected": 0.7742006778717041,
-      "logps/chosen": -582.1048583984375,
-      "logps/rejected": -819.34033203125,
-      "loss": 0.4007,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -2.9853100776672363,
-      "rewards/margins": 2.4650139808654785,
-      "rewards/rejected": -5.450324058532715,
-      "step": 560
-    },
-    {
-      "epoch": 0.65649294558019,
-      "grad_norm": 27.543745008054035,
-      "learning_rate": 1.5911068067978818e-07,
-      "logits/chosen": -0.05375183746218681,
-      "logits/rejected": 1.1043269634246826,
-      "logps/chosen": -581.2166748046875,
-      "logps/rejected": -818.9441528320312,
-      "loss": 0.3971,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": -2.8354713916778564,
-      "rewards/margins": 2.655651330947876,
-      "rewards/rejected": -5.491122245788574,
-      "step": 570
-    },
-    {
-      "epoch": 0.6680103656780881,
-      "grad_norm": 22.093767953647365,
-      "learning_rate": 1.4981855017728197e-07,
-      "logits/chosen": 0.0580272376537323,
-      "logits/rejected": 0.7513723373413086,
-      "logps/chosen": -571.5791625976562,
-      "logps/rejected": -858.3342895507812,
-      "loss": 0.3701,
-      "rewards/accuracies": 0.784375011920929,
-      "rewards/chosen": -3.0100650787353516,
-      "rewards/margins": 2.6902260780334473,
-      "rewards/rejected": -5.700291633605957,
-      "step": 580
-    },
-    {
-      "epoch": 0.6795277857759862,
-      "grad_norm": 36.73163562183304,
-      "learning_rate": 1.406884985556804e-07,
-      "logits/chosen": -0.005457936320453882,
-      "logits/rejected": 1.047271490097046,
-      "logps/chosen": -635.6920166015625,
-      "logps/rejected": -881.1370849609375,
-      "loss": 0.3825,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -3.439662456512451,
-      "rewards/margins": 2.575695037841797,
-      "rewards/rejected": -6.01535701751709,
-      "step": 590
-    },
-    {
-      "epoch": 0.6910452058738843,
-      "grad_norm": 30.080057939243627,
-      "learning_rate": 1.3173529689837354e-07,
-      "logits/chosen": -0.23538751900196075,
-      "logits/rejected": 0.9952915906906128,
-      "logps/chosen": -625.2637939453125,
-      "logps/rejected": -905.7393798828125,
-      "loss": 0.4031,
-      "rewards/accuracies": 0.765625,
-      "rewards/chosen": -3.336890459060669,
-      "rewards/margins": 3.089966058731079,
-      "rewards/rejected": -6.42685604095459,
-      "step": 600
-    },
-    {
-      "epoch": 0.6910452058738843,
-      "eval_logits/chosen": -0.26048585772514343,
-      "eval_logits/rejected": 0.6162645220756531,
-      "eval_logps/chosen": -748.3095703125,
-      "eval_logps/rejected": -1143.1573486328125,
-      "eval_loss": 0.3190823495388031,
-      "eval_rewards/accuracies": 0.820067286491394,
-      "eval_rewards/chosen": -4.573415279388428,
-      "eval_rewards/margins": 3.9572343826293945,
-      "eval_rewards/rejected": -8.530649185180664,
-      "eval_runtime": 651.1572,
-      "eval_samples_per_second": 10.944,
-      "eval_steps_per_second": 0.342,
-      "step": 600
-    },
-    {
-      "epoch": 0.7025626259717823,
-      "grad_norm": 28.901245277836818,
-      "learning_rate": 1.2297343017146726e-07,
-      "logits/chosen": 0.07719476521015167,
-      "logits/rejected": 1.148842453956604,
-      "logps/chosen": -615.670166015625,
-      "logps/rejected": -902.8016357421875,
-      "loss": 0.385,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -3.2692692279815674,
-      "rewards/margins": 2.9544837474823,
-      "rewards/rejected": -6.223752975463867,
-      "step": 610
-    },
-    {
-      "epoch": 0.7140800460696803,
-      "grad_norm": 26.881220630663055,
-      "learning_rate": 1.1441707378923474e-07,
-      "logits/chosen": 0.3414779305458069,
-      "logits/rejected": 1.1920559406280518,
-      "logps/chosen": -611.7634887695312,
-      "logps/rejected": -883.4708862304688,
-      "loss": 0.4032,
-      "rewards/accuracies": 0.7593749761581421,
-      "rewards/chosen": -3.267723798751831,
-      "rewards/margins": 2.798133373260498,
-      "rewards/rejected": -6.065857410430908,
-      "step": 620
-    },
-    {
-      "epoch": 0.7255974661675785,
-      "grad_norm": 22.92522846442678,
-      "learning_rate": 1.06080070680377e-07,
-      "logits/chosen": 0.059290122240781784,
-      "logits/rejected": 1.0623096227645874,
-      "logps/chosen": -614.2824096679688,
-      "logps/rejected": -868.0498046875,
-      "loss": 0.372,
-      "rewards/accuracies": 0.7718750238418579,
-      "rewards/chosen": -3.2304539680480957,
-      "rewards/margins": 2.7317616939544678,
-      "rewards/rejected": -5.962214946746826,
-      "step": 630
-    },
-    {
-      "epoch": 0.7371148862654765,
-      "grad_norm": 18.474464704704374,
-      "learning_rate": 9.797590889219587e-08,
-      "logits/chosen": -0.07347230613231659,
-      "logits/rejected": 0.7878081798553467,
-      "logps/chosen": -598.407958984375,
-      "logps/rejected": -922.5660400390625,
-      "loss": 0.3733,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": -3.226865768432617,
-      "rewards/margins": 3.285538911819458,
-      "rewards/rejected": -6.5124053955078125,
-      "step": 640
-    },
-    {
-      "epoch": 0.7486323063633746,
-      "grad_norm": 25.105406106031534,
-      "learning_rate": 9.011769976891367e-08,
-      "logits/chosen": 0.06855427473783493,
-      "logits/rejected": 1.2701406478881836,
-      "logps/chosen": -594.861083984375,
-      "logps/rejected": -820.2781372070312,
-      "loss": 0.3929,
-      "rewards/accuracies": 0.828125,
-      "rewards/chosen": -2.9540488719940186,
-      "rewards/margins": 2.544384241104126,
-      "rewards/rejected": -5.4984331130981445,
-      "step": 650
-    },
-    {
-      "epoch": 0.7601497264612727,
-      "grad_norm": 25.930812393377074,
-      "learning_rate": 8.251815673944218e-08,
-      "logits/chosen": -0.13862136006355286,
-      "logits/rejected": 0.950897216796875,
-      "logps/chosen": -660.083740234375,
-      "logps/rejected": -984.3076171875,
-      "loss": 0.3798,
-      "rewards/accuracies": 0.815625011920929,
-      "rewards/chosen": -3.4895882606506348,
-      "rewards/margins": 3.4641425609588623,
-      "rewards/rejected": -6.953730583190918,
-      "step": 660
-    },
-    {
-      "epoch": 0.7716671465591708,
-      "grad_norm": 22.848572550568402,
-      "learning_rate": 7.518957474892148e-08,
-      "logits/chosen": 0.03879556804895401,
-      "logits/rejected": 0.8222616314888,
-      "logps/chosen": -593.1844482421875,
-      "logps/rejected": -868.5235595703125,
-      "loss": 0.3716,
-      "rewards/accuracies": 0.809374988079071,
-      "rewards/chosen": -3.1576333045959473,
-      "rewards/margins": 2.791321277618408,
-      "rewards/rejected": -5.9489545822143555,
-      "step": 670
-    },
-    {
-      "epoch": 0.7831845666570688,
-      "grad_norm": 37.77871708341422,
-      "learning_rate": 6.814381036730274e-08,
-      "logits/chosen": -0.06809209287166595,
-      "logits/rejected": 0.9388583898544312,
-      "logps/chosen": -602.1769409179688,
-      "logps/rejected": -918.8854370117188,
-      "loss": 0.4027,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -3.232111692428589,
-      "rewards/margins": 3.247992753982544,
-      "rewards/rejected": -6.480103969573975,
-      "step": 680
-    },
-    {
-      "epoch": 0.7947019867549668,
-      "grad_norm": 25.302552395916596,
-      "learning_rate": 6.139226260715872e-08,
-      "logits/chosen": -0.06320186704397202,
-      "logits/rejected": 0.8823334574699402,
-      "logps/chosen": -625.1529541015625,
-      "logps/rejected": -898.3558349609375,
-      "loss": 0.3655,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -3.5077052116394043,
-      "rewards/margins": 2.779940366744995,
-      "rewards/rejected": -6.2876458168029785,
-      "step": 690
-    },
-    {
-      "epoch": 0.806219406852865,
-      "grad_norm": 41.487105287447704,
-      "learning_rate": 5.4945854481754734e-08,
-      "logits/chosen": 0.07060976326465607,
-      "logits/rejected": 0.9207429885864258,
-      "logps/chosen": -644.9542236328125,
-      "logps/rejected": -981.1370849609375,
-      "loss": 0.4007,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -3.7041306495666504,
-      "rewards/margins": 3.448993682861328,
-      "rewards/rejected": -7.1531243324279785,
-      "step": 700
-    },
-    {
-      "epoch": 0.806219406852865,
-      "eval_logits/chosen": -0.4981551170349121,
-      "eval_logits/rejected": 0.44106799364089966,
-      "eval_logps/chosen": -753.01123046875,
-      "eval_logps/rejected": -1189.425048828125,
-      "eval_loss": 0.31710898876190186,
-      "eval_rewards/accuracies": 0.8178251385688782,
-      "eval_rewards/chosen": -4.620431900024414,
-      "eval_rewards/margins": 4.372895240783691,
-      "eval_rewards/rejected": -8.993328094482422,
-      "eval_runtime": 653.0396,
-      "eval_samples_per_second": 10.912,
-      "eval_steps_per_second": 0.341,
-      "step": 700
-    },
-    {
-      "epoch": 0.817736826950763,
-      "grad_norm": 26.15798738128027,
-      "learning_rate": 4.881501533321605e-08,
-      "logits/chosen": -0.3350176513195038,
-      "logits/rejected": 0.5944274663925171,
-      "logps/chosen": -611.4078369140625,
-      "logps/rejected": -894.845703125,
-      "loss": 0.3819,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": -3.2478299140930176,
-      "rewards/margins": 2.9104466438293457,
-      "rewards/rejected": -6.158276557922363,
-      "step": 710
-    },
-    {
-      "epoch": 0.8292542470486611,
-      "grad_norm": 28.210401445519196,
-      "learning_rate": 4.300966395938377e-08,
-      "logits/chosen": -0.47553783655166626,
-      "logits/rejected": 0.6052624583244324,
-      "logps/chosen": -642.4817504882812,
-      "logps/rejected": -950.6018676757812,
-      "loss": 0.3724,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -3.420116901397705,
-      "rewards/margins": 3.2143654823303223,
-      "rewards/rejected": -6.634482383728027,
-      "step": 720
-    },
-    {
-      "epoch": 0.8407716671465592,
-      "grad_norm": 27.28999144486062,
-      "learning_rate": 3.7539192566655246e-08,
-      "logits/chosen": -0.0816282406449318,
-      "logits/rejected": 0.8518702387809753,
-      "logps/chosen": -626.6390991210938,
-      "logps/rejected": -941.8958129882812,
-      "loss": 0.3713,
-      "rewards/accuracies": 0.8343750238418579,
-      "rewards/chosen": -3.378054141998291,
-      "rewards/margins": 3.2282519340515137,
-      "rewards/rejected": -6.606306552886963,
-      "step": 730
-    },
-    {
-      "epoch": 0.8522890872444573,
-      "grad_norm": 27.71621255798267,
-      "learning_rate": 3.24124515747731e-08,
-      "logits/chosen": -0.0028346062172204256,
-      "logits/rejected": 1.1290369033813477,
-      "logps/chosen": -672.173828125,
-      "logps/rejected": -975.3580322265625,
-      "loss": 0.374,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": -3.6781773567199707,
-      "rewards/margins": 3.3185067176818848,
-      "rewards/rejected": -6.996683597564697,
-      "step": 740
-    },
-    {
-      "epoch": 0.8638065073423553,
-      "grad_norm": 35.482960030400996,
-      "learning_rate": 2.763773529814506e-08,
-      "logits/chosen": 0.17448297142982483,
-      "logits/rejected": 0.9923737645149231,
-      "logps/chosen": -603.94970703125,
-      "logps/rejected": -925.9880981445312,
-      "loss": 0.3918,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -3.315547466278076,
-      "rewards/margins": 3.1224138736724854,
-      "rewards/rejected": -6.437961578369141,
-      "step": 750
-    },
-    {
-      "epoch": 0.8753239274402533,
-      "grad_norm": 28.184713620117034,
-      "learning_rate": 2.3222768526860698e-08,
-      "logits/chosen": 0.0647897943854332,
-      "logits/rejected": 0.9855157136917114,
-      "logps/chosen": -613.9244995117188,
-      "logps/rejected": -901.00927734375,
-      "loss": 0.3741,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -3.254974365234375,
-      "rewards/margins": 2.7708938121795654,
-      "rewards/rejected": -6.0258684158325195,
-      "step": 760
-    },
-    {
-      "epoch": 0.8868413475381515,
-      "grad_norm": 35.13633269103924,
-      "learning_rate": 1.9174694029115146e-08,
-      "logits/chosen": 0.19886977970600128,
-      "logits/rejected": 1.013934850692749,
-      "logps/chosen": -620.3436279296875,
-      "logps/rejected": -958.3701171875,
-      "loss": 0.3682,
-      "rewards/accuracies": 0.8218749761581421,
-      "rewards/chosen": -3.4876530170440674,
-      "rewards/margins": 3.2890784740448,
-      "rewards/rejected": -6.776731967926025,
-      "step": 770
-    },
-    {
-      "epoch": 0.8983587676360495,
-      "grad_norm": 29.350577487943855,
-      "learning_rate": 1.5500060995258134e-08,
-      "logits/chosen": 0.12428224086761475,
-      "logits/rejected": 1.2418944835662842,
-      "logps/chosen": -604.94873046875,
-      "logps/rejected": -891.98828125,
-      "loss": 0.37,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -3.2872474193573,
-      "rewards/margins": 2.9589405059814453,
-      "rewards/rejected": -6.24618673324585,
-      "step": 780
-    },
-    {
-      "epoch": 0.9098761877339476,
-      "grad_norm": 31.77954056090223,
-      "learning_rate": 1.2204814442165812e-08,
-      "logits/chosen": 0.1471497118473053,
-      "logits/rejected": 1.0471051931381226,
-      "logps/chosen": -657.8414306640625,
-      "logps/rejected": -977.1556396484375,
-      "loss": 0.3992,
-      "rewards/accuracies": 0.784375011920929,
-      "rewards/chosen": -3.6439871788024902,
-      "rewards/margins": 3.291074752807617,
-      "rewards/rejected": -6.935061454772949,
-      "step": 790
-    },
-    {
-      "epoch": 0.9213936078318457,
-      "grad_norm": 35.231363022526715,
-      "learning_rate": 9.294285595075669e-09,
-      "logits/chosen": 0.23517772555351257,
-      "logits/rejected": 1.1635137796401978,
-      "logps/chosen": -621.228515625,
-      "logps/rejected": -941.3455200195312,
-      "loss": 0.3644,
-      "rewards/accuracies": 0.7906249761581421,
-      "rewards/chosen": -3.437223434448242,
-      "rewards/margins": 3.3359901905059814,
-      "rewards/rejected": -6.7732133865356445,
-      "step": 800
-    },
-    {
-      "epoch": 0.9213936078318457,
-      "eval_logits/chosen": -0.3096068501472473,
-      "eval_logits/rejected": 0.6049354672431946,
-      "eval_logps/chosen": -755.9322509765625,
-      "eval_logps/rejected": -1192.5621337890625,
-      "eval_loss": 0.31517288088798523,
-      "eval_rewards/accuracies": 0.818385660648346,
-      "eval_rewards/chosen": -4.649641990661621,
-      "eval_rewards/margins": 4.37505578994751,
-      "eval_rewards/rejected": -9.024698257446289,
-      "eval_runtime": 652.0187,
-      "eval_samples_per_second": 10.929,
-      "eval_steps_per_second": 0.342,
-      "step": 800
-    },
-    {
-      "epoch": 0.9329110279297438,
-      "grad_norm": 35.70619984366826,
-      "learning_rate": 6.773183262446914e-09,
-      "logits/chosen": 0.08351641893386841,
-      "logits/rejected": 1.0455710887908936,
-      "logps/chosen": -619.4977416992188,
-      "logps/rejected": -918.3001098632812,
-      "loss": 0.4056,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -3.3363006114959717,
-      "rewards/margins": 3.10974383354187,
-      "rewards/rejected": -6.446043968200684,
-      "step": 810
-    },
-    {
-      "epoch": 0.9444284480276418,
-      "grad_norm": 32.90474966984876,
-      "learning_rate": 4.645586217799452e-09,
-      "logits/chosen": -0.05233382433652878,
-      "logits/rejected": 0.976836085319519,
-      "logps/chosen": -630.252685546875,
-      "logps/rejected": -968.9739379882812,
-      "loss": 0.3685,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": -3.2664294242858887,
-      "rewards/margins": 3.563570022583008,
-      "rewards/rejected": -6.8299994468688965,
-      "step": 820
-    },
-    {
-      "epoch": 0.9559458681255398,
-      "grad_norm": 36.38359169566316,
-      "learning_rate": 2.9149366008568987e-09,
-      "logits/chosen": 0.14797405898571014,
-      "logits/rejected": 0.9976932406425476,
-      "logps/chosen": -601.341552734375,
-      "logps/rejected": -791.478271484375,
-      "loss": 0.4137,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -3.2392711639404297,
-      "rewards/margins": 2.041738986968994,
-      "rewards/rejected": -5.281010150909424,
-      "step": 830
-    },
-    {
-      "epoch": 0.967463288223438,
-      "grad_norm": 36.38873535658025,
-      "learning_rate": 1.5840343486700215e-09,
-      "logits/chosen": 0.11059533059597015,
-      "logits/rejected": 1.2648974657058716,
-      "logps/chosen": -640.0452880859375,
-      "logps/rejected": -975.3658447265625,
-      "loss": 0.394,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -3.4839179515838623,
-      "rewards/margins": 3.5232937335968018,
-      "rewards/rejected": -7.007212162017822,
-      "step": 840
-    },
-    {
-      "epoch": 0.978980708321336,
-      "grad_norm": 40.1741036497201,
-      "learning_rate": 6.550326657293881e-10,
-      "logits/chosen": 0.27081722021102905,
-      "logits/rejected": 1.2972664833068848,
-      "logps/chosen": -605.1519775390625,
-      "logps/rejected": -885.68896484375,
-      "loss": 0.4039,
-      "rewards/accuracies": 0.7906249761581421,
-      "rewards/chosen": -3.3234386444091797,
-      "rewards/margins": 2.970566511154175,
-      "rewards/rejected": -6.294005870819092,
-      "step": 850
-    },
-    {
-      "epoch": 0.9904981284192341,
-      "grad_norm": 31.693047329975887,
-      "learning_rate": 1.2943454039654467e-10,
-      "logits/chosen": 0.14183056354522705,
-      "logits/rejected": 1.1139782667160034,
-      "logps/chosen": -605.5335693359375,
-      "logps/rejected": -855.60595703125,
-      "loss": 0.3858,
-      "rewards/accuracies": 0.753125011920929,
-      "rewards/chosen": -3.16903018951416,
-      "rewards/margins": 2.619706869125366,
-      "rewards/rejected": -5.7887372970581055,
-      "step": 860
-    },
-    {
-      "epoch": 0.9997120644975526,
-      "step": 868,
+      "epoch": 0.9980806142034548,
+      "step": 390,
       "total_flos": 0.0,
       "train_loss": 0.0,
-      "train_runtime": 0.0211,
-      "train_samples_per_second": 5273498.215,
-      "train_steps_per_second": 41188.083
+      "train_runtime": 0.0175,
+      "train_samples_per_second": 2864609.543,
+      "train_steps_per_second": 22343.954
     }
   ],
   "logging_steps": 10,
-  "max_steps": 868,
+  "max_steps": 390,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 100,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f946c235b2bc5024b00947701378b4ca0f3c75fcc712ad7686fafcc17070ef0
+oid sha256:988550f06a2decddde38e6e9382b161be46e22d760b56961ec99e98f752818d8
 size 7480