Model save

README.md

@@ -3,16 +3,10 @@ library_name: transformers
 license: llama3.2
 base_model: tanliboy/llama-3.2-3b-sft
 tags:
-- alignment-handbook
-- trl
-- dpo
-- generated_from_trainer
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
-- HuggingFaceH4/orca_dpo_pairs
 model-index:
 - name: llama-3.2-3b-dpo
   results: []
@@ -23,17 +17,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # llama-3.2-3b-dpo
 
-This model is a fine-tuned version of [tanliboy/llama-3.2-3b-sft](https://huggingface.co/tanliboy/llama-3.2-3b-sft) on the HuggingFaceH4/ultrafeedback_binarized and the HuggingFaceH4/orca_dpo_pairs datasets.
+This model is a fine-tuned version of [tanliboy/llama-3.2-3b-sft](https://huggingface.co/tanliboy/llama-3.2-3b-sft) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4863
-- Rewards/chosen: -1.4532
-- Rewards/rejected: -2.6152
-- Rewards/accuracies: 0.7215
-- Rewards/margins: 1.1620
-- Logps/rejected: -561.0516
-- Logps/chosen: -483.2492
-- Logits/rejected: 0.5144
-- Logits/chosen: 0.3599
+- Loss: 0.6284
+- Rewards/chosen: 0.8504
+- Rewards/rejected: -3.7058
+- Rewards/accuracies: 0.7437
+- Rewards/margins: 4.5562
+- Logps/rejected: -368.9125
+- Logps/chosen: -337.3143
+- Logits/rejected: 0.4571
+- Logits/chosen: 0.3820
 
 ## Model description
 
@@ -52,7 +46,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 5e-07
+- learning_rate: 1e-06
 - train_batch_size: 4
 - eval_batch_size: 4
 - seed: 42
@@ -63,18 +57,19 @@ The following hyperparameters were used during training:
 - total_eval_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
-- lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 1
+- lr_scheduler_warmup_ratio: 0.03
+- num_epochs: 3
 
 ### Training results
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6115        | 0.1741 | 100  | 0.6000          | -0.3357        | -0.5534          | 0.6329             | 0.2176          | -354.8722      | -371.5030    | 0.8149          | 0.6445        |
-| 0.5416        | 0.3483 | 200  | 0.5221          | -1.2141        | -1.9406          | 0.6741             | 0.7265          | -493.5951      | -459.3420    | 0.6298          | 0.4810        |
-| 0.5158        | 0.5224 | 300  | 0.5048          | -1.3617        | -2.3705          | 0.7057             | 1.0088          | -536.5855      | -474.1037    | 0.4784          | 0.3320        |
-| 0.501         | 0.6966 | 400  | 0.4906          | -1.4306        | -2.5214          | 0.7152             | 1.0908          | -551.6774      | -480.9883    | 0.5108          | 0.3615        |
-| 0.4806        | 0.8707 | 500  | 0.4864          | -1.4549        | -2.6071          | 0.7247             | 1.1522          | -560.2480      | -483.4225    | 0.5181          | 0.3637        |
+| 0.5801        | 0.4739 | 100  | 0.6840          | 0.6485         | -2.9389          | 0.6899             | 3.5875          | -361.2435      | -339.3325    | 0.6783          | 0.6103        |
+| 0.537         | 0.9479 | 200  | 0.6514          | 0.2045         | -4.0315          | 0.7278             | 4.2360          | -372.1696      | -343.7731    | 0.5648          | 0.4948        |
+| 0.4787        | 1.4218 | 300  | 0.6387          | 0.4099         | -3.9882          | 0.7215             | 4.3981          | -371.7361      | -341.7187    | 0.5326          | 0.4589        |
+| 0.4559        | 1.8957 | 400  | 0.6332          | 0.7690         | -3.6688          | 0.7342             | 4.4379          | -368.5425      | -338.1277    | 0.4841          | 0.4110        |
+| 0.4028        | 2.3697 | 500  | 0.6289          | 0.7479         | -3.8379          | 0.7405             | 4.5857          | -370.2327      | -338.3392    | 0.4475          | 0.3731        |
+| 0.4029        | 2.8436 | 600  | 0.6284          | 0.8504         | -3.7058          | 0.7437             | 4.5562          | -368.9125      | -337.3143    | 0.4571          | 0.3820        |
 
 
 ### Framework versions

all_results.json

@@ -1,5 +1,5 @@
 {
-    "epoch": 0.999564649542882,
+    "epoch": 3.0,
     "eval_logits/chosen": 0.3599458634853363,
     "eval_logits/rejected": 0.514390230178833,
     "eval_logps/chosen": -483.24920654296875,
@@ -14,9 +14,9 @@
     "eval_samples_per_second": 44.381,
     "eval_steps_per_second": 1.402,
     "total_flos": 0.0,
-    "train_loss": 0.5408797243330952,
-    "train_runtime": 4250.8819,
-    "train_samples": 73493,
-    "train_samples_per_second": 17.289,
-    "train_steps_per_second": 0.135
+    "train_loss": 0.5009220597491634,
+    "train_runtime": 6227.6413,
+    "train_samples": 26990,
+    "train_samples_per_second": 13.002,
+    "train_steps_per_second": 0.102
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a15a798615ab3d1c74b309a9b52b739cf50a081ce1afcb020fd57eaea81a92b
+oid sha256:a949d0d551d426eda481df55c9ce61d9e9f886cf169e5144df329731c5587075
 size 4965799096

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17e2fdba19d954562a5f4d4ce7307ec582677840c6cdc9bb8072338b63a699d2
+oid sha256:34d352336d2f3d15b0fdace3f3e6e2ba4213d9a93cc7ff41fc3f50a8dd9beca7
 size 1459729952

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.999564649542882,
+    "epoch": 3.0,
     "total_flos": 0.0,
-    "train_loss": 0.5408797243330952,
-    "train_runtime": 4250.8819,
-    "train_samples": 73493,
-    "train_samples_per_second": 17.289,
-    "train_steps_per_second": 0.135
+    "train_loss": 0.5009220597491634,
+    "train_runtime": 6227.6413,
+    "train_samples": 26990,
+    "train_samples_per_second": 13.002,
+    "train_steps_per_second": 0.102
 }

trainer_state.json

@@ -1,22 +1,22 @@
 {
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 0.999564649542882,
+  "best_metric": 0.6289177536964417,
+  "best_model_checkpoint": "models/llama-3.2-3b-sft-dpo/checkpoint-500",
+  "epoch": 3.0,
   "eval_steps": 100,
-  "global_step": 574,
+  "global_step": 633,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0017414018284719198,
-      "grad_norm": 3.3411689380276015,
-      "learning_rate": 8.620689655172413e-09,
-      "logits/chosen": 0.4204842150211334,
-      "logits/rejected": 0.797350287437439,
-      "logps/chosen": -397.1048889160156,
-      "logps/rejected": -273.9348449707031,
-      "loss": 0.6931,
+      "epoch": 0.004739336492890996,
+      "grad_norm": 18.306584799400138,
+      "learning_rate": 5.2631578947368416e-08,
+      "logits/chosen": 1.1032867431640625,
+      "logits/rejected": 1.1176480054855347,
+      "logps/chosen": -175.54205322265625,
+      "logps/rejected": -196.64266967773438,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,954 +24,1060 @@
       "step": 1
     },
     {
-      "epoch": 0.0174140182847192,
-      "grad_norm": 3.2818454280142397,
-      "learning_rate": 8.620689655172414e-08,
-      "logits/chosen": 0.6469573974609375,
-      "logits/rejected": 0.773016631603241,
-      "logps/chosen": -312.58782958984375,
-      "logps/rejected": -294.86376953125,
-      "loss": 0.6932,
-      "rewards/accuracies": 0.4305555522441864,
-      "rewards/chosen": -0.0005739983171224594,
-      "rewards/margins": 4.1095463529927656e-05,
-      "rewards/rejected": -0.0006150936824269593,
+      "epoch": 0.04739336492890995,
+      "grad_norm": 18.19518017806804,
+      "learning_rate": 5.263157894736842e-07,
+      "logits/chosen": 0.6209686994552612,
+      "logits/rejected": 0.7449740171432495,
+      "logps/chosen": -350.8912658691406,
+      "logps/rejected": -307.96142578125,
+      "loss": 0.9979,
+      "rewards/accuracies": 0.4861111044883728,
+      "rewards/chosen": 0.00011829059076262638,
+      "rewards/margins": 0.016186419874429703,
+      "rewards/rejected": -0.016068127006292343,
       "step": 10
     },
     {
-      "epoch": 0.0348280365694384,
-      "grad_norm": 3.251396782118138,
-      "learning_rate": 1.7241379310344828e-07,
-      "logits/chosen": 0.7936784029006958,
-      "logits/rejected": 0.7803818583488464,
-      "logps/chosen": -298.35137939453125,
-      "logps/rejected": -303.12591552734375,
-      "loss": 0.6931,
-      "rewards/accuracies": 0.46875,
-      "rewards/chosen": -0.0012876250548288226,
-      "rewards/margins": -0.00017336892778985202,
-      "rewards/rejected": -0.0011142559815198183,
+      "epoch": 0.0947867298578199,
+      "grad_norm": 15.415652807377189,
+      "learning_rate": 9.99993455114332e-07,
+      "logits/chosen": 0.9229280352592468,
+      "logits/rejected": 0.8609384298324585,
+      "logps/chosen": -252.894775390625,
+      "logps/rejected": -263.6702575683594,
+      "loss": 0.9588,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.0033816881477832794,
+      "rewards/margins": 0.16803663969039917,
+      "rewards/rejected": -0.164654940366745,
       "step": 20
     },
     {
-      "epoch": 0.052242054854157595,
-      "grad_norm": 3.046183099717941,
-      "learning_rate": 2.586206896551724e-07,
-      "logits/chosen": 0.7200006246566772,
-      "logits/rejected": 0.7392618060112,
-      "logps/chosen": -327.70428466796875,
-      "logps/rejected": -313.39422607421875,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.00010628718882799149,
-      "rewards/margins": 0.0027407719753682613,
-      "rewards/rejected": -0.00263448478654027,
+      "epoch": 0.14218009478672985,
+      "grad_norm": 12.850588595957225,
+      "learning_rate": 9.992082761369566e-07,
+      "logits/chosen": 0.8715411424636841,
+      "logits/rejected": 0.8170267343521118,
+      "logps/chosen": -296.8494567871094,
+      "logps/rejected": -305.7926025390625,
+      "loss": 0.8133,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.6128842830657959,
+      "rewards/margins": 1.1374889612197876,
+      "rewards/rejected": -0.5246046781539917,
       "step": 30
     },
     {
-      "epoch": 0.0696560731388768,
-      "grad_norm": 3.340902174817753,
-      "learning_rate": 3.4482758620689656e-07,
-      "logits/chosen": 0.7511029839515686,
-      "logits/rejected": 0.8245828747749329,
-      "logps/chosen": -295.2296142578125,
-      "logps/rejected": -329.61358642578125,
-      "loss": 0.6901,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": 0.003507365705445409,
-      "rewards/margins": 0.0062782615423202515,
-      "rewards/rejected": -0.0027708951383829117,
+      "epoch": 0.1895734597156398,
+      "grad_norm": 14.501186311778227,
+      "learning_rate": 9.971164749660148e-07,
+      "logits/chosen": 0.9155582189559937,
+      "logits/rejected": 0.9567469358444214,
+      "logps/chosen": -313.08514404296875,
+      "logps/rejected": -309.0679626464844,
+      "loss": 0.7405,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.23792271316051483,
+      "rewards/margins": 2.1163926124572754,
+      "rewards/rejected": -1.878469467163086,
       "step": 40
     },
     {
-      "epoch": 0.087070091423596,
-      "grad_norm": 3.0385013323870447,
-      "learning_rate": 4.310344827586206e-07,
-      "logits/chosen": 0.7283368110656738,
-      "logits/rejected": 0.7367040514945984,
-      "logps/chosen": -288.52020263671875,
-      "logps/rejected": -298.375,
-      "loss": 0.6846,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.012100273743271828,
-      "rewards/margins": 0.021190276369452477,
-      "rewards/rejected": -0.009090004488825798,
+      "epoch": 0.23696682464454977,
+      "grad_norm": 11.740811645701724,
+      "learning_rate": 9.937235266586424e-07,
+      "logits/chosen": 0.6986435651779175,
+      "logits/rejected": 0.8309999704360962,
+      "logps/chosen": -319.8310852050781,
+      "logps/rejected": -317.59918212890625,
+      "loss": 0.6552,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.6028285622596741,
+      "rewards/margins": 3.663621425628662,
+      "rewards/rejected": -3.060793161392212,
       "step": 50
     },
     {
-      "epoch": 0.10448410970831519,
-      "grad_norm": 2.8950641460333264,
-      "learning_rate": 4.999814661783161e-07,
-      "logits/chosen": 0.7964831590652466,
-      "logits/rejected": 0.8126887083053589,
-      "logps/chosen": -291.2521057128906,
-      "logps/rejected": -293.8636779785156,
-      "loss": 0.6749,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": 0.023344680666923523,
-      "rewards/margins": 0.03975962847471237,
-      "rewards/rejected": -0.0164149459451437,
+      "epoch": 0.2843601895734597,
+      "grad_norm": 14.434952077378005,
+      "learning_rate": 9.890383118800284e-07,
+      "logits/chosen": 0.7444020509719849,
+      "logits/rejected": 0.7484663724899292,
+      "logps/chosen": -327.59576416015625,
+      "logps/rejected": -349.929931640625,
+      "loss": 0.6285,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.3002261221408844,
+      "rewards/margins": 3.5275771617889404,
+      "rewards/rejected": -3.227351427078247,
       "step": 60
     },
     {
-      "epoch": 0.12189812799303439,
-      "grad_norm": 3.225678300931263,
-      "learning_rate": 4.993330709158879e-07,
-      "logits/chosen": 0.863906979560852,
-      "logits/rejected": 0.9115025401115417,
-      "logps/chosen": -317.2396240234375,
-      "logps/rejected": -319.9203186035156,
-      "loss": 0.6645,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.026739483699202538,
-      "rewards/margins": 0.04891226813197136,
-      "rewards/rejected": -0.02217279188334942,
+      "epoch": 0.33175355450236965,
+      "grad_norm": 10.030890442911925,
+      "learning_rate": 9.830730936592615e-07,
+      "logits/chosen": 0.7815200090408325,
+      "logits/rejected": 0.7069059610366821,
+      "logps/chosen": -252.94921875,
+      "logps/rejected": -323.2224426269531,
+      "loss": 0.6106,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 1.3401187658309937,
+      "rewards/margins": 5.26017427444458,
+      "rewards/rejected": -3.920055866241455,
       "step": 70
     },
     {
-      "epoch": 0.1393121462777536,
-      "grad_norm": 3.4663859481682495,
-      "learning_rate": 4.977607307587086e-07,
-      "logits/chosen": 0.6847103834152222,
-      "logits/rejected": 0.8103203773498535,
-      "logps/chosen": -331.44964599609375,
-      "logps/rejected": -311.62713623046875,
-      "loss": 0.6447,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": 0.003575064241886139,
-      "rewards/margins": 0.11137789487838745,
-      "rewards/rejected": -0.10780283063650131,
+      "epoch": 0.3791469194312796,
+      "grad_norm": 12.131364583934603,
+      "learning_rate": 9.758434852922123e-07,
+      "logits/chosen": 0.7100412249565125,
+      "logits/rejected": 0.6621907353401184,
+      "logps/chosen": -271.33331298828125,
+      "logps/rejected": -328.0660705566406,
+      "loss": 0.59,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.908360481262207,
+      "rewards/margins": 4.926724910736084,
+      "rewards/rejected": -4.018364429473877,
       "step": 80
     },
     {
-      "epoch": 0.1567261645624728,
-      "grad_norm": 3.1824594089594873,
-      "learning_rate": 4.952702722730485e-07,
-      "logits/chosen": 0.6765316724777222,
-      "logits/rejected": 0.885266125202179,
-      "logps/chosen": -344.87274169921875,
-      "logps/rejected": -334.42303466796875,
-      "loss": 0.6273,
-      "rewards/accuracies": 0.6812499761581421,
-      "rewards/chosen": -0.05379386991262436,
-      "rewards/margins": 0.2033984214067459,
-      "rewards/rejected": -0.25719231367111206,
+      "epoch": 0.4265402843601896,
+      "grad_norm": 11.822232959802975,
+      "learning_rate": 9.673684094754685e-07,
+      "logits/chosen": 0.6003296375274658,
+      "logits/rejected": 0.6765642762184143,
+      "logps/chosen": -293.85015869140625,
+      "logps/rejected": -305.929443359375,
+      "loss": 0.586,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 1.231705904006958,
+      "rewards/margins": 4.982685089111328,
+      "rewards/rejected": -3.750978946685791,
       "step": 90
     },
     {
-      "epoch": 0.174140182847192,
-      "grad_norm": 3.792697956177081,
-      "learning_rate": 4.918709242643563e-07,
-      "logits/chosen": 0.6538839936256409,
-      "logits/rejected": 0.75486820936203,
-      "logps/chosen": -327.5845642089844,
-      "logps/rejected": -333.3156433105469,
-      "loss": 0.6115,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.18015703558921814,
-      "rewards/margins": 0.2053672820329666,
-      "rewards/rejected": -0.38552433252334595,
+      "epoch": 0.47393364928909953,
+      "grad_norm": 9.616291876594419,
+      "learning_rate": 9.576700487782773e-07,
+      "logits/chosen": 0.6642001867294312,
+      "logits/rejected": 0.6596721410751343,
+      "logps/chosen": -326.2373046875,
+      "logps/rejected": -381.3326110839844,
+      "loss": 0.5801,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 1.7316535711288452,
+      "rewards/margins": 6.260350704193115,
+      "rewards/rejected": -4.5286970138549805,
       "step": 100
     },
     {
-      "epoch": 0.174140182847192,
-      "eval_logits/chosen": 0.6445267200469971,
-      "eval_logits/rejected": 0.8148620128631592,
-      "eval_logps/chosen": -371.50299072265625,
-      "eval_logps/rejected": -354.8721618652344,
-      "eval_loss": 0.599998414516449,
-      "eval_rewards/accuracies": 0.6329113841056824,
-      "eval_rewards/chosen": -0.3357231020927429,
-      "eval_rewards/margins": 0.21764038503170013,
-      "eval_rewards/rejected": -0.553363561630249,
-      "eval_runtime": 57.4976,
-      "eval_samples_per_second": 43.48,
-      "eval_steps_per_second": 1.374,
+      "epoch": 0.47393364928909953,
+      "eval_logits/chosen": 0.610289990901947,
+      "eval_logits/rejected": 0.6783497929573059,
+      "eval_logps/chosen": -339.33251953125,
+      "eval_logps/rejected": -361.24346923828125,
+      "eval_loss": 0.6839759349822998,
+      "eval_rewards/accuracies": 0.6898733973503113,
+      "eval_rewards/chosen": 0.6485355496406555,
+      "eval_rewards/margins": 3.587477684020996,
+      "eval_rewards/rejected": -2.9389421939849854,
+      "eval_runtime": 76.922,
+      "eval_samples_per_second": 32.5,
+      "eval_steps_per_second": 1.027,
       "step": 100
     },
     {
-      "epoch": 0.19155420113191118,
-      "grad_norm": 3.786481240905226,
-      "learning_rate": 4.875752835783956e-07,
-      "logits/chosen": 0.531981348991394,
-      "logits/rejected": 0.6186498403549194,
-      "logps/chosen": -389.45556640625,
-      "logps/rejected": -391.052734375,
-      "loss": 0.5967,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -0.2725273370742798,
-      "rewards/margins": 0.3243584632873535,
-      "rewards/rejected": -0.5968858003616333,
+      "epoch": 0.5213270142180095,
+      "grad_norm": 11.519611398516883,
+      "learning_rate": 9.467737875821367e-07,
+      "logits/chosen": 0.659843385219574,
+      "logits/rejected": 0.6010033488273621,
+      "logps/chosen": -293.62200927734375,
+      "logps/rejected": -334.9098205566406,
+      "loss": 0.5742,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 1.1434353590011597,
+      "rewards/margins": 5.331825256347656,
+      "rewards/rejected": -4.188389301300049,
       "step": 110
     },
     {
-      "epoch": 0.20896821941663038,
-      "grad_norm": 4.181016924469078,
-      "learning_rate": 4.823992684215516e-07,
-      "logits/chosen": 0.48549872636795044,
-      "logits/rejected": 0.6067858338356018,
-      "logps/chosen": -375.8733825683594,
-      "logps/rejected": -405.65240478515625,
-      "loss": 0.5621,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -0.5050081014633179,
-      "rewards/margins": 0.46570420265197754,
-      "rewards/rejected": -0.9707123041152954,
+      "epoch": 0.5687203791469194,
+      "grad_norm": 10.75922014108817,
+      "learning_rate": 9.347081456399957e-07,
+      "logits/chosen": 0.6637296676635742,
+      "logits/rejected": 0.5958945155143738,
+      "logps/chosen": -272.2585144042969,
+      "logps/rejected": -393.41949462890625,
+      "loss": 0.5821,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.9803568124771118,
+      "rewards/margins": 6.413501739501953,
+      "rewards/rejected": -5.433144569396973,
       "step": 120
     },
     {
-      "epoch": 0.22638223770134958,
-      "grad_norm": 4.23287712214465,
-      "learning_rate": 4.7636205937328664e-07,
-      "logits/chosen": 0.5461466908454895,
-      "logits/rejected": 0.7331717610359192,
-      "logps/chosen": -405.69677734375,
-      "logps/rejected": -414.6841735839844,
-      "loss": 0.5759,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -0.6262551546096802,
-      "rewards/margins": 0.4949806332588196,
-      "rewards/rejected": -1.1212358474731445,
+      "epoch": 0.6161137440758294,
+      "grad_norm": 11.497074098204886,
+      "learning_rate": 9.215047034289715e-07,
+      "logits/chosen": 0.6836856603622437,
+      "logits/rejected": 0.6638469696044922,
+      "logps/chosen": -275.0943603515625,
+      "logps/rejected": -332.6889343261719,
+      "loss": 0.5752,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 1.4476346969604492,
+      "rewards/margins": 6.094024658203125,
+      "rewards/rejected": -4.646389961242676,
       "step": 130
     },
     {
-      "epoch": 0.24379625598606877,
-      "grad_norm": 5.73858119497669,
-      "learning_rate": 4.69486028309334e-07,
-      "logits/chosen": 0.5876033902168274,
-      "logits/rejected": 0.6097403168678284,
-      "logps/chosen": -379.1561279296875,
-      "logps/rejected": -430.0502014160156,
-      "loss": 0.5551,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -0.7041950225830078,
-      "rewards/margins": 0.4728410840034485,
-      "rewards/rejected": -1.1770360469818115,
+      "epoch": 0.6635071090047393,
+      "grad_norm": 9.658859904375,
+      "learning_rate": 9.07198019491959e-07,
+      "logits/chosen": 0.61662757396698,
+      "logits/rejected": 0.5779851675033569,
+      "logps/chosen": -272.382080078125,
+      "logps/rejected": -355.6089172363281,
+      "loss": 0.5468,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.8889511227607727,
+      "rewards/margins": 5.594452857971191,
+      "rewards/rejected": -4.705502510070801,
       "step": 140
     },
     {
-      "epoch": 0.26121027427078797,
-      "grad_norm": 6.624624616809953,
-      "learning_rate": 4.61796655499015e-07,
-      "logits/chosen": 0.4104432165622711,
-      "logits/rejected": 0.6294438242912292,
-      "logps/chosen": -416.1566467285156,
-      "logps/rejected": -426.4996032714844,
-      "loss": 0.5562,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.7248625755310059,
-      "rewards/margins": 0.550578236579895,
-      "rewards/rejected": -1.2754409313201904,
+      "epoch": 0.7109004739336493,
+      "grad_norm": 10.07652231167762,
+      "learning_rate": 8.918255399844853e-07,
+      "logits/chosen": 0.5373108983039856,
+      "logits/rejected": 0.654308021068573,
+      "logps/chosen": -330.0559997558594,
+      "logps/rejected": -349.55224609375,
+      "loss": 0.5738,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.3335852324962616,
+      "rewards/margins": 4.550914287567139,
+      "rewards/rejected": -4.217329502105713,
       "step": 150
     },
     {
-      "epoch": 0.2786242925555072,
-      "grad_norm": 6.025999930652061,
-      "learning_rate": 4.5332243518389136e-07,
-      "logits/chosen": 0.45941343903541565,
-      "logits/rejected": 0.6137918829917908,
-      "logps/chosen": -421.73095703125,
-      "logps/rejected": -463.8855895996094,
-      "loss": 0.5323,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -0.8197100758552551,
-      "rewards/margins": 0.7435392141342163,
-      "rewards/rejected": -1.5632489919662476,
+      "epoch": 0.7582938388625592,
+      "grad_norm": 8.965490487953566,
+      "learning_rate": 8.754275006635572e-07,
+      "logits/chosen": 0.565764844417572,
+      "logits/rejected": 0.539226233959198,
+      "logps/chosen": -269.29742431640625,
+      "logps/rejected": -355.60589599609375,
+      "loss": 0.5997,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.5406277179718018,
+      "rewards/margins": 5.479567527770996,
+      "rewards/rejected": -4.938939571380615,
       "step": 160
     },
     {
-      "epoch": 0.29603831084022636,
-      "grad_norm": 6.778470707105329,
-      "learning_rate": 4.4409476998764364e-07,
-      "logits/chosen": 0.4422511160373688,
-      "logits/rejected": 0.636109471321106,
-      "logps/chosen": -439.1053161621094,
-      "logps/rejected": -487.4310607910156,
-      "loss": 0.5382,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.0064823627471924,
-      "rewards/margins": 0.7615920305252075,
-      "rewards/rejected": -1.7680743932724,
+      "epoch": 0.8056872037914692,
+      "grad_norm": 9.437674903727038,
+      "learning_rate": 8.580468215750391e-07,
+      "logits/chosen": 0.6932438611984253,
+      "logits/rejected": 0.636594831943512,
+      "logps/chosen": -296.7684631347656,
+      "logps/rejected": -367.45318603515625,
+      "loss": 0.5783,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": 1.147369146347046,
+      "rewards/margins": 5.5389909744262695,
+      "rewards/rejected": -4.391622066497803,
       "step": 170
     },
     {
-      "epoch": 0.3134523291249456,
-      "grad_norm": 5.155340194107305,
-      "learning_rate": 4.3414785454846093e-07,
-      "logits/chosen": 0.4746861457824707,
-      "logits/rejected": 0.49304714798927307,
-      "logps/chosen": -429.70355224609375,
-      "logps/rejected": -487.3233337402344,
-      "loss": 0.5377,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.9950485229492188,
-      "rewards/margins": 0.7225963473320007,
-      "rewards/rejected": -1.7176446914672852,
+      "epoch": 0.8530805687203792,
+      "grad_norm": 8.5658002946873,
+      "learning_rate": 8.39728994715202e-07,
+      "logits/chosen": 0.6020892858505249,
+      "logits/rejected": 0.5168766379356384,
+      "logps/chosen": -288.558349609375,
+      "logps/rejected": -348.62640380859375,
+      "loss": 0.5531,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": 0.6757786870002747,
+      "rewards/margins": 5.149857997894287,
+      "rewards/rejected": -4.474079132080078,
       "step": 180
     },
     {
-      "epoch": 0.3308663474096648,
-      "grad_norm": 7.739271405731777,
-      "learning_rate": 4.235185488051585e-07,
-      "logits/chosen": 0.4361630082130432,
-      "logits/rejected": 0.5531889200210571,
-      "logps/chosen": -457.5230407714844,
-      "logps/rejected": -510.4833068847656,
-      "loss": 0.5269,
+      "epoch": 0.9004739336492891,
+      "grad_norm": 11.065263225689659,
+      "learning_rate": 8.20521964960477e-07,
+      "logits/chosen": 0.6599653363227844,
+      "logits/rejected": 0.6458830237388611,
+      "logps/chosen": -289.4867858886719,
+      "logps/rejected": -342.56243896484375,
+      "loss": 0.5439,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.0585914850234985,
-      "rewards/margins": 0.7955325245857239,
-      "rewards/rejected": -1.8541240692138672,
+      "rewards/chosen": 1.274778962135315,
+      "rewards/margins": 6.3435516357421875,
+      "rewards/rejected": -5.068772792816162,
       "step": 190
     },
     {
-      "epoch": 0.348280365694384,
-      "grad_norm": 6.703891062008893,
-      "learning_rate": 4.1224624140658336e-07,
-      "logits/chosen": 0.4384649395942688,
-      "logits/rejected": 0.5880831480026245,
-      "logps/chosen": -461.74053955078125,
-      "logps/rejected": -518.26806640625,
-      "loss": 0.5416,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.2263882160186768,
-      "rewards/margins": 0.9702332615852356,
-      "rewards/rejected": -2.1966214179992676,
+      "epoch": 0.9478672985781991,
+      "grad_norm": 8.426424572195439,
+      "learning_rate": 8.0047600457707e-07,
+      "logits/chosen": 0.6277160048484802,
+      "logits/rejected": 0.6192003488540649,
+      "logps/chosen": -318.033447265625,
+      "logps/rejected": -377.3500061035156,
+      "loss": 0.537,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 1.3354051113128662,
+      "rewards/margins": 6.755140781402588,
+      "rewards/rejected": -5.419735431671143,
       "step": 200
     },
     {
-      "epoch": 0.348280365694384,
-      "eval_logits/chosen": 0.48104235529899597,
-      "eval_logits/rejected": 0.6297945976257324,
-      "eval_logps/chosen": -459.3420104980469,
-      "eval_logps/rejected": -493.5951232910156,
-      "eval_loss": 0.5221381187438965,
-      "eval_rewards/accuracies": 0.6740506291389465,
-      "eval_rewards/chosen": -1.214113473892212,
-      "eval_rewards/margins": 0.7264798283576965,
-      "eval_rewards/rejected": -1.9405934810638428,
-      "eval_runtime": 56.9644,
-      "eval_samples_per_second": 43.887,
-      "eval_steps_per_second": 1.387,
+      "epoch": 0.9478672985781991,
+      "eval_logits/chosen": 0.494819700717926,
+      "eval_logits/rejected": 0.5648438930511475,
+      "eval_logps/chosen": -343.7730712890625,
+      "eval_logps/rejected": -372.1695861816406,
+      "eval_loss": 0.6514427661895752,
+      "eval_rewards/accuracies": 0.7278481125831604,
+      "eval_rewards/chosen": 0.20448331534862518,
+      "eval_rewards/margins": 4.236032485961914,
+      "eval_rewards/rejected": -4.031548976898193,
+      "eval_runtime": 74.0508,
+      "eval_samples_per_second": 33.761,
+      "eval_steps_per_second": 1.067,
       "step": 200
     },
     {
-      "epoch": 0.3656943839791032,
-      "grad_norm": 6.9743912275745075,
-      "learning_rate": 4.003727037504676e-07,
-      "logits/chosen": 0.5726699829101562,
-      "logits/rejected": 0.6551696062088013,
-      "logps/chosen": -398.76287841796875,
-      "logps/rejected": -471.94219970703125,
-      "loss": 0.525,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.0725631713867188,
-      "rewards/margins": 0.7981334924697876,
-      "rewards/rejected": -1.8706966638565063,
+      "epoch": 0.995260663507109,
+      "grad_norm": 9.878709661135902,
+      "learning_rate": 7.796435816388898e-07,
+      "logits/chosen": 0.6760674118995667,
+      "logits/rejected": 0.6518660187721252,
+      "logps/chosen": -284.24749755859375,
+      "logps/rejected": -363.0601501464844,
+      "loss": 0.554,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.6821473836898804,
+      "rewards/margins": 6.51880407333374,
+      "rewards/rejected": -5.8366570472717285,
       "step": 210
     },
     {
-      "epoch": 0.38310840226382237,
-      "grad_norm": 6.816210344544152,
-      "learning_rate": 3.879419351926115e-07,
-      "logits/chosen": 0.48985353112220764,
-      "logits/rejected": 0.5055981874465942,
-      "logps/chosen": -444.0646057128906,
-      "logps/rejected": -542.5759887695312,
-      "loss": 0.5119,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.1069988012313843,
-      "rewards/margins": 1.0880978107452393,
-      "rewards/rejected": -2.195096731185913,
+      "epoch": 1.042654028436019,
+      "grad_norm": 10.875728154843127,
+      "learning_rate": 7.580792226981954e-07,
+      "logits/chosen": 0.5221652984619141,
+      "logits/rejected": 0.44479990005493164,
+      "logps/chosen": -281.39190673828125,
+      "logps/rejected": -370.33941650390625,
+      "loss": 0.4911,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 2.0442254543304443,
+      "rewards/margins": 7.068573951721191,
+      "rewards/rejected": -5.024348258972168,
       "step": 220
     },
     {
-      "epoch": 0.4005224205485416,
-      "grad_norm": 6.1503561448910595,
-      "learning_rate": 3.75e-07,
-      "logits/chosen": 0.40112408995628357,
-      "logits/rejected": 0.4424813389778137,
-      "logps/chosen": -434.7146911621094,
-      "logps/rejected": -532.7242431640625,
-      "loss": 0.5219,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": -1.1246516704559326,
-      "rewards/margins": 1.0327972173690796,
-      "rewards/rejected": -2.1574490070343018,
+      "epoch": 1.0900473933649288,
+      "grad_norm": 10.04148994728917,
+      "learning_rate": 7.358393700684032e-07,
+      "logits/chosen": 0.5540430545806885,
+      "logits/rejected": 0.5128260850906372,
+      "logps/chosen": -279.4583435058594,
+      "logps/rejected": -350.32684326171875,
+      "loss": 0.5022,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.9357398152351379,
+      "rewards/margins": 5.9159369468688965,
+      "rewards/rejected": -4.980198383331299,
       "step": 230
     },
     {
-      "epoch": 0.41793643883326076,
-      "grad_norm": 6.948961329300032,
-      "learning_rate": 3.615948566520498e-07,
-      "logits/chosen": 0.2614263594150543,
-      "logits/rejected": 0.316015362739563,
-      "logps/chosen": -478.199462890625,
-      "logps/rejected": -555.8576049804688,
-      "loss": 0.5478,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.3383628129959106,
-      "rewards/margins": 0.8917714953422546,
-      "rewards/rejected": -2.2301342487335205,
+      "epoch": 1.1374407582938388,
+      "grad_norm": 11.466420945945197,
+      "learning_rate": 7.129822340926043e-07,
+      "logits/chosen": 0.5252267122268677,
+      "logits/rejected": 0.6392233371734619,
+      "logps/chosen": -300.5268859863281,
+      "logps/rejected": -328.5356750488281,
+      "loss": 0.4908,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 1.1534405946731567,
+      "rewards/margins": 6.1857991218566895,
+      "rewards/rejected": -5.032358169555664,
       "step": 240
     },
     {
-      "epoch": 0.43535045711798,
-      "grad_norm": 6.214549773976368,
-      "learning_rate": 3.4777618012253895e-07,
-      "logits/chosen": 0.5206897854804993,
-      "logits/rejected": 0.4764159321784973,
-      "logps/chosen": -427.69720458984375,
-      "logps/rejected": -528.8837280273438,
-      "loss": 0.5137,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.1115195751190186,
-      "rewards/margins": 0.9832460284233093,
-      "rewards/rejected": -2.0947654247283936,
+      "epoch": 1.1848341232227488,
+      "grad_norm": 9.714339627017372,
+      "learning_rate": 6.895676407844586e-07,
+      "logits/chosen": 0.5342652797698975,
+      "logits/rejected": 0.5475658178329468,
+      "logps/chosen": -275.02972412109375,
+      "logps/rejected": -325.74993896484375,
+      "loss": 0.4508,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 2.0915255546569824,
+      "rewards/margins": 6.8750715255737305,
+      "rewards/rejected": -4.783546447753906,
       "step": 250
     },
     {
-      "epoch": 0.45276447540269915,
-      "grad_norm": 10.426891895491888,
-      "learning_rate": 3.3359517780078315e-07,
-      "logits/chosen": 0.31379422545433044,
-      "logits/rejected": 0.4527011811733246,
-      "logps/chosen": -460.69390869140625,
-      "logps/rejected": -530.2111206054688,
-      "loss": 0.521,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.0801621675491333,
-      "rewards/margins": 0.9811126589775085,
-      "rewards/rejected": -2.061274766921997,
+      "epoch": 1.2322274881516588,
+      "grad_norm": 8.702659887264469,
+      "learning_rate": 6.656568752402521e-07,
+      "logits/chosen": 0.4584909975528717,
+      "logits/rejected": 0.5478152632713318,
+      "logps/chosen": -314.6927185058594,
+      "logps/rejected": -357.88226318359375,
+      "loss": 0.4621,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 1.3858213424682617,
+      "rewards/margins": 6.8659563064575195,
+      "rewards/rejected": -5.480134963989258,
       "step": 260
     },
     {
-      "epoch": 0.4701784936874184,
-      "grad_norm": 6.486890040446789,
-      "learning_rate": 3.191043997341929e-07,
-      "logits/chosen": 0.41157540678977966,
-      "logits/rejected": 0.48239168524742126,
-      "logps/chosen": -449.7867736816406,
-      "logps/rejected": -511.8810119628906,
-      "loss": 0.5079,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.1862131357192993,
-      "rewards/margins": 0.9392406344413757,
-      "rewards/rejected": -2.1254539489746094,
+      "epoch": 1.2796208530805688,
+      "grad_norm": 10.924278197277149,
+      "learning_rate": 6.413125212319663e-07,
+      "logits/chosen": 0.6362992525100708,
+      "logits/rejected": 0.6484791040420532,
+      "logps/chosen": -285.7840270996094,
+      "logps/rejected": -360.7676086425781,
+      "loss": 0.4712,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.0224599838256836,
+      "rewards/margins": 7.362783908843994,
+      "rewards/rejected": -5.3403239250183105,
       "step": 270
     },
     {
-      "epoch": 0.48759251197213754,
-      "grad_norm": 6.17919759443457,
-      "learning_rate": 3.0435754389538925e-07,
-      "logits/chosen": 0.4492722153663635,
-      "logits/rejected": 0.477822482585907,
-      "logps/chosen": -405.44488525390625,
-      "logps/rejected": -496.3724060058594,
-      "loss": 0.5069,
+      "epoch": 1.3270142180094786,
+      "grad_norm": 9.286266066829205,
+      "learning_rate": 6.165982974012104e-07,
+      "logits/chosen": 0.48062658309936523,
+      "logits/rejected": 0.4873732626438141,
+      "logps/chosen": -345.07586669921875,
+      "logps/rejected": -393.88165283203125,
+      "loss": 0.4628,
       "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -1.1837854385375977,
-      "rewards/margins": 0.9472991824150085,
-      "rewards/rejected": -2.13108491897583,
+      "rewards/chosen": 1.449973225593567,
+      "rewards/margins": 7.039644718170166,
+      "rewards/rejected": -5.589670658111572,
       "step": 280
     },
     {
-      "epoch": 0.5050065302568568,
-      "grad_norm": 6.347632610711749,
-      "learning_rate": 2.8940925719549335e-07,
-      "logits/chosen": 0.35942569375038147,
-      "logits/rejected": 0.4725337624549866,
-      "logps/chosen": -454.9664611816406,
-      "logps/rejected": -541.79052734375,
-      "loss": 0.5089,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.3150675296783447,
-      "rewards/margins": 1.0351736545562744,
-      "rewards/rejected": -2.350241184234619,
+      "epoch": 1.3744075829383886,
+      "grad_norm": 9.83819564198541,
+      "learning_rate": 5.915788904827553e-07,
+      "logits/chosen": 0.43026304244995117,
+      "logits/rejected": 0.459343820810318,
+      "logps/chosen": -294.733154296875,
+      "logps/rejected": -363.80340576171875,
+      "loss": 0.4507,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 1.6585981845855713,
+      "rewards/margins": 6.437933444976807,
+      "rewards/rejected": -4.779335021972656,
       "step": 290
     },
     {
-      "epoch": 0.5224205485415759,
-      "grad_norm": 9.01871162756941,
-      "learning_rate": 2.7431493298096725e-07,
-      "logits/chosen": 0.3180525004863739,
-      "logits/rejected": 0.4156918525695801,
-      "logps/chosen": -477.7978515625,
-      "logps/rejected": -595.4729614257812,
-      "loss": 0.5158,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.484802007675171,
-      "rewards/margins": 1.233532190322876,
-      "rewards/rejected": -2.718334197998047,
+      "epoch": 1.4218009478672986,
+      "grad_norm": 8.577071743246128,
+      "learning_rate": 5.663197859941938e-07,
+      "logits/chosen": 0.6086027026176453,
+      "logits/rejected": 0.6251193881034851,
+      "logps/chosen": -262.66644287109375,
+      "logps/rejected": -320.42974853515625,
+      "loss": 0.4787,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 1.298060655593872,
+      "rewards/margins": 6.941515922546387,
+      "rewards/rejected": -5.643455505371094,
       "step": 300
     },
     {
-      "epoch": 0.5224205485415759,
-      "eval_logits/chosen": 0.3319605886936188,
-      "eval_logits/rejected": 0.4784182906150818,
-      "eval_logps/chosen": -474.1037292480469,
-      "eval_logps/rejected": -536.5855102539062,
-      "eval_loss": 0.5047817826271057,
-      "eval_rewards/accuracies": 0.7056962251663208,
-      "eval_rewards/chosen": -1.3617302179336548,
-      "eval_rewards/margins": 1.0087664127349854,
-      "eval_rewards/rejected": -2.3704965114593506,
-      "eval_runtime": 56.8943,
-      "eval_samples_per_second": 43.941,
-      "eval_steps_per_second": 1.389,
+      "epoch": 1.4218009478672986,
+      "eval_logits/chosen": 0.45885032415390015,
+      "eval_logits/rejected": 0.5325651168823242,
+      "eval_logps/chosen": -341.7187194824219,
+      "eval_logps/rejected": -371.7361145019531,
+      "eval_loss": 0.6386769413948059,
+      "eval_rewards/accuracies": 0.7215189933776855,
+      "eval_rewards/chosen": 0.40991881489753723,
+      "eval_rewards/margins": 4.398120880126953,
+      "eval_rewards/rejected": -3.98820161819458,
+      "eval_runtime": 72.3153,
+      "eval_samples_per_second": 34.571,
+      "eval_steps_per_second": 1.092,
       "step": 300
     },
     {
-      "epoch": 0.5398345668262952,
-      "grad_norm": 9.655940482231035,
-      "learning_rate": 2.5913050576441473e-07,
-      "logits/chosen": 0.28561896085739136,
-      "logits/rejected": 0.4566105902194977,
-      "logps/chosen": -485.2098083496094,
-      "logps/rejected": -578.7564086914062,
-      "loss": 0.4937,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.3072640895843506,
-      "rewards/margins": 1.2091175317764282,
-      "rewards/rejected": -2.5163815021514893,
+      "epoch": 1.4691943127962086,
+      "grad_norm": 12.642599504555136,
+      "learning_rate": 5.408870968348749e-07,
+      "logits/chosen": 0.46862930059432983,
+      "logits/rejected": 0.45317015051841736,
+      "logps/chosen": -269.1434631347656,
+      "logps/rejected": -348.3428955078125,
+      "loss": 0.4684,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 1.3798935413360596,
+      "rewards/margins": 6.562399864196777,
+      "rewards/rejected": -5.182506561279297,
       "step": 310
     },
     {
-      "epoch": 0.5572485851110144,
-      "grad_norm": 6.61161140792094,
-      "learning_rate": 2.439122439500026e-07,
-      "logits/chosen": 0.2611474394798279,
-      "logits/rejected": 0.4488650858402252,
-      "logps/chosen": -484.05462646484375,
-      "logps/rejected": -537.9361572265625,
-      "loss": 0.499,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.2471102476119995,
-      "rewards/margins": 1.0742133855819702,
-      "rewards/rejected": -2.3213236331939697,
+      "epoch": 1.5165876777251186,
+      "grad_norm": 9.79584839845262,
+      "learning_rate": 5.153473902427354e-07,
+      "logits/chosen": 0.47858723998069763,
+      "logits/rejected": 0.5644794702529907,
+      "logps/chosen": -321.48345947265625,
+      "logps/rejected": -343.6278991699219,
+      "loss": 0.4803,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 1.1607013940811157,
+      "rewards/margins": 5.799595832824707,
+      "rewards/rejected": -4.638894557952881,
       "step": 320
     },
     {
-      "epoch": 0.5746626033957336,
-      "grad_norm": 7.51197628735834,
-      "learning_rate": 2.2871654132159104e-07,
-      "logits/chosen": 0.2992505431175232,
-      "logits/rejected": 0.41670387983322144,
-      "logps/chosen": -504.9584045410156,
-      "logps/rejected": -583.96923828125,
-      "loss": 0.5024,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -1.4169723987579346,
-      "rewards/margins": 1.1618797779083252,
-      "rewards/rejected": -2.5788521766662598,
+      "epoch": 1.5639810426540284,
+      "grad_norm": 8.875212778872154,
+      "learning_rate": 4.897675135619516e-07,
+      "logits/chosen": 0.47927242517471313,
+      "logits/rejected": 0.605729341506958,
+      "logps/chosen": -296.8520812988281,
+      "logps/rejected": -339.26220703125,
+      "loss": 0.48,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 1.206688404083252,
+      "rewards/margins": 6.4211745262146,
+      "rewards/rejected": -5.214486598968506,
       "step": 330
     },
     {
-      "epoch": 0.5920766216804527,
-      "grad_norm": 7.130014982854212,
-      "learning_rate": 2.1359970806624884e-07,
-      "logits/chosen": 0.2984461784362793,
-      "logits/rejected": 0.363652765750885,
-      "logps/chosen": -478.5108337402344,
-      "logps/rejected": -577.00048828125,
-      "loss": 0.5178,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.4051122665405273,
-      "rewards/margins": 0.9606796503067017,
-      "rewards/rejected": -2.3657917976379395,
+      "epoch": 1.6113744075829384,
+      "grad_norm": 9.788751062324735,
+      "learning_rate": 4.642144192774429e-07,
+      "logits/chosen": 0.6517030000686646,
+      "logits/rejected": 0.6343492269515991,
+      "logps/chosen": -256.8311767578125,
+      "logps/rejected": -318.10504150390625,
+      "loss": 0.4687,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 1.4574129581451416,
+      "rewards/margins": 7.180891513824463,
+      "rewards/rejected": -5.723478317260742,
       "step": 340
     },
     {
-      "epoch": 0.6094906399651719,
-      "grad_norm": 7.111868751712803,
-      "learning_rate": 1.9861776210754986e-07,
-      "logits/chosen": 0.4120015501976013,
-      "logits/rejected": 0.5323067903518677,
-      "logps/chosen": -501.5419921875,
-      "logps/rejected": -561.2122802734375,
-      "loss": 0.5137,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.5511784553527832,
-      "rewards/margins": 1.0202207565307617,
-      "rewards/rejected": -2.571399211883545,
+      "epoch": 1.6587677725118484,
+      "grad_norm": 8.123068784558978,
+      "learning_rate": 4.387549897741825e-07,
+      "logits/chosen": 0.43539008498191833,
+      "logits/rejected": 0.4823547303676605,
+      "logps/chosen": -322.7386474609375,
+      "logps/rejected": -349.6393127441406,
+      "loss": 0.4903,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 1.6534090042114258,
+      "rewards/margins": 6.494222164154053,
+      "rewards/rejected": -4.840813159942627,
       "step": 350
     },
     {
-      "epoch": 0.6269046582498912,
-      "grad_norm": 8.698410343780237,
-      "learning_rate": 1.8382622152190158e-07,
-      "logits/chosen": 0.3698151707649231,
-      "logits/rejected": 0.4354040026664734,
-      "logps/chosen": -461.7838439941406,
-      "logps/rejected": -519.1609497070312,
-      "loss": 0.4968,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.2663956880569458,
-      "rewards/margins": 0.9138597249984741,
-      "rewards/rejected": -2.18025541305542,
+      "epoch": 1.7061611374407581,
+      "grad_norm": 10.106462346167355,
+      "learning_rate": 4.1345586227998634e-07,
+      "logits/chosen": 0.4860106110572815,
+      "logits/rejected": 0.48908883333206177,
+      "logps/chosen": -289.710693359375,
+      "logps/rejected": -384.22686767578125,
+      "loss": 0.446,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.587738275527954,
+      "rewards/margins": 7.2089128494262695,
+      "rewards/rejected": -5.6211748123168945,
       "step": 360
     },
     {
-      "epoch": 0.6443186765346104,
-      "grad_norm": 7.210125130445963,
-      "learning_rate": 1.692798988071385e-07,
-      "logits/chosen": 0.34133654832839966,
-      "logits/rejected": 0.43571311235427856,
-      "logps/chosen": -427.91424560546875,
-      "logps/rejected": -496.55206298828125,
-      "loss": 0.5107,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.1491998434066772,
-      "rewards/margins": 0.9772473573684692,
-      "rewards/rejected": -2.1264472007751465,
+      "epoch": 1.7535545023696684,
+      "grad_norm": 10.81635763601606,
+      "learning_rate": 3.883832544499735e-07,
+      "logits/chosen": 0.5913195013999939,
+      "logits/rejected": 0.5606914758682251,
+      "logps/chosen": -292.9503173828125,
+      "logps/rejected": -390.93878173828125,
+      "loss": 0.4592,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 1.614689588546753,
+      "rewards/margins": 6.656731605529785,
+      "rewards/rejected": -5.042041301727295,
       "step": 370
     },
     {
-      "epoch": 0.6617326948193296,
-      "grad_norm": 6.808319521539246,
-      "learning_rate": 1.5503269776575362e-07,
-      "logits/chosen": 0.4110635221004486,
-      "logits/rejected": 0.3942858576774597,
-      "logps/chosen": -437.7587890625,
-      "logps/rejected": -536.278564453125,
-      "loss": 0.4949,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.3278793096542358,
-      "rewards/margins": 0.8986042141914368,
-      "rewards/rejected": -2.2264838218688965,
+      "epoch": 1.8009478672985781,
+      "grad_norm": 10.495084061438284,
+      "learning_rate": 3.636027910492114e-07,
+      "logits/chosen": 0.4658740162849426,
+      "logits/rejected": 0.5308722257614136,
+      "logps/chosen": -305.28753662109375,
+      "logps/rejected": -352.7513122558594,
+      "loss": 0.4648,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": 1.0712064504623413,
+      "rewards/margins": 6.167966365814209,
+      "rewards/rejected": -5.096759796142578,
       "step": 380
     },
     {
-      "epoch": 0.6791467131040487,
-      "grad_norm": 5.0994020171282886,
-      "learning_rate": 1.411374137554522e-07,
-      "logits/chosen": 0.21764016151428223,
-      "logits/rejected": 0.3392140865325928,
-      "logps/chosen": -478.3634338378906,
-      "logps/rejected": -557.5911865234375,
-      "loss": 0.5066,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.41581130027771,
-      "rewards/margins": 1.0703743696212769,
-      "rewards/rejected": -2.4861857891082764,
+      "epoch": 1.8483412322274881,
+      "grad_norm": 11.413974134819627,
+      "learning_rate": 3.3917933218718566e-07,
+      "logits/chosen": 0.6185089349746704,
+      "logits/rejected": 0.6838531494140625,
+      "logps/chosen": -284.1628112792969,
+      "logps/rejected": -333.17657470703125,
+      "loss": 0.4426,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 1.4776874780654907,
+      "rewards/margins": 6.398137092590332,
+      "rewards/rejected": -4.920449733734131,
       "step": 390
     },
     {
-      "epoch": 0.696560731388768,
-      "grad_norm": 5.872014380298252,
-      "learning_rate": 1.2764553804722867e-07,
-      "logits/chosen": 0.3027791380882263,
-      "logits/rejected": 0.3619641661643982,
-      "logps/chosen": -507.347900390625,
-      "logps/rejected": -654.2447509765625,
-      "loss": 0.501,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -1.6098072528839111,
-      "rewards/margins": 1.409834384918213,
-      "rewards/rejected": -3.019641399383545,
+      "epoch": 1.8957345971563981,
+      "grad_norm": 9.664147195442332,
+      "learning_rate": 3.151768035536698e-07,
+      "logits/chosen": 0.6407091617584229,
+      "logits/rejected": 0.6542560458183289,
+      "logps/chosen": -284.20037841796875,
+      "logps/rejected": -345.27880859375,
+      "loss": 0.4559,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 2.0247559547424316,
+      "rewards/margins": 7.09304141998291,
+      "rewards/rejected": -5.0682854652404785,
       "step": 400
     },
     {
-      "epoch": 0.696560731388768,
-      "eval_logits/chosen": 0.36147814989089966,
-      "eval_logits/rejected": 0.5108113884925842,
-      "eval_logps/chosen": -480.98834228515625,
-      "eval_logps/rejected": -551.6774291992188,
-      "eval_loss": 0.4905659258365631,
-      "eval_rewards/accuracies": 0.7151898741722107,
-      "eval_rewards/chosen": -1.4305766820907593,
-      "eval_rewards/margins": 1.09083890914917,
-      "eval_rewards/rejected": -2.5214157104492188,
-      "eval_runtime": 56.8059,
-      "eval_samples_per_second": 44.01,
-      "eval_steps_per_second": 1.391,
+      "epoch": 1.8957345971563981,
+      "eval_logits/chosen": 0.41101595759391785,
+      "eval_logits/rejected": 0.4840773642063141,
+      "eval_logps/chosen": -338.1277160644531,
+      "eval_logps/rejected": -368.54248046875,
+      "eval_loss": 0.6332134008407593,
+      "eval_rewards/accuracies": 0.7341772317886353,
+      "eval_rewards/chosen": 0.7690173983573914,
+      "eval_rewards/margins": 4.437857151031494,
+      "eval_rewards/rejected": -3.668839931488037,
+      "eval_runtime": 72.5998,
+      "eval_samples_per_second": 34.435,
+      "eval_steps_per_second": 1.088,
       "step": 400
     },
     {
-      "epoch": 0.7139747496734872,
-      "grad_norm": 7.0264805606792855,
-      "learning_rate": 1.1460706701595385e-07,
-      "logits/chosen": 0.3922923505306244,
-      "logits/rejected": 0.43147850036621094,
-      "logps/chosen": -446.358154296875,
-      "logps/rejected": -545.4576416015625,
-      "loss": 0.5176,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.2909159660339355,
-      "rewards/margins": 1.0999126434326172,
-      "rewards/rejected": -2.3908286094665527,
+      "epoch": 1.943127962085308,
+      "grad_norm": 10.263641095491934,
+      "learning_rate": 2.9165802910033603e-07,
+      "logits/chosen": 0.5565508604049683,
+      "logits/rejected": 0.5877315402030945,
+      "logps/chosen": -328.7551574707031,
+      "logps/rejected": -364.5121154785156,
+      "loss": 0.4644,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": 1.852020502090454,
+      "rewards/margins": 6.0383710861206055,
+      "rewards/rejected": -4.186350345611572,
       "step": 410
     },
     {
-      "epoch": 0.7313887679582064,
-      "grad_norm": 7.312076711655367,
-      "learning_rate": 1.0207031687054663e-07,
-      "logits/chosen": 0.3949499726295471,
-      "logits/rejected": 0.4116601347923279,
-      "logps/chosen": -459.15606689453125,
-      "logps/rejected": -557.7392578125,
-      "loss": 0.4935,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -1.3280662298202515,
-      "rewards/margins": 1.038793683052063,
-      "rewards/rejected": -2.3668599128723145,
+      "epoch": 1.9905213270142181,
+      "grad_norm": 8.889403142715599,
+      "learning_rate": 2.686845666060415e-07,
+      "logits/chosen": 0.5102426409721375,
+      "logits/rejected": 0.43454083800315857,
+      "logps/chosen": -271.08160400390625,
+      "logps/rejected": -369.26458740234375,
+      "loss": 0.461,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 1.6376615762710571,
+      "rewards/margins": 7.588493347167969,
+      "rewards/rejected": -5.950831413269043,
       "step": 420
     },
     {
-      "epoch": 0.7488027862429255,
-      "grad_norm": 8.57419806945796,
-      "learning_rate": 9.008174461027723e-08,
-      "logits/chosen": 0.18675744533538818,
-      "logits/rejected": 0.36219197511672974,
-      "logps/chosen": -502.193603515625,
-      "logps/rejected": -595.4912719726562,
-      "loss": 0.4973,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": -1.3678879737854004,
-      "rewards/margins": 1.271985650062561,
-      "rewards/rejected": -2.639873743057251,
+      "epoch": 2.037914691943128,
+      "grad_norm": 7.4495856256114195,
+      "learning_rate": 2.4631654655618287e-07,
+      "logits/chosen": 0.37354058027267456,
+      "logits/rejected": 0.4436867833137512,
+      "logps/chosen": -310.15802001953125,
+      "logps/rejected": -382.03253173828125,
+      "loss": 0.3945,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 1.8288238048553467,
+      "rewards/margins": 7.114483833312988,
+      "rewards/rejected": -5.2856597900390625,
       "step": 430
     },
     {
-      "epoch": 0.7662168045276447,
-      "grad_norm": 7.593562856079916,
-      "learning_rate": 7.86857758706802e-08,
-      "logits/chosen": 0.391216903924942,
-      "logits/rejected": 0.4270710051059723,
-      "logps/chosen": -434.24713134765625,
-      "logps/rejected": -548.2892456054688,
-      "loss": 0.4988,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.390061378479004,
-      "rewards/margins": 0.9570953249931335,
-      "rewards/rejected": -2.3471570014953613,
+      "epoch": 2.085308056872038,
+      "grad_norm": 8.829254132221473,
+      "learning_rate": 2.2461251475783155e-07,
+      "logits/chosen": 0.5162326693534851,
+      "logits/rejected": 0.4021889567375183,
+      "logps/chosen": -288.923095703125,
+      "logps/rejected": -389.34979248046875,
+      "loss": 0.3748,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 1.8741111755371094,
+      "rewards/margins": 7.6665802001953125,
+      "rewards/rejected": -5.792468547821045,
       "step": 440
     },
     {
-      "epoch": 0.783630822812364,
-      "grad_norm": 9.701744270423196,
-      "learning_rate": 6.792464029702102e-08,
-      "logits/chosen": 0.40113019943237305,
-      "logits/rejected": 0.47713106870651245,
-      "logps/chosen": -432.3943786621094,
-      "logps/rejected": -561.9862670898438,
-      "loss": 0.4717,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": -1.339185357093811,
-      "rewards/margins": 1.3107116222381592,
-      "rewards/rejected": -2.6498968601226807,
+      "epoch": 2.132701421800948,
+      "grad_norm": 8.156529944948277,
+      "learning_rate": 2.0362927910258986e-07,
+      "logits/chosen": 0.45688456296920776,
+      "logits/rejected": 0.4526469111442566,
+      "logps/chosen": -253.50131225585938,
+      "logps/rejected": -349.1957702636719,
+      "loss": 0.4147,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.0875327587127686,
+      "rewards/margins": 8.09435749053955,
+      "rewards/rejected": -6.006823539733887,
       "step": 450
     },
     {
-      "epoch": 0.8010448410970832,
-      "grad_norm": 8.67365127688629,
-      "learning_rate": 5.7838215055366954e-08,
-      "logits/chosen": 0.3222309350967407,
-      "logits/rejected": 0.4205148220062256,
-      "logps/chosen": -430.9859924316406,
-      "logps/rejected": -525.9786376953125,
-      "loss": 0.4975,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.3063308000564575,
-      "rewards/margins": 0.9936330914497375,
-      "rewards/rejected": -2.2999637126922607,
+      "epoch": 2.1800947867298577,
+      "grad_norm": 7.824692642426332,
+      "learning_rate": 1.8342176087824573e-07,
+      "logits/chosen": 0.4325633645057678,
+      "logits/rejected": 0.3565566837787628,
+      "logps/chosen": -284.46624755859375,
+      "logps/rejected": -372.12091064453125,
+      "loss": 0.3992,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 1.8221031427383423,
+      "rewards/margins": 7.619426727294922,
+      "rewards/rejected": -5.797322750091553,
       "step": 460
     },
     {
-      "epoch": 0.8184588593818024,
-      "grad_norm": 8.357079349594828,
-      "learning_rate": 4.846387706115931e-08,
-      "logits/chosen": 0.30623525381088257,
-      "logits/rejected": 0.3187192380428314,
-      "logps/chosen": -481.569580078125,
-      "logps/rejected": -676.182373046875,
-      "loss": 0.4907,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": -1.4792513847351074,
-      "rewards/margins": 2.092211961746216,
-      "rewards/rejected": -3.5714633464813232,
+      "epoch": 2.227488151658768,
+      "grad_norm": 13.407256371457692,
+      "learning_rate": 1.6404285101840565e-07,
+      "logits/chosen": 0.3386808931827545,
+      "logits/rejected": 0.47734910249710083,
+      "logps/chosen": -331.7251892089844,
+      "logps/rejected": -367.4866638183594,
+      "loss": 0.3822,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 1.9130542278289795,
+      "rewards/margins": 7.692631721496582,
+      "rewards/rejected": -5.779577732086182,
       "step": 470
     },
     {
-      "epoch": 0.8358728776665215,
-      "grad_norm": 7.391474804049522,
-      "learning_rate": 3.9836364472876555e-08,
-      "logits/chosen": 0.32915499806404114,
-      "logits/rejected": 0.469025194644928,
-      "logps/chosen": -479.5523376464844,
-      "logps/rejected": -578.352294921875,
-      "loss": 0.5104,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.3519645929336548,
-      "rewards/margins": 1.1279245615005493,
-      "rewards/rejected": -2.479889392852783,
+      "epoch": 2.2748815165876777,
+      "grad_norm": 10.86707059625683,
+      "learning_rate": 1.455432716663517e-07,
+      "logits/chosen": 0.36686116456985474,
+      "logits/rejected": 0.48829737305641174,
+      "logps/chosen": -285.77008056640625,
+      "logps/rejected": -328.3174743652344,
+      "loss": 0.4089,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 1.7794748544692993,
+      "rewards/margins": 6.214818477630615,
+      "rewards/rejected": -4.435343265533447,
       "step": 480
     },
     {
-      "epoch": 0.8532868959512407,
-      "grad_norm": 6.99188787539793,
-      "learning_rate": 3.198764796404807e-08,
-      "logits/chosen": 0.3297652006149292,
-      "logits/rejected": 0.5077934265136719,
-      "logps/chosen": -462.5154724121094,
-      "logps/rejected": -589.4317016601562,
-      "loss": 0.4871,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.5567705631256104,
-      "rewards/margins": 1.4095462560653687,
-      "rewards/rejected": -2.9663164615631104,
+      "epoch": 2.322274881516588,
+      "grad_norm": 9.830177502454013,
+      "learning_rate": 1.2797144341546883e-07,
+      "logits/chosen": 0.3986554741859436,
+      "logits/rejected": 0.44396382570266724,
+      "logps/chosen": -321.13818359375,
+      "logps/rejected": -390.934326171875,
+      "loss": 0.4219,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 1.6029850244522095,
+      "rewards/margins": 7.5643768310546875,
+      "rewards/rejected": -5.961391448974609,
       "step": 490
     },
     {
-      "epoch": 0.87070091423596,
-      "grad_norm": 6.675925102880909,
-      "learning_rate": 2.494681225064066e-08,
-      "logits/chosen": 0.25453683733940125,
-      "logits/rejected": 0.31085866689682007,
-      "logps/chosen": -464.4830017089844,
-      "logps/rejected": -590.9197387695312,
-      "loss": 0.4806,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -1.5319304466247559,
-      "rewards/margins": 1.2942126989364624,
-      "rewards/rejected": -2.8261430263519287,
+      "epoch": 2.3696682464454977,
+      "grad_norm": 9.42905977432162,
+      "learning_rate": 1.1137335857372043e-07,
+      "logits/chosen": 0.4437794089317322,
+      "logits/rejected": 0.42870789766311646,
+      "logps/chosen": -287.81451416015625,
+      "logps/rejected": -374.01873779296875,
+      "loss": 0.4028,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.1330111026763916,
+      "rewards/margins": 7.767390251159668,
+      "rewards/rejected": -5.6343793869018555,
       "step": 500
     },
     {
-      "epoch": 0.87070091423596,
-      "eval_logits/chosen": 0.36373192071914673,
-      "eval_logits/rejected": 0.5181233286857605,
-      "eval_logps/chosen": -483.4225158691406,
-      "eval_logps/rejected": -560.248046875,
-      "eval_loss": 0.4863789975643158,
-      "eval_rewards/accuracies": 0.7246835231781006,
-      "eval_rewards/chosen": -1.4549182653427124,
-      "eval_rewards/margins": 1.1522036790847778,
-      "eval_rewards/rejected": -2.6071219444274902,
-      "eval_runtime": 57.0421,
-      "eval_samples_per_second": 43.827,
-      "eval_steps_per_second": 1.385,
+      "epoch": 2.3696682464454977,
+      "eval_logits/chosen": 0.3730663061141968,
+      "eval_logits/rejected": 0.4475269019603729,
+      "eval_logps/chosen": -338.3392028808594,
+      "eval_logps/rejected": -370.232666015625,
+      "eval_loss": 0.6289177536964417,
+      "eval_rewards/accuracies": 0.7405063509941101,
+      "eval_rewards/chosen": 0.7478683590888977,
+      "eval_rewards/margins": 4.585729122161865,
+      "eval_rewards/rejected": -3.8378612995147705,
+      "eval_runtime": 73.3012,
+      "eval_samples_per_second": 34.106,
+      "eval_steps_per_second": 1.078,
       "step": 500
     },
     {
-      "epoch": 0.8881149325206792,
-      "grad_norm": 7.8390265932386445,
-      "learning_rate": 1.8739948312837012e-08,
-      "logits/chosen": 0.2649977207183838,
-      "logits/rejected": 0.36522507667541504,
-      "logps/chosen": -476.8448791503906,
-      "logps/rejected": -576.7594604492188,
-      "loss": 0.4867,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": -1.3562562465667725,
-      "rewards/margins": 1.2045637369155884,
-      "rewards/rejected": -2.5608201026916504,
+      "epoch": 2.4170616113744074,
+      "grad_norm": 10.06462647313331,
+      "learning_rate": 9.579246078389403e-08,
+      "logits/chosen": 0.5295278429985046,
+      "logits/rejected": 0.43623122572898865,
+      "logps/chosen": -258.68963623046875,
+      "logps/rejected": -339.7721252441406,
+      "loss": 0.3858,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 1.592254400253296,
+      "rewards/margins": 7.2217698097229,
+      "rewards/rejected": -5.629514694213867,
       "step": 510
     },
     {
-      "epoch": 0.9055289508053983,
-      "grad_norm": 8.11481406815297,
-      "learning_rate": 1.3390056710597647e-08,
-      "logits/chosen": 0.4297245144844055,
-      "logits/rejected": 0.45524507761001587,
-      "logps/chosen": -461.51593017578125,
-      "logps/rejected": -598.4251708984375,
-      "loss": 0.493,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": -1.5076782703399658,
-      "rewards/margins": 1.4993212223052979,
-      "rewards/rejected": -3.0069994926452637,
+      "epoch": 2.4644549763033177,
+      "grad_norm": 9.022052721765009,
+      "learning_rate": 8.126953131469228e-08,
+      "logits/chosen": 0.44106584787368774,
+      "logits/rejected": 0.39466392993927,
+      "logps/chosen": -303.3637390136719,
+      "logps/rejected": -370.74114990234375,
+      "loss": 0.4143,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 1.8263496160507202,
+      "rewards/margins": 7.823184013366699,
+      "rewards/rejected": -5.996834754943848,
       "step": 520
     },
     {
-      "epoch": 0.9229429690901175,
-      "grad_norm": 13.292030068430211,
-      "learning_rate": 8.916962351285363e-09,
-      "logits/chosen": 0.37979334592819214,
-      "logits/rejected": 0.4294493794441223,
-      "logps/chosen": -445.51361083984375,
-      "logps/rejected": -526.9692993164062,
-      "loss": 0.5026,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": -1.3705123662948608,
-      "rewards/margins": 0.9539819955825806,
-      "rewards/rejected": -2.3244943618774414,
+      "epoch": 2.5118483412322274,
+      "grad_norm": 8.021054640921763,
+      "learning_rate": 6.784258232029472e-08,
+      "logits/chosen": 0.3634105622768402,
+      "logits/rejected": 0.3859165608882904,
+      "logps/chosen": -307.2467041015625,
+      "logps/rejected": -376.1995849609375,
+      "loss": 0.3822,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 2.497091770172119,
+      "rewards/margins": 7.9943437576293945,
+      "rewards/rejected": -5.497252464294434,
       "step": 530
     },
     {
-      "epoch": 0.9403569873748368,
-      "grad_norm": 7.831877125937131,
-      "learning_rate": 5.337241025194728e-09,
-      "logits/chosen": 0.3343893885612488,
-      "logits/rejected": 0.41524171829223633,
-      "logps/chosen": -470.4335021972656,
-      "logps/rejected": -616.5140380859375,
-      "loss": 0.4923,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": -1.6075522899627686,
-      "rewards/margins": 1.6572930812835693,
-      "rewards/rejected": -3.264845609664917,
+      "epoch": 2.5592417061611377,
+      "grad_norm": 10.013425700067337,
+      "learning_rate": 5.554675734776665e-08,
+      "logits/chosen": 0.5024563074111938,
+      "logits/rejected": 0.5056658387184143,
+      "logps/chosen": -276.1619567871094,
+      "logps/rejected": -368.4447021484375,
+      "loss": 0.4035,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 1.820339560508728,
+      "rewards/margins": 8.141976356506348,
+      "rewards/rejected": -6.321636199951172,
       "step": 540
     },
     {
-      "epoch": 0.957771005659556,
-      "grad_norm": 6.987219814527263,
-      "learning_rate": 2.664157981222437e-09,
-      "logits/chosen": 0.20853932201862335,
-      "logits/rejected": 0.2358679324388504,
-      "logps/chosen": -495.81256103515625,
-      "logps/rejected": -627.6561889648438,
-      "loss": 0.4959,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.4556350708007812,
-      "rewards/margins": 1.3767093420028687,
-      "rewards/rejected": -2.8323445320129395,
+      "epoch": 2.6066350710900474,
+      "grad_norm": 9.209955480260117,
+      "learning_rate": 4.4414239352730867e-08,
+      "logits/chosen": 0.42310771346092224,
+      "logits/rejected": 0.48689502477645874,
+      "logps/chosen": -313.3210754394531,
+      "logps/rejected": -351.4210205078125,
+      "loss": 0.406,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.1306679248809814,
+      "rewards/margins": 7.7195258140563965,
+      "rewards/rejected": -5.588858127593994,
       "step": 550
     },
     {
-      "epoch": 0.9751850239442751,
-      "grad_norm": 8.183878735354227,
-      "learning_rate": 9.076187702954652e-10,
-      "logits/chosen": 0.2713332772254944,
-      "logits/rejected": 0.35139814019203186,
-      "logps/chosen": -478.97515869140625,
-      "logps/rejected": -582.2161254882812,
-      "loss": 0.4943,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -1.5067349672317505,
-      "rewards/margins": 1.3343360424041748,
-      "rewards/rejected": -2.8410706520080566,
+      "epoch": 2.654028436018957,
+      "grad_norm": 9.959818332708023,
+      "learning_rate": 3.447416646405632e-08,
+      "logits/chosen": 0.5685544610023499,
+      "logits/rejected": 0.5256290435791016,
+      "logps/chosen": -287.7798156738281,
+      "logps/rejected": -380.33685302734375,
+      "loss": 0.4009,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 1.8459497690200806,
+      "rewards/margins": 7.295513153076172,
+      "rewards/rejected": -5.449563503265381,
       "step": 560
     },
     {
-      "epoch": 0.9925990422289943,
-      "grad_norm": 7.7106115387615395,
-      "learning_rate": 7.413253871516035e-11,
-      "logits/chosen": 0.25796160101890564,
-      "logits/rejected": 0.3311242461204529,
-      "logps/chosen": -475.98175048828125,
-      "logps/rejected": -590.1162109375,
-      "loss": 0.4887,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": -1.4784862995147705,
-      "rewards/margins": 1.1988673210144043,
-      "rewards/rejected": -2.677353620529175,
+      "epoch": 2.7014218009478674,
+      "grad_norm": 8.593809820816018,
+      "learning_rate": 2.575255571804391e-08,
+      "logits/chosen": 0.41258078813552856,
+      "logits/rejected": 0.4132450222969055,
+      "logps/chosen": -287.94476318359375,
+      "logps/rejected": -369.03656005859375,
+      "loss": 0.4,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 1.5231783390045166,
+      "rewards/margins": 7.392594814300537,
+      "rewards/rejected": -5.8694167137146,
       "step": 570
     },
     {
-      "epoch": 0.999564649542882,
-      "step": 574,
+      "epoch": 2.748815165876777,
+      "grad_norm": 9.646946039027634,
+      "learning_rate": 1.8272234961725084e-08,
+      "logits/chosen": 0.48128992319107056,
+      "logits/rejected": 0.4887717366218567,
+      "logps/chosen": -303.7729797363281,
+      "logps/rejected": -359.5372314453125,
+      "loss": 0.3912,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.173060655593872,
+      "rewards/margins": 8.012847900390625,
+      "rewards/rejected": -5.839787006378174,
+      "step": 580
+    },
+    {
+      "epoch": 2.7962085308056874,
+      "grad_norm": 11.09612482230785,
+      "learning_rate": 1.2052783103508102e-08,
+      "logits/chosen": 0.5081132650375366,
+      "logits/rejected": 0.5602059364318848,
+      "logps/chosen": -270.61737060546875,
+      "logps/rejected": -335.85577392578125,
+      "loss": 0.3991,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 1.619431495666504,
+      "rewards/margins": 6.8268561363220215,
+      "rewards/rejected": -5.207424163818359,
+      "step": 590
+    },
+    {
+      "epoch": 2.843601895734597,
+      "grad_norm": 8.273064520857158,
+      "learning_rate": 7.1104788675613315e-09,
+      "logits/chosen": 0.32943224906921387,
+      "logits/rejected": 0.4085375666618347,
+      "logps/chosen": -288.88995361328125,
+      "logps/rejected": -364.12860107421875,
+      "loss": 0.4029,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.0637223720550537,
+      "rewards/margins": 7.937726020812988,
+      "rewards/rejected": -5.874002933502197,
+      "step": 600
+    },
+    {
+      "epoch": 2.843601895734597,
+      "eval_logits/chosen": 0.38198891282081604,
+      "eval_logits/rejected": 0.45711585879325867,
+      "eval_logps/chosen": -337.3143310546875,
+      "eval_logps/rejected": -368.9125061035156,
+      "eval_loss": 0.6283919215202332,
+      "eval_rewards/accuracies": 0.7436708807945251,
+      "eval_rewards/chosen": 0.8503568768501282,
+      "eval_rewards/margins": 4.556199073791504,
+      "eval_rewards/rejected": -3.7058422565460205,
+      "eval_runtime": 73.7958,
+      "eval_samples_per_second": 33.877,
+      "eval_steps_per_second": 1.071,
+      "step": 600
+    },
+    {
+      "epoch": 2.890995260663507,
+      "grad_norm": 9.238913123295514,
+      "learning_rate": 3.4582581860612137e-09,
+      "logits/chosen": 0.43385523557662964,
+      "logits/rejected": 0.43230634927749634,
+      "logps/chosen": -292.0911865234375,
+      "logps/rejected": -353.61590576171875,
+      "loss": 0.3884,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": 1.989989995956421,
+      "rewards/margins": 6.724064826965332,
+      "rewards/rejected": -4.734074115753174,
+      "step": 610
+    },
+    {
+      "epoch": 2.938388625592417,
+      "grad_norm": 9.407237089972764,
+      "learning_rate": 1.1056803408273085e-09,
+      "logits/chosen": 0.48387131094932556,
+      "logits/rejected": 0.4587581753730774,
+      "logps/chosen": -282.6869201660156,
+      "logps/rejected": -344.5205078125,
+      "loss": 0.4089,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 1.806133508682251,
+      "rewards/margins": 7.467283725738525,
+      "rewards/rejected": -5.661149978637695,
+      "step": 620
+    },
+    {
+      "epoch": 2.985781990521327,
+      "grad_norm": 8.481488205996529,
+      "learning_rate": 5.890294296428955e-11,
+      "logits/chosen": 0.44664233922958374,
+      "logits/rejected": 0.5504810810089111,
+      "logps/chosen": -319.47119140625,
+      "logps/rejected": -348.36090087890625,
+      "loss": 0.3848,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 2.1828243732452393,
+      "rewards/margins": 6.884246826171875,
+      "rewards/rejected": -4.701422214508057,
+      "step": 630
+    },
+    {
+      "epoch": 3.0,
+      "step": 633,
       "total_flos": 0.0,
-      "train_loss": 0.5408797243330952,
-      "train_runtime": 4250.8819,
-      "train_samples_per_second": 17.289,
-      "train_steps_per_second": 0.135
+      "train_loss": 0.5009220597491634,
+      "train_runtime": 6227.6413,
+      "train_samples_per_second": 13.002,
+      "train_steps_per_second": 0.102
     }
   ],
   "logging_steps": 10,
-  "max_steps": 574,
+  "max_steps": 633,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 3,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -979,8 +1085,8 @@
         "should_epoch_stop": false,
         "should_evaluate": false,
         "should_log": false,
-        "should_save": false,
-        "should_training_stop": false
+        "should_save": true,
+        "should_training_stop": true
       },
       "attributes": {}
     }