End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.397261410782448e-05,
-    "train_runtime": 331.6611,
+    "train_loss": 4.647485911846161e-06,
+    "train_runtime": 435.1003,
     "train_samples": 160,
-    "train_samples_per_second": 0.965,
-    "train_steps_per_second": 0.06
+    "train_samples_per_second": 0.735,
+    "train_steps_per_second": 0.046
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a9b7aeccb5585dec47b287ded621768218ecb4111e020a6f03cd1a09ebf3d74
+oid sha256:af61ab8ba806dfe651878c6b360aff73748022a03d1c39dabe81ede054b886c4
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.397261410782448e-05,
-    "train_runtime": 331.6611,
+    "train_loss": 4.647485911846161e-06,
+    "train_runtime": 435.1003,
     "train_samples": 160,
-    "train_samples_per_second": 0.965,
-    "train_steps_per_second": 0.06
+    "train_samples_per_second": 0.735,
+    "train_steps_per_second": 0.046
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 257.84375,
+      "completion_length": 421.84375,
       "epoch": 0.1,
-      "grad_norm": 71.31194305419922,
+      "grad_norm": 7.744514465332031,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": -0.0,
-      "reward": 0.6542720094294054,
-      "reward_std": 0.671761574325501,
-      "rewards/concensus_correctness_reward_func": 0.109375,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.3874333486892283,
+      "reward_std": 0.8486921527073719,
+      "rewards/concensus_correctness_reward_func": 0.03393750078976154,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.11333451081009116,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3380895941518247,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.05656249821186066,
+      "rewards/xmlcount_reward_func": -0.04709374811500311,
       "step": 2
     },
     {
-      "completion_length": 388.75,
+      "completion_length": 391.84375,
       "epoch": 0.2,
-      "grad_norm": 48.57069778442383,
-      "kl": 0.0015563661654596217,
+      "grad_norm": 5.300667762756348,
+      "kl": 0.0019505722484609578,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 1.34490574139636,
-      "reward_std": 1.4263234985410236,
+      "reward": 1.3110849247314036,
+      "reward_std": 1.6469321683980525,
       "rewards/concensus_correctness_reward_func": 0.625,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.13059324386995286,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3452412204351276,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.08931249612942338,
+      "rewards/xmlcount_reward_func": 0.21584375109523535,
       "step": 4
     },
     {
-      "completion_length": 355.125,
+      "completion_length": 394.46875,
       "epoch": 0.3,
-      "grad_norm": 11.513766288757324,
-      "kl": 0.0018346092207366382,
+      "grad_norm": 46.67064666748047,
+      "kl": 0.0019083435508946422,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 0.3823581865581218,
-      "reward_std": 0.31888675036452696,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.3914839383214712,
+      "reward_std": 1.3497825153172016,
+      "rewards/concensus_correctness_reward_func": 0.05831250175833702,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.20129567771800794,
+      "rewards/question_recreation_reward_func": 0.31582766558858566,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11856249894481152,
+      "rewards/xmlcount_reward_func": -0.10765624674968421,
       "step": 6
     },
     {
-      "completion_length": 332.78125,
+      "completion_length": 411.25,
       "epoch": 0.4,
-      "grad_norm": 5.0440673828125,
-      "kl": 0.0017456002333346987,
+      "grad_norm": 16.974180221557617,
+      "kl": 0.00413768243015511,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 0.7418190783355385,
-      "reward_std": 0.6216794796055183,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.5314379204064608,
+      "reward_std": 0.5463541564531624,
+      "rewards/concensus_correctness_reward_func": 0.0018749999580904841,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.16697532951366156,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.2779379215789959,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07484374335035682,
+      "rewards/xmlcount_reward_func": 0.0641250021290034,
       "step": 8
     },
     {
-      "completion_length": 367.375,
+      "completion_length": 385.9375,
       "epoch": 0.5,
-      "grad_norm": 15.639695167541504,
-      "kl": 0.0024030036183830816,
+      "grad_norm": 11.258064270019531,
+      "kl": 0.002582737106422428,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 0.8275067815557122,
-      "reward_std": 0.9870248432271183,
-      "rewards/concensus_correctness_reward_func": 0.0599374994635582,
+      "reward": 2.051718756556511,
+      "reward_std": 0.7069938564673066,
+      "rewards/concensus_correctness_reward_func": 1.3228750005364418,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.09656929643824697,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4470312101766467,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.04600000171922147,
+      "rewards/xmlcount_reward_func": 0.0318124967161566,
       "step": 10
     },
     {
-      "completion_length": 377.71875,
+      "completion_length": 446.59375,
       "epoch": 0.6,
-      "grad_norm": 8.872749328613281,
-      "kl": 0.003477493522950681,
+      "grad_norm": 4.753056526184082,
+      "kl": 0.004790044869878329,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 0.5521137183532119,
-      "reward_std": 0.9777508705155924,
-      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
+      "reward": 0.8615806391462684,
+      "reward_std": 0.8717811293900013,
+      "rewards/concensus_correctness_reward_func": 0.08181250095367432,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.20373869099421427,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.474549381993711,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.27412500232458115,
+      "rewards/xmlcount_reward_func": -0.0072812524158507586,
       "step": 12
     },
     {
-      "completion_length": 343.59375,
+      "completion_length": 401.125,
       "epoch": 0.7,
-      "grad_norm": 5.5229268074035645,
-      "kl": 0.002653340348842903,
+      "grad_norm": 6.981433868408203,
+      "kl": 0.0025986589171225205,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 1.1060769520699978,
-      "reward_std": 0.8273453344590962,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 2.256255905609578,
+      "reward_std": 2.391559364972636,
+      "rewards/concensus_correctness_reward_func": 1.361625000834465,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.875,
-      "rewards/question_recreation_reward_func": 0.18898318015271798,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.35309968960064,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.04209375102072954,
+      "rewards/xmlcount_reward_func": 0.22903124894946814,
       "step": 14
     },
     {
-      "completion_length": 424.90625,
+      "completion_length": 395.125,
       "epoch": 0.8,
-      "grad_norm": 19.185876846313477,
-      "kl": 0.27459875560816727,
+      "grad_norm": 7.406398773193359,
+      "kl": 0.008011827812879346,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0003,
-      "reward": 1.4826910288538784,
-      "reward_std": 0.36641132296063006,
-      "rewards/concensus_correctness_reward_func": 1.25,
+      "loss": 0.0,
+      "reward": 0.8543199766427279,
+      "reward_std": 1.0620909905992448,
+      "rewards/concensus_correctness_reward_func": 0.13006250001490116,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.14225358597468585,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5446637291461229,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.03456250127055682,
+      "rewards/xmlcount_reward_func": -0.00790624599903822,
       "step": 16
     },
     {
-      "completion_length": 284.46875,
+      "completion_length": 370.03125,
       "epoch": 0.9,
-      "grad_norm": 40.3567008972168,
-      "kl": 0.026134049770917045,
+      "grad_norm": 56.691341400146484,
+      "kl": 0.011987929798124242,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 2.7290455140173435,
-      "reward_std": 1.5447391173802316,
-      "rewards/concensus_correctness_reward_func": 1.9351250007748604,
+      "reward": 1.4162302482873201,
+      "reward_std": 1.8340731484349817,
+      "rewards/concensus_correctness_reward_func": 0.6328125,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.16785799397621304,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4869490059791133,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.04793750052340329,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.23396874405443668,
       "step": 18
     },
     {
-      "completion_length": 350.09375,
+      "completion_length": 429.40625,
       "epoch": 1.0,
-      "grad_norm": 52.060001373291016,
-      "kl": 0.025269348172514583,
+      "grad_norm": 5.524323463439941,
+      "kl": 0.00813784722413402,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 1.9592706635594368,
-      "reward_std": 2.636638791358564,
-      "rewards/concensus_correctness_reward_func": 1.25,
+      "reward": 1.216457948088646,
+      "reward_std": 0.9376593017950654,
+      "rewards/concensus_correctness_reward_func": 0.2383749932050705,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.19583316356875002,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.44552044360898435,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07593749847728759,
+      "rewards/xmlcount_reward_func": 0.2825624970719218,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.397261410782448e-05,
-      "train_runtime": 331.6611,
-      "train_samples_per_second": 0.965,
-      "train_steps_per_second": 0.06
+      "train_loss": 4.647485911846161e-06,
+      "train_runtime": 435.1003,
+      "train_samples_per_second": 0.735,
+      "train_steps_per_second": 0.046
     }
   ],
   "logging_steps": 2,