End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.00017831340264820028,
-    "train_runtime": 174.6087,
+    "train_loss": 0.0014741608130862005,
+    "train_runtime": 143.5727,
     "train_samples": 28,
-    "train_samples_per_second": 1.833,
-    "train_steps_per_second": 0.115
+    "train_samples_per_second": 2.229,
+    "train_steps_per_second": 0.139
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:286dbc3abc4a0ce900aaa9b4434ab2b7c1c72f67ac901030428dcaccac70a1d0
+oid sha256:fb53acbd6f3e84dbb8f349a179e0aa331f58df2e36407978880c793ecaacb2f9
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.00017831340264820028,
-    "train_runtime": 174.6087,
+    "train_loss": 0.0014741608130862005,
+    "train_runtime": 143.5727,
     "train_samples": 28,
-    "train_samples_per_second": 1.833,
-    "train_steps_per_second": 0.115
+    "train_samples_per_second": 2.229,
+    "train_steps_per_second": 0.139
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 284.0,
+      "completion_length": 221.9375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 10.0138578414917,
+      "grad_norm": 9.694320678710938,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 1.54826570302248,
-      "reward_std": 0.6247149528935552,
-      "rewards/concensus_correctness_reward_func": 0.11962500214576721,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 2.8578853271901608,
+      "reward_std": 1.0625475882552564,
+      "rewards/concensus_correctness_reward_func": 0.5570000037550926,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5772032062523067,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6873852936550975,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.5701874950900674,
+      "rewards/strict_format_reward_func": 0.109375,
+      "rewards/xmlcount_reward_func": 0.7541250037029386,
       "step": 2
     },
     {
-      "completion_length": 174.375,
+      "completion_length": 136.83333333333334,
       "epoch": 1.0,
-      "grad_norm": 8.450321197509766,
-      "kl": 0.007973578428694358,
+      "grad_norm": 5.852896690368652,
+      "kl": 0.0063058519347881275,
       "learning_rate": 4.864543104251586e-07,
-      "loss": -0.0,
-      "reward": 3.7705226093530655,
-      "reward_std": 0.45547457427407306,
-      "rewards/concensus_correctness_reward_func": 0.9129999950528145,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.0,
+      "reward": 3.7386062343915305,
+      "reward_std": 1.4028123530248802,
+      "rewards/concensus_correctness_reward_func": 0.70066666106383,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.5226477482356131,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.6852311789989471,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.16666666666666666,
-      "rewards/xmlcount_reward_func": 1.0848750198880832,
+      "rewards/strict_format_reward_func": 0.3333333333333333,
+      "rewards/xmlcount_reward_func": 1.1860416730244954,
       "step": 4
     },
     {
-      "completion_length": 175.4375,
+      "completion_length": 146.78125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 12.835233688354492,
-      "kl": 0.02209460710582789,
+      "grad_norm": 10.775361061096191,
+      "kl": 0.022688565848511644,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 3.522875487804413,
-      "reward_std": 0.8763596473727375,
-      "rewards/concensus_correctness_reward_func": 0.8311874978244305,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 4.566311419010162,
+      "reward_std": 1.5661333670868771,
+      "rewards/concensus_correctness_reward_func": 1.4004999957978725,
+      "rewards/consensus_reward_func": 0.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5801567230373621,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.7594364183023572,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.15625,
-      "rewards/xmlcount_reward_func": 1.080281250178814,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 1.1719999946653843,
       "step": 6
     },
     {
-      "completion_length": 210.75,
+      "completion_length": 142.91666666666666,
       "epoch": 2.0,
-      "grad_norm": 9.03157901763916,
-      "kl": 0.0420371776757141,
+      "grad_norm": 9.037089347839355,
+      "kl": 0.31922435003798455,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0,
-      "reward": 4.086789906024933,
-      "reward_std": 1.799154872695605,
-      "rewards/concensus_correctness_reward_func": 1.3994166478514671,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "loss": 0.0002,
+      "reward": 4.1081428329149885,
+      "reward_std": 1.029661375455665,
+      "rewards/concensus_correctness_reward_func": 0.7650833452741305,
+      "rewards/consensus_reward_func": 0.9166666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.711248300348719,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6767677310854197,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 1.121958335240682,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.1871250073115032,
       "step": 8
     },
     {
-      "completion_length": 247.625,
+      "completion_length": 143.09375,
       "epoch": 2.571428571428571,
-      "grad_norm": 7.818336009979248,
-      "kl": 0.020400665380293503,
+      "grad_norm": 11.110583305358887,
+      "kl": 0.5744109276565723,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0,
-      "reward": 3.089905232191086,
-      "reward_std": 1.3913947939872742,
-      "rewards/concensus_correctness_reward_func": 0.4646250009536743,
-      "rewards/consensus_reward_func": 0.4375,
+      "loss": 0.0006,
+      "reward": 3.682307220995426,
+      "reward_std": 1.1703917261911556,
+      "rewards/concensus_correctness_reward_func": 0.7889375016093254,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.6601552572101355,
+      "rewards/question_recreation_reward_func": 0.6156822107732296,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.0588750019669533,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 1.1058125048875809,
       "step": 10
     },
     {
-      "completion_length": 168.70833333333334,
+      "completion_length": 136.41666666666666,
       "epoch": 3.0,
-      "grad_norm": 8.348956108093262,
-      "kl": 0.539873332871745,
+      "grad_norm": 5.076348304748535,
+      "kl": 2.3429599769879133,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0004,
-      "reward": 4.927634050448735,
-      "reward_std": 2.4490899654726186,
-      "rewards/concensus_correctness_reward_func": 1.6257499977946281,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.0018,
+      "reward": 4.40448385477066,
+      "reward_std": 1.6560175865888596,
+      "rewards/concensus_correctness_reward_func": 0.8267500052849451,
+      "rewards/consensus_reward_func": 1.1666666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.7438007940848669,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.8022754968454441,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.20833333333333334,
-      "rewards/xmlcount_reward_func": 1.183083325624466,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.2129583358764648,
       "step": 12
     },
     {
-      "completion_length": 184.28125,
+      "completion_length": 169.6875,
       "epoch": 3.571428571428571,
-      "grad_norm": 21.000173568725586,
-      "kl": 0.048413707700092345,
+      "grad_norm": 8.082115173339844,
+      "kl": 7.364466468396131,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0,
-      "reward": 3.79239871352911,
-      "reward_std": 2.138681993819773,
-      "rewards/concensus_correctness_reward_func": 1.1563750263303518,
-      "rewards/consensus_reward_func": 0.5625,
+      "loss": 0.0074,
+      "reward": 4.209167793393135,
+      "reward_std": 1.530743383220397,
+      "rewards/concensus_correctness_reward_func": 0.805249996483326,
+      "rewards/consensus_reward_func": 1.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6480862642638385,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.6893240767531097,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 1.0035624951124191,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.1677187494933605,
       "step": 14
     },
     {
-      "completion_length": 192.95833333333334,
+      "completion_length": 151.83333333333334,
       "epoch": 4.0,
-      "grad_norm": 19.743518829345703,
-      "kl": 0.5323486616058896,
+      "grad_norm": 23.623441696166992,
+      "kl": 5.0384774438571185,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0004,
-      "reward": 3.238221829136213,
-      "reward_std": 1.3768471541504066,
-      "rewards/concensus_correctness_reward_func": 0.5742499902844429,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "loss": 0.0038,
+      "reward": 3.1199777324994407,
+      "reward_std": 1.2431689854711294,
+      "rewards/concensus_correctness_reward_func": 0.32775000234444934,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.7081801050032178,
+      "rewards/question_recreation_reward_func": 0.714102791932722,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.10416666666666667,
-      "rewards/xmlcount_reward_func": 1.0182916621367137,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.1822916666666667,
       "step": 16
     },
     {
-      "completion_length": 212.65625,
+      "completion_length": 202.25,
       "epoch": 4.571428571428571,
-      "grad_norm": 11.760193824768066,
-      "kl": 0.6820494926068932,
+      "grad_norm": 9.800145149230957,
+      "kl": 0.6018333853571676,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0007,
-      "reward": 3.831192933022976,
-      "reward_std": 1.4825926979538053,
-      "rewards/concensus_correctness_reward_func": 0.8002499993890524,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.0006,
+      "reward": 3.698561042547226,
+      "reward_std": 1.2764807561179623,
+      "rewards/concensus_correctness_reward_func": 0.6848750058561563,
+      "rewards/consensus_reward_func": 0.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.7534429049119353,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.7144985762424767,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.15625,
-      "rewards/xmlcount_reward_func": 1.0587499998509884,
+      "rewards/strict_format_reward_func": 0.203125,
+      "rewards/xmlcount_reward_func": 1.096062496304512,
       "step": 18
     },
     {
-      "completion_length": 183.875,
+      "completion_length": 162.91666666666666,
       "epoch": 5.0,
-      "grad_norm": 7.98806095123291,
-      "kl": 0.24098504100887416,
+      "grad_norm": 7.49417781829834,
+      "kl": 0.5289312032206605,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0002,
-      "reward": 3.720976173877716,
-      "reward_std": 1.3116380800590075,
-      "rewards/concensus_correctness_reward_func": 0.6259166647990545,
-      "rewards/consensus_reward_func": 0.9166666666666666,
+      "loss": 0.0004,
+      "reward": 4.249745080868403,
+      "reward_std": 1.1143968345907826,
+      "rewards/concensus_correctness_reward_func": 0.8489166696866354,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.822184552748998,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.7353284160296122,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.22916666666666666,
-      "rewards/xmlcount_reward_func": 1.1270416676998138,
+      "rewards/strict_format_reward_func": 0.2916666666666667,
+      "rewards/xmlcount_reward_func": 1.2071666717529297,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.00017831340264820028,
-      "train_runtime": 174.6087,
-      "train_samples_per_second": 1.833,
-      "train_steps_per_second": 0.115
+      "train_loss": 0.0014741608130862005,
+      "train_runtime": 143.5727,
+      "train_samples_per_second": 2.229,
+      "train_steps_per_second": 0.139
     }
   ],
   "logging_steps": 2,