End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.0714679956436157e-05,
-    "train_runtime": 211.3782,
+    "train_loss": 1.5345588326454163e-05,
+    "train_runtime": 186.0778,
     "train_samples": 28,
-    "train_samples_per_second": 1.514,
-    "train_steps_per_second": 0.095
+    "train_samples_per_second": 1.72,
+    "train_steps_per_second": 0.107
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0529e08086d0006286b720da89f94de6a3564bc9fc8220e5fa3a4a512929d4f9
+oid sha256:2d9355abf0949b352f575b75515237817961fe630104b738fbbfeae8c0232c41
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.0714679956436157e-05,
-    "train_runtime": 211.3782,
+    "train_loss": 1.5345588326454163e-05,
+    "train_runtime": 186.0778,
     "train_samples": 28,
-    "train_samples_per_second": 1.514,
-    "train_steps_per_second": 0.095
+    "train_samples_per_second": 1.72,
+    "train_steps_per_second": 0.107
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 264.9375,
+      "completion_length": 241.1875,
       "epoch": 0.5714285714285714,
-      "grad_norm": 8.294731140136719,
+      "grad_norm": 8.503501892089844,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 2.3257929645478725,
-      "reward_std": 2.239821334835142,
-      "rewards/concensus_correctness_reward_func": 0.819812498986721,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.489977477118373,
+      "reward_std": 0.9155122159281746,
+      "rewards/concensus_correctness_reward_func": 0.18056249991059303,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5426992317661643,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.44560248544439673,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5882812500931323,
+      "rewards/xmlcount_reward_func": 0.5513125089928508,
       "step": 2
     },
     {
-      "completion_length": 209.08333333333334,
+      "completion_length": 173.625,
       "epoch": 1.0,
-      "grad_norm": 8.786028861999512,
-      "kl": 0.0020196451854038364,
+      "grad_norm": 11.55927562713623,
+      "kl": 0.003464061524330949,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.6236170927683513,
-      "reward_std": 0.8473314394553503,
-      "rewards/concensus_correctness_reward_func": 0.07999999821186066,
+      "reward": 1.540897677342097,
+      "reward_std": 0.5160543769598007,
+      "rewards/concensus_correctness_reward_func": 0.0503333310286204,
       "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.4209504547373702,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5737726578178505,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.706000010172526,
+      "rewards/xmlcount_reward_func": 0.8334583342075348,
       "step": 4
     },
     {
-      "completion_length": 259.5,
+      "completion_length": 176.03125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 8.249679565429688,
-      "kl": 0.007281162346771453,
+      "grad_norm": 7.549230098724365,
+      "kl": 0.006556663851370104,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.6952194906771183,
-      "reward_std": 0.9508220433490351,
-      "rewards/concensus_correctness_reward_func": 0.293187502771616,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.971242893487215,
+      "reward_std": 1.369354895039578,
+      "rewards/concensus_correctness_reward_func": 0.53575000166893,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5035007460974157,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.413930396316573,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6485312506556511,
+      "rewards/xmlcount_reward_func": 0.584062498062849,
       "step": 6
     },
     {
-      "completion_length": 243.29166666666666,
+      "completion_length": 191.875,
       "epoch": 2.0,
-      "grad_norm": 9.076318740844727,
-      "kl": 0.00907895618972058,
+      "grad_norm": 7.489903926849365,
+      "kl": 0.008484401817743977,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.9068866223096848,
-      "reward_std": 1.3555283701668184,
-      "rewards/concensus_correctness_reward_func": 0.314833328127861,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 2.7487063656250634,
+      "reward_std": 1.0709232110530138,
+      "rewards/concensus_correctness_reward_func": 0.549750010172526,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.47217828532059986,
+      "rewards/final_correctness_reward_func": 0.4166666666666667,
+      "rewards/question_recreation_reward_func": 0.5391230415552855,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7865416804949442,
+      "rewards/xmlcount_reward_func": 0.7431666528185209,
       "step": 8
     },
     {
-      "completion_length": 241.375,
+      "completion_length": 202.375,
       "epoch": 2.571428571428571,
-      "grad_norm": 7.0489912033081055,
-      "kl": 0.009360351294162683,
+      "grad_norm": 10.167320251464844,
+      "kl": 0.013743098577833734,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 2.0282553620636463,
-      "reward_std": 1.4322956603136845,
-      "rewards/concensus_correctness_reward_func": 0.4737500064074993,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.5999224437400699,
+      "reward_std": 0.8269358775578439,
+      "rewards/concensus_correctness_reward_func": 0.1863749995827675,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.44641162687912583,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3886724393814802,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7330937590450048,
+      "rewards/xmlcount_reward_func": 0.712375009432435,
       "step": 10
     },
     {
-      "completion_length": 299.7916666666667,
+      "completion_length": 187.70833333333334,
       "epoch": 3.0,
-      "grad_norm": 5.941826820373535,
-      "kl": 0.010387163997317353,
+      "grad_norm": 5.543355941772461,
+      "kl": 0.03392461900754521,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 2.6202968458334603,
-      "reward_std": 1.383658204227686,
-      "rewards/concensus_correctness_reward_func": 0.49033333857854206,
+      "reward": 3.602676714460055,
+      "reward_std": 2.3583224962155023,
+      "rewards/concensus_correctness_reward_func": 1.3790833701690037,
       "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6505051727096239,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.5984267567594846,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7294583469629288,
+      "rewards/xmlcount_reward_func": 0.9584999680519104,
       "step": 12
     },
     {
-      "completion_length": 228.71875,
+      "completion_length": 215.75,
       "epoch": 3.571428571428571,
-      "grad_norm": 6.403939723968506,
-      "kl": 0.012776212432072498,
+      "grad_norm": 8.53206729888916,
+      "kl": 0.05182240842259489,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0,
-      "reward": 1.9797073863446712,
-      "reward_std": 0.8076981548219919,
-      "rewards/concensus_correctness_reward_func": 0.24943750351667404,
-      "rewards/consensus_reward_func": 0.25,
+      "loss": 0.0001,
+      "reward": 2.0001366659998894,
+      "reward_std": 1.3096730313263834,
+      "rewards/concensus_correctness_reward_func": 0.3738125078380108,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5630198871949688,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.40176167245954275,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6672499943524599,
+      "rewards/xmlcount_reward_func": 0.724562500603497,
       "step": 14
     },
     {
-      "completion_length": 212.375,
+      "completion_length": 167.625,
       "epoch": 4.0,
-      "grad_norm": 4.981140613555908,
-      "kl": 0.04448071836183468,
+      "grad_norm": 6.399363994598389,
+      "kl": 0.015622085076756775,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.6369245201349258,
-      "reward_std": 0.6018119080302616,
-      "rewards/concensus_correctness_reward_func": 0.07975000143051147,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 2.192226921518644,
+      "reward_std": 1.3540961171189945,
+      "rewards/concensus_correctness_reward_func": 0.3948333313067754,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.45667453358570737,
+      "rewards/question_recreation_reward_func": 0.4708102932199836,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9338333259026209,
+      "rewards/xmlcount_reward_func": 0.9099166691303253,
       "step": 16
     },
     {
-      "completion_length": 192.09375,
+      "completion_length": 159.125,
       "epoch": 4.571428571428571,
-      "grad_norm": 8.520259857177734,
-      "kl": 0.018384715731372125,
+      "grad_norm": 18.8550968170166,
+      "kl": 0.02504031316493638,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.2643840201199055,
-      "reward_std": 1.6802739538252354,
-      "rewards/concensus_correctness_reward_func": 0.48006249964237213,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 2.1071279235184193,
+      "reward_std": 1.0606713661400136,
+      "rewards/concensus_correctness_reward_func": 0.29612500593066216,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.4296340200235136,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.40919047803618014,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.8390624951571226,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.9018124938011169,
       "step": 18
     },
     {
-      "completion_length": 238.66666666666666,
+      "completion_length": 205.95833333333334,
       "epoch": 5.0,
-      "grad_norm": 6.646514892578125,
-      "kl": 0.013058107443309078,
+      "grad_norm": 5.739038944244385,
+      "kl": 0.01997669351597627,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.9825004537900288,
-      "reward_std": 1.0364690441638231,
-      "rewards/concensus_correctness_reward_func": 0.34199999769528705,
+      "reward": 2.0862623304128647,
+      "reward_std": 0.9388213828206062,
+      "rewards/concensus_correctness_reward_func": 0.5483333269755045,
       "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.4312920483450095,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5659290192027887,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7092083295186361,
+      "rewards/xmlcount_reward_func": 0.6386666645606359,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.0714679956436157e-05,
-      "train_runtime": 211.3782,
-      "train_samples_per_second": 1.514,
-      "train_steps_per_second": 0.095
+      "train_loss": 1.5345588326454163e-05,
+      "train_runtime": 186.0778,
+      "train_samples_per_second": 1.72,
+      "train_steps_per_second": 0.107
     }
   ],
   "logging_steps": 2,