End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.564687460515415e-05,
-    "train_runtime": 980.676,
+    "train_loss": 4.6607106924057e-06,
+    "train_runtime": 528.4481,
     "train_samples": 28,
-    "train_samples_per_second": 0.326,
-    "train_steps_per_second": 0.02
+    "train_samples_per_second": 0.606,
+    "train_steps_per_second": 0.038
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ead068b87254bc4f9a7a7a66aacde33320f964e2156e71845a531ed56424c59d
+oid sha256:61ab4f847ebc3d81a28f6bed48fd929a834a56ceb7613054608622428531a563
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 4.564687460515415e-05,
-    "train_runtime": 980.676,
+    "train_loss": 4.6607106924057e-06,
+    "train_runtime": 528.4481,
     "train_samples": 28,
-    "train_samples_per_second": 0.326,
-    "train_steps_per_second": 0.02
+    "train_samples_per_second": 0.606,
+    "train_steps_per_second": 0.038
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 341.40625,
+      "completion_length": 291.71875,
       "epoch": 0.5714285714285714,
-      "grad_norm": 7.184085369110107,
+      "grad_norm": 5.5191755294799805,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.464074700139463,
-      "reward_std": 0.92922607017681,
-      "rewards/concensus_correctness_reward_func": 0.17987500131130219,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.5525079816579819,
+      "reward_std": 1.210987733094953,
+      "rewards/concensus_correctness_reward_func": 0.3031249977648258,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.47863720334134996,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.4316017273813486,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6180624926928431,
+      "rewards/xmlcount_reward_func": 0.42715625558048487,
       "step": 2
     },
     {
-      "completion_length": 279.9583333333333,
+      "completion_length": 187.91666666666666,
       "epoch": 1.0,
-      "grad_norm": 4.18076753616333,
-      "kl": 0.0011171183869009838,
+      "grad_norm": 3.369981050491333,
+      "kl": 0.0018891085104163115,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.6553137799104054,
-      "reward_std": 0.6190369529649615,
-      "rewards/concensus_correctness_reward_func": 0.31700000166893005,
+      "reward": 1.733353316783905,
+      "reward_std": 0.8906166382754842,
+      "rewards/concensus_correctness_reward_func": 0.1579166700442632,
       "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.48139710041383904,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.477644973124067,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6069166734814644,
+      "rewards/xmlcount_reward_func": 0.7644583284854889,
       "step": 4
     },
     {
-      "completion_length": 297.34375,
+      "completion_length": 226.65625,
       "epoch": 1.5714285714285714,
-      "grad_norm": 4.952084064483643,
-      "kl": 0.0020245183277438628,
+      "grad_norm": 5.8894524574279785,
+      "kl": 0.003135249629849568,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.609711330384016,
-      "reward_std": 1.102993049658835,
-      "rewards/concensus_correctness_reward_func": 0.171875,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.976506132632494,
+      "reward_std": 1.3569977073930204,
+      "rewards/concensus_correctness_reward_func": 0.3553749993443489,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5468363286490785,
+      "rewards/question_recreation_reward_func": 0.524193589692004,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5784999970346689,
+      "rewards/xmlcount_reward_func": 0.6594374980777502,
       "step": 6
     },
     {
-      "completion_length": 343.25,
+      "completion_length": 240.20833333333334,
       "epoch": 2.0,
-      "grad_norm": 4.897860527038574,
-      "kl": 0.49013493678648956,
+      "grad_norm": 4.913305759429932,
+      "kl": 0.003807866412292545,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0004,
-      "reward": 1.4871363962690036,
-      "reward_std": 0.7363182871292034,
-      "rewards/concensus_correctness_reward_func": 0.09016666809717815,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "loss": 0.0,
+      "reward": 1.831045498450597,
+      "reward_std": 0.44153941185989726,
+      "rewards/concensus_correctness_reward_func": 0.010416666666666666,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6520947298655907,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.5784205005814632,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5782083186010519,
+      "rewards/xmlcount_reward_func": 0.8255416651566824,
       "step": 8
     },
     {
-      "completion_length": 315.46875,
+      "completion_length": 225.03125,
       "epoch": 2.571428571428571,
-      "grad_norm": 6.122135162353516,
-      "kl": 0.04317952653946122,
+      "grad_norm": 8.09153938293457,
+      "kl": 0.007186611874203663,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.5414855852723122,
-      "reward_std": 0.9473953179549426,
-      "rewards/concensus_correctness_reward_func": 0.2148125022649765,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.4572835750877857,
+      "reward_std": 1.1924903796752915,
+      "rewards/concensus_correctness_reward_func": 0.12212499976158142,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4897355728899129,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.3975648246705532,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.586937508545816,
+      "rewards/xmlcount_reward_func": 0.6875937469303608,
       "step": 10
     },
     {
-      "completion_length": 275.8333333333333,
+      "completion_length": 167.0,
       "epoch": 3.0,
-      "grad_norm": 4.760578632354736,
-      "kl": 0.004625927947927266,
+      "grad_norm": 4.35850715637207,
+      "kl": 0.009038338272754723,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.510415772596995,
-      "reward_std": 0.6962736174464226,
-      "rewards/concensus_correctness_reward_func": 0.010416666666666666,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 2.172236522038778,
+      "reward_std": 1.3352671524820228,
+      "rewards/concensus_correctness_reward_func": 0.32199999690055847,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.656624094893535,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5439448431134224,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.7392083406448364,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.8062916770577431,
       "step": 12
     },
     {
-      "completion_length": 292.21875,
+      "completion_length": 176.78125,
       "epoch": 3.571428571428571,
-      "grad_norm": 5.914809703826904,
-      "kl": 0.005078462898381986,
+      "grad_norm": 6.705428123474121,
+      "kl": 0.0063926830916898325,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.0314752999693155,
-      "reward_std": 1.125947633292526,
-      "rewards/concensus_correctness_reward_func": 0.3634374998509884,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.621943838894367,
+      "reward_std": 1.5366092347539961,
+      "rewards/concensus_correctness_reward_func": 0.43143749982118607,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5771003246773034,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.7003813628107309,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.653437509201467,
+      "rewards/xmlcount_reward_func": 0.8026250153779984,
       "step": 14
     },
     {
-      "completion_length": 292.0416666666667,
+      "completion_length": 205.91666666666666,
       "epoch": 4.0,
-      "grad_norm": 4.0226874351501465,
-      "kl": 0.00821993917634245,
+      "grad_norm": 3.3670480251312256,
+      "kl": 0.007393967578536831,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.0891437853376071,
-      "reward_std": 0.5855259736999869,
-      "rewards/concensus_correctness_reward_func": 0.07975000143051147,
+      "reward": 1.4719673146804173,
+      "reward_std": 0.7381301919619242,
+      "rewards/concensus_correctness_reward_func": 0.0742499977350235,
       "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3388104323918621,
+      "rewards/question_recreation_reward_func": 0.571050651371479,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5872499917944273,
+      "rewards/xmlcount_reward_func": 0.7433333223064741,
       "step": 16
     },
     {
-      "completion_length": 207.78125,
+      "completion_length": 174.0,
       "epoch": 4.571428571428571,
-      "grad_norm": 6.2369184494018555,
-      "kl": 0.023525779928604607,
+      "grad_norm": 8.952384948730469,
+      "kl": 0.0078432568552671,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.7136339340358973,
-      "reward_std": 0.6242597099626437,
-      "rewards/concensus_correctness_reward_func": 0.06787500157952309,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 2.2049832567572594,
+      "reward_std": 1.262722612824291,
+      "rewards/concensus_correctness_reward_func": 0.35725000500679016,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6303839252796024,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5689207338728011,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7653750218451023,
+      "rewards/xmlcount_reward_func": 0.7788124959915876,
       "step": 18
     },
     {
-      "completion_length": 344.7083333333333,
+      "completion_length": 207.25,
       "epoch": 5.0,
-      "grad_norm": 3.5628011226654053,
-      "kl": 0.005101005168398842,
+      "grad_norm": 4.44536828994751,
+      "kl": 0.00663318302637587,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.4887317071358364,
-      "reward_std": 0.9142672990759214,
-      "rewards/concensus_correctness_reward_func": 0.2234999934832255,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 1.7914074063301086,
+      "reward_std": 0.7179869742443165,
+      "rewards/concensus_correctness_reward_func": 0.08216666678587596,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5464816769817844,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.5659907013177872,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3854166691501935,
+      "rewards/xmlcount_reward_func": 0.8932499984900156,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 4.564687460515415e-05,
-      "train_runtime": 980.676,
-      "train_samples_per_second": 0.326,
-      "train_steps_per_second": 0.02
+      "train_loss": 4.6607106924057e-06,
+      "train_runtime": 528.4481,
+      "train_samples_per_second": 0.606,
+      "train_steps_per_second": 0.038
     }
   ],
   "logging_steps": 2,