End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.6667490601539613e-06,
-    "train_runtime": 286.5218,
+    "train_loss": 2.26292759180069e-06,
+    "train_runtime": 300.3933,
     "train_samples": 28,
-    "train_samples_per_second": 1.117,
-    "train_steps_per_second": 0.07
+    "train_samples_per_second": 1.065,
+    "train_steps_per_second": 0.067
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c2c89d705c156cf36af2b06df2ac5fb863664f6e7857de5dfaa0539e8bccbc8
+oid sha256:4bfbbe167c301cf7070b1a4c6166425f441f35553d065dc65144d3b395da1533
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.6667490601539613e-06,
-    "train_runtime": 286.5218,
+    "train_loss": 2.26292759180069e-06,
+    "train_runtime": 300.3933,
     "train_samples": 28,
-    "train_samples_per_second": 1.117,
-    "train_steps_per_second": 0.07
+    "train_samples_per_second": 1.065,
+    "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 393.875,
+      "completion_length": 409.5625,
       "epoch": 0.5714285714285714,
-      "grad_norm": 5.421616554260254,
+      "grad_norm": 7.3488874435424805,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 1.0247059089597315,
-      "reward_std": 0.7572404937818646,
-      "rewards/concensus_correctness_reward_func": 0.06962500140070915,
-      "rewards/consensus_reward_func": 0.0625,
+      "loss": -0.0,
+      "reward": 1.911701524630189,
+      "reward_std": 1.9608921660110354,
+      "rewards/concensus_correctness_reward_func": 0.6924999989569187,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5236434224061668,
+      "rewards/question_recreation_reward_func": 0.502670262940228,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.24393749237060547,
+      "rewards/xmlcount_reward_func": 0.40403125062584877,
       "step": 2
     },
     {
-      "completion_length": 487.0,
+      "completion_length": 328.0833333333333,
       "epoch": 1.0,
-      "grad_norm": 2.7857470512390137,
-      "kl": 0.0008117672841763124,
+      "grad_norm": 2.5143661499023438,
+      "kl": 0.000962535836151801,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.3484513219445944,
-      "reward_std": 1.1112091614243884,
-      "rewards/concensus_correctness_reward_func": 0.07625000178813934,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 1.6085879653692245,
+      "reward_std": 0.4928823659817378,
+      "rewards/concensus_correctness_reward_func": 0.07999999821186066,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.48057635376850766,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.5609629477063814,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.20829167465368906,
+      "rewards/xmlcount_reward_func": 0.8009583279490471,
       "step": 4
     },
     {
-      "completion_length": 359.84375,
+      "completion_length": 399.75,
       "epoch": 1.5714285714285714,
-      "grad_norm": 6.515714168548584,
-      "kl": 0.0013933880654803943,
+      "grad_norm": 5.2374091148376465,
+      "kl": 0.0016545295457035536,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.9008149541914463,
-      "reward_std": 1.8278396958485246,
-      "rewards/concensus_correctness_reward_func": 0.7128749992698431,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.8188971318304539,
+      "reward_std": 1.099338564556092,
+      "rewards/concensus_correctness_reward_func": 0.6851250007748604,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.4973774557001889,
+      "rewards/question_recreation_reward_func": 0.5636470918543637,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5030625062063336,
+      "rewards/xmlcount_reward_func": 0.5076250061392784,
       "step": 6
     },
     {
-      "completion_length": 368.125,
+      "completion_length": 369.0833333333333,
       "epoch": 2.0,
-      "grad_norm": 7.336918830871582,
-      "kl": 0.0021219167295688144,
+      "grad_norm": 2.376291275024414,
+      "kl": 0.0015570806475201,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.9288174286484718,
-      "reward_std": 1.5418372388618689,
-      "rewards/concensus_correctness_reward_func": 0.84375,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 2.6855219999949136,
+      "reward_std": 2.0642324524621167,
+      "rewards/concensus_correctness_reward_func": 0.9224166671435038,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.3940257113426924,
+      "rewards/question_recreation_reward_func": 0.6277303335567316,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3577083336810271,
+      "rewards/xmlcount_reward_func": 0.6353750017782053,
       "step": 8
     },
     {
-      "completion_length": 362.625,
+      "completion_length": 424.96875,
       "epoch": 2.571428571428571,
-      "grad_norm": 4.1312360763549805,
-      "kl": 0.002214349529822357,
+      "grad_norm": 3.9620633125305176,
+      "kl": 0.002315260087925708,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.187094490043819,
-      "reward_std": 0.8256354574114084,
-      "rewards/concensus_correctness_reward_func": 0.11406250298023224,
+      "reward": 1.3290682332590222,
+      "reward_std": 0.747250135987997,
+      "rewards/concensus_correctness_reward_func": 0.07956250011920929,
       "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.41878195432946086,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.41284948121756315,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.40424999594688416,
+      "rewards/xmlcount_reward_func": 0.524156260304153,
       "step": 10
     },
     {
-      "completion_length": 392.5416666666667,
+      "completion_length": 423.0833333333333,
       "epoch": 3.0,
-      "grad_norm": 55.360015869140625,
-      "kl": 0.0031522798332540938,
+      "grad_norm": 2.795499563217163,
+      "kl": 0.003039477189304307,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 2.772899887834986,
-      "reward_std": 2.1842014212161303,
-      "rewards/concensus_correctness_reward_func": 0.8591666668653488,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 1.7231739660104115,
+      "reward_std": 1.0104351732879877,
+      "rewards/concensus_correctness_reward_func": 0.16308333228031793,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6666666666666666,
-      "rewards/question_recreation_reward_func": 0.5807332390298446,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5202989131212234,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.49966666599114734,
+      "rewards/xmlcount_reward_func": 0.37312500551342964,
       "step": 12
     },
     {
-      "completion_length": 424.40625,
+      "completion_length": 377.09375,
       "epoch": 3.571428571428571,
-      "grad_norm": 4.0036187171936035,
-      "kl": 0.004003107293101493,
+      "grad_norm": 13.31064510345459,
+      "kl": 0.0036816853971686214,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.7733455048874021,
-      "reward_std": 1.3317391743184999,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.623508368153125,
+      "reward_std": 0.8322483664378524,
+      "rewards/concensus_correctness_reward_func": 0.10193750099278986,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.6190330386161804,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5396646414883435,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.22068751440383494,
+      "rewards/xmlcount_reward_func": 0.669406246393919,
       "step": 14
     },
     {
-      "completion_length": 316.7916666666667,
+      "completion_length": 344.9583333333333,
       "epoch": 4.0,
-      "grad_norm": 2.858476161956787,
-      "kl": 0.0032398014154750854,
+      "grad_norm": 4.11659574508667,
+      "kl": 0.005116298232072343,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.2890068783114355,
-      "reward_std": 0.7038339432328939,
-      "rewards/concensus_correctness_reward_func": 0.03716666748126348,
+      "reward": 1.2915391772985458,
+      "reward_std": 0.7148025852317611,
+      "rewards/concensus_correctness_reward_func": 0.03758333375056585,
       "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.5088402436425289,
+      "rewards/question_recreation_reward_func": 0.4621641853203376,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.49299999823172885,
+      "rewards/xmlcount_reward_func": 0.5417916551232338,
       "step": 16
     },
     {
-      "completion_length": 359.25,
+      "completion_length": 501.90625,
       "epoch": 4.571428571428571,
-      "grad_norm": 56.07101821899414,
-      "kl": 0.007982015507877804,
+      "grad_norm": 4.820127487182617,
+      "kl": 0.0034084110739058815,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.153754443046637,
-      "reward_std": 1.8314161598682404,
-      "rewards/concensus_correctness_reward_func": 0.8815000038594007,
+      "reward": 1.9342150054872036,
+      "reward_std": 2.3277871979225893,
+      "rewards/concensus_correctness_reward_func": 0.6885625012218952,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.3999731181538664,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5643087650532834,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30978125240653753,
+      "rewards/xmlcount_reward_func": 0.18134376220405102,
       "step": 18
     },
     {
-      "completion_length": 368.7916666666667,
+      "completion_length": 415.9166666666667,
       "epoch": 5.0,
-      "grad_norm": 3.375103235244751,
-      "kl": 0.005311911751050502,
+      "grad_norm": 3.886725902557373,
+      "kl": 0.003820983639646632,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 2.4130918135245643,
-      "reward_std": 1.8152965704600017,
-      "rewards/concensus_correctness_reward_func": 0.8591666668653488,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 1.0419001132249832,
+      "reward_std": 0.8572910024474064,
+      "rewards/concensus_correctness_reward_func": 0.08016666769981384,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.4933835205932458,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.471900115100046,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.72720834116141,
+      "rewards/xmlcount_reward_func": 0.3231666736925642,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.6667490601539613e-06,
-      "train_runtime": 286.5218,
-      "train_samples_per_second": 1.117,
-      "train_steps_per_second": 0.07
+      "train_loss": 2.26292759180069e-06,
+      "train_runtime": 300.3933,
+      "train_samples_per_second": 1.065,
+      "train_steps_per_second": 0.067
     }
   ],
   "logging_steps": 2,