End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.867377773123735e-06,
-    "train_runtime": 309.0031,
-    "train_samples": 30,
-    "train_samples_per_second": 1.036,
-    "train_steps_per_second": 0.065
+    "train_loss": 1.5609432011842728e-05,
+    "train_runtime": 322.3942,
+    "train_samples": 28,
+    "train_samples_per_second": 0.993,
+    "train_steps_per_second": 0.062
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c677ce11671f271851ea72230f28d6829ff5c1aa813fb74610e8b1e36ea22626
+oid sha256:7e769a31a3ce1ce306521f0823087df26c36a571ca0abdae87ce0d8e52a395bc
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.867377773123735e-06,
-    "train_runtime": 309.0031,
-    "train_samples": 30,
-    "train_samples_per_second": 1.036,
-    "train_steps_per_second": 0.065
+    "train_loss": 1.5609432011842728e-05,
+    "train_runtime": 322.3942,
+    "train_samples": 28,
+    "train_samples_per_second": 0.993,
+    "train_steps_per_second": 0.062
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 296.65625,
-      "epoch": 0.5333333333333333,
-      "grad_norm": 5.975367069244385,
+      "completion_length": 507.25,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 6.7352681159973145,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 0.4923781892284751,
-      "reward_std": 0.5103055639192462,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "loss": -0.0,
+      "reward": 0.3999821678735316,
+      "reward_std": 0.7925753965755575,
+      "rewards/concensus_correctness_reward_func": 0.06943749636411667,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3379719341173768,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.3292009161086753,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.15440625371411443,
+      "rewards/xmlcount_reward_func": -0.1861562550693634,
       "step": 2
     },
     {
-      "completion_length": 295.0,
+      "completion_length": 463.7083333333333,
       "epoch": 1.0,
-      "grad_norm": 8.55418586730957,
-      "kl": 0.0011205757016016702,
+      "grad_norm": 4.924514293670654,
+      "kl": 0.012235193547288267,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.24337514423366105,
-      "reward_std": 0.3472900068653481,
+      "reward": 0.3449036317567031,
+      "reward_std": 0.7224312257664375,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1831965765782765,
+      "rewards/question_recreation_reward_func": 0.2850702985500296,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.060178570449352264,
+      "rewards/xmlcount_reward_func": 0.0598333366215229,
       "step": 4
     },
     {
-      "completion_length": 392.1875,
-      "epoch": 1.5333333333333332,
-      "grad_norm": 6.873995780944824,
-      "kl": 0.0011324490551487543,
+      "completion_length": 460.71875,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 60.659767150878906,
+      "kl": 0.008520981671608752,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.4639145736582577,
-      "reward_std": 0.555704839178361,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.4180720462463796,
+      "reward_std": 0.6416869102977216,
+      "rewards/concensus_correctness_reward_func": 0.012812499888241291,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.23878957249689847,
+      "rewards/question_recreation_reward_func": 0.35766580735798925,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2251249966211617,
+      "rewards/xmlcount_reward_func": -0.014906252268701792,
       "step": 6
     },
     {
-      "completion_length": 342.5,
+      "completion_length": 346.0416666666667,
       "epoch": 2.0,
-      "grad_norm": 8.925894737243652,
-      "kl": 0.00151446635468996,
+      "grad_norm": 5.058046340942383,
+      "kl": 0.006551954245272403,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.5655032782150167,
-      "reward_std": 0.6307864111981222,
-      "rewards/concensus_correctness_reward_func": 0.07071428639548165,
+      "reward": 0.4580930570761363,
+      "reward_std": 0.369822606133918,
+      "rewards/concensus_correctness_reward_func": 0.010416666666666666,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.288253280866359,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3284680585687359,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1351071453786322,
+      "rewards/xmlcount_reward_func": 0.11920833339293797,
       "step": 8
     },
     {
-      "completion_length": 297.53125,
-      "epoch": 2.533333333333333,
-      "grad_norm": 38.51108932495117,
-      "kl": 0.0015030291870061774,
+      "completion_length": 496.0,
+      "epoch": 2.571428571428571,
+      "grad_norm": 48.087276458740234,
+      "kl": 0.00559661753140972,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.49919217044953257,
-      "reward_std": 0.8162963903305354,
-      "rewards/concensus_correctness_reward_func": 0.12256249785423279,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.8703444199636579,
+      "reward_std": 1.5510720520978794,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.19547343289013952,
+      "rewards/question_recreation_reward_func": 0.25175065133953467,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.006343749817460775,
+      "rewards/xmlcount_reward_func": -0.13140625273808837,
       "step": 10
     },
     {
-      "completion_length": 459.14285714285717,
+      "completion_length": 414.7083333333333,
       "epoch": 3.0,
-      "grad_norm": 4.779883861541748,
-      "kl": 0.00666219020578345,
+      "grad_norm": 3.388296127319336,
+      "kl": 0.015154626812242592,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.5990122389048338,
-      "reward_std": 0.6876163507944771,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.3922838255142172,
+      "reward_std": 0.5539494124241173,
+      "rewards/concensus_correctness_reward_func": 0.01774999996026357,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.14285714285714285,
-      "rewards/question_recreation_reward_func": 0.31111936563892023,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.1093214319220611,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.31607550382614136,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.02487500508626302,
       "step": 12
     },
     {
-      "completion_length": 406.0,
-      "epoch": 3.533333333333333,
-      "grad_norm": 8.839569091796875,
-      "kl": 0.0020622438369173324,
+      "completion_length": 447.15625,
+      "epoch": 3.571428571428571,
+      "grad_norm": 7.227327346801758,
+      "kl": 0.03378651111233921,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.6188934349920601,
-      "reward_std": 0.46599482698366046,
+      "reward": 0.3113585321698338,
+      "reward_std": 0.31600811146199703,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.4561746867839247,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.20320227718912065,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.03771875286474824,
+      "rewards/xmlcount_reward_func": 0.10815624774841126,
       "step": 14
     },
     {
-      "completion_length": 301.39285714285717,
+      "completion_length": 385.9583333333333,
       "epoch": 4.0,
-      "grad_norm": 9.801114082336426,
-      "kl": 0.001455888821510598,
+      "grad_norm": 12.35800838470459,
+      "kl": 0.009904773668192016,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.5090827822153058,
-      "reward_std": 0.47882825615150587,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.7000408694148064,
+      "reward_std": 0.8576791263185441,
+      "rewards/concensus_correctness_reward_func": 0.020250000059604645,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.14285714285714285,
-      "rewards/question_recreation_reward_func": 0.23311848738895996,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4997075231124957,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13310714278902328,
+      "rewards/xmlcount_reward_func": 0.09675000980496407,
       "step": 16
     },
     {
-      "completion_length": 315.84375,
-      "epoch": 4.533333333333333,
-      "grad_norm": 18.273168563842773,
-      "kl": 0.0030655029258923605,
+      "completion_length": 379.34375,
+      "epoch": 4.571428571428571,
+      "grad_norm": 6.773362636566162,
+      "kl": 0.0719214873661258,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0,
-      "reward": 0.4697370844660327,
-      "reward_std": 0.5033468106412329,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "loss": 0.0001,
+      "reward": 0.8918995666317642,
+      "reward_std": 0.6417970298789442,
+      "rewards/concensus_correctness_reward_func": 0.012812499888241291,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.22579959442373365,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.3090245572384447,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.22831250133458525,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3825624920427799,
       "step": 18
     },
     {
-      "completion_length": 514.0714285714286,
+      "completion_length": 439.5416666666667,
       "epoch": 5.0,
-      "grad_norm": 5.46905517578125,
-      "kl": 0.0017352707842032292,
+      "grad_norm": 10.787079811096191,
+      "kl": 0.00297006058099214,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.2908818700483867,
-      "reward_std": 0.49696402663331746,
+      "reward": 0.14036849016944566,
+      "reward_std": 0.6102742572741894,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2124175831143345,
+      "rewards/question_recreation_reward_func": 0.2782851659382383,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07846428560359138,
+      "rewards/xmlcount_reward_func": -0.13791666583468518,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.867377773123735e-06,
-      "train_runtime": 309.0031,
-      "train_samples_per_second": 1.036,
-      "train_steps_per_second": 0.065
+      "train_loss": 1.5609432011842728e-05,
+      "train_runtime": 322.3942,
+      "train_samples_per_second": 0.993,
+      "train_steps_per_second": 0.062
     }
   ],
   "logging_steps": 2,