End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 69070.94929174794,
-    "train_runtime": 1492.4283,
+    "train_loss": 296549.3844699778,
+    "train_runtime": 1501.8809,
     "train_samples": 140,
-    "train_samples_per_second": 1.072,
+    "train_samples_per_second": 1.065,
     "train_steps_per_second": 0.067
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60f95d9dca8dab5a3bbe962bd31abceb4e646ea910d56401b65c9e5bbb65a2f5
+oid sha256:7aaa9065df806024fdc8b6235eb388fe611870af778efd05d97eff23f4eb2a52
 size 4996670464

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2812ae866c6020426eb8b6dd447cce7d4d1747315cf87bbcf91b987b63db1433
+oid sha256:3124cc9c290006d42f9e8ff285c36d3a31712f9601e7c939cc305a738cb9539c
 size 1178224960

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 69070.94929174794,
-    "train_runtime": 1492.4283,
+    "train_loss": 296549.3844699778,
+    "train_runtime": 1501.8809,
     "train_samples": 140,
-    "train_samples_per_second": 1.072,
+    "train_samples_per_second": 1.065,
     "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -10,962 +10,962 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 296.65625,
+      "completion_length": 258.78125,
       "epoch": 0.11428571428571428,
-      "grad_norm": 6.421307563781738,
+      "grad_norm": 2.054772138595581,
       "kl": 0.0,
       "learning_rate": 1.6666666666666665e-07,
       "loss": 0.0,
-      "reward": 16.115779522806406,
-      "reward_std": 1.6200225981592666,
-      "rewards/concensus_correctness_reward_func": 11.875,
-      "rewards/consensus_reward_func": 1.1875,
+      "reward": 17.02327122539282,
+      "reward_std": 4.40558909252286,
+      "rewards/concensus_correctness_reward_func": 12.5,
+      "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.8294045682996511,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.7546773618087173,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 0.6613750024698675,
+      "rewards/xmlcount_reward_func": 0.8310937508940697,
       "step": 2
     },
     {
-      "completion_length": 287.375,
+      "completion_length": 279.90625,
       "epoch": 0.22857142857142856,
-      "grad_norm": 83.90865325927734,
-      "kl": 0.060066412363084964,
+      "grad_norm": 6.912466049194336,
+      "kl": 0.01094070820545312,
       "learning_rate": 5e-07,
-      "loss": 0.0001,
-      "reward": 22.926603138446808,
-      "reward_std": 1.1492585969390348,
-      "rewards/concensus_correctness_reward_func": 16.875,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 0.0,
+      "reward": 23.899269431829453,
+      "reward_std": 0.04921933158766478,
+      "rewards/concensus_correctness_reward_func": 17.5,
+      "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4375,
-      "rewards/question_recreation_reward_func": 0.9539468586444855,
+      "rewards/final_correctness_reward_func": 1.75,
+      "rewards/question_recreation_reward_func": 0.9344256119802594,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.484375,
-      "rewards/xmlcount_reward_func": 1.23828125,
+      "rewards/xmlcount_reward_func": 1.23046875,
       "step": 4
     },
     {
-      "completion_length": 303.34375,
+      "completion_length": 318.5625,
       "epoch": 0.34285714285714286,
-      "grad_norm": 8.03852367401123,
-      "kl": 0.08650782657787204,
+      "grad_norm": 5.637369632720947,
+      "kl": 83747384.09356707,
       "learning_rate": 4.994757065594279e-07,
-      "loss": 0.0001,
-      "reward": 19.984008461236954,
-      "reward_std": 0.1418806642468553,
-      "rewards/concensus_correctness_reward_func": 13.75,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 83747.3828,
+      "reward": 18.383561819791794,
+      "reward_std": 0.45949314400786534,
+      "rewards/concensus_correctness_reward_func": 12.5,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.9683833792805672,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.9421556144952774,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/xmlcount_reward_func": 1.23828125,
       "step": 6
     },
     {
-      "completion_length": 287.90625,
+      "completion_length": 293.25,
       "epoch": 0.45714285714285713,
-      "grad_norm": 5.897157669067383,
-      "kl": 0.2046783430268988,
+      "grad_norm": 6.48016881942749,
+      "kl": 0.10146056092344224,
       "learning_rate": 4.979050253066063e-07,
-      "loss": 0.0002,
-      "reward": 21.05784034729004,
-      "reward_std": 0.1875998378091026,
-      "rewards/concensus_correctness_reward_func": 15.0,
+      "loss": 0.0001,
+      "reward": 24.90006184577942,
+      "reward_std": 0.250804515555501,
+      "rewards/concensus_correctness_reward_func": 18.75,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.9328403398394585,
+      "rewards/final_correctness_reward_func": 1.5,
+      "rewards/question_recreation_reward_func": 0.9156866073608398,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.5,
+      "rewards/strict_format_reward_func": 0.484375,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 8
     },
     {
-      "completion_length": 307.34375,
+      "completion_length": 294.90625,
       "epoch": 0.5714285714285714,
-      "grad_norm": 10.724071502685547,
-      "kl": 0.43954612920060754,
+      "grad_norm": 16.164697647094727,
+      "kl": 0.09648153139278293,
       "learning_rate": 4.952945442245597e-07,
-      "loss": 0.0004,
-      "reward": 24.9216285943985,
-      "reward_std": 0.23980266088619828,
+      "loss": 0.0001,
+      "reward": 25.115739941596985,
+      "reward_std": 0.14409029902890325,
       "rewards/concensus_correctness_reward_func": 18.75,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.9216288048774004,
+      "rewards/final_correctness_reward_func": 1.6875,
+      "rewards/question_recreation_reward_func": 0.9477710761129856,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.5,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/xmlcount_reward_func": 1.24609375,
       "step": 10
     },
     {
-      "completion_length": 288.09375,
+      "completion_length": 285.53125,
       "epoch": 0.6857142857142857,
-      "grad_norm": 78.01028442382812,
-      "kl": 3.3898711418733,
+      "grad_norm": 0.182708278298378,
+      "kl": 0.16675707220565528,
       "learning_rate": 4.916552125781528e-07,
-      "loss": 0.0034,
-      "reward": 25.49212944507599,
-      "reward_std": 0.004952086288540158,
-      "rewards/concensus_correctness_reward_func": 18.75,
+      "loss": 0.0002,
+      "reward": 26.597843527793884,
+      "reward_std": 0.002685512910829857,
+      "rewards/concensus_correctness_reward_func": 20.0,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 2.0,
-      "rewards/question_recreation_reward_func": 0.9921293407678604,
+      "rewards/final_correctness_reward_func": 1.875,
+      "rewards/question_recreation_reward_func": 0.9728433899581432,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 12
     },
     {
-      "completion_length": 288.09375,
+      "completion_length": 308.71875,
       "epoch": 0.8,
-      "grad_norm": 5.335823059082031,
-      "kl": 0.6103032417595387,
+      "grad_norm": 6.084612846374512,
+      "kl": 0.21913069766014814,
       "learning_rate": 4.870022949890676e-07,
-      "loss": 0.0005,
-      "reward": 22.442598521709442,
-      "reward_std": 0.2463319645394222,
-      "rewards/concensus_correctness_reward_func": 16.25,
+      "loss": 0.0002,
+      "reward": 21.113146513700485,
+      "reward_std": 0.13240044617487,
+      "rewards/concensus_correctness_reward_func": 15.0,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.9621299877762794,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.9442716278135777,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.484375,
-      "rewards/xmlcount_reward_func": 1.24609375,
+      "rewards/xmlcount_reward_func": 1.2470000013709068,
       "step": 14
     },
     {
-      "completion_length": 302.65625,
+      "completion_length": 333.25,
       "epoch": 0.9142857142857143,
-      "grad_norm": 555077.625,
-      "kl": 19965.11735479813,
+      "grad_norm": 6157868.5,
+      "kl": 288729.23049705196,
       "learning_rate": 4.81355307410676e-07,
-      "loss": 19.9651,
-      "reward": 20.95334878563881,
-      "reward_std": 0.4493027157150209,
-      "rewards/concensus_correctness_reward_func": 15.0,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 288.7293,
+      "reward": 19.089906871318817,
+      "reward_std": 1.0611610219348222,
+      "rewards/concensus_correctness_reward_func": 13.125,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.9466612227261066,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.9974069446325302,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.225437507033348,
+      "rewards/strict_format_reward_func": 0.46875,
+      "rewards/xmlcount_reward_func": 1.248750001192093,
       "step": 16
     },
     {
-      "completion_length": 301.5,
+      "completion_length": 285.5,
       "epoch": 1.0,
-      "grad_norm": 0.38694965839385986,
-      "kl": 0.24521661549806595,
+      "grad_norm": 78.33558654785156,
+      "kl": 4402.47263197725,
       "learning_rate": 4.747379352713488e-07,
-      "loss": 0.0002,
-      "reward": 21.118529717127483,
-      "reward_std": 0.12700148997828364,
-      "rewards/concensus_correctness_reward_func": 15.0,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 3.3019,
+      "reward": 20.285069465637207,
+      "reward_std": 1.364617843956997,
+      "rewards/concensus_correctness_reward_func": 14.166666666666666,
+      "rewards/consensus_reward_func": 1.9166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.9935297220945358,
+      "rewards/final_correctness_reward_func": 1.5,
+      "rewards/question_recreation_reward_func": 0.9986111124356588,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.4583333333333333,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/xmlcount_reward_func": 1.2447916666666667,
       "step": 18
     },
     {
-      "completion_length": 296.09375,
+      "completion_length": 302.59375,
       "epoch": 1.1142857142857143,
-      "grad_norm": 5.091805458068848,
-      "kl": 0.8085636631585658,
+      "grad_norm": 3.6053833961486816,
+      "kl": 0.13355864235199988,
       "learning_rate": 4.6717793412953776e-07,
-      "loss": 0.0009,
-      "reward": 21.177861392498016,
-      "reward_std": 0.09898363525644527,
-      "rewards/concensus_correctness_reward_func": 15.0,
+      "loss": 0.0001,
+      "reward": 19.868769019842148,
+      "reward_std": 0.008435817871941254,
+      "rewards/concensus_correctness_reward_func": 13.75,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.9473926387727261,
+      "rewards/final_correctness_reward_func": 1.375,
+      "rewards/question_recreation_reward_func": 0.9937690384685993,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.484375,
-      "rewards/xmlcount_reward_func": 1.24609375,
+      "rewards/strict_format_reward_func": 0.5,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 20
     },
     {
-      "completion_length": 299.8125,
+      "completion_length": 298.9375,
       "epoch": 1.2285714285714286,
-      "grad_norm": 35.016902923583984,
-      "kl": 5.345846755430102,
+      "grad_norm": 4.034364700317383,
+      "kl": 0.284994178917259,
       "learning_rate": 4.5870701325731773e-07,
-      "loss": 0.0053,
-      "reward": 24.928703397512436,
-      "reward_std": 0.32053298623941373,
+      "loss": 0.0003,
+      "reward": 25.217251539230347,
+      "reward_std": 0.1768817222182406,
       "rewards/concensus_correctness_reward_func": 18.75,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.9355159923434258,
+      "rewards/final_correctness_reward_func": 1.75,
+      "rewards/question_recreation_reward_func": 0.9985014833509922,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.227562502026558,
+      "rewards/strict_format_reward_func": 0.46875,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 22
     },
     {
-      "completion_length": 299.625,
+      "completion_length": 295.875,
       "epoch": 1.342857142857143,
-      "grad_norm": 3.4500892162323,
-      "kl": 0.48288927460089326,
+      "grad_norm": 19.670671463012695,
+      "kl": 93.65684759290889,
       "learning_rate": 4.4936070264068016e-07,
-      "loss": 0.0005,
-      "reward": 26.40441143512726,
-      "reward_std": 0.20266110051306896,
-      "rewards/concensus_correctness_reward_func": 20.0,
+      "loss": 0.0937,
+      "reward": 25.219618678092957,
+      "reward_std": 0.2108936388976872,
+      "rewards/concensus_correctness_reward_func": 18.75,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 1.75,
-      "rewards/question_recreation_reward_func": 0.974724005907774,
+      "rewards/question_recreation_reward_func": 0.9891498424112797,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.2109375,
+      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/xmlcount_reward_func": 1.24609375,
       "step": 24
     },
     {
-      "completion_length": 285.65625,
+      "completion_length": 289.5,
       "epoch": 1.457142857142857,
-      "grad_norm": 61298.04296875,
-      "kl": 15362.55208538007,
+      "grad_norm": 118071795712.0,
+      "kl": 14642561024.712172,
       "learning_rate": 4.391782039544238e-07,
-      "loss": 15.3626,
-      "reward": 22.46716809272766,
-      "reward_std": 0.24589635042502778,
-      "rewards/concensus_correctness_reward_func": 16.25,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 14642560.0,
+      "reward": 23.393190026283264,
+      "reward_std": 1.1594447159441188,
+      "rewards/concensus_correctness_reward_func": 16.875,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.625,
-      "rewards/question_recreation_reward_func": 0.919355308637023,
+      "rewards/final_correctness_reward_func": 1.875,
+      "rewards/question_recreation_reward_func": 0.987252376973629,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.2196875028312206,
+      "rewards/strict_format_reward_func": 0.46875,
+      "rewards/xmlcount_reward_func": 1.2496875077486038,
       "step": 26
     },
     {
-      "completion_length": 284.6875,
+      "completion_length": 290.4375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 10.184200286865234,
-      "kl": 287324.80912376195,
+      "grad_norm": 985.9644165039062,
+      "kl": 47.3938269498758,
       "learning_rate": 4.282022261367073e-07,
-      "loss": 287.3248,
-      "reward": 21.207891523838043,
-      "reward_std": 0.3488070629828144,
+      "loss": 0.0474,
+      "reward": 20.982531785964966,
+      "reward_std": 0.09451750945299864,
       "rewards/concensus_correctness_reward_func": 15.0,
-      "rewards/consensus_reward_func": 2.0,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.9461727738380432,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.9512817114591599,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.24609375,
+      "rewards/strict_format_reward_func": 0.46875,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 28
     },
     {
-      "completion_length": 292.75,
+      "completion_length": 291.75,
       "epoch": 1.6857142857142857,
-      "grad_norm": 8.854732513427734,
-      "kl": 15780.277490709908,
+      "grad_norm": 14.219393730163574,
+      "kl": 5697.919686453417,
       "learning_rate": 4.1647880625292027e-07,
-      "loss": 15.7803,
-      "reward": 22.25498914718628,
-      "reward_std": 0.4585948936874047,
-      "rewards/concensus_correctness_reward_func": 16.25,
+      "loss": 5.6979,
+      "reward": 24.938138127326965,
+      "reward_std": 0.5503595303162001,
+      "rewards/concensus_correctness_reward_func": 18.75,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.8799891024827957,
+      "rewards/final_correctness_reward_func": 1.5625,
+      "rewards/question_recreation_reward_func": 0.9167943075299263,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.5,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/xmlcount_reward_func": 1.2244687490165234,
       "step": 30
     },
     {
-      "completion_length": 301.5,
+      "completion_length": 320.78125,
       "epoch": 1.8,
-      "grad_norm": 5.105992794036865,
-      "kl": 0.41865785513073206,
+      "grad_norm": 111691.4140625,
+      "kl": 9744.625750856474,
       "learning_rate": 4.040571164002318e-07,
-      "loss": 0.0004,
-      "reward": 21.198615819215775,
-      "reward_std": 0.18745470189242042,
-      "rewards/concensus_correctness_reward_func": 15.0,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 9.7446,
+      "reward": 23.627057909965515,
+      "reward_std": 0.3214102545171045,
+      "rewards/concensus_correctness_reward_func": 17.5,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.9798658639192581,
+      "rewards/final_correctness_reward_func": 1.6875,
+      "rewards/question_recreation_reward_func": 0.9628704935312271,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/xmlcount_reward_func": 1.1329374983906746,
       "step": 32
     },
     {
-      "completion_length": 291.5625,
+      "completion_length": 288.90625,
       "epoch": 1.9142857142857141,
-      "grad_norm": 70842.7109375,
-      "kl": 7595.819510911126,
+      "grad_norm": 5.710204124450684,
+      "kl": 35.09870433434844,
       "learning_rate": 3.909892574627266e-07,
-      "loss": 7.5958,
-      "reward": 21.041768729686737,
-      "reward_std": 0.25126288196770474,
-      "rewards/concensus_correctness_reward_func": 15.0,
+      "loss": 0.0351,
+      "reward": 19.93251895904541,
+      "reward_std": 0.09295251258299686,
+      "rewards/concensus_correctness_reward_func": 13.75,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.9792686924338341,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.9950189106166363,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 34
     },
     {
-      "completion_length": 298.25,
+      "completion_length": 312.9583333333333,
       "epoch": 2.0,
-      "grad_norm": 329.4862365722656,
-      "kl": 316224.1397752762,
+      "grad_norm": 186.2655029296875,
+      "kl": 8339.249363450954,
       "learning_rate": 3.773300405821908e-07,
-      "loss": 237.1681,
-      "reward": 22.495349248250324,
-      "reward_std": 0.5391187722949932,
+      "loss": 6.2544,
+      "reward": 22.710465768973034,
+      "reward_std": 0.40588310376430553,
       "rewards/concensus_correctness_reward_func": 16.666666666666668,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1666666666666667,
-      "rewards/question_recreation_reward_func": 0.9120159248510996,
+      "rewards/final_correctness_reward_func": 1.5,
+      "rewards/question_recreation_reward_func": 0.8562991668780645,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.5,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/strict_format_reward_func": 0.4583333333333333,
+      "rewards/xmlcount_reward_func": 1.2291666666666667,
       "step": 36
     },
     {
-      "completion_length": 278.0625,
+      "completion_length": 286.53125,
       "epoch": 2.1142857142857143,
-      "grad_norm": 83.9132308959961,
-      "kl": 274299.0737755578,
+      "grad_norm": 10.10280704498291,
+      "kl": 1.4235679027624428,
       "learning_rate": 3.6313675726113475e-07,
-      "loss": 274.2991,
-      "reward": 26.602285027503967,
-      "reward_std": 0.20196314249187708,
-      "rewards/concensus_correctness_reward_func": 20.0,
+      "loss": 0.0014,
+      "reward": 25.085916996002197,
+      "reward_std": 0.01389117946382612,
+      "rewards/concensus_correctness_reward_func": 18.75,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.875,
-      "rewards/question_recreation_reward_func": 0.9772850759327412,
+      "rewards/final_correctness_reward_func": 1.625,
+      "rewards/question_recreation_reward_func": 0.9609169252216816,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 38
     },
     {
-      "completion_length": 290.9375,
+      "completion_length": 307.84375,
       "epoch": 2.2285714285714286,
-      "grad_norm": 13.199821472167969,
-      "kl": 16.826391340699047,
+      "grad_norm": 301.5379333496094,
+      "kl": 31.047572038602084,
       "learning_rate": 3.484689390623218e-07,
-      "loss": 0.0168,
-      "reward": 23.80302059650421,
-      "reward_std": 0.4077005465514958,
-      "rewards/concensus_correctness_reward_func": 17.5,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.031,
+      "reward": 24.409271508455276,
+      "reward_std": 1.3443972589448094,
+      "rewards/concensus_correctness_reward_func": 18.125,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.625,
-      "rewards/question_recreation_reward_func": 0.9601456187665462,
+      "rewards/final_correctness_reward_func": 1.75,
+      "rewards/question_recreation_reward_func": 0.9250839278101921,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.2491250038146973,
+      "rewards/strict_format_reward_func": 0.453125,
+      "rewards/xmlcount_reward_func": 1.2185625061392784,
       "step": 40
     },
     {
-      "completion_length": 305.0625,
+      "completion_length": 321.125,
       "epoch": 2.342857142857143,
-      "grad_norm": 8.125014305114746,
-      "kl": 1.4574856441468,
+      "grad_norm": 9.44746208190918,
+      "kl": 99998651.09828581,
       "learning_rate": 3.3338810791270517e-07,
-      "loss": 0.0014,
-      "reward": 19.75320541858673,
-      "reward_std": 0.3490199828884215,
+      "loss": 99998.6562,
+      "reward": 19.91300740838051,
+      "reward_std": 0.06640663610232878,
       "rewards/concensus_correctness_reward_func": 13.75,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.9407055526971817,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.9950386732816696,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.5,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/xmlcount_reward_func": 1.24609375,
       "step": 42
     },
     {
-      "completion_length": 296.46875,
+      "completion_length": 297.0,
       "epoch": 2.4571428571428573,
-      "grad_norm": 0.7579712867736816,
-      "kl": 96.8114567277953,
+      "grad_norm": 4.234458923339844,
+      "kl": 26.058129234821536,
       "learning_rate": 3.179575180590857e-07,
-      "loss": 0.0968,
-      "reward": 22.325815856456757,
-      "reward_std": 0.2327170339121949,
-      "rewards/concensus_correctness_reward_func": 16.25,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.0261,
+      "reward": 20.36888739466667,
+      "reward_std": 1.2362263132818043,
+      "rewards/concensus_correctness_reward_func": 14.375,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.9508159533143044,
+      "rewards/question_recreation_reward_func": 0.9313873276114464,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 44
     },
     {
-      "completion_length": 301.59375,
+      "completion_length": 293.375,
       "epoch": 2.571428571428571,
-      "grad_norm": 5.712331771850586,
-      "kl": 6.341034076642245,
+      "grad_norm": 3.131242513656616,
+      "kl": 0.3580109770409763,
       "learning_rate": 3.022418907578188e-07,
-      "loss": 0.0063,
-      "reward": 22.558997809886932,
-      "reward_std": 0.0432420757424552,
+      "loss": 0.0004,
+      "reward": 22.51879781484604,
+      "reward_std": 0.1030593290925026,
       "rewards/concensus_correctness_reward_func": 16.25,
-      "rewards/consensus_reward_func": 2.0,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.625,
-      "rewards/question_recreation_reward_func": 0.9808727987110615,
+      "rewards/final_correctness_reward_func": 1.6875,
+      "rewards/question_recreation_reward_func": 0.9875477105379105,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
+      "rewards/strict_format_reward_func": 0.46875,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 46
     },
     {
-      "completion_length": 302.1875,
+      "completion_length": 303.6875,
       "epoch": 2.685714285714286,
-      "grad_norm": 44952662016.0,
-      "kl": 3445322040.5881023,
+      "grad_norm": 7157400.5,
+      "kl": 723253.104146285,
       "learning_rate": 2.863071428113726e-07,
-      "loss": 3445322.0,
-      "reward": 21.52135854959488,
-      "reward_std": 1.3011465523632069,
+      "loss": 723.2531,
+      "reward": 21.64236208796501,
+      "reward_std": 1.3292767723178258,
       "rewards/concensus_correctness_reward_func": 15.625,
       "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.8964210189878941,
+      "rewards/final_correctness_reward_func": 1.375,
+      "rewards/question_recreation_reward_func": 0.9548620991408825,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.5,
-      "rewards/xmlcount_reward_func": 1.249937504529953,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 48
     },
     {
-      "completion_length": 288.5,
+      "completion_length": 288.4375,
       "epoch": 2.8,
-      "grad_norm": 4259.3798828125,
-      "kl": 223.73643614712637,
+      "grad_norm": 3.812501907348633,
+      "kl": 0.1941389513667673,
       "learning_rate": 2.7022011009035107e-07,
-      "loss": 0.2237,
-      "reward": 23.679662585258484,
-      "reward_std": 0.1834919499233365,
+      "loss": 0.0002,
+      "reward": 23.754722118377686,
+      "reward_std": 0.13451411115238443,
       "rewards/concensus_correctness_reward_func": 17.5,
-      "rewards/consensus_reward_func": 2.0,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.960912674665451,
+      "rewards/final_correctness_reward_func": 1.6875,
+      "rewards/question_recreation_reward_func": 0.9734721444547176,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.46875,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 50
     },
     {
-      "completion_length": 306.4375,
+      "completion_length": 273.25,
       "epoch": 2.914285714285714,
-      "grad_norm": 204.8453826904297,
-      "kl": 32238.16137995734,
+      "grad_norm": 64386.68359375,
+      "kl": 4703.566962761339,
       "learning_rate": 2.540482672006254e-07,
-      "loss": 32.2382,
-      "reward": 20.900875002145767,
-      "reward_std": 0.08672392391599715,
-      "rewards/concensus_correctness_reward_func": 15.0,
+      "loss": 4.7036,
+      "reward": 23.86061406135559,
+      "reward_std": 0.11497652903926792,
+      "rewards/concensus_correctness_reward_func": 17.5,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.9594687074422836,
+      "rewards/final_correctness_reward_func": 1.6875,
+      "rewards/question_recreation_reward_func": 0.9387390580959618,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.23828125,
+      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 52
     },
     {
-      "completion_length": 306.5416666666667,
+      "completion_length": 283.375,
       "epoch": 3.0,
-      "grad_norm": 671869.25,
-      "kl": 71707.22513961916,
+      "grad_norm": 4.72637939453125,
+      "kl": 0.41162373684346676,
       "learning_rate": 2.37859444471388e-07,
-      "loss": 53.7804,
-      "reward": 21.561778783798218,
-      "reward_std": 0.2654109305391709,
-      "rewards/concensus_correctness_reward_func": 15.0,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.0003,
+      "reward": 22.2163348197937,
+      "reward_std": 1.4251218068723877,
+      "rewards/concensus_correctness_reward_func": 15.833333333333334,
+      "rewards/consensus_reward_func": 1.9166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.8333333333333333,
-      "rewards/question_recreation_reward_func": 0.978445569674174,
+      "rewards/final_correctness_reward_func": 1.75,
+      "rewards/question_recreation_reward_func": 0.9663348992665609,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 54
     },
     {
-      "completion_length": 284.25,
+      "completion_length": 288.875,
       "epoch": 3.1142857142857143,
-      "grad_norm": 75.00627899169922,
-      "kl": 7.327355385874398,
+      "grad_norm": 8.240327835083008,
+      "kl": 0.11788215674459934,
       "learning_rate": 2.2172154345117894e-07,
-      "loss": 0.0073,
-      "reward": 23.936051309108734,
-      "reward_std": 0.0883883461356163,
+      "loss": 0.0001,
+      "reward": 23.904327988624573,
+      "reward_std": 0.30620702836313285,
       "rewards/concensus_correctness_reward_func": 17.5,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 1.6875,
-      "rewards/question_recreation_reward_func": 0.9985513240098953,
+      "rewards/question_recreation_reward_func": 0.9668280705809593,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 56
     },
     {
-      "completion_length": 285.03125,
+      "completion_length": 286.1875,
       "epoch": 3.2285714285714286,
-      "grad_norm": 6.350292205810547,
-      "kl": 0.6016469232272357,
+      "grad_norm": 5.542027950286865,
+      "kl": 0.11034615035168827,
       "learning_rate": 2.0570225210519433e-07,
-      "loss": 0.0006,
-      "reward": 21.1969113945961,
-      "reward_std": 0.05649599526077509,
+      "loss": 0.0001,
+      "reward": 21.113164007663727,
+      "reward_std": 0.19153568047477165,
       "rewards/concensus_correctness_reward_func": 15.0,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.982067696750164,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.941288948059082,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.24609375,
+      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 58
     },
     {
-      "completion_length": 297.875,
+      "completion_length": 293.96875,
       "epoch": 3.342857142857143,
-      "grad_norm": 4.946191787719727,
-      "kl": 2931.4463935733074,
+      "grad_norm": 89873.234375,
+      "kl": 15506.7046823774,
       "learning_rate": 1.8986876090843664e-07,
-      "loss": 2.9315,
-      "reward": 21.175804615020752,
-      "reward_std": 0.1754202425290714,
-      "rewards/concensus_correctness_reward_func": 15.0,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 15.5067,
+      "reward": 24.321931093931198,
+      "reward_std": 0.9067758475139271,
+      "rewards/concensus_correctness_reward_func": 18.238062500953674,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.9570545256137848,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.9588685538619757,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.46875,
+      "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 60
     },
     {
-      "completion_length": 296.6875,
+      "completion_length": 292.65625,
       "epoch": 3.4571428571428573,
-      "grad_norm": 4.3400092124938965,
-      "kl": 647.1872735361103,
+      "grad_norm": 1.29377019405365,
+      "kl": 2.4666087054647505,
       "learning_rate": 1.7428748102551234e-07,
-      "loss": 0.6472,
-      "reward": 21.385672003030777,
-      "reward_std": 0.11072899930877611,
+      "loss": 0.0025,
+      "reward": 21.342330873012543,
+      "reward_std": 0.17704137448163237,
       "rewards/concensus_correctness_reward_func": 15.0,
-      "rewards/consensus_reward_func": 2.0,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.6875,
-      "rewards/question_recreation_reward_func": 0.9950470179319382,
+      "rewards/final_correctness_reward_func": 1.75,
+      "rewards/question_recreation_reward_func": 0.9985808059573174,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
+      "rewards/strict_format_reward_func": 0.46875,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 62
     },
     {
-      "completion_length": 292.375,
+      "completion_length": 296.125,
       "epoch": 3.571428571428571,
-      "grad_norm": 4.24850606918335,
-      "kl": 0.27754142810590565,
+      "grad_norm": 11.036781311035156,
+      "kl": 1.3272000113502145,
       "learning_rate": 1.5902376575912814e-07,
-      "loss": 0.0003,
-      "reward": 22.218331456184387,
-      "reward_std": 0.17698911238403525,
+      "loss": 0.0013,
+      "reward": 22.402726590633392,
+      "reward_std": 0.0906209279637551,
       "rewards/concensus_correctness_reward_func": 16.25,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.9683314673602581,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.9652266465127468,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 64
     },
     {
-      "completion_length": 300.125,
+      "completion_length": 311.25,
       "epoch": 3.685714285714286,
-      "grad_norm": 5.008735179901123,
-      "kl": 0.1533073673490435,
+      "grad_norm": 3.4472224712371826,
+      "kl": 0.8945652563124895,
       "learning_rate": 1.4414163643562753e-07,
-      "loss": 0.0002,
-      "reward": 22.805782973766327,
-      "reward_std": 0.09155728336190805,
+      "loss": 0.0009,
+      "reward": 22.61855262517929,
+      "reward_std": 0.18265954998787493,
       "rewards/concensus_correctness_reward_func": 16.25,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.8125,
-      "rewards/question_recreation_reward_func": 0.993282962590456,
+      "rewards/final_correctness_reward_func": 1.625,
+      "rewards/question_recreation_reward_func": 0.9935526624321938,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 66
     },
     {
-      "completion_length": 324.21875,
+      "completion_length": 322.75,
       "epoch": 3.8,
-      "grad_norm": 4.079814434051514,
-      "kl": 240703.86267980258,
+      "grad_norm": 6.844115734100342,
+      "kl": 3.179185423068702,
       "learning_rate": 1.2970351387729872e-07,
-      "loss": 240.7039,
-      "reward": 22.386139184236526,
-      "reward_std": 0.144500538790453,
-      "rewards/concensus_correctness_reward_func": 16.25,
+      "loss": 0.0032,
+      "reward": 21.179388463497162,
+      "reward_std": 0.09523822067421861,
+      "rewards/concensus_correctness_reward_func": 15.0,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 1.4375,
-      "rewards/question_recreation_reward_func": 0.9681704863905907,
+      "rewards/question_recreation_reward_func": 0.9918884485960007,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.484375,
-      "rewards/xmlcount_reward_func": 1.24609375,
+      "rewards/strict_format_reward_func": 0.5,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 68
     },
     {
-      "completion_length": 291.84375,
+      "completion_length": 280.5,
       "epoch": 3.914285714285714,
-      "grad_norm": 5.8456339836120605,
-      "kl": 9508.670825830894,
+      "grad_norm": 5.8618083000183105,
+      "kl": 7725.971832116949,
       "learning_rate": 1.1576995658775404e-07,
-      "loss": 9.5088,
-      "reward": 24.96782159805298,
-      "reward_std": 0.35711817086848896,
-      "rewards/concensus_correctness_reward_func": 18.75,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 7.726,
+      "reward": 24.525205433368683,
+      "reward_std": 1.1531662662309827,
+      "rewards/concensus_correctness_reward_func": 18.125,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.9571027122437954,
+      "rewards/final_correctness_reward_func": 1.8125,
+      "rewards/question_recreation_reward_func": 0.935486514121294,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.229468747973442,
+      "rewards/xmlcount_reward_func": 1.2459687516093254,
       "step": 70
     },
     {
-      "completion_length": 309.75,
+      "completion_length": 295.375,
       "epoch": 4.0,
-      "grad_norm": 4.866423606872559,
-      "kl": 4863924.029457786,
+      "grad_norm": 4.205502986907959,
+      "kl": 5170.305763981615,
       "learning_rate": 1.0239940674851941e-07,
-      "loss": 3647.9429,
-      "reward": 24.461654980977375,
-      "reward_std": 0.09552274364978075,
+      "loss": 3.8777,
+      "reward": 24.69401526451111,
+      "reward_std": 0.29310186601166305,
       "rewards/concensus_correctness_reward_func": 18.333333333333332,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.9409046818812689,
+      "rewards/final_correctness_reward_func": 1.6666666666666667,
+      "rewards/question_recreation_reward_func": 0.9440152446428934,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.249916672706604,
+      "rewards/strict_format_reward_func": 0.5,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 72
     },
     {
-      "completion_length": 313.125,
+      "completion_length": 322.75,
       "epoch": 4.114285714285714,
-      "grad_norm": 7.353360652923584,
-      "kl": 21.889807341387495,
+      "grad_norm": 853897.4375,
+      "kl": 93995.61682448094,
       "learning_rate": 8.964794509221507e-08,
-      "loss": 0.0218,
-      "reward": 23.785234093666077,
-      "reward_std": 0.10294873413658934,
-      "rewards/concensus_correctness_reward_func": 17.5,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 93.9956,
+      "reward": 21.796031087636948,
+      "reward_std": 1.1461751838796772,
+      "rewards/concensus_correctness_reward_func": 15.625,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.9727341197431087,
+      "rewards/final_correctness_reward_func": 1.5,
+      "rewards/question_recreation_reward_func": 0.9835311211645603,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 74
     },
     {
-      "completion_length": 312.0625,
+      "completion_length": 310.625,
       "epoch": 4.228571428571429,
-      "grad_norm": 3.4897100925445557,
-      "kl": 3379667.7199420603,
+      "grad_norm": 6.561783790588379,
+      "kl": 0.6868871822953224,
       "learning_rate": 7.756905568047392e-08,
-      "loss": 3379.668,
-      "reward": 24.401052832603455,
-      "reward_std": 1.1398758271243423,
-      "rewards/concensus_correctness_reward_func": 18.249875009059906,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 0.0007,
+      "reward": 23.608940601348877,
+      "reward_std": 0.1481841884815367,
+      "rewards/concensus_correctness_reward_func": 17.5,
+      "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.9491151906549931,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.9526904746890068,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.484375,
-      "rewards/xmlcount_reward_func": 1.2176874987781048,
+      "rewards/strict_format_reward_func": 0.46875,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 76
     },
     {
-      "completion_length": 277.21875,
+      "completion_length": 300.6875,
       "epoch": 4.3428571428571425,
-      "grad_norm": 5.596811294555664,
-      "kl": 2.9425320012960583,
+      "grad_norm": 5.61808967590332,
+      "kl": 0.17497208586428314,
       "learning_rate": 6.621340157319996e-08,
-      "loss": 0.0029,
-      "reward": 20.854586482048035,
-      "reward_std": 0.05484232149319723,
+      "loss": 0.0002,
+      "reward": 20.960274010896683,
+      "reward_std": 0.12753743107896298,
       "rewards/concensus_correctness_reward_func": 15.0,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.9405239969491959,
+      "rewards/final_correctness_reward_func": 1.375,
+      "rewards/question_recreation_reward_func": 0.9174302890896797,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.2109375,
+      "rewards/xmlcount_reward_func": 1.2147187516093254,
       "step": 78
     },
     {
-      "completion_length": 270.59375,
+      "completion_length": 269.59375,
       "epoch": 4.457142857142857,
-      "grad_norm": 92.10065460205078,
-      "kl": 3.8375826531555504,
+      "grad_norm": 58.665802001953125,
+      "kl": 5.802749904803932,
       "learning_rate": 5.5628612330087724e-08,
-      "loss": 0.0038,
-      "reward": 23.8216050863266,
-      "reward_std": 0.2481758954236284,
-      "rewards/concensus_correctness_reward_func": 17.5,
+      "loss": 0.0058,
+      "reward": 26.747634530067444,
+      "reward_std": 0.0016583941760472953,
+      "rewards/concensus_correctness_reward_func": 20.0,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.625,
-      "rewards/question_recreation_reward_func": 0.9661363810300827,
+      "rewards/final_correctness_reward_func": 2.0,
+      "rewards/question_recreation_reward_func": 0.9976345337927341,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.484375,
-      "rewards/xmlcount_reward_func": 1.24609375,
+      "rewards/strict_format_reward_func": 0.5,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 80
     },
     {
-      "completion_length": 311.625,
+      "completion_length": 315.21875,
       "epoch": 4.571428571428571,
-      "grad_norm": 3.580958843231201,
-      "kl": 0.21251056378241628,
+      "grad_norm": 0.19868668913841248,
+      "kl": 131.63193395012058,
       "learning_rate": 4.5859084235697235e-08,
-      "loss": 0.0002,
-      "reward": 17.167713046073914,
-      "reward_std": 0.24630744382739067,
-      "rewards/concensus_correctness_reward_func": 11.25,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.1316,
+      "reward": 16.382025599479675,
+      "reward_std": 1.1490484923124313,
+      "rewards/concensus_correctness_reward_func": 10.625,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.9645880535244942,
+      "rewards/question_recreation_reward_func": 0.9757755696773529,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
+      "rewards/strict_format_reward_func": 0.46875,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 82
     },
     {
-      "completion_length": 281.8125,
+      "completion_length": 291.96875,
       "epoch": 4.685714285714286,
-      "grad_norm": 36.02040481567383,
-      "kl": 3.439565767766908,
+      "grad_norm": 1.1480889320373535,
+      "kl": 0.19505003839731216,
       "learning_rate": 3.6945794086007705e-08,
-      "loss": 0.0034,
-      "reward": 22.68697714805603,
-      "reward_std": 0.19833910652960185,
+      "loss": 0.0002,
+      "reward": 22.737855315208435,
+      "reward_std": 0.0,
       "rewards/concensus_correctness_reward_func": 16.25,
-      "rewards/consensus_reward_func": 2.0,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.75,
-      "rewards/question_recreation_reward_func": 0.9860397130250931,
+      "rewards/final_correctness_reward_func": 1.875,
+      "rewards/question_recreation_reward_func": 0.9878552854061127,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.2478125020861626,
+      "rewards/strict_format_reward_func": 0.5,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 84
     },
     {
-      "completion_length": 299.8125,
+      "completion_length": 284.625,
       "epoch": 4.8,
-      "grad_norm": 5.582298755645752,
-      "kl": 0.19541561254300177,
+      "grad_norm": 9.016759872436523,
+      "kl": 0.27029623091220856,
       "learning_rate": 2.892612731749414e-08,
-      "loss": 0.0002,
-      "reward": 23.891813337802887,
-      "reward_std": 0.14955170825123787,
-      "rewards/concensus_correctness_reward_func": 17.5,
+      "loss": 0.0003,
+      "reward": 25.095996618270874,
+      "reward_std": 0.38329675246495754,
+      "rewards/concensus_correctness_reward_func": 18.75,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.6875,
-      "rewards/question_recreation_reward_func": 0.954313363879919,
+      "rewards/final_correctness_reward_func": 1.625,
+      "rewards/question_recreation_reward_func": 0.9905278831720352,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.5,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/xmlcount_reward_func": 1.24609375,
       "step": 86
     },
     {
-      "completion_length": 298.875,
+      "completion_length": 282.96875,
       "epoch": 4.914285714285715,
-      "grad_norm": 194.55441284179688,
-      "kl": 16.51644215453416,
+      "grad_norm": 10.288511276245117,
+      "kl": 1.007630041684024,
       "learning_rate": 2.183372119961499e-08,
-      "loss": 0.0165,
-      "reward": 23.978663206100464,
-      "reward_std": 0.18283734598662704,
+      "loss": 0.001,
+      "reward": 23.806805968284607,
+      "reward_std": 0.08782566487207077,
       "rewards/concensus_correctness_reward_func": 17.5,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.75,
-      "rewards/question_recreation_reward_func": 0.978663295507431,
+      "rewards/final_correctness_reward_func": 1.5625,
+      "rewards/question_recreation_reward_func": 0.9943058528006077,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 88
     },
     {
-      "completion_length": 281.1666666666667,
+      "completion_length": 288.9166666666667,
       "epoch": 5.0,
-      "grad_norm": 2.8502633571624756,
-      "kl": 1.1907670572400093,
+      "grad_norm": 5.103769779205322,
+      "kl": 0.5060152485966682,
       "learning_rate": 1.5698323748414122e-08,
-      "loss": 0.0009,
-      "reward": 22.561877250671387,
-      "reward_std": 0.26481426507234573,
-      "rewards/concensus_correctness_reward_func": 16.666666666666668,
+      "loss": 0.0004,
+      "reward": 24.75821030139923,
+      "reward_std": 0.1651654613087885,
+      "rewards/concensus_correctness_reward_func": 18.333333333333332,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1666666666666667,
-      "rewards/question_recreation_reward_func": 0.999377171198527,
+      "rewards/final_correctness_reward_func": 1.75,
+      "rewards/question_recreation_reward_func": 0.9873770674069723,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4791666666666667,
+      "rewards/strict_format_reward_func": 0.4375,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 90
     },
     {
-      "completion_length": 305.125,
+      "completion_length": 312.25,
       "epoch": 5.114285714285714,
-      "grad_norm": 228.8664093017578,
-      "kl": 25.48564671073109,
+      "grad_norm": 4.315695762634277,
+      "kl": 0.2543930097017437,
       "learning_rate": 1.054566895300324e-08,
-      "loss": 0.0255,
-      "reward": 25.125020027160645,
-      "reward_std": 0.11629757171976962,
-      "rewards/concensus_correctness_reward_func": 18.75,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.0003,
+      "reward": 24.557986617088318,
+      "reward_std": 1.1104227881878614,
+      "rewards/concensus_correctness_reward_func": 18.125,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.6875,
-      "rewards/question_recreation_reward_func": 0.9687698371708393,
+      "rewards/final_correctness_reward_func": 1.8125,
+      "rewards/question_recreation_reward_func": 0.9994240589439869,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/strict_format_reward_func": 0.453125,
+      "rewards/xmlcount_reward_func": 1.2304375022649765,
       "step": 92
     },
     {
-      "completion_length": 310.375,
+      "completion_length": 297.375,
       "epoch": 5.228571428571429,
-      "grad_norm": 1273.266357421875,
-      "kl": 99.00182915199548,
+      "grad_norm": 4.21738338470459,
+      "kl": 0.24928702949546278,
       "learning_rate": 6.397368838268496e-09,
-      "loss": 0.099,
-      "reward": 15.815190017223358,
-      "reward_std": 0.18272748742310796,
-      "rewards/concensus_correctness_reward_func": 10.0,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.0002,
+      "reward": 19.507496863603592,
+      "reward_std": 0.33748881984502077,
+      "rewards/concensus_correctness_reward_func": 13.75,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.125,
-      "rewards/question_recreation_reward_func": 0.9714402034878731,
+      "rewards/final_correctness_reward_func": 1.25,
+      "rewards/question_recreation_reward_func": 0.9494031816720963,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/xmlcount_reward_func": 1.214343748986721,
       "step": 94
     },
     {
-      "completion_length": 294.28125,
+      "completion_length": 286.0,
       "epoch": 5.3428571428571425,
-      "grad_norm": 25.452686309814453,
-      "kl": 1.9639836233109236,
+      "grad_norm": 4.452499866485596,
+      "kl": 7.8268896921072155,
       "learning_rate": 3.2708228165273244e-09,
-      "loss": 0.002,
-      "reward": 23.699071168899536,
-      "reward_std": 0.41938550118356943,
-      "rewards/concensus_correctness_reward_func": 17.5,
+      "loss": 0.0078,
+      "reward": 22.384023070335388,
+      "reward_std": 0.30442836828297004,
+      "rewards/concensus_correctness_reward_func": 16.25,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.964696180075407,
+      "rewards/final_correctness_reward_func": 1.4375,
+      "rewards/question_recreation_reward_func": 0.9465231820940971,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/strict_format_reward_func": 0.5,
       "rewards/xmlcount_reward_func": 1.25,
       "step": 96
     },
     {
-      "completion_length": 259.5625,
+      "completion_length": 264.4375,
       "epoch": 5.457142857142857,
-      "grad_norm": 2.797135591506958,
-      "kl": 6.031112845055759,
+      "grad_norm": 2.327564001083374,
+      "kl": 0.21026335144415498,
       "learning_rate": 1.1791447083465133e-09,
-      "loss": 0.006,
-      "reward": 21.275031089782715,
-      "reward_std": 0.09077301478828304,
+      "loss": 0.0002,
+      "reward": 21.16586148738861,
+      "reward_std": 0.29173441627062857,
       "rewards/concensus_correctness_reward_func": 15.0,
-      "rewards/consensus_reward_func": 2.0,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.9937809370458126,
+      "rewards/final_correctness_reward_func": 1.625,
+      "rewards/question_recreation_reward_func": 0.968142680823803,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.46875,
-      "rewards/xmlcount_reward_func": 1.25,
+      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/xmlcount_reward_func": 1.2133437506854534,
       "step": 98
     },
     {
-      "completion_length": 289.28125,
+      "completion_length": 284.6875,
       "epoch": 5.571428571428571,
-      "grad_norm": 4.234410762786865,
-      "kl": 0.2854203868191689,
+      "grad_norm": 7.0255889892578125,
+      "kl": 0.2286427365615964,
       "learning_rate": 1.3110773862126667e-10,
-      "loss": 0.0003,
-      "reward": 26.528146028518677,
-      "reward_std": 0.21690074453363195,
+      "loss": 0.0002,
+      "reward": 26.42949390411377,
+      "reward_std": 0.40112065349239856,
       "rewards/concensus_correctness_reward_func": 20.0,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.875,
-      "rewards/question_recreation_reward_func": 0.9520210325717926,
+      "rewards/final_correctness_reward_func": 1.75,
+      "rewards/question_recreation_reward_func": 0.9490251876413822,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.2480000033974648,
+      "rewards/strict_format_reward_func": 0.484375,
+      "rewards/xmlcount_reward_func": 1.24609375,
       "step": 100
     },
     {
       "epoch": 5.571428571428571,
       "step": 100,
       "total_flos": 0.0,
-      "train_loss": 69070.94929174794,
-      "train_runtime": 1492.4283,
-      "train_samples_per_second": 1.072,
+      "train_loss": 296549.3844699778,
+      "train_runtime": 1501.8809,
+      "train_samples_per_second": 1.065,
       "train_steps_per_second": 0.067
     }
   ],