End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.005916564127692254,
-    "train_runtime": 674.0869,
+    "train_loss": 5.491919757317737,
+    "train_runtime": 671.7848,
     "train_samples": 72,
-    "train_samples_per_second": 1.187,
+    "train_samples_per_second": 1.191,
     "train_steps_per_second": 0.074
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:593ee440b09725d9acb23b2622980636c66fd339f28aa1e1bda5ad9b564de4eb
+oid sha256:f9591b7f4c6657c764961e248f7d66b28b4009b8d30657471519d7a283035b1b
 size 4996670464

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20935d86192d99dbc3c37a29f99ae92ed2c2e534cec5f1396cb3497338f93080
+oid sha256:b495ec090ba0f8d34eafe59eefa10a2bf7b82a591e1a355c539e74dc16503a6f
 size 1178224960

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.005916564127692254,
-    "train_runtime": 674.0869,
+    "train_loss": 5.491919757317737,
+    "train_runtime": 671.7848,
     "train_samples": 72,
-    "train_samples_per_second": 1.187,
+    "train_samples_per_second": 1.191,
     "train_steps_per_second": 0.074
 }

trainer_state.json

@@ -10,487 +10,487 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 292.375,
+      "completion_length": 251.09375,
       "epoch": 0.2222222222222222,
-      "grad_norm": 3.8107857704162598,
+      "grad_norm": 3.406686544418335,
       "kl": 0.0,
       "learning_rate": 2e-07,
       "loss": -0.0,
-      "reward": 4.110599463805556,
-      "reward_std": 3.714366073545534,
-      "rewards/concensus_correctness_reward_func": 2.0961249992251396,
+      "reward": 5.090359630994499,
+      "reward_std": 4.374027599813417,
+      "rewards/concensus_correctness_reward_func": 2.586999997496605,
       "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5882245441898704,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.5857971757650375,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.47312500420957804,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.6363125052303076,
       "step": 2
     },
     {
-      "completion_length": 242.84375,
+      "completion_length": 238.53125,
       "epoch": 0.4444444444444444,
-      "grad_norm": 4.162234306335449,
-      "kl": 0.002089262896333821,
+      "grad_norm": 6.297623157501221,
+      "kl": 0.002095213483698899,
       "learning_rate": 6e-07,
       "loss": 0.0,
-      "reward": 6.153819195926189,
-      "reward_std": 3.73072009999305,
-      "rewards/concensus_correctness_reward_func": 3.3734999974258244,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 6.276864215731621,
+      "reward_std": 4.376051411032677,
+      "rewards/concensus_correctness_reward_func": 3.2183125205338,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.7112567536532879,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.6349266925826669,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.7878125039860606,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 1.0642500249668956,
       "step": 4
     },
     {
-      "completion_length": 238.96875,
+      "completion_length": 247.1875,
       "epoch": 0.6666666666666666,
-      "grad_norm": 3.1884305477142334,
-      "kl": 0.03603306884178892,
+      "grad_norm": 5.020749568939209,
+      "kl": 0.01812579674879089,
       "learning_rate": 1e-06,
-      "loss": 0.0001,
-      "reward": 6.223309241235256,
-      "reward_std": 4.281767612181284,
-      "rewards/concensus_correctness_reward_func": 3.5337499850429595,
-      "rewards/consensus_reward_func": 0.625,
+      "loss": 0.0,
+      "reward": 8.028940327465534,
+      "reward_std": 7.026336292154156,
+      "rewards/concensus_correctness_reward_func": 4.982750025577843,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.6908091912046075,
+      "rewards/final_correctness_reward_func": 0.875,
+      "rewards/question_recreation_reward_func": 0.6356277531012893,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.8581249974668026,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.9418125078082085,
       "step": 6
     },
     {
-      "completion_length": 268.625,
+      "completion_length": 264.5625,
       "epoch": 0.8888888888888888,
-      "grad_norm": 3.539409875869751,
-      "kl": 0.4308171756565571,
+      "grad_norm": 6.246143341064453,
+      "kl": 1.4362198570743203,
       "learning_rate": 9.95134034370785e-07,
-      "loss": 0.0004,
-      "reward": 2.3415392953902483,
-      "reward_std": 1.1331623401492834,
-      "rewards/concensus_correctness_reward_func": 0.2723749913275242,
-      "rewards/consensus_reward_func": 0.5,
+      "loss": 0.0014,
+      "reward": 5.521899528801441,
+      "reward_std": 3.0373885043663904,
+      "rewards/concensus_correctness_reward_func": 2.975437503308058,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.5665392577648163,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5394933708012104,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.5494999946095049,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 1.0225937701761723,
       "step": 8
     },
     {
-      "completion_length": 258.5625,
+      "completion_length": 253.09375,
       "epoch": 1.1111111111111112,
-      "grad_norm": 54.83183288574219,
-      "kl": 15.701219761976972,
+      "grad_norm": 379351.5,
+      "kl": 121253.2733656764,
       "learning_rate": 9.806308479691594e-07,
-      "loss": 0.0157,
-      "reward": 4.375889036804438,
-      "reward_std": 3.6191295210737735,
-      "rewards/concensus_correctness_reward_func": 2.0978124998509884,
-      "rewards/consensus_reward_func": 0.5625,
+      "loss": 121.2533,
+      "reward": 6.7504779398441315,
+      "reward_std": 3.2276243461528793,
+      "rewards/concensus_correctness_reward_func": 3.5170000011567026,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.5807640934363008,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.6782278986647725,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5723125000949949,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.8990000244230032,
       "step": 10
     },
     {
-      "completion_length": 224.5,
+      "completion_length": 246.3125,
       "epoch": 1.3333333333333333,
-      "grad_norm": 5.148858547210693,
-      "kl": 2.3633982008323073,
+      "grad_norm": 11.476463317871094,
+      "kl": 53.28857711702585,
       "learning_rate": 9.567727288213004e-07,
-      "loss": 0.0024,
-      "reward": 5.041572377085686,
-      "reward_std": 3.1810707906261086,
-      "rewards/concensus_correctness_reward_func": 2.3565000272355974,
-      "rewards/consensus_reward_func": 0.625,
+      "loss": 0.0533,
+      "reward": 7.5567396730184555,
+      "reward_std": 6.025143433362246,
+      "rewards/concensus_correctness_reward_func": 4.465624971315265,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.4960412373766303,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.43192729796282947,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.923406271263957,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 1.0029375031590462,
       "step": 12
     },
     {
-      "completion_length": 244.71875,
+      "completion_length": 262.9375,
       "epoch": 1.5555555555555556,
-      "grad_norm": 4.224116802215576,
-      "kl": 1.5940971969394013,
+      "grad_norm": 108.78337860107422,
+      "kl": 15830.661798972636,
       "learning_rate": 9.240240480782129e-07,
-      "loss": 0.0016,
-      "reward": 3.845541685819626,
-      "reward_std": 0.7413416516501456,
-      "rewards/concensus_correctness_reward_func": 1.4198125060647726,
-      "rewards/consensus_reward_func": 0.6875,
+      "loss": 15.8307,
+      "reward": 4.460910737514496,
+      "reward_std": 2.0786191248334944,
+      "rewards/concensus_correctness_reward_func": 1.170062493532896,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5501666869968176,
+      "rewards/final_correctness_reward_func": 0.9375,
+      "rewards/question_recreation_reward_func": 0.6721919253468513,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.7974375048652291,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 1.087406262755394,
       "step": 14
     },
     {
-      "completion_length": 288.03125,
+      "completion_length": 228.65625,
       "epoch": 1.7777777777777777,
-      "grad_norm": 3.2477917671203613,
-      "kl": 1.24803132773377,
+      "grad_norm": 17.862974166870117,
+      "kl": 24.956144016236067,
       "learning_rate": 8.83022221559489e-07,
-      "loss": 0.0012,
-      "reward": 3.5894551295787096,
-      "reward_std": 1.4415434753755108,
-      "rewards/concensus_correctness_reward_func": 1.0913125132210553,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.025,
+      "reward": 6.531906917691231,
+      "reward_std": 5.046906062096241,
+      "rewards/concensus_correctness_reward_func": 3.738874990493059,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5858613271266222,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.6465007420629263,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.8341562564019114,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.9902812633663416,
       "step": 16
     },
     {
-      "completion_length": 277.90625,
+      "completion_length": 251.90625,
       "epoch": 2.0,
-      "grad_norm": 3.8915557861328125,
-      "kl": 1.672196735162288,
+      "grad_norm": 4.168570041656494,
+      "kl": 5.209025803487748,
       "learning_rate": 8.34565303179429e-07,
-      "loss": 0.0017,
-      "reward": 5.043356254696846,
-      "reward_std": 3.8196976413019,
-      "rewards/concensus_correctness_reward_func": 2.2441875115036964,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.0052,
+      "reward": 7.48027828335762,
+      "reward_std": 3.668259933590889,
+      "rewards/concensus_correctness_reward_func": 4.085375004447997,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.6289812638424337,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.7168719079345465,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6701875114813447,
+      "rewards/xmlcount_reward_func": 0.9905312536284328,
       "step": 18
     },
     {
-      "completion_length": 252.84375,
+      "completion_length": 253.46875,
       "epoch": 2.2222222222222223,
-      "grad_norm": 4.681396484375,
-      "kl": 0.7993139801546931,
+      "grad_norm": 4.598588943481445,
+      "kl": 0.6110749203944579,
       "learning_rate": 7.795964517353733e-07,
-      "loss": 0.0008,
-      "reward": 4.347612947225571,
-      "reward_std": 3.5975783905014396,
-      "rewards/concensus_correctness_reward_func": 1.8726249998435378,
-      "rewards/consensus_reward_func": 0.4375,
+      "loss": 0.0006,
+      "reward": 6.335133410990238,
+      "reward_std": 4.133349603740498,
+      "rewards/concensus_correctness_reward_func": 3.3612499944865704,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.4997691195458174,
+      "rewards/final_correctness_reward_func": 0.875,
+      "rewards/question_recreation_reward_func": 0.6452896627597511,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.865843765437603,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.8754687653854489,
       "step": 20
     },
     {
-      "completion_length": 230.5625,
+      "completion_length": 246.40625,
       "epoch": 2.4444444444444446,
-      "grad_norm": 14.132287979125977,
-      "kl": 3.0374708212912083,
+      "grad_norm": 4.547008514404297,
+      "kl": 0.8581339719239622,
       "learning_rate": 7.191855733945386e-07,
-      "loss": 0.003,
-      "reward": 6.949155829846859,
-      "reward_std": 3.3791947152931243,
-      "rewards/concensus_correctness_reward_func": 3.738687491742894,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0009,
+      "reward": 3.972074016928673,
+      "reward_std": 2.218664333457127,
+      "rewards/concensus_correctness_reward_func": 1.2331875103991479,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.611030868254602,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.6385114891454577,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.7556875105947256,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 1.0222500003874302,
       "step": 22
     },
     {
-      "completion_length": 265.5625,
+      "completion_length": 314.75,
       "epoch": 2.6666666666666665,
-      "grad_norm": 4.178548336029053,
-      "kl": 1.368599567329511,
+      "grad_norm": 3.5269718170166016,
+      "kl": 1.1453959059435874,
       "learning_rate": 6.545084971874736e-07,
-      "loss": 0.0014,
-      "reward": 3.9503697529435158,
-      "reward_std": 2.410870131352567,
-      "rewards/concensus_correctness_reward_func": 1.6398749821819365,
-      "rewards/consensus_reward_func": 0.6875,
+      "loss": 0.0011,
+      "reward": 5.517209440469742,
+      "reward_std": 2.4003249094821513,
+      "rewards/concensus_correctness_reward_func": 2.6363124921917915,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.576807199511677,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.5380531689152122,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.6555625096370932,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.9678437616676092,
       "step": 24
     },
     {
-      "completion_length": 255.40625,
+      "completion_length": 242.59375,
       "epoch": 2.888888888888889,
-      "grad_norm": 3.538485050201416,
-      "kl": 1.5496129877865314,
+      "grad_norm": 3.7082345485687256,
+      "kl": 1.661696835188195,
       "learning_rate": 5.868240888334652e-07,
-      "loss": 0.0015,
-      "reward": 4.10240344889462,
-      "reward_std": 2.6716376403346658,
-      "rewards/concensus_correctness_reward_func": 1.5082499934360385,
-      "rewards/consensus_reward_func": 0.4375,
+      "loss": 0.0017,
+      "reward": 7.0335357040166855,
+      "reward_std": 5.964661547914147,
+      "rewards/concensus_correctness_reward_func": 4.0771874990314245,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.6146223489195108,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.6281919404864311,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.8701562639325857,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 1.0000312514603138,
       "step": 26
     },
     {
-      "completion_length": 241.5625,
+      "completion_length": 233.21875,
       "epoch": 3.111111111111111,
-      "grad_norm": 3.752775192260742,
-      "kl": 3.886347927618772,
+      "grad_norm": 25.148460388183594,
+      "kl": 5.4034789321012795,
       "learning_rate": 5.174497483512505e-07,
-      "loss": 0.0039,
-      "reward": 3.3237821646034718,
-      "reward_std": 2.1524549201130867,
-      "rewards/concensus_correctness_reward_func": 0.8466250016354024,
-      "rewards/consensus_reward_func": 0.5625,
+      "loss": 0.0054,
+      "reward": 5.528537306934595,
+      "reward_std": 2.2288794559426606,
+      "rewards/concensus_correctness_reward_func": 2.6711875112960115,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.5080321566201746,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5974122565239668,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.7972500007599592,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.9161875313147902,
       "step": 28
     },
     {
-      "completion_length": 261.8125,
+      "completion_length": 257.6875,
       "epoch": 3.3333333333333335,
-      "grad_norm": 5.829784870147705,
-      "kl": 3.451839253306389,
+      "grad_norm": 15.775626182556152,
+      "kl": 4.37631535762921,
       "learning_rate": 4.477357683661733e-07,
-      "loss": 0.0035,
-      "reward": 2.6074002254754305,
-      "reward_std": 1.677693044708576,
-      "rewards/concensus_correctness_reward_func": 0.9816249692812562,
-      "rewards/consensus_reward_func": 0.1875,
+      "loss": 0.0044,
+      "reward": 7.878351733088493,
+      "reward_std": 3.3786554981488734,
+      "rewards/concensus_correctness_reward_func": 5.094187501817942,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5892127025872469,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.5230704261921346,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.5678125042468309,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.9642187729477882,
       "step": 30
     },
     {
-      "completion_length": 267.4375,
+      "completion_length": 227.34375,
       "epoch": 3.5555555555555554,
-      "grad_norm": 22.223487854003906,
-      "kl": 83.22811474930495,
+      "grad_norm": 3.913177967071533,
+      "kl": 1.274317068979144,
       "learning_rate": 3.790390522001662e-07,
-      "loss": 0.0832,
-      "reward": 5.953872742131352,
-      "reward_std": 1.9447148024337366,
-      "rewards/concensus_correctness_reward_func": 3.6568749928846955,
-      "rewards/consensus_reward_func": 0.5625,
+      "loss": 0.0013,
+      "reward": 8.299735330045223,
+      "reward_std": 3.323359102010727,
+      "rewards/concensus_correctness_reward_func": 5.080312505364418,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.614747672341764,
+      "rewards/final_correctness_reward_func": 0.875,
+      "rewards/question_recreation_reward_func": 0.6301415413618088,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.4166250051930547,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 1.0267812612000853,
       "step": 32
     },
     {
-      "completion_length": 252.8125,
+      "completion_length": 292.15625,
       "epoch": 3.7777777777777777,
-      "grad_norm": 3.262629508972168,
-      "kl": 3.2397216241806746,
+      "grad_norm": 11.773321151733398,
+      "kl": 7.366791445761919,
       "learning_rate": 3.1269670329204393e-07,
-      "loss": 0.0032,
-      "reward": 5.505353234708309,
-      "reward_std": 2.5832796641625464,
-      "rewards/concensus_correctness_reward_func": 3.007625007769093,
-      "rewards/consensus_reward_func": 0.625,
+      "loss": 0.0074,
+      "reward": 7.669991716742516,
+      "reward_std": 4.413828293792903,
+      "rewards/concensus_correctness_reward_func": 4.837812501937151,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.6153219498228282,
+      "rewards/final_correctness_reward_func": 0.875,
+      "rewards/question_recreation_reward_func": 0.5424916381016374,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.7261562403291464,
+      "rewards/xmlcount_reward_func": 0.8209374994039536,
       "step": 34
     },
     {
-      "completion_length": 263.75,
+      "completion_length": 251.59375,
       "epoch": 4.0,
-      "grad_norm": 3.4170873165130615,
-      "kl": 5.119970758911222,
+      "grad_norm": 9.20654582977295,
+      "kl": 3.2344849393703043,
       "learning_rate": 2.500000000000001e-07,
-      "loss": 0.0051,
-      "reward": 5.552348531782627,
-      "reward_std": 3.542175215203315,
-      "rewards/concensus_correctness_reward_func": 2.4716250132769346,
-      "rewards/consensus_reward_func": 0.875,
+      "loss": 0.0032,
+      "reward": 6.5783121436834335,
+      "reward_std": 4.831312101276126,
+      "rewards/concensus_correctness_reward_func": 3.0734374783933163,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.7155673063825816,
+      "rewards/final_correctness_reward_func": 1.125,
+      "rewards/question_recreation_reward_func": 0.6645933054387569,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7401562570594251,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 1.0121562704443932,
       "step": 36
     },
     {
-      "completion_length": 236.34375,
+      "completion_length": 296.84375,
       "epoch": 4.222222222222222,
-      "grad_norm": 3.966975212097168,
-      "kl": 2.2727794479578733,
+      "grad_norm": 6.073974609375,
+      "kl": 80.8798265401274,
       "learning_rate": 1.9216926233717084e-07,
-      "loss": 0.0023,
-      "reward": 2.9746797680854797,
-      "reward_std": 1.6573905128752813,
-      "rewards/concensus_correctness_reward_func": 0.8823749981820583,
-      "rewards/consensus_reward_func": 0.375,
+      "loss": 0.0809,
+      "reward": 4.182305257767439,
+      "reward_std": 3.0763283036649227,
+      "rewards/concensus_correctness_reward_func": 1.8618750125169754,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5670548444613814,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.6596803162246943,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.8065000018104911,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.7232500202953815,
       "step": 38
     },
     {
-      "completion_length": 282.34375,
+      "completion_length": 250.09375,
       "epoch": 4.444444444444445,
-      "grad_norm": 3.61830735206604,
-      "kl": 0.557787942700088,
+      "grad_norm": 8.074630737304688,
+      "kl": 2.947148013394326,
       "learning_rate": 1.4033009983067452e-07,
-      "loss": 0.0006,
-      "reward": 4.611605238169432,
-      "reward_std": 3.382820954779163,
-      "rewards/concensus_correctness_reward_func": 2.163499998860061,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.0029,
+      "reward": 7.208126068115234,
+      "reward_std": 4.182072307477938,
+      "rewards/concensus_correctness_reward_func": 3.5688749849796295,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.531542734708637,
+      "rewards/final_correctness_reward_func": 0.9375,
+      "rewards/question_recreation_reward_func": 0.6139386314898729,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.697812516707927,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 1.0721875205636024,
       "step": 40
     },
     {
-      "completion_length": 249.75,
+      "completion_length": 223.28125,
       "epoch": 4.666666666666667,
-      "grad_norm": 5.680083274841309,
-      "kl": 1.7294750094879419,
+      "grad_norm": 4.93925666809082,
+      "kl": 2.2123069388326257,
       "learning_rate": 9.549150281252632e-08,
-      "loss": 0.0017,
-      "reward": 4.764165852218866,
-      "reward_std": 4.05824239552021,
-      "rewards/concensus_correctness_reward_func": 2.169937494210899,
-      "rewards/consensus_reward_func": 0.375,
+      "loss": 0.0022,
+      "reward": 7.967550210654736,
+      "reward_std": 6.516119306630571,
+      "rewards/concensus_correctness_reward_func": 4.553937453776598,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.5612284038215876,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.6051440089941025,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.7986249960958958,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 1.105343785136938,
       "step": 42
     },
     {
-      "completion_length": 325.9375,
+      "completion_length": 261.0,
       "epoch": 4.888888888888889,
-      "grad_norm": 27.005922317504883,
-      "kl": 2.3825969519093633,
+      "grad_norm": 3.927772045135498,
+      "kl": 1.7613378865644336,
       "learning_rate": 5.8526203570536504e-08,
-      "loss": 0.0024,
-      "reward": 3.219888724386692,
-      "reward_std": 2.2487800465896726,
-      "rewards/concensus_correctness_reward_func": 0.9441250010859221,
-      "rewards/consensus_reward_func": 0.5,
+      "loss": 0.0018,
+      "reward": 7.037954144179821,
+      "reward_std": 4.669055305188522,
+      "rewards/concensus_correctness_reward_func": 3.980187527835369,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7558574602007866,
+      "rewards/final_correctness_reward_func": 1.125,
+      "rewards/question_recreation_reward_func": 0.6027041245251894,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.7542812507599592,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.892562497407198,
       "step": 44
     },
     {
-      "completion_length": 271.21875,
+      "completion_length": 234.75,
       "epoch": 5.111111111111111,
-      "grad_norm": 3.743121385574341,
-      "kl": 0.6370164407417178,
+      "grad_norm": 5.521337985992432,
+      "kl": 6.4816615069285035,
       "learning_rate": 3.015368960704584e-08,
-      "loss": 0.0006,
-      "reward": 5.291439572349191,
-      "reward_std": 3.2558085079072043,
-      "rewards/concensus_correctness_reward_func": 2.495437517296523,
-      "rewards/consensus_reward_func": 0.5,
+      "loss": 0.0065,
+      "reward": 8.22635594010353,
+      "reward_std": 3.674573255237192,
+      "rewards/concensus_correctness_reward_func": 5.032687483006157,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.7084708036854863,
+      "rewards/question_recreation_reward_func": 0.5293248100206256,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.8062812462449074,
+      "rewards/xmlcount_reward_func": 1.0080937594175339,
       "step": 46
     },
     {
-      "completion_length": 255.59375,
+      "completion_length": 294.0,
       "epoch": 5.333333333333333,
-      "grad_norm": 4.239270210266113,
-      "kl": 9.081267803441733,
+      "grad_norm": 3.950500249862671,
+      "kl": 3.034157019108534,
       "learning_rate": 1.0926199633097154e-08,
-      "loss": 0.0091,
-      "reward": 3.829737439751625,
-      "reward_std": 2.9596789788920432,
-      "rewards/concensus_correctness_reward_func": 1.5678749959915876,
+      "loss": 0.003,
+      "reward": 7.7403726652264595,
+      "reward_std": 5.27158590964973,
+      "rewards/concensus_correctness_reward_func": 4.709000015282072,
       "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.4738624934107065,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.7043102057650685,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8505000225268304,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.998937513679266,
       "step": 48
     },
     {
-      "completion_length": 243.875,
+      "completion_length": 237.59375,
       "epoch": 5.555555555555555,
-      "grad_norm": 11.775144577026367,
-      "kl": 2.5097127612680197,
+      "grad_norm": 8.28918743133545,
+      "kl": 5.900157635100186,
       "learning_rate": 1.217974870087901e-09,
-      "loss": 0.0025,
-      "reward": 3.7787333577871323,
-      "reward_std": 1.0139204831793904,
-      "rewards/concensus_correctness_reward_func": 1.442687500268221,
-      "rewards/consensus_reward_func": 0.375,
+      "loss": 0.0059,
+      "reward": 8.886989444494247,
+      "reward_std": 5.726772859226912,
+      "rewards/concensus_correctness_reward_func": 5.724687514826655,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.5674208542332053,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.7998750088736415,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.5425831722095609,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.078125,
+      "rewards/xmlcount_reward_func": 0.8540937826037407,
       "step": 50
     },
     {
       "epoch": 5.555555555555555,
       "step": 50,
       "total_flos": 0.0,
-      "train_loss": 0.005916564127692254,
-      "train_runtime": 674.0869,
-      "train_samples_per_second": 1.187,
+      "train_loss": 5.491919757317737,
+      "train_runtime": 671.7848,
+      "train_samples_per_second": 1.191,
       "train_steps_per_second": 0.074
     }
   ],