End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.018054345073833248,
-    "train_runtime": 1357.263,
-    "train_samples": 140,
-    "train_samples_per_second": 1.179,
-    "train_steps_per_second": 0.074
+    "train_loss": 5.246203290880657e-06,
+    "train_runtime": 3714.3127,
+    "train_samples": 48,
+    "train_samples_per_second": 0.431,
+    "train_steps_per_second": 0.027
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c63c6ce3ce02d94e50a0f1d5d9c98db03b918fde96b79778ff70084beaad495
+oid sha256:da4263e7a35562370a51d0fa373be9cc7d793b22ac7adabe400565ca4c12e9b4
 size 4996670464

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f05b1466c485e49eb663e61e433c4cecdae40d3c64ac97a04c91578b92c3e43
+oid sha256:820a3848e15044629f249d7c069f6631abe495b0c8c46d64e65d1d8e91b39678
 size 1178224960

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.018054345073833248,
-    "train_runtime": 1357.263,
-    "train_samples": 140,
-    "train_samples_per_second": 1.179,
-    "train_steps_per_second": 0.074
+    "train_loss": 5.246203290880657e-06,
+    "train_runtime": 3714.3127,
+    "train_samples": 48,
+    "train_samples_per_second": 0.431,
+    "train_steps_per_second": 0.027
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.571428571428571,
+  "epoch": 16.666666666666668,
   "eval_steps": 500,
   "global_step": 100,
   "is_hyper_param_search": false,
@@ -10,969 +10,969 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 238.8125,
-      "epoch": 0.11428571428571428,
-      "grad_norm": 10.970682144165039,
+      "completion_length": 248.21875,
+      "epoch": 0.3333333333333333,
+      "grad_norm": 3.4614758491516113,
       "kl": 0.0,
       "learning_rate": 1.6666666666666665e-07,
       "loss": -0.0,
-      "reward": 12.407915402203798,
-      "reward_std": 5.188350445881952,
-      "rewards/concensus_correctness_reward_func": 8.488562531769276,
-      "rewards/consensus_reward_func": 0.9375,
+      "reward": 1.7851608581840992,
+      "reward_std": 1.0083980893250555,
+      "rewards/concensus_correctness_reward_func": 0.12012499943375587,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.6283841012045741,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.518067104741931,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 0.8534687550272793,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.4125937600620091,
       "step": 2
     },
     {
-      "completion_length": 251.6875,
-      "epoch": 0.22857142857142856,
-      "grad_norm": 7.923824310302734,
-      "kl": 0.042037057399284095,
+      "completion_length": 273.46875,
+      "epoch": 0.6666666666666666,
+      "grad_norm": 2.2614688873291016,
+      "kl": 0.000583945713515277,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 14.51514945924282,
-      "reward_std": 5.186308401403949,
-      "rewards/concensus_correctness_reward_func": 9.763374998001382,
-      "rewards/consensus_reward_func": 1.25,
+      "reward": 2.26899578794837,
+      "reward_std": 0.8781831180676818,
+      "rewards/concensus_correctness_reward_func": 0.18174999952316284,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4375,
-      "rewards/question_recreation_reward_func": 0.8561495095491409,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.6493395259603858,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 0.989375002682209,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.6410312540829182,
       "step": 4
     },
     {
-      "completion_length": 255.4375,
-      "epoch": 0.34285714285714286,
-      "grad_norm": 118.1331558227539,
-      "kl": 15.88943462620955,
+      "completion_length": 230.65625,
+      "epoch": 1.0,
+      "grad_norm": 2.344890594482422,
+      "kl": 0.0007161081903177546,
       "learning_rate": 4.994757065594279e-07,
-      "loss": 0.0163,
-      "reward": 10.717048183083534,
-      "reward_std": 6.401097939996816,
-      "rewards/concensus_correctness_reward_func": 6.5562499817460775,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0,
+      "reward": 1.7817438933998346,
+      "reward_std": 0.8162978617474437,
+      "rewards/concensus_correctness_reward_func": 0.09124999865889549,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.125,
-      "rewards/question_recreation_reward_func": 0.714673223963473,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4062751979799941,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 1.1336250007152557,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.5498437532223761,
       "step": 6
     },
     {
-      "completion_length": 266.34375,
-      "epoch": 0.45714285714285713,
-      "grad_norm": 10.96125602722168,
-      "kl": 2.4939099808689207,
+      "completion_length": 219.8125,
+      "epoch": 1.3333333333333333,
+      "grad_norm": 26.660634994506836,
+      "kl": 0.0012114797100366559,
       "learning_rate": 4.979050253066063e-07,
-      "loss": 0.0025,
-      "reward": 13.457202315330505,
-      "reward_std": 5.08114674501121,
-      "rewards/concensus_correctness_reward_func": 8.9798124730587,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": -0.0,
+      "reward": 1.8910054825246334,
+      "reward_std": 1.1408569859668205,
+      "rewards/concensus_correctness_reward_func": 0.12249999865889549,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.6849835254251957,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.536755473469384,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 0.9642812530510128,
+      "rewards/strict_format_reward_func": 0.078125,
+      "rewards/xmlcount_reward_func": 0.5911250002682209,
       "step": 8
     },
     {
-      "completion_length": 252.03125,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 18.904144287109375,
-      "kl": 4.680454459274188,
+      "completion_length": 202.46875,
+      "epoch": 1.6666666666666665,
+      "grad_norm": 7.760759353637695,
+      "kl": 0.0012618956752703525,
       "learning_rate": 4.952945442245597e-07,
-      "loss": 0.0047,
-      "reward": 16.20142674446106,
-      "reward_std": 8.446806251784437,
-      "rewards/concensus_correctness_reward_func": 11.267187535762787,
-      "rewards/consensus_reward_func": 1.125,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.6875,
-      "rewards/question_recreation_reward_func": 0.7198330629616976,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.089406255632639,
+      "loss": 0.0,
+      "reward": 1.9051898624747992,
+      "reward_std": 0.958464287687093,
+      "rewards/concensus_correctness_reward_func": 0.12012499943375587,
+      "rewards/consensus_reward_func": 0.4375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.3846898500341922,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.09375,
+      "rewards/xmlcount_reward_func": 0.6660000011324883,
       "step": 10
     },
     {
-      "completion_length": 234.09375,
-      "epoch": 0.6857142857142857,
-      "grad_norm": 18.74327850341797,
-      "kl": 6.5579799057450145,
+      "completion_length": 336.28125,
+      "epoch": 2.0,
+      "grad_norm": 2.4356420040130615,
+      "kl": 0.0008406055276282132,
       "learning_rate": 4.916552125781528e-07,
-      "loss": 0.0066,
-      "reward": 17.783124715089798,
-      "reward_std": 8.461443929263623,
-      "rewards/concensus_correctness_reward_func": 12.696437515318394,
-      "rewards/consensus_reward_func": 1.1875,
+      "loss": 0.0,
+      "reward": 1.8280094247311354,
+      "reward_std": 1.094328472390771,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.8125,
-      "rewards/question_recreation_reward_func": 0.6896556532010436,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5948844254016876,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.1470312476158142,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.28237500321120024,
       "step": 12
     },
     {
-      "completion_length": 269.53125,
-      "epoch": 0.8,
-      "grad_norm": 6.917476654052734,
-      "kl": 4.977367766899988,
+      "completion_length": 282.8125,
+      "epoch": 2.3333333333333335,
+      "grad_norm": 2.0542495250701904,
+      "kl": 0.0009237846843461739,
       "learning_rate": 4.870022949890676e-07,
-      "loss": 0.005,
-      "reward": 15.972103998064995,
-      "reward_std": 2.9697941527701914,
-      "rewards/concensus_correctness_reward_func": 11.044250033795834,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0,
+      "reward": 1.8283372893929482,
+      "reward_std": 1.189666131976992,
+      "rewards/concensus_correctness_reward_func": 0.23618749901652336,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.8037290628999472,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.6880872882902622,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 0.952250000089407,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.15406249510124326,
       "step": 14
     },
     {
-      "completion_length": 270.6875,
-      "epoch": 0.9142857142857143,
-      "grad_norm": 15.063738822937012,
-      "kl": 2.8824040475301445,
+      "completion_length": 266.84375,
+      "epoch": 2.6666666666666665,
+      "grad_norm": 3.6135895252227783,
+      "kl": 0.0011918679920199793,
       "learning_rate": 4.81355307410676e-07,
-      "loss": 0.0029,
-      "reward": 10.288215219974518,
-      "reward_std": 5.413245497271419,
-      "rewards/concensus_correctness_reward_func": 6.029687514528632,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 1.9164189714938402,
+      "reward_std": 1.160076865926385,
+      "rewards/concensus_correctness_reward_func": 0.11756250262260437,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9375,
-      "rewards/question_recreation_reward_func": 0.7746213864884339,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.48829398211091757,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1089062541723251,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.513687499333173,
       "step": 16
     },
     {
-      "completion_length": 233.95833333333334,
-      "epoch": 1.0,
-      "grad_norm": 4.807560443878174,
-      "kl": 1.358936990921696,
+      "completion_length": 238.6875,
+      "epoch": 3.0,
+      "grad_norm": 2.7586710453033447,
+      "kl": 0.0012445114989532158,
       "learning_rate": 4.747379352713488e-07,
-      "loss": 0.001,
-      "reward": 16.22363926966985,
-      "reward_std": 4.749512891595562,
-      "rewards/concensus_correctness_reward_func": 11.275166720151901,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "loss": 0.0,
+      "reward": 1.9129376076161861,
+      "reward_std": 0.9141782382503152,
+      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5833333333333333,
-      "rewards/question_recreation_reward_func": 0.6966391950845718,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5800001341849566,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2916666666666667,
-      "rewards/xmlcount_reward_func": 1.210166667898496,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5698124971240759,
       "step": 18
     },
     {
-      "completion_length": 247.25,
-      "epoch": 1.1142857142857143,
-      "grad_norm": 75.82022857666016,
-      "kl": 8.28540600137785,
+      "completion_length": 243.90625,
+      "epoch": 3.3333333333333335,
+      "grad_norm": 3.191770553588867,
+      "kl": 0.0015616379932907876,
       "learning_rate": 4.6717793412953776e-07,
-      "loss": 0.0083,
-      "reward": 15.897803097963333,
-      "reward_std": 3.59038737998344,
-      "rewards/concensus_correctness_reward_func": 11.287875011563301,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0,
+      "reward": 1.7438320461660624,
+      "reward_std": 1.1812182366847992,
+      "rewards/concensus_correctness_reward_func": 0.1197500005364418,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.125,
-      "rewards/question_recreation_reward_func": 0.7908340934664011,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.40089456969872117,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.0847187526524067,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3950625052675605,
       "step": 20
     },
     {
-      "completion_length": 243.03125,
-      "epoch": 1.2285714285714286,
-      "grad_norm": 18.962554931640625,
-      "kl": 5.726443055551499,
+      "completion_length": 295.375,
+      "epoch": 3.6666666666666665,
+      "grad_norm": 2.1044507026672363,
+      "kl": 0.002216623950516805,
       "learning_rate": 4.5870701325731773e-07,
-      "loss": 0.0057,
-      "reward": 15.181419372558594,
-      "reward_std": 5.393561449236586,
-      "rewards/concensus_correctness_reward_func": 10.06800001859665,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0,
+      "reward": 2.3217686265707016,
+      "reward_std": 0.9629865860333666,
+      "rewards/concensus_correctness_reward_func": 0.12012499943375587,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.6875,
-      "rewards/question_recreation_reward_func": 0.7863881841767579,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5311123521532863,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.0926562510430813,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.5142812496051192,
       "step": 22
     },
     {
-      "completion_length": 246.6875,
-      "epoch": 1.342857142857143,
-      "grad_norm": 8.005983352661133,
-      "kl": 2.3502084868960083,
+      "completion_length": 250.9375,
+      "epoch": 4.0,
+      "grad_norm": 2.842360496520996,
+      "kl": 0.001575836147821974,
       "learning_rate": 4.4936070264068016e-07,
-      "loss": 0.0023,
-      "reward": 15.992204293608665,
-      "reward_std": 5.12271255068481,
-      "rewards/concensus_correctness_reward_func": 11.144812501966953,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 0.0,
+      "reward": 1.8550556376576424,
+      "reward_std": 1.1311865192838013,
+      "rewards/concensus_correctness_reward_func": 0.11993750184774399,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.7768920734524727,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6037118844687939,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.0079999975860119,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.36578124947845936,
       "step": 24
     },
     {
-      "completion_length": 221.375,
-      "epoch": 1.457142857142857,
-      "grad_norm": 5.74846887588501,
-      "kl": 11.304446186637506,
+      "completion_length": 236.375,
+      "epoch": 4.333333333333333,
+      "grad_norm": 3.5260348320007324,
+      "kl": 0.0018173266253143083,
       "learning_rate": 4.391782039544238e-07,
-      "loss": 0.0113,
-      "reward": 13.14157024025917,
-      "reward_std": 4.419819911709055,
-      "rewards/concensus_correctness_reward_func": 8.400812551379204,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 2.022034127265215,
+      "reward_std": 0.9448904548771679,
+      "rewards/concensus_correctness_reward_func": 0.26981250010430813,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.7510077767074108,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.410690407268703,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.208500012755394,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.34153125202283263,
       "step": 26
     },
     {
-      "completion_length": 239.21875,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 13.846333503723145,
-      "kl": 2.687789380317554,
+      "completion_length": 222.1875,
+      "epoch": 4.666666666666667,
+      "grad_norm": 2.8555054664611816,
+      "kl": 0.0025639020168455318,
       "learning_rate": 4.282022261367073e-07,
-      "loss": 0.0027,
-      "reward": 13.192271679639816,
-      "reward_std": 8.560021809767932,
-      "rewards/concensus_correctness_reward_func": 8.766687527298927,
-      "rewards/consensus_reward_func": 0.9375,
+      "loss": 0.0,
+      "reward": 1.850969212129712,
+      "reward_std": 0.8895461307838559,
+      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.6983654154464602,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5744379386305809,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.1803437545895576,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5134062475990504,
       "step": 28
     },
     {
-      "completion_length": 268.40625,
-      "epoch": 1.6857142857142857,
-      "grad_norm": 10.3473482131958,
-      "kl": 2.4997252374887466,
+      "completion_length": 312.1875,
+      "epoch": 5.0,
+      "grad_norm": 2.6812455654144287,
+      "kl": 0.002034817836829461,
       "learning_rate": 4.1647880625292027e-07,
-      "loss": 0.0025,
-      "reward": 13.871409982442856,
-      "reward_std": 7.89060926809907,
-      "rewards/concensus_correctness_reward_func": 9.331625029444695,
-      "rewards/consensus_reward_func": 1.1875,
+      "loss": 0.0,
+      "reward": 2.235242122784257,
+      "reward_std": 0.903150615748018,
+      "rewards/concensus_correctness_reward_func": 0.302437499165535,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.7168787051923573,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.701929610222578,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 1.0416562482714653,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.48087499663233757,
       "step": 30
     },
     {
-      "completion_length": 270.625,
-      "epoch": 1.8,
-      "grad_norm": 125.35323333740234,
-      "kl": 15.523216519504786,
+      "completion_length": 222.03125,
+      "epoch": 5.333333333333333,
+      "grad_norm": 3.4555599689483643,
+      "kl": 0.0031993754600989632,
       "learning_rate": 4.040571164002318e-07,
-      "loss": 0.0155,
-      "reward": 14.418130666017532,
-      "reward_std": 5.059405547566712,
-      "rewards/concensus_correctness_reward_func": 9.623312469571829,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0,
+      "reward": 2.742512159049511,
+      "reward_std": 1.1281160314101726,
+      "rewards/concensus_correctness_reward_func": 0.42487499490380287,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1875,
-      "rewards/question_recreation_reward_func": 0.7698495015501976,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.6586684007197618,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.0562187489122152,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.5964687503874302,
       "step": 32
     },
     {
-      "completion_length": 278.625,
-      "epoch": 1.9142857142857141,
-      "grad_norm": 41.78948974609375,
-      "kl": 22.502449576277286,
+      "completion_length": 289.875,
+      "epoch": 5.666666666666667,
+      "grad_norm": 2.337428092956543,
+      "kl": 0.0029264501317811664,
       "learning_rate": 3.909892574627266e-07,
-      "loss": 0.0225,
-      "reward": 14.04726266860962,
-      "reward_std": 4.064370384789072,
-      "rewards/concensus_correctness_reward_func": 9.231374958530068,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0,
+      "reward": 1.9419155293144286,
+      "reward_std": 1.168605322483927,
+      "rewards/concensus_correctness_reward_func": 0.30256250128149986,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.7898254841566086,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4774467754177749,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.0416875034570694,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.31815625075250864,
       "step": 34
     },
     {
-      "completion_length": 279.5416666666667,
-      "epoch": 2.0,
-      "grad_norm": 9.201582908630371,
-      "kl": 21.56579526141286,
+      "completion_length": 271.75,
+      "epoch": 6.0,
+      "grad_norm": 3.06026291847229,
+      "kl": 0.0034197392051282804,
       "learning_rate": 3.773300405821908e-07,
-      "loss": 0.0162,
-      "reward": 14.157723685105642,
-      "reward_std": 4.17470802180469,
-      "rewards/concensus_correctness_reward_func": 9.812166665991148,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0,
+      "reward": 2.5837525203824043,
+      "reward_std": 1.1307801827788353,
+      "rewards/concensus_correctness_reward_func": 0.29768750071525574,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.7103902654101452,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.54522127751261,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.14583333333333334,
-      "rewards/xmlcount_reward_func": 0.8226666711270809,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.3970937414560467,
       "step": 36
     },
     {
-      "completion_length": 222.8125,
-      "epoch": 2.1142857142857143,
-      "grad_norm": 36.22496032714844,
-      "kl": 16.87672139937058,
+      "completion_length": 273.96875,
+      "epoch": 6.333333333333333,
+      "grad_norm": 6.918361186981201,
+      "kl": 0.0025484783691354096,
       "learning_rate": 3.6313675726113475e-07,
-      "loss": 0.0169,
-      "reward": 14.74327240884304,
-      "reward_std": 4.779272536048666,
-      "rewards/concensus_correctness_reward_func": 10.158499985933304,
-      "rewards/consensus_reward_func": 0.9375,
+      "loss": 0.0,
+      "reward": 1.684893824160099,
+      "reward_std": 1.0457911072298884,
+      "rewards/concensus_correctness_reward_func": 0.1797499991953373,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.7058975584805012,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5752063244581223,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.1132500022649765,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3674375033006072,
       "step": 38
     },
     {
-      "completion_length": 246.65625,
-      "epoch": 2.2285714285714286,
-      "grad_norm": 19.6318359375,
-      "kl": 16.881314451806247,
+      "completion_length": 268.4375,
+      "epoch": 6.666666666666667,
+      "grad_norm": 2.9002139568328857,
+      "kl": 0.002866584218281787,
       "learning_rate": 3.484689390623218e-07,
-      "loss": 0.0169,
-      "reward": 18.013662189245224,
-      "reward_std": 7.812059601201327,
-      "rewards/concensus_correctness_reward_func": 13.051250010728836,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0,
+      "reward": 1.900494983419776,
+      "reward_std": 1.2571525508537889,
+      "rewards/concensus_correctness_reward_func": 0.24512499943375587,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.6924745952710509,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.6020261840894818,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 1.1293125040829182,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3033437514677644,
       "step": 40
     },
     {
-      "completion_length": 272.125,
-      "epoch": 2.342857142857143,
-      "grad_norm": 622.0595092773438,
-      "kl": 40.245274308137596,
+      "completion_length": 292.96875,
+      "epoch": 7.0,
+      "grad_norm": 2.595947742462158,
+      "kl": 0.0034651760361157358,
       "learning_rate": 3.3338810791270517e-07,
-      "loss": 0.0402,
-      "reward": 15.975400440394878,
-      "reward_std": 3.027749645523727,
-      "rewards/concensus_correctness_reward_func": 11.107500043697655,
-      "rewards/consensus_reward_func": 1.4375,
+      "loss": 0.0,
+      "reward": 2.582839421927929,
+      "reward_std": 0.8570395507849753,
+      "rewards/concensus_correctness_reward_func": 0.42537499964237213,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.8080566711723804,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.5414331587962806,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 0.9660937523003668,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.5222812481224537,
       "step": 42
     },
     {
-      "completion_length": 230.40625,
-      "epoch": 2.4571428571428573,
-      "grad_norm": 12.730220794677734,
-      "kl": 2.831924519967288,
+      "completion_length": 277.78125,
+      "epoch": 7.333333333333333,
+      "grad_norm": 3.5444931983947754,
+      "kl": 0.005151846205990296,
       "learning_rate": 3.179575180590857e-07,
-      "loss": 0.0028,
-      "reward": 14.358340233564377,
-      "reward_std": 7.420220456435345,
-      "rewards/concensus_correctness_reward_func": 9.464937542332336,
-      "rewards/consensus_reward_func": 1.1875,
+      "loss": 0.0,
+      "reward": 2.2039641477167606,
+      "reward_std": 1.085899718105793,
+      "rewards/concensus_correctness_reward_func": 0.1823749952018261,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.7924965554848313,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4583704025717452,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.132156252861023,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.6100937575101852,
       "step": 44
     },
     {
-      "completion_length": 252.6875,
-      "epoch": 2.571428571428571,
-      "grad_norm": 91.50186920166016,
-      "kl": 13.465573271270841,
+      "completion_length": 255.9375,
+      "epoch": 7.666666666666667,
+      "grad_norm": 2.611354112625122,
+      "kl": 0.004227373341564089,
       "learning_rate": 3.022418907578188e-07,
-      "loss": 0.0135,
-      "reward": 10.495669901371002,
-      "reward_std": 3.2124593601038214,
-      "rewards/concensus_correctness_reward_func": 6.154187520965934,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 1.8945876602083445,
+      "reward_std": 0.9345793006941676,
+      "rewards/concensus_correctness_reward_func": 0.18268750235438347,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1875,
-      "rewards/question_recreation_reward_func": 0.7502010772004724,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.56027518119663,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 1.1069062501192093,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4484999952837825,
       "step": 46
     },
     {
-      "completion_length": 267.3125,
-      "epoch": 2.685714285714286,
-      "grad_norm": 9.117658615112305,
-      "kl": 2.676259968895465,
+      "completion_length": 211.78125,
+      "epoch": 8.0,
+      "grad_norm": 7.4239501953125,
+      "kl": 0.005186592134123202,
       "learning_rate": 2.863071428113726e-07,
-      "loss": 0.0027,
-      "reward": 13.41846364736557,
-      "reward_std": 4.830206839018501,
-      "rewards/concensus_correctness_reward_func": 9.071687553077936,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 2.3263955637812614,
+      "reward_std": 0.9710670886561275,
+      "rewards/concensus_correctness_reward_func": 0.05962499976158142,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.7668074369430542,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6223018001765013,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 0.9393437597900629,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.5975937452167273,
       "step": 48
     },
     {
-      "completion_length": 268.9375,
-      "epoch": 2.8,
-      "grad_norm": 6.602773666381836,
-      "kl": 3.5302634860854596,
+      "completion_length": 237.625,
+      "epoch": 8.333333333333334,
+      "grad_norm": 3.110980272293091,
+      "kl": 0.005397260436438955,
       "learning_rate": 2.7022011009035107e-07,
-      "loss": 0.0035,
-      "reward": 13.124091684818268,
-      "reward_std": 6.698563320926041,
-      "rewards/concensus_correctness_reward_func": 8.646312475204468,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0,
+      "reward": 2.3635387681424618,
+      "reward_std": 1.1881022350862622,
+      "rewards/concensus_correctness_reward_func": 0.1797499991953373,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.7686855588108301,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4961012676358223,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.109375,
-      "rewards/xmlcount_reward_func": 1.0372187588363886,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.6095625087618828,
       "step": 50
     },
     {
-      "completion_length": 249.09375,
-      "epoch": 2.914285714285714,
-      "grad_norm": 8879.2197265625,
-      "kl": 443.5015660626814,
+      "completion_length": 302.375,
+      "epoch": 8.666666666666666,
+      "grad_norm": 3.359147787094116,
+      "kl": 0.00526040667318739,
       "learning_rate": 2.540482672006254e-07,
-      "loss": 0.4435,
-      "reward": 14.807004809379578,
-      "reward_std": 6.6539650214836,
-      "rewards/concensus_correctness_reward_func": 10.309812501072884,
-      "rewards/consensus_reward_func": 1.1875,
+      "loss": 0.0,
+      "reward": 1.951256174594164,
+      "reward_std": 1.4674215724226087,
+      "rewards/concensus_correctness_reward_func": 0.14731250144541264,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.7173797897994518,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5990999331697822,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.0141874998807907,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.29859375115484,
       "step": 52
     },
     {
-      "completion_length": 238.20833333333334,
-      "epoch": 3.0,
-      "grad_norm": 5.6875457763671875,
-      "kl": 43.41115608314673,
+      "completion_length": 271.34375,
+      "epoch": 9.0,
+      "grad_norm": 2.3752264976501465,
+      "kl": 0.006538017390994355,
       "learning_rate": 2.37859444471388e-07,
-      "loss": 0.0326,
-      "reward": 12.497497618198395,
-      "reward_std": 4.959660264973839,
-      "rewards/concensus_correctness_reward_func": 8.096666658918062,
-      "rewards/consensus_reward_func": 0.9166666666666666,
+      "loss": 0.0,
+      "reward": 2.0346078351140022,
+      "reward_std": 0.7186543988646008,
+      "rewards/concensus_correctness_reward_func": 0.24512499943375587,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.6404145161310831,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5727328350767493,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2708333333333333,
-      "rewards/xmlcount_reward_func": 1.0729166616996129,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5761250024661422,
       "step": 54
     },
     {
-      "completion_length": 226.96875,
-      "epoch": 3.1142857142857143,
-      "grad_norm": 13.502339363098145,
-      "kl": 14.871396910399199,
+      "completion_length": 258.5625,
+      "epoch": 9.333333333333334,
+      "grad_norm": 2.415621519088745,
+      "kl": 0.0068657751398859546,
       "learning_rate": 2.2172154345117894e-07,
-      "loss": 0.0149,
-      "reward": 14.69212443381548,
-      "reward_std": 6.670687978621572,
-      "rewards/concensus_correctness_reward_func": 10.009812474250793,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0,
+      "reward": 2.7013516426086426,
+      "reward_std": 1.1713198693469167,
+      "rewards/concensus_correctness_reward_func": 0.17987500131130219,
+      "rewards/consensus_reward_func": 1.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.7185621028766036,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5597578389570117,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 0.9793750094249845,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.5867187539115548,
       "step": 56
     },
     {
-      "completion_length": 251.40625,
-      "epoch": 3.2285714285714286,
-      "grad_norm": 12.282088279724121,
-      "kl": 18.559334562858567,
+      "completion_length": 250.4375,
+      "epoch": 9.666666666666666,
+      "grad_norm": 3.3267598152160645,
+      "kl": 0.007265938082127832,
       "learning_rate": 2.0570225210519433e-07,
-      "loss": 0.0186,
-      "reward": 15.513990081846714,
-      "reward_std": 3.7251646753866225,
-      "rewards/concensus_correctness_reward_func": 10.890124998986721,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 0.0,
+      "reward": 1.7129965675994754,
+      "reward_std": 0.9574196808971465,
+      "rewards/concensus_correctness_reward_func": 0.24262499809265137,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.6494588730856776,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5204653043183498,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 1.0837812572717667,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4811562584945932,
       "step": 58
     },
     {
-      "completion_length": 278.84375,
-      "epoch": 3.342857142857143,
-      "grad_norm": 12.170891761779785,
-      "kl": 4.615750857628882,
+      "completion_length": 294.5625,
+      "epoch": 10.0,
+      "grad_norm": 3.0382344722747803,
+      "kl": 0.007241527950100135,
       "learning_rate": 1.8986876090843664e-07,
-      "loss": 0.0046,
-      "reward": 13.320735141634941,
-      "reward_std": 6.643354088068008,
-      "rewards/concensus_correctness_reward_func": 8.784562550485134,
-      "rewards/consensus_reward_func": 1.0625,
+      "loss": 0.0,
+      "reward": 1.9392794030718505,
+      "reward_std": 1.0110920977240312,
+      "rewards/concensus_correctness_reward_func": 0.11993750184774399,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.6847666085232049,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.500998193398118,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 1.0545312482863665,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.42771875113248825,
       "step": 60
     },
     {
-      "completion_length": 246.78125,
-      "epoch": 3.4571428571428573,
-      "grad_norm": 9.369564056396484,
-      "kl": 4.713114297017455,
+      "completion_length": 222.125,
+      "epoch": 10.333333333333334,
+      "grad_norm": 3.4102041721343994,
+      "kl": 0.00738686349359341,
       "learning_rate": 1.7428748102551234e-07,
-      "loss": 0.0047,
-      "reward": 13.258399233222008,
-      "reward_std": 6.423583036288619,
-      "rewards/concensus_correctness_reward_func": 9.021875010803342,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0,
+      "reward": 1.7328537330031395,
+      "reward_std": 0.9404821525095031,
+      "rewards/concensus_correctness_reward_func": 0.1223749965429306,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9375,
-      "rewards/question_recreation_reward_func": 0.7111177369952202,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.45538498694077134,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.103531252592802,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.5613437537103891,
       "step": 62
     },
     {
-      "completion_length": 263.75,
-      "epoch": 3.571428571428571,
-      "grad_norm": 34.83134078979492,
-      "kl": 8.73400528007187,
+      "completion_length": 322.46875,
+      "epoch": 10.666666666666666,
+      "grad_norm": 2.5114853382110596,
+      "kl": 0.005600389951723628,
       "learning_rate": 1.5902376575912814e-07,
-      "loss": 0.0087,
-      "reward": 13.156703546643257,
-      "reward_std": 3.433466176968068,
-      "rewards/concensus_correctness_reward_func": 8.570312479510903,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0,
+      "reward": 2.345261871814728,
+      "reward_std": 1.4083554986864328,
+      "rewards/concensus_correctness_reward_func": 0.35993750020861626,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1875,
-      "rewards/question_recreation_reward_func": 0.7510160449892282,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.621855610050261,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.1322499997913837,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4103437541052699,
       "step": 64
     },
     {
-      "completion_length": 248.53125,
-      "epoch": 3.685714285714286,
-      "grad_norm": 7.867584228515625,
-      "kl": 2.0124676286941394,
+      "completion_length": 238.65625,
+      "epoch": 11.0,
+      "grad_norm": 2.935357093811035,
+      "kl": 0.00783520688128192,
       "learning_rate": 1.4414163643562753e-07,
-      "loss": 0.002,
-      "reward": 16.860622122883797,
-      "reward_std": 5.908110194373876,
-      "rewards/concensus_correctness_reward_func": 11.929625045508146,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 0.0,
+      "reward": 2.1366525441408157,
+      "reward_std": 1.0672186479205266,
+      "rewards/concensus_correctness_reward_func": 0.11743750050663948,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.7619347143918276,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.588996303267777,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.1690624989569187,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.6802187506109476,
       "step": 66
     },
     {
-      "completion_length": 264.8125,
-      "epoch": 3.8,
-      "grad_norm": 6.784933567047119,
-      "kl": 1.5569924684241414,
+      "completion_length": 266.90625,
+      "epoch": 11.333333333333334,
+      "grad_norm": 2.8767664432525635,
+      "kl": 0.006433373549953103,
       "learning_rate": 1.2970351387729872e-07,
-      "loss": 0.0016,
-      "reward": 12.743360340595245,
-      "reward_std": 2.557829722936731,
-      "rewards/concensus_correctness_reward_func": 8.357437543570995,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 1.6148663735948503,
+      "reward_std": 1.3225993164815009,
+      "rewards/concensus_correctness_reward_func": 0.20812500081956387,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.6942979600280523,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5250538997352123,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.0041249990463257,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.3504374986514449,
       "step": 68
     },
     {
-      "completion_length": 261.5625,
-      "epoch": 3.914285714285714,
-      "grad_norm": 10.075136184692383,
-      "kl": 2.913988582789898,
+      "completion_length": 250.21875,
+      "epoch": 11.666666666666666,
+      "grad_norm": 4.7036638259887695,
+      "kl": 0.007804472814314067,
       "learning_rate": 1.1576995658775404e-07,
-      "loss": 0.0029,
-      "reward": 15.701346069574356,
-      "reward_std": 4.226778008276597,
-      "rewards/concensus_correctness_reward_func": 10.768500030040741,
-      "rewards/consensus_reward_func": 1.1875,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.6875,
-      "rewards/question_recreation_reward_func": 0.7689711172133684,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.0388750080019236,
+      "loss": 0.0,
+      "reward": 2.2799468226730824,
+      "reward_std": 1.166436342522502,
+      "rewards/concensus_correctness_reward_func": 0.30262500047683716,
+      "rewards/consensus_reward_func": 0.6875,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5988530963659286,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.47221875097602606,
       "step": 70
     },
     {
-      "completion_length": 242.375,
-      "epoch": 4.0,
-      "grad_norm": 7.251352787017822,
-      "kl": 2.0374175421893597,
+      "completion_length": 248.9375,
+      "epoch": 12.0,
+      "grad_norm": 2.7038285732269287,
+      "kl": 0.00650077304453589,
       "learning_rate": 1.0239940674851941e-07,
-      "loss": 0.0015,
-      "reward": 16.076478640238445,
-      "reward_std": 6.51126005128026,
-      "rewards/concensus_correctness_reward_func": 11.290750041604042,
-      "rewards/consensus_reward_func": 1.3333333333333333,
+      "loss": 0.0,
+      "reward": 1.7189936628565192,
+      "reward_std": 1.1028610868379474,
+      "rewards/concensus_correctness_reward_func": 0.11999999731779099,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.7505618532498678,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.4415561552159488,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.22916666666666666,
-      "rewards/xmlcount_reward_func": 0.9726666783293089,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4543124968186021,
       "step": 72
     },
     {
-      "completion_length": 273.78125,
-      "epoch": 4.114285714285714,
-      "grad_norm": 11.66654109954834,
-      "kl": 5.751777551136911,
+      "completion_length": 241.9375,
+      "epoch": 12.333333333333334,
+      "grad_norm": 3.6059765815734863,
+      "kl": 0.007852518698200583,
       "learning_rate": 8.964794509221507e-08,
-      "loss": 0.0058,
-      "reward": 12.40931786596775,
-      "reward_std": 4.601450197398663,
-      "rewards/concensus_correctness_reward_func": 8.060937531292439,
-      "rewards/consensus_reward_func": 0.9375,
+      "loss": 0.0,
+      "reward": 1.7060329876840115,
+      "reward_std": 1.2234434876590967,
+      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4375,
-      "rewards/question_recreation_reward_func": 0.6974428757093847,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4156580241397023,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 1.0571874994784594,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.44912500213831663,
       "step": 74
     },
     {
-      "completion_length": 280.625,
-      "epoch": 4.228571428571429,
-      "grad_norm": 8.615840911865234,
-      "kl": 2.3532106273341924,
+      "completion_length": 286.28125,
+      "epoch": 12.666666666666666,
+      "grad_norm": 2.444171190261841,
+      "kl": 0.00814603897742927,
       "learning_rate": 7.756905568047392e-08,
-      "loss": 0.0024,
-      "reward": 12.607302218675613,
-      "reward_std": 4.357940239366144,
-      "rewards/concensus_correctness_reward_func": 7.97631249576807,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 0.0,
+      "reward": 2.122521236538887,
+      "reward_std": 0.9578379703452811,
+      "rewards/concensus_correctness_reward_func": 0.42274999618530273,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.8087085485458374,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.504114959621802,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.009781249333173,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.6175312478444539,
       "step": 76
     },
     {
-      "completion_length": 228.125,
-      "epoch": 4.3428571428571425,
-      "grad_norm": 15.023426055908203,
-      "kl": 4.856777695240453,
+      "completion_length": 234.6875,
+      "epoch": 13.0,
+      "grad_norm": 3.0294716358184814,
+      "kl": 0.008441783153102733,
       "learning_rate": 6.621340157319996e-08,
-      "loss": 0.0049,
-      "reward": 15.21293680369854,
-      "reward_std": 7.808012545632664,
-      "rewards/concensus_correctness_reward_func": 10.372124960646033,
-      "rewards/consensus_reward_func": 1.1875,
+      "loss": 0.0,
+      "reward": 2.081810735166073,
+      "reward_std": 0.9143577099894173,
+      "rewards/concensus_correctness_reward_func": 0.125,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.7565617831423879,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5170919904485345,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.0998749984428287,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.37721875379793346,
       "step": 78
     },
     {
-      "completion_length": 228.3125,
-      "epoch": 4.457142857142857,
-      "grad_norm": 14.391371726989746,
-      "kl": 25.115882573183626,
+      "completion_length": 224.40625,
+      "epoch": 13.333333333333334,
+      "grad_norm": 3.4747705459594727,
+      "kl": 0.01025624800240621,
       "learning_rate": 5.5628612330087724e-08,
-      "loss": 0.0251,
-      "reward": 10.238258555531502,
-      "reward_std": 6.636154420673847,
-      "rewards/concensus_correctness_reward_func": 5.875812470912933,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.0,
+      "reward": 1.9557546898722649,
+      "reward_std": 0.9802739145234227,
+      "rewards/concensus_correctness_reward_func": 0.24450000002980232,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.625,
-      "rewards/question_recreation_reward_func": 0.6746960403397679,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.562535947188735,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.0627500042319298,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4924687468446791,
       "step": 80
     },
     {
-      "completion_length": 297.6875,
-      "epoch": 4.571428571428571,
-      "grad_norm": 9.444145202636719,
-      "kl": 2.323721103835851,
+      "completion_length": 234.75,
+      "epoch": 13.666666666666666,
+      "grad_norm": 2.7190427780151367,
+      "kl": 0.008365696892724372,
       "learning_rate": 4.5859084235697235e-08,
-      "loss": 0.0024,
-      "reward": 14.212048962712288,
-      "reward_std": 5.437735598199652,
-      "rewards/concensus_correctness_reward_func": 9.653124989941716,
-      "rewards/consensus_reward_func": 1.4375,
+      "loss": 0.0,
+      "reward": 1.6996353343129158,
+      "reward_std": 0.6642087557120249,
+      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1875,
-      "rewards/question_recreation_reward_func": 0.7724551558494568,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.395666572265327,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 0.9739687610417604,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.5095937536098063,
       "step": 82
     },
     {
-      "completion_length": 263.28125,
-      "epoch": 4.685714285714286,
-      "grad_norm": 6.39396858215332,
-      "kl": 7.9060614665504545,
+      "completion_length": 263.96875,
+      "epoch": 14.0,
+      "grad_norm": 2.663147449493408,
+      "kl": 0.010309035773389041,
       "learning_rate": 3.6945794086007705e-08,
-      "loss": 0.0079,
-      "reward": 16.380697011947632,
-      "reward_std": 3.2155018125195056,
-      "rewards/concensus_correctness_reward_func": 11.199437521398067,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0,
+      "reward": 2.0628331936895847,
+      "reward_std": 1.0619396911934018,
+      "rewards/concensus_correctness_reward_func": 0.17756250128149986,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.854978347197175,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5063332146964967,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.1856562532484531,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.5195625014603138,
       "step": 84
     },
     {
-      "completion_length": 264.5,
-      "epoch": 4.8,
-      "grad_norm": 8.364204406738281,
-      "kl": 5.299565837252885,
+      "completion_length": 231.1875,
+      "epoch": 14.333333333333334,
+      "grad_norm": 3.154367685317993,
+      "kl": 0.01306345833290834,
       "learning_rate": 2.892612731749414e-08,
-      "loss": 0.0053,
-      "reward": 15.947876825928688,
-      "reward_std": 6.744944950565696,
-      "rewards/concensus_correctness_reward_func": 11.113375023007393,
-      "rewards/consensus_reward_func": 1.4375,
+      "loss": 0.0,
+      "reward": 2.1949321180582047,
+      "reward_std": 1.4137339515145868,
+      "rewards/concensus_correctness_reward_func": 0.18025000020861626,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.697533005848527,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.49090082419570535,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 0.9963437505066395,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.5550312586128712,
       "step": 86
     },
     {
-      "completion_length": 239.28125,
-      "epoch": 4.914285714285715,
-      "grad_norm": 20.019237518310547,
-      "kl": 6.356791608501226,
+      "completion_length": 315.9375,
+      "epoch": 14.666666666666666,
+      "grad_norm": 2.249851703643799,
+      "kl": 0.006411108406609856,
       "learning_rate": 2.183372119961499e-08,
-      "loss": 0.0064,
-      "reward": 17.015481919050217,
-      "reward_std": 3.0891955395927653,
-      "rewards/concensus_correctness_reward_func": 12.235625021159649,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 0.0,
+      "reward": 2.2250705547630787,
+      "reward_std": 1.5259972442872822,
+      "rewards/concensus_correctness_reward_func": 0.39806249737739563,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.6681380509398878,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5338830836117268,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 1.0960937510244548,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.3087499989196658,
       "step": 88
     },
     {
-      "completion_length": 260.2916666666667,
-      "epoch": 5.0,
-      "grad_norm": 5.369960784912109,
-      "kl": 7.677339844405651,
+      "completion_length": 261.09375,
+      "epoch": 15.0,
+      "grad_norm": 2.743177652359009,
+      "kl": 0.00881043096887879,
       "learning_rate": 1.5698323748414122e-08,
-      "loss": 0.0058,
-      "reward": 12.575575411319733,
-      "reward_std": 9.121821681658426,
-      "rewards/concensus_correctness_reward_func": 8.391249996920427,
-      "rewards/consensus_reward_func": 0.9166666666666666,
+      "loss": 0.0,
+      "reward": 1.895367180928588,
+      "reward_std": 1.2597154856775887,
+      "rewards/concensus_correctness_reward_func": 0.18012499809265137,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.7138255399962267,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5660547083243728,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.125,
-      "rewards/xmlcount_reward_func": 0.9288333433990678,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.35231249686330557,
       "step": 90
     },
     {
-      "completion_length": 239.96875,
-      "epoch": 5.114285714285714,
-      "grad_norm": 96.5102310180664,
-      "kl": 18.3592914538458,
+      "completion_length": 277.90625,
+      "epoch": 15.333333333333334,
+      "grad_norm": 3.3928024768829346,
+      "kl": 0.006757803043001331,
       "learning_rate": 1.054566895300324e-08,
-      "loss": 0.0184,
-      "reward": 13.847605228424072,
-      "reward_std": 6.045313289389014,
-      "rewards/concensus_correctness_reward_func": 9.380875006318092,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 1.7474502064287663,
+      "reward_std": 0.7529050658922642,
+      "rewards/concensus_correctness_reward_func": 0.1798749975860119,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4375,
-      "rewards/question_recreation_reward_func": 0.6726677799597383,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5421689683571458,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 0.981562502682209,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.337906246073544,
       "step": 92
     },
     {
-      "completion_length": 285.28125,
-      "epoch": 5.228571428571429,
-      "grad_norm": 99.03144073486328,
-      "kl": 15.879234082996845,
+      "completion_length": 304.09375,
+      "epoch": 15.666666666666666,
+      "grad_norm": 2.197697162628174,
+      "kl": 0.006906057009473443,
       "learning_rate": 6.397368838268496e-09,
-      "loss": 0.0159,
-      "reward": 9.960642769932747,
-      "reward_std": 5.102133866865188,
-      "rewards/concensus_correctness_reward_func": 6.02468744199723,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 0.0,
+      "reward": 1.6495681330561638,
+      "reward_std": 0.6983778811700176,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9375,
-      "rewards/question_recreation_reward_func": 0.7212677141651511,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5284118615090847,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.15625,
-      "rewards/xmlcount_reward_func": 0.8084374981699511,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.48290624911896884,
       "step": 94
     },
     {
-      "completion_length": 240.71875,
-      "epoch": 5.3428571428571425,
-      "grad_norm": 52.954837799072266,
-      "kl": 20.95912204636261,
+      "completion_length": 269.59375,
+      "epoch": 16.0,
+      "grad_norm": 2.6010444164276123,
+      "kl": 0.011520398882566951,
       "learning_rate": 3.2708228165273244e-09,
-      "loss": 0.021,
-      "reward": 15.641045704483986,
-      "reward_std": 3.0330299666675273,
-      "rewards/concensus_correctness_reward_func": 10.918187517672777,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0,
+      "reward": 2.7417864352464676,
+      "reward_std": 1.6308739269152284,
+      "rewards/concensus_correctness_reward_func": 0.30268750339746475,
+      "rewards/consensus_reward_func": 1.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4375,
-      "rewards/question_recreation_reward_func": 0.5993270333856344,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5818177200853825,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.076656258199364,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.4354062539059669,
       "step": 96
     },
     {
-      "completion_length": 234.71875,
-      "epoch": 5.457142857142857,
-      "grad_norm": 16.95273208618164,
-      "kl": 11.928820116794668,
+      "completion_length": 256.625,
+      "epoch": 16.333333333333332,
+      "grad_norm": 3.064824104309082,
+      "kl": 0.013104435958666727,
       "learning_rate": 1.1791447083465133e-09,
-      "loss": 0.0119,
-      "reward": 12.215655967593193,
-      "reward_std": 8.51226019859314,
-      "rewards/concensus_correctness_reward_func": 7.525312505662441,
-      "rewards/consensus_reward_func": 0.875,
+      "loss": 0.0,
+      "reward": 2.3778811804950237,
+      "reward_std": 1.2875230926729273,
+      "rewards/concensus_correctness_reward_func": 0.18262499943375587,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.8116247765719891,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.44909992604516447,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.1912187524139881,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.6836562557145953,
       "step": 98
     },
     {
-      "completion_length": 220.0625,
-      "epoch": 5.571428571428571,
-      "grad_norm": 21.187820434570312,
-      "kl": 11.71281518554315,
+      "completion_length": 251.875,
+      "epoch": 16.666666666666668,
+      "grad_norm": 5.960361957550049,
+      "kl": 0.014634610328357667,
       "learning_rate": 1.3110773862126667e-10,
-      "loss": 0.0117,
-      "reward": 13.504412293434143,
-      "reward_std": 6.072454041801393,
-      "rewards/concensus_correctness_reward_func": 9.202812563627958,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0,
+      "reward": 1.9424380734562874,
+      "reward_std": 1.420625472906977,
+      "rewards/concensus_correctness_reward_func": 0.24249999597668648,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.6520686270669103,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.38812559354119003,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.040156252682209,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.5149374911561608,
       "step": 100
     },
     {
-      "epoch": 5.571428571428571,
+      "epoch": 16.666666666666668,
       "step": 100,
       "total_flos": 0.0,
-      "train_loss": 0.018054345073833248,
-      "train_runtime": 1357.263,
-      "train_samples_per_second": 1.179,
-      "train_steps_per_second": 0.074
+      "train_loss": 5.246203290880657e-06,
+      "train_runtime": 3714.3127,
+      "train_samples_per_second": 0.431,
+      "train_steps_per_second": 0.027
     }
   ],
   "logging_steps": 2,
   "max_steps": 100,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 6,
+  "num_train_epochs": 17,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81edfe9f3c2f35464bd2b7f022611d889bd2ecffa44d2c1040267960882f5651
+oid sha256:929dae0ec6f304f24eac715a1449fdddd810343375eb432e5ee668c04ef01240
 size 5944