End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.1608919289756158e-05,
-    "train_runtime": 2917.0763,
-    "train_samples": 140,
-    "train_samples_per_second": 0.548,
-    "train_steps_per_second": 0.034
+    "train_loss": 1.5718676149845123e-05,
+    "train_runtime": 4262.7362,
+    "train_samples": 48,
+    "train_samples_per_second": 0.375,
+    "train_steps_per_second": 0.023
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:22c31451dc66d5752513620bc1dd4c04c566b0b1a06a85a1a797c79cd0cd56aa
+oid sha256:7aa4f6bae2562b59ce5b4c50ed5ba43bb6c7a6490d3bdbfd840200784bb86db8
 size 4996670464

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27b7048459ff44f8eeaa9ed4b95162ad5a92c389ca99ed4a8f94a4af0012401f
+oid sha256:e5d2dcd5763357799ad9114059b0028a01c6683f147409faa0f31aafbd95ece0
 size 1178224960

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.1608919289756158e-05,
-    "train_runtime": 2917.0763,
-    "train_samples": 140,
-    "train_samples_per_second": 0.548,
-    "train_steps_per_second": 0.034
+    "train_loss": 1.5718676149845123e-05,
+    "train_runtime": 4262.7362,
+    "train_samples": 48,
+    "train_samples_per_second": 0.375,
+    "train_steps_per_second": 0.023
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.571428571428571,
+  "epoch": 16.666666666666668,
   "eval_steps": 500,
   "global_step": 100,
   "is_hyper_param_search": false,
@@ -10,969 +10,969 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 286.78125,
-      "epoch": 0.11428571428571428,
-      "grad_norm": 2.276369333267212,
+      "completion_length": 227.8125,
+      "epoch": 0.3333333333333333,
+      "grad_norm": 6.339727878570557,
       "kl": 0.0,
       "learning_rate": 1.6666666666666665e-07,
-      "loss": 0.0,
-      "reward": 4.382780752843246,
-      "reward_std": 2.351492334040813,
-      "rewards/concensus_correctness_reward_func": 2.5035625000018626,
-      "rewards/consensus_reward_func": 0.375,
+      "loss": -0.0,
+      "reward": 1.5849424004554749,
+      "reward_std": 1.131471099331975,
+      "rewards/concensus_correctness_reward_func": 0.12262500077486038,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.5795618183910847,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.46509863436222076,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2996562537737191,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.35659374902024865,
       "step": 2
     },
     {
-      "completion_length": 252.75,
-      "epoch": 0.22857142857142856,
-      "grad_norm": 2.464397430419922,
-      "kl": 0.0005955380293016788,
+      "completion_length": 287.125,
+      "epoch": 0.6666666666666666,
+      "grad_norm": 2.283590793609619,
+      "kl": 0.000743766751838848,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 3.0856650918722153,
-      "reward_std": 0.7508293140563183,
-      "rewards/concensus_correctness_reward_func": 1.2791875004186295,
-      "rewards/consensus_reward_func": 0.125,
+      "loss": 0.0,
+      "reward": 2.0471834875643253,
+      "reward_std": 0.9305327897891402,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5572276492603123,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.6308709642617032,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.6086250022053719,
+      "rewards/xmlcount_reward_func": 0.46318750735372305,
       "step": 4
     },
     {
-      "completion_length": 281.625,
-      "epoch": 0.34285714285714286,
-      "grad_norm": 2.6032321453094482,
-      "kl": 0.0007702869843342341,
+      "completion_length": 286.03125,
+      "epoch": 1.0,
+      "grad_norm": 2.7924423217773438,
+      "kl": 0.0008395667373406468,
       "learning_rate": 4.994757065594279e-07,
       "loss": 0.0,
-      "reward": 1.5275531343650073,
-      "reward_std": 0.9591646681074053,
-      "rewards/concensus_correctness_reward_func": 0.07531250268220901,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.7317106202244759,
+      "reward_std": 1.3122332114144228,
+      "rewards/concensus_correctness_reward_func": 0.12025000154972076,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.528365645557642,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.4853668725118041,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.45512500917539,
+      "rewards/xmlcount_reward_func": 0.21984374802559614,
       "step": 6
     },
     {
-      "completion_length": 329.8125,
-      "epoch": 0.45714285714285713,
-      "grad_norm": 3.317532539367676,
-      "kl": 0.000785435793659417,
+      "completion_length": 289.90625,
+      "epoch": 1.3333333333333333,
+      "grad_norm": 2.7986364364624023,
+      "kl": 0.0008700692887941841,
       "learning_rate": 4.979050253066063e-07,
       "loss": 0.0,
-      "reward": 5.570158363319933,
-      "reward_std": 1.5328934689750895,
-      "rewards/concensus_correctness_reward_func": 3.344875006005168,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 1.865300026256591,
+      "reward_std": 0.8366383262909949,
+      "rewards/concensus_correctness_reward_func": 0.06006250157952309,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.6130958534777164,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6006125509738922,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.19031249126419425,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4389999993145466,
       "step": 8
     },
     {
-      "completion_length": 275.65625,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 2.6608853340148926,
-      "kl": 0.0007642239979759324,
+      "completion_length": 224.3125,
+      "epoch": 1.6666666666666665,
+      "grad_norm": 3.4794700145721436,
+      "kl": 0.0009347902014269494,
       "learning_rate": 4.952945442245597e-07,
-      "loss": -0.0,
-      "reward": 4.114271555095911,
-      "reward_std": 1.842139216605574,
-      "rewards/concensus_correctness_reward_func": 2.100500001106411,
-      "rewards/consensus_reward_func": 0.375,
+      "loss": 0.0,
+      "reward": 1.8756496086716652,
+      "reward_std": 1.0868838177993894,
+      "rewards/concensus_correctness_reward_func": 0.08568750135600567,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.471396672539413,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4599620746448636,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.6517500090412796,
+      "rewards/xmlcount_reward_func": 0.5018750056624413,
       "step": 10
     },
     {
-      "completion_length": 322.71875,
-      "epoch": 0.6857142857142857,
-      "grad_norm": 2.0212087631225586,
-      "kl": 0.0009263881365768611,
+      "completion_length": 296.28125,
+      "epoch": 2.0,
+      "grad_norm": 3.0765440464019775,
+      "kl": 0.0012358422500255983,
       "learning_rate": 4.916552125781528e-07,
       "loss": 0.0,
-      "reward": 3.2796499207615852,
-      "reward_std": 2.9198597713839263,
-      "rewards/concensus_correctness_reward_func": 1.3640624983236194,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.9579493142664433,
+      "reward_std": 0.8908882063115016,
+      "rewards/concensus_correctness_reward_func": 0.12025000154972076,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.63699368853122,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.465324345510453,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5285937548615038,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4817500030621886,
       "step": 12
     },
     {
-      "completion_length": 273.03125,
-      "epoch": 0.8,
-      "grad_norm": 2.2424488067626953,
-      "kl": 0.0011271409348410089,
+      "completion_length": 280.90625,
+      "epoch": 2.3333333333333335,
+      "grad_norm": 2.9266109466552734,
+      "kl": 0.0017369047309330199,
       "learning_rate": 4.870022949890676e-07,
       "loss": 0.0,
-      "reward": 3.058269999921322,
-      "reward_std": 2.4546356331557035,
-      "rewards/concensus_correctness_reward_func": 1.011687501159031,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 1.4219924416393042,
+      "reward_std": 0.8281801359262317,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.5196137428283691,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.450836188509129,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4644687445834279,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.34615624509751797,
       "step": 14
     },
     {
-      "completion_length": 271.21875,
-      "epoch": 0.9142857142857143,
-      "grad_norm": 3.093533515930176,
-      "kl": 0.0015523226065852214,
+      "completion_length": 243.0625,
+      "epoch": 2.6666666666666665,
+      "grad_norm": 3.0913712978363037,
+      "kl": 0.0019520393034326844,
       "learning_rate": 4.81355307410676e-07,
       "loss": 0.0,
-      "reward": 1.6757539696991444,
-      "reward_std": 0.8273154485941632,
-      "rewards/concensus_correctness_reward_func": 0.14224999770522118,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 2.2805784731172025,
+      "reward_std": 1.2899235817894805,
+      "rewards/concensus_correctness_reward_func": 0.24512499943375587,
+      "rewards/consensus_reward_func": 0.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.562660242896527,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.47473472240380943,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.5177187529043294,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.451343753375113,
       "step": 16
     },
     {
-      "completion_length": 278.0416666666667,
-      "epoch": 1.0,
-      "grad_norm": 1.7071589231491089,
-      "kl": 0.0019262856318770598,
+      "completion_length": 267.40625,
+      "epoch": 3.0,
+      "grad_norm": 3.0766594409942627,
+      "kl": 0.0020293756315368228,
       "learning_rate": 4.747379352713488e-07,
       "loss": 0.0,
-      "reward": 2.2802133609851203,
-      "reward_std": 1.2267065872438252,
-      "rewards/concensus_correctness_reward_func": 0.23658333408335844,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 1.9329118076711893,
+      "reward_std": 1.2741229943931103,
+      "rewards/concensus_correctness_reward_func": 0.12256250157952309,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6666666666666666,
-      "rewards/question_recreation_reward_func": 0.5471299886703491,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.619443034986034,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.4756666754838079,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.23778124805539846,
       "step": 18
     },
     {
-      "completion_length": 283.15625,
-      "epoch": 1.1142857142857143,
-      "grad_norm": 2.894879102706909,
-      "kl": 0.0032310091046383604,
+      "completion_length": 236.09375,
+      "epoch": 3.3333333333333335,
+      "grad_norm": 2.809358596801758,
+      "kl": 0.0026170795026700944,
       "learning_rate": 4.6717793412953776e-07,
       "loss": 0.0,
-      "reward": 3.1169432625174522,
-      "reward_std": 2.748181344475597,
-      "rewards/concensus_correctness_reward_func": 1.3761250004172325,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.8976972922682762,
+      "reward_std": 1.0421031441655941,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.4786620208178647,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5437285574153066,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5121562429703772,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.512593756429851,
       "step": 20
     },
     {
-      "completion_length": 262.40625,
-      "epoch": 1.2285714285714286,
-      "grad_norm": 2.271083116531372,
-      "kl": 0.003482333617284894,
+      "completion_length": 265.0625,
+      "epoch": 3.6666666666666665,
+      "grad_norm": 3.7295589447021484,
+      "kl": 0.003362223884323612,
       "learning_rate": 4.5870701325731773e-07,
       "loss": 0.0,
-      "reward": 3.1201551258563995,
-      "reward_std": 1.7736396370455623,
-      "rewards/concensus_correctness_reward_func": 1.0093125184066594,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.9837484806776047,
+      "reward_std": 0.753805372864008,
+      "rewards/concensus_correctness_reward_func": 0.11993750184774399,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.6677489534486085,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.4247172431787476,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.48996875435113907,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5015937560237944,
       "step": 22
     },
     {
-      "completion_length": 281.71875,
-      "epoch": 1.342857142857143,
-      "grad_norm": 2.927028179168701,
-      "kl": 0.004172757333435584,
+      "completion_length": 237.6875,
+      "epoch": 4.0,
+      "grad_norm": 2.7614455223083496,
+      "kl": 0.0042717494507087395,
       "learning_rate": 4.4936070264068016e-07,
       "loss": 0.0,
-      "reward": 1.7949492074549198,
-      "reward_std": 0.6816417840309441,
-      "rewards/concensus_correctness_reward_func": 0.09593750163912773,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.828284303745022,
+      "reward_std": 0.7124169690505369,
+      "rewards/concensus_correctness_reward_func": 0.05743750184774399,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.573636656627059,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.7025655592733528,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.5785000016912818,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.5057812501909211,
       "step": 24
     },
     {
-      "completion_length": 240.0625,
-      "epoch": 1.457142857142857,
-      "grad_norm": 2.403028726577759,
-      "kl": 0.0059904284426011145,
+      "completion_length": 296.71875,
+      "epoch": 4.333333333333333,
+      "grad_norm": 3.3926703929901123,
+      "kl": 0.0044053339806851,
       "learning_rate": 4.391782039544238e-07,
       "loss": 0.0,
-      "reward": 2.1940407678484917,
-      "reward_std": 0.970807102508843,
-      "rewards/concensus_correctness_reward_func": 0.160500000230968,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.3961677476763725,
+      "reward_std": 1.0368912005797029,
+      "rewards/concensus_correctness_reward_func": 0.024687500670552254,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5637594992294908,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.46885524597018957,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.7510312646627426,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.230750004760921,
       "step": 26
     },
     {
-      "completion_length": 291.59375,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 2.285961866378784,
-      "kl": 0.005350364925106987,
+      "completion_length": 208.375,
+      "epoch": 4.666666666666667,
+      "grad_norm": 3.8391273021698,
+      "kl": 0.005014390684664249,
       "learning_rate": 4.282022261367073e-07,
       "loss": 0.0,
-      "reward": 1.8530698530375957,
-      "reward_std": 0.817692642332986,
-      "rewards/concensus_correctness_reward_func": 0.17443750286474824,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.6386928837746382,
+      "reward_std": 0.5791225910652429,
+      "rewards/concensus_correctness_reward_func": 0.125,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5304135773330927,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6325937514193356,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5378491813316941,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.44459374831058085,
       "step": 28
     },
     {
-      "completion_length": 279.8125,
-      "epoch": 1.6857142857142857,
-      "grad_norm": 2.314671754837036,
-      "kl": 0.007266657623404171,
+      "completion_length": 285.625,
+      "epoch": 5.0,
+      "grad_norm": 3.2744433879852295,
+      "kl": 0.004670602691476233,
       "learning_rate": 4.1647880625292027e-07,
       "loss": 0.0,
-      "reward": 4.273585867136717,
-      "reward_std": 2.3483394776121713,
-      "rewards/concensus_correctness_reward_func": 2.1670625028200448,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.5773469675332308,
+      "reward_std": 0.8148748113308102,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.479585743509233,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5454094994347543,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.4863125025294721,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.3756875009275973,
       "step": 30
     },
     {
-      "completion_length": 284.1875,
-      "epoch": 1.8,
-      "grad_norm": 2.3709328174591064,
-      "kl": 0.0072359980258625,
+      "completion_length": 265.375,
+      "epoch": 5.333333333333333,
+      "grad_norm": 3.7803711891174316,
+      "kl": 0.0071189729642355815,
       "learning_rate": 4.040571164002318e-07,
       "loss": 0.0,
-      "reward": 2.3310193195939064,
-      "reward_std": 1.6677571701875422,
-      "rewards/concensus_correctness_reward_func": 0.6925625018775463,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.915034051053226,
+      "reward_std": 1.0130134378559887,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.7243943102657795,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5950653096660972,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.41406250291038305,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.42934375285403803,
       "step": 32
     },
     {
-      "completion_length": 304.84375,
-      "epoch": 1.9142857142857141,
-      "grad_norm": 2.7485358715057373,
-      "kl": 0.007408478311845101,
+      "completion_length": 257.96875,
+      "epoch": 5.666666666666667,
+      "grad_norm": 2.1448609828948975,
+      "kl": 0.006875298960949294,
       "learning_rate": 3.909892574627266e-07,
       "loss": 0.0,
-      "reward": 2.814433066174388,
-      "reward_std": 2.1739278100430965,
-      "rewards/concensus_correctness_reward_func": 1.0306875060778111,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 2.0118120573461056,
+      "reward_std": 1.1375641755294055,
+      "rewards/concensus_correctness_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.6048393156379461,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.4547183125978336,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30390625447034836,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5414687562733889,
       "step": 34
     },
     {
-      "completion_length": 329.5,
-      "epoch": 2.0,
-      "grad_norm": 1.5283843278884888,
-      "kl": 0.008974243615133068,
+      "completion_length": 228.03125,
+      "epoch": 6.0,
+      "grad_norm": 4.327014923095703,
+      "kl": 0.010515401692828164,
       "learning_rate": 3.773300405821908e-07,
       "loss": 0.0,
-      "reward": 5.211016138394673,
-      "reward_std": 2.3310749906425676,
-      "rewards/concensus_correctness_reward_func": 2.7943333238363266,
-      "rewards/consensus_reward_func": 0.75,
+      "reward": 2.3596832640469074,
+      "reward_std": 0.8677473589777946,
+      "rewards/concensus_correctness_reward_func": 0.29768750071525574,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5833333333333334,
-      "rewards/question_recreation_reward_func": 0.4819745300337672,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5928395111113787,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.041666666666666664,
-      "rewards/xmlcount_reward_func": 0.5597083357473215,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.6566562494263053,
       "step": 36
     },
     {
-      "completion_length": 306.46875,
-      "epoch": 2.1142857142857143,
-      "grad_norm": 2.286961555480957,
-      "kl": 0.01197911988128908,
+      "completion_length": 255.125,
+      "epoch": 6.333333333333333,
+      "grad_norm": 3.2016897201538086,
+      "kl": 0.010390633338829502,
       "learning_rate": 3.6313675726113475e-07,
       "loss": 0.0,
-      "reward": 2.252480274066329,
-      "reward_std": 1.2174878492951393,
-      "rewards/concensus_correctness_reward_func": 0.23100000014528632,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 2.1920023262500763,
+      "reward_std": 1.4953037183731794,
+      "rewards/concensus_correctness_reward_func": 0.18418750166893005,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5501990381162614,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5606585624627769,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.078125,
-      "rewards/xmlcount_reward_func": 0.5806562546640635,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3690312569960952,
       "step": 38
     },
     {
-      "completion_length": 280.3125,
-      "epoch": 2.2285714285714286,
-      "grad_norm": 1.9673209190368652,
-      "kl": 0.010138543642824516,
+      "completion_length": 255.21875,
+      "epoch": 6.666666666666667,
+      "grad_norm": 2.821410894393921,
+      "kl": 0.011521625056047924,
       "learning_rate": 3.484689390623218e-07,
       "loss": 0.0,
-      "reward": 2.7239319160580635,
-      "reward_std": 0.854443228803575,
-      "rewards/concensus_correctness_reward_func": 1.2991249989718199,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 2.26532906293869,
+      "reward_std": 1.2260184331098571,
+      "rewards/concensus_correctness_reward_func": 0.30037499964237213,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.6282755881547928,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6025790590792894,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.4059062581509352,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.4405000088736415,
       "step": 40
     },
     {
-      "completion_length": 258.71875,
-      "epoch": 2.342857142857143,
-      "grad_norm": 4.803712844848633,
-      "kl": 0.01296805081074126,
+      "completion_length": 296.125,
+      "epoch": 7.0,
+      "grad_norm": 2.361708641052246,
+      "kl": 0.009184279042528942,
       "learning_rate": 3.3338810791270517e-07,
       "loss": 0.0,
-      "reward": 3.26824863627553,
-      "reward_std": 0.7542198827723041,
-      "rewards/concensus_correctness_reward_func": 1.3208750002086163,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.8683023676276207,
+      "reward_std": 0.931499857455492,
+      "rewards/concensus_correctness_reward_func": 0.11750000342726707,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.4741549016907811,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5745836496353149,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.6450937520712614,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.3168437508866191,
       "step": 42
     },
     {
-      "completion_length": 297.875,
-      "epoch": 2.4571428571428573,
-      "grad_norm": 2.1548197269439697,
-      "kl": 0.010247149417409673,
+      "completion_length": 264.34375,
+      "epoch": 7.333333333333333,
+      "grad_norm": 3.3224129676818848,
+      "kl": 0.01854561164509505,
       "learning_rate": 3.179575180590857e-07,
       "loss": 0.0,
-      "reward": 3.1603624299168587,
-      "reward_std": 3.0717462142929435,
-      "rewards/concensus_correctness_reward_func": 1.3996250029304065,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.4564557429403067,
+      "reward_std": 0.8835583210457116,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.432518573012203,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5099557400681078,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3750937534496188,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.41524998657405376,
       "step": 44
     },
     {
-      "completion_length": 291.1875,
-      "epoch": 2.571428571428571,
-      "grad_norm": 2.1110994815826416,
-      "kl": 0.009947843587724492,
+      "completion_length": 302.34375,
+      "epoch": 7.666666666666667,
+      "grad_norm": 3.5825905799865723,
+      "kl": 0.013279616308864206,
       "learning_rate": 3.022418907578188e-07,
       "loss": 0.0,
-      "reward": 2.907536911778152,
-      "reward_std": 2.789290243992582,
-      "rewards/concensus_correctness_reward_func": 1.2540624998509884,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 2.223274264484644,
+      "reward_std": 0.9707561411778443,
+      "rewards/concensus_correctness_reward_func": 0.22681250050663948,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.6130369543097913,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5176492994651198,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4154375079087913,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5256875129416585,
       "step": 46
     },
     {
-      "completion_length": 312.21875,
-      "epoch": 2.685714285714286,
-      "grad_norm": 2.2388100624084473,
-      "kl": 0.012466569722164422,
+      "completion_length": 284.5,
+      "epoch": 8.0,
+      "grad_norm": 3.046675682067871,
+      "kl": 0.014580188668332994,
       "learning_rate": 2.863071428113726e-07,
       "loss": 0.0,
-      "reward": 3.700060784816742,
-      "reward_std": 1.4210271453484893,
-      "rewards/concensus_correctness_reward_func": 1.5416875053197145,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 2.079888518899679,
+      "reward_std": 1.115025261300616,
+      "rewards/concensus_correctness_reward_func": 0.125,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5754670230671763,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6062635215930641,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.6297812489792705,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.37987500336021185,
       "step": 48
     },
     {
-      "completion_length": 330.78125,
-      "epoch": 2.8,
-      "grad_norm": 3.0799405574798584,
-      "kl": 0.013010555092478171,
+      "completion_length": 283.40625,
+      "epoch": 8.333333333333334,
+      "grad_norm": 4.013537883758545,
+      "kl": 0.01412903075106442,
       "learning_rate": 2.7022011009035107e-07,
       "loss": 0.0,
-      "reward": 2.5463776104152203,
-      "reward_std": 1.2827392800245434,
-      "rewards/concensus_correctness_reward_func": 0.39993750234134495,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 1.7887509390711784,
+      "reward_std": 1.0781924333423376,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.6346588416490704,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4950321572832763,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.44928124244324863,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.46796874795109034,
       "step": 50
     },
     {
-      "completion_length": 258.96875,
-      "epoch": 2.914285714285714,
-      "grad_norm": 3.135465383529663,
-      "kl": 0.017136874987045303,
+      "completion_length": 255.75,
+      "epoch": 8.666666666666666,
+      "grad_norm": 3.4840548038482666,
+      "kl": 0.020143478643149137,
       "learning_rate": 2.540482672006254e-07,
       "loss": 0.0,
-      "reward": 5.302450358867645,
-      "reward_std": 4.525662333006039,
-      "rewards/concensus_correctness_reward_func": 2.898500017821789,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.2514243982732296,
+      "reward_std": 0.9967716456158087,
+      "rewards/concensus_correctness_reward_func": 0.14731250144541264,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.49360665678977966,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.7038306472823024,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.5665937489829957,
+      "rewards/strict_format_reward_func": 0.09375,
+      "rewards/xmlcount_reward_func": 0.5565312498365529,
       "step": 52
     },
     {
-      "completion_length": 257.5416666666667,
-      "epoch": 3.0,
-      "grad_norm": 1.707671880722046,
-      "kl": 0.016812068953489263,
+      "completion_length": 257.25,
+      "epoch": 9.0,
+      "grad_norm": 3.285977602005005,
+      "kl": 0.020005275029689074,
       "learning_rate": 2.37859444471388e-07,
       "loss": 0.0,
-      "reward": 2.3912815153598785,
-      "reward_std": 1.0656018268394594,
-      "rewards/concensus_correctness_reward_func": 0.21558333188295364,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.8586036376655102,
+      "reward_std": 0.8674497168976814,
+      "rewards/concensus_correctness_reward_func": 0.23737500235438347,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.5453648641705513,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4530098957475275,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.6095000064621369,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.4807187654078007,
       "step": 54
     },
     {
-      "completion_length": 333.34375,
-      "epoch": 3.1142857142857143,
-      "grad_norm": 1.6752287149429321,
-      "kl": 0.013997640533489175,
+      "completion_length": 240.28125,
+      "epoch": 9.333333333333334,
+      "grad_norm": 3.7038044929504395,
+      "kl": 0.018931806669570506,
       "learning_rate": 2.2172154345117894e-07,
       "loss": 0.0,
-      "reward": 5.073978617787361,
-      "reward_std": 1.2313855392858386,
-      "rewards/concensus_correctness_reward_func": 2.745937497355044,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.623201709240675,
+      "reward_std": 1.2911038948222995,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.590072276070714,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.48360797856003046,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.6598437521606684,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.3607187531888485,
       "step": 56
     },
     {
-      "completion_length": 278.4375,
-      "epoch": 3.2285714285714286,
-      "grad_norm": 2.5768773555755615,
-      "kl": 0.017602251144126058,
+      "completion_length": 269.84375,
+      "epoch": 9.666666666666666,
+      "grad_norm": 2.881014585494995,
+      "kl": 0.017237395339179784,
       "learning_rate": 2.0570225210519433e-07,
       "loss": 0.0,
-      "reward": 2.7841400876641273,
-      "reward_std": 1.728901147376746,
-      "rewards/concensus_correctness_reward_func": 0.8728750348091125,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.9306830489076674,
+      "reward_std": 1.3888351377099752,
+      "rewards/concensus_correctness_reward_func": 0.24249999597668648,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5445462940260768,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5115580353885889,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.7260937541723251,
+      "rewards/xmlcount_reward_func": 0.4735000031068921,
       "step": 58
     },
     {
-      "completion_length": 306.53125,
-      "epoch": 3.342857142857143,
-      "grad_norm": 2.259335517883301,
-      "kl": 0.015203700924757868,
+      "completion_length": 249.15625,
+      "epoch": 10.0,
+      "grad_norm": 3.1154470443725586,
+      "kl": 0.02515559794846922,
       "learning_rate": 1.8986876090843664e-07,
       "loss": 0.0,
-      "reward": 3.736787687987089,
-      "reward_std": 1.3046259782277048,
-      "rewards/concensus_correctness_reward_func": 1.5642499991226941,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 2.0984230153262615,
+      "reward_std": 1.3092318717390299,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5504127382300794,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5064230696298182,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.497125007212162,
+      "rewards/strict_format_reward_func": 0.078125,
+      "rewards/xmlcount_reward_func": 0.5787499956786633,
       "step": 60
     },
     {
-      "completion_length": 304.9375,
-      "epoch": 3.4571428571428573,
-      "grad_norm": 2.193737268447876,
-      "kl": 0.0159946876228787,
+      "completion_length": 253.25,
+      "epoch": 10.333333333333334,
+      "grad_norm": 3.137545347213745,
+      "kl": 0.024528241163352504,
       "learning_rate": 1.7428748102551234e-07,
       "loss": 0.0,
-      "reward": 2.9109396040439606,
-      "reward_std": 2.4176445261691697,
-      "rewards/concensus_correctness_reward_func": 0.8153750021010637,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.8578170202672482,
+      "reward_std": 1.1804295498877764,
+      "rewards/concensus_correctness_reward_func": 0.059812501072883606,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.6774083143100142,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.4470045296475291,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.43378125317394733,
+      "rewards/xmlcount_reward_func": 0.7416250004898757,
       "step": 62
     },
     {
-      "completion_length": 239.0625,
-      "epoch": 3.571428571428571,
-      "grad_norm": 2.8111119270324707,
-      "kl": 0.019410541572142392,
+      "completion_length": 223.3125,
+      "epoch": 10.666666666666666,
+      "grad_norm": 3.1050543785095215,
+      "kl": 0.02882619173033163,
       "learning_rate": 1.5902376575912814e-07,
       "loss": 0.0,
-      "reward": 3.376887336373329,
-      "reward_std": 2.5654359097243287,
-      "rewards/concensus_correctness_reward_func": 1.3211875017732382,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.743748527020216,
+      "reward_std": 0.6933234713651473,
+      "rewards/concensus_correctness_reward_func": 0.20481249876320362,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5570748917525634,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4549047634936869,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.6705000004731119,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4902812475338578,
       "step": 64
     },
     {
-      "completion_length": 290.0,
-      "epoch": 3.685714285714286,
-      "grad_norm": 2.42885160446167,
-      "kl": 0.020226228778483346,
+      "completion_length": 237.8125,
+      "epoch": 11.0,
+      "grad_norm": 3.4488277435302734,
+      "kl": 0.024283534032292664,
       "learning_rate": 1.4414163643562753e-07,
       "loss": 0.0,
-      "reward": 2.901755426079035,
-      "reward_std": 1.9001993554411456,
-      "rewards/concensus_correctness_reward_func": 0.9014999968931079,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.9523196145892143,
+      "reward_std": 1.559489093720913,
+      "rewards/concensus_correctness_reward_func": 0.14818750135600567,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.530786651186645,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.6100937562296167,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5911321062594652,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.43175000394694507,
       "step": 66
     },
     {
-      "completion_length": 291.90625,
-      "epoch": 3.8,
-      "grad_norm": 2.7818875312805176,
-      "kl": 0.015350552916061133,
+      "completion_length": 285.21875,
+      "epoch": 11.333333333333334,
+      "grad_norm": 3.1919057369232178,
+      "kl": 0.029086316528264433,
       "learning_rate": 1.2970351387729872e-07,
       "loss": 0.0,
-      "reward": 3.8678977601230145,
-      "reward_std": 3.3767369369015796,
-      "rewards/concensus_correctness_reward_func": 1.9805624762084335,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 1.9048342034220695,
+      "reward_std": 1.2717823022976518,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5122728096321225,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5372404857771471,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.40631250059232116,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.4324687570333481,
       "step": 68
     },
     {
-      "completion_length": 260.375,
-      "epoch": 3.914285714285714,
-      "grad_norm": 2.0774481296539307,
-      "kl": 0.018419178784824908,
+      "completion_length": 274.03125,
+      "epoch": 11.666666666666666,
+      "grad_norm": 4.114351749420166,
+      "kl": 0.028069639985915273,
       "learning_rate": 1.1576995658775404e-07,
       "loss": 0.0,
-      "reward": 2.9608126636594534,
-      "reward_std": 2.178357046446763,
-      "rewards/concensus_correctness_reward_func": 0.8325625006109476,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 1.8863378204405308,
+      "reward_std": 1.3141623558476567,
+      "rewards/concensus_correctness_reward_func": 0.12256250157952309,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.49425020324997604,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.5402499944902956,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5985253136605024,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.078125,
+      "rewards/xmlcount_reward_func": 0.3215000149793923,
       "step": 70
     },
     {
-      "completion_length": 297.1666666666667,
-      "epoch": 4.0,
-      "grad_norm": 1.390352487564087,
-      "kl": 0.015421232635465762,
+      "completion_length": 248.75,
+      "epoch": 12.0,
+      "grad_norm": 2.4701993465423584,
+      "kl": 0.020335440058261156,
       "learning_rate": 1.0239940674851941e-07,
       "loss": 0.0,
-      "reward": 2.524449576934179,
-      "reward_std": 1.8518620263785124,
-      "rewards/concensus_correctness_reward_func": 0.8333333333333334,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 2.2941419184207916,
+      "reward_std": 0.8937435210682452,
+      "rewards/concensus_correctness_reward_func": 0.11487500369548798,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.5952413380146027,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.5048919152468443,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.34587500027070445,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.627500012749806,
       "step": 72
     },
     {
-      "completion_length": 251.84375,
-      "epoch": 4.114285714285714,
-      "grad_norm": 2.091481924057007,
-      "kl": 0.01848960021743551,
+      "completion_length": 220.09375,
+      "epoch": 12.333333333333334,
+      "grad_norm": 2.9127869606018066,
+      "kl": 0.022209461370948702,
       "learning_rate": 8.964794509221507e-08,
       "loss": 0.0,
-      "reward": 3.833787925541401,
-      "reward_std": 3.6238025003112853,
-      "rewards/concensus_correctness_reward_func": 1.9853749983012676,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 2.0042398422956467,
+      "reward_std": 1.136198466643691,
+      "rewards/concensus_correctness_reward_func": 0.24262500181794167,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.4506316084880382,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.49439611518755555,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.5540312556549907,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5015937518328428,
       "step": 74
     },
     {
-      "completion_length": 289.90625,
-      "epoch": 4.228571428571429,
-      "grad_norm": 2.230874538421631,
-      "kl": 0.01755688912817277,
+      "completion_length": 273.3125,
+      "epoch": 12.666666666666666,
+      "grad_norm": 2.498856782913208,
+      "kl": 0.01810361386742443,
       "learning_rate": 7.756905568047392e-08,
       "loss": 0.0,
-      "reward": 4.188707336783409,
-      "reward_std": 3.5413391031324863,
-      "rewards/concensus_correctness_reward_func": 2.0821875180117786,
+      "reward": 1.7342982944101095,
+      "reward_std": 1.2671478418633342,
+      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
       "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.5773635879158974,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.575642024166882,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.5760312499478459,
+      "rewards/xmlcount_reward_func": 0.39553124993108213,
       "step": 76
     },
     {
-      "completion_length": 292.03125,
-      "epoch": 4.3428571428571425,
-      "grad_norm": 2.0806541442871094,
-      "kl": 0.019203898555133492,
+      "completion_length": 239.71875,
+      "epoch": 13.0,
+      "grad_norm": 2.6426241397857666,
+      "kl": 0.027558521192986518,
       "learning_rate": 6.621340157319996e-08,
       "loss": 0.0,
-      "reward": 4.9116186555475,
-      "reward_std": 2.788472333922982,
-      "rewards/concensus_correctness_reward_func": 2.791312505491078,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 2.0297871977090836,
+      "reward_std": 0.9375762529671192,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.5585249215364456,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.643943477421999,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.5461562548298389,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.4663437455892563,
       "step": 78
     },
     {
-      "completion_length": 286.4375,
-      "epoch": 4.457142857142857,
-      "grad_norm": 2.314354181289673,
-      "kl": 0.020370854035718367,
+      "completion_length": 217.0,
+      "epoch": 13.333333333333334,
+      "grad_norm": 2.7676427364349365,
+      "kl": 0.03532541490858421,
       "learning_rate": 5.5628612330087724e-08,
       "loss": 0.0,
-      "reward": 3.2957633957266808,
-      "reward_std": 1.2075208893074887,
-      "rewards/concensus_correctness_reward_func": 0.5571249965578318,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 2.5324832424521446,
+      "reward_std": 1.1609387751668692,
+      "rewards/concensus_correctness_reward_func": 0.1401250008493662,
+      "rewards/consensus_reward_func": 1.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.46320093097165227,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.4988894378184341,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.8379375012591481,
+      "rewards/xmlcount_reward_func": 0.5184687459841371,
       "step": 80
     },
     {
-      "completion_length": 320.75,
-      "epoch": 4.571428571428571,
-      "grad_norm": 1.8077600002288818,
-      "kl": 0.01631612313212827,
+      "completion_length": 263.84375,
+      "epoch": 13.666666666666666,
+      "grad_norm": 3.3539531230926514,
+      "kl": 0.02452456980245188,
       "learning_rate": 4.5859084235697235e-08,
       "loss": 0.0,
-      "reward": 1.9219548534601927,
-      "reward_std": 0.7796940823318437,
-      "rewards/concensus_correctness_reward_func": 0.15187500603497028,
+      "reward": 1.9322984656319022,
+      "reward_std": 1.3757213475182652,
+      "rewards/concensus_correctness_reward_func": 0.12262500077486038,
       "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5343610807321966,
+      "rewards/question_recreation_reward_func": 0.5088922204449773,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.5169687578454614,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.5664062486030161,
       "step": 82
     },
     {
-      "completion_length": 272.25,
-      "epoch": 4.685714285714286,
-      "grad_norm": 2.141181707382202,
-      "kl": 0.017336227814666927,
+      "completion_length": 258.9375,
+      "epoch": 14.0,
+      "grad_norm": 3.50706148147583,
+      "kl": 0.02300187383661978,
       "learning_rate": 3.6945794086007705e-08,
       "loss": 0.0,
-      "reward": 4.1498072892427444,
-      "reward_std": 3.9383774576708674,
-      "rewards/concensus_correctness_reward_func": 2.1485625030472875,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 2.368156984448433,
+      "reward_std": 1.27269869716838,
+      "rewards/concensus_correctness_reward_func": 0.23487500101327896,
+      "rewards/consensus_reward_func": 0.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5559322023764253,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.6125319767743349,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.24218749906867743,
+      "rewards/xmlcount_reward_func": 0.42700000666081905,
       "step": 84
     },
     {
-      "completion_length": 263.3125,
-      "epoch": 4.8,
-      "grad_norm": 3.1173408031463623,
-      "kl": 0.022343544085742906,
+      "completion_length": 238.8125,
+      "epoch": 14.333333333333334,
+      "grad_norm": 2.6380765438079834,
+      "kl": 0.027566006290726364,
       "learning_rate": 2.892612731749414e-08,
       "loss": 0.0,
-      "reward": 4.092738252133131,
-      "reward_std": 1.894249181394116,
-      "rewards/concensus_correctness_reward_func": 2.0336249992251396,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 2.055258920416236,
+      "reward_std": 0.6858638301491737,
+      "rewards/concensus_correctness_reward_func": 0.18012499809265137,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5459569627419114,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5494776804698631,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.5756562538444996,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5600312501192093,
       "step": 86
     },
     {
-      "completion_length": 257.625,
-      "epoch": 4.914285714285715,
-      "grad_norm": 2.271671772003174,
-      "kl": 0.019497435074299574,
+      "completion_length": 283.0,
+      "epoch": 14.666666666666666,
+      "grad_norm": 2.740476369857788,
+      "kl": 0.02960860973689705,
       "learning_rate": 2.183372119961499e-08,
       "loss": 0.0,
-      "reward": 2.9152057096362114,
-      "reward_std": 2.1247581483912654,
-      "rewards/concensus_correctness_reward_func": 0.7478749994188547,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.8798525519669056,
+      "reward_std": 1.1110668628825806,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5662681483663619,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.7729375008493662,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5739150438457727,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.37081251526251435,
       "step": 88
     },
     {
-      "completion_length": 295.5833333333333,
-      "epoch": 5.0,
-      "grad_norm": 2.0402109622955322,
-      "kl": 0.023389367774749797,
+      "completion_length": 242.3125,
+      "epoch": 15.0,
+      "grad_norm": 2.513399124145508,
+      "kl": 0.026559468533378094,
       "learning_rate": 1.5698323748414122e-08,
       "loss": 0.0,
-      "reward": 4.139378140370051,
-      "reward_std": 2.417986176908016,
-      "rewards/concensus_correctness_reward_func": 1.5169999947150548,
-      "rewards/consensus_reward_func": 0.8333333333333334,
+      "reward": 2.03849926404655,
+      "reward_std": 0.964166424702853,
+      "rewards/concensus_correctness_reward_func": 0.14824999682605267,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.628961397955815,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5135617647320032,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.660083344206214,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.5016875043511391,
       "step": 90
     },
     {
-      "completion_length": 284.21875,
-      "epoch": 5.114285714285714,
-      "grad_norm": 2.9972126483917236,
-      "kl": 0.020673706487286836,
+      "completion_length": 205.65625,
+      "epoch": 15.333333333333334,
+      "grad_norm": 3.1790051460266113,
+      "kl": 0.036513498693238944,
       "learning_rate": 1.054566895300324e-08,
       "loss": 0.0,
-      "reward": 5.346356093883514,
-      "reward_std": 3.1238869512453675,
-      "rewards/concensus_correctness_reward_func": 2.7256875003222376,
+      "reward": 2.030654199421406,
+      "reward_std": 1.0498718353919685,
+      "rewards/concensus_correctness_reward_func": 0.08824999816715717,
       "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.5663561150431633,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6112791877239943,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6168124983087182,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.596749996766448,
       "step": 92
     },
     {
-      "completion_length": 320.9375,
-      "epoch": 5.228571428571429,
-      "grad_norm": 2.07313871383667,
-      "kl": 0.015393795300042257,
+      "completion_length": 263.0,
+      "epoch": 15.666666666666666,
+      "grad_norm": 3.719067096710205,
+      "kl": 0.021271194782457314,
       "learning_rate": 6.397368838268496e-09,
       "loss": 0.0,
-      "reward": 4.471762063913047,
-      "reward_std": 2.338888173457235,
-      "rewards/concensus_correctness_reward_func": 2.6311249914579093,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.2683150228112936,
+      "reward_std": 1.4738470809534192,
+      "rewards/concensus_correctness_reward_func": 0.36025000363588333,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.6448872559703887,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5795337841846049,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.44575000391341746,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.5941562601365149,
       "step": 94
     },
     {
-      "completion_length": 299.75,
-      "epoch": 5.3428571428571425,
-      "grad_norm": 1.8763506412506104,
-      "kl": 0.019261252775322646,
+      "completion_length": 257.71875,
+      "epoch": 16.0,
+      "grad_norm": 5.480099201202393,
+      "kl": 0.028246068861335516,
       "learning_rate": 3.2708228165273244e-09,
       "loss": 0.0,
-      "reward": 4.042540363967419,
-      "reward_std": 1.5998169761151075,
-      "rewards/concensus_correctness_reward_func": 2.1019999987911433,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 2.2440423257648945,
+      "reward_std": 0.9810051110107452,
+      "rewards/concensus_correctness_reward_func": 0.18512500077486038,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.602071626111865,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4930110676214099,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.4478437500074506,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.5190312538761646,
       "step": 96
     },
     {
-      "completion_length": 261.5625,
-      "epoch": 5.457142857142857,
-      "grad_norm": 2.159895420074463,
-      "kl": 0.023489647486712784,
+      "completion_length": 239.34375,
+      "epoch": 16.333333333333332,
+      "grad_norm": 3.926326274871826,
+      "kl": 0.03365567361470312,
       "learning_rate": 1.1791447083465133e-09,
       "loss": 0.0,
-      "reward": 2.6311225779354572,
-      "reward_std": 1.2659746335702948,
-      "rewards/concensus_correctness_reward_func": 0.3871250026859343,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.9388289339840412,
+      "reward_std": 0.958217971608974,
+      "rewards/concensus_correctness_reward_func": 0.11999999731779099,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.6012788028456271,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4502664606552571,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.6895937495864928,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.6185624990612268,
       "step": 98
     },
     {
-      "completion_length": 256.0,
-      "epoch": 5.571428571428571,
-      "grad_norm": 2.200911045074463,
-      "kl": 0.0240181177505292,
+      "completion_length": 271.6875,
+      "epoch": 16.666666666666668,
+      "grad_norm": 2.2361977100372314,
+      "kl": 0.02408603549702093,
       "learning_rate": 1.3110773862126667e-10,
       "loss": 0.0,
-      "reward": 4.581089836545289,
-      "reward_std": 4.262082199566066,
-      "rewards/concensus_correctness_reward_func": 2.612312503159046,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.8076389655470848,
+      "reward_std": 1.001751037707436,
+      "rewards/concensus_correctness_reward_func": 0.2303125038743019,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.5605585183948278,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5585764544084668,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.5644687572494149,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.4718750088359229,
       "step": 100
     },
     {
-      "epoch": 5.571428571428571,
+      "epoch": 16.666666666666668,
       "step": 100,
       "total_flos": 0.0,
-      "train_loss": 1.1608919289756158e-05,
-      "train_runtime": 2917.0763,
-      "train_samples_per_second": 0.548,
-      "train_steps_per_second": 0.034
+      "train_loss": 1.5718676149845123e-05,
+      "train_runtime": 4262.7362,
+      "train_samples_per_second": 0.375,
+      "train_steps_per_second": 0.023
     }
   ],
   "logging_steps": 2,
   "max_steps": 100,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 6,
+  "num_train_epochs": 17,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03d6f864640ff2f7874aaff157b6b0c8d4f48abf640964a2ea7de104211b2b07
+oid sha256:f771f4e41b930eadf03c7a0abf7ec0d72660f696af7340eebd672e2ccbbd1025
 size 5944