End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.685158779058838e-06,
-    "train_runtime": 302.846,
-    "train_samples": 40,
-    "train_samples_per_second": 1.057,
-    "train_steps_per_second": 0.066
+    "train_loss": 1.3907960237702355e-05,
+    "train_runtime": 296.6974,
+    "train_samples": 28,
+    "train_samples_per_second": 1.079,
+    "train_steps_per_second": 0.067
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b87cd5798bae1a120f63152fc9e8affa23b3526c63ecbd89bdad24995061441d
+oid sha256:668b1c6f18310538c4cbc396b18174ae67c78ea0278d324c8903770a2e0ef8fe
 size 4996670464

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d9e76e5ffe675f74cf685e40de4c9fc726f1670b431cdb8b443f4bf4dd7fc0d
+oid sha256:ac89b813fd1680d221ebaa5cbcbf47d5de7c5cb985e1322fe320c3b491cf0a78
 size 1178224960

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.685158779058838e-06,
-    "train_runtime": 302.846,
-    "train_samples": 40,
-    "train_samples_per_second": 1.057,
-    "train_steps_per_second": 0.066
+    "train_loss": 1.3907960237702355e-05,
+    "train_runtime": 296.6974,
+    "train_samples": 28,
+    "train_samples_per_second": 1.079,
+    "train_steps_per_second": 0.067
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4.0,
+  "epoch": 5.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 229.40625,
-      "epoch": 0.4,
-      "grad_norm": 2.872025966644287,
+      "completion_length": 305.0,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 3.332977056503296,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 3.4632836803793907,
-      "reward_std": 1.2683328343555331,
-      "rewards/concensus_correctness_reward_func": 0.8254375010728836,
-      "rewards/consensus_reward_func": 0.8125,
+      "reward": 4.415860116481781,
+      "reward_std": 1.516854090616107,
+      "rewards/concensus_correctness_reward_func": 1.8175624976865947,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.6735961344093084,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.7081100679934025,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.09375,
-      "rewards/xmlcount_reward_func": 0.6830000188201666,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.8276875019073486,
       "step": 2
     },
     {
-      "completion_length": 249.84375,
-      "epoch": 0.8,
-      "grad_norm": 4.029212474822998,
-      "kl": 0.0013401164214883465,
+      "completion_length": 261.5833333333333,
+      "epoch": 1.0,
+      "grad_norm": 2.668982982635498,
+      "kl": 0.0025756577160791494,
       "learning_rate": 4.864543104251586e-07,
       "loss": -0.0,
-      "reward": 4.485839881002903,
-      "reward_std": 1.8941392286214978,
-      "rewards/concensus_correctness_reward_func": 1.227875005453825,
-      "rewards/consensus_reward_func": 1.1875,
+      "reward": 5.931286990642548,
+      "reward_std": 1.4188148624186094,
+      "rewards/concensus_correctness_reward_func": 1.8613333503405254,
+      "rewards/consensus_reward_func": 1.1666666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.6534336768090725,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.6925787776708603,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.09375,
-      "rewards/xmlcount_reward_func": 0.8857812620699406,
+      "rewards/strict_format_reward_func": 0.125,
+      "rewards/xmlcount_reward_func": 1.0857083400090535,
       "step": 4
     },
     {
-      "completion_length": 249.0,
-      "epoch": 1.2,
-      "grad_norm": 3.2268619537353516,
-      "kl": 0.0020504993153735995,
+      "completion_length": 291.03125,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 2.482466697692871,
+      "kl": 0.005271486799756531,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 4.873421519994736,
-      "reward_std": 1.395420724556061,
-      "rewards/concensus_correctness_reward_func": 1.4667499959468842,
-      "rewards/consensus_reward_func": 1.1875,
+      "reward": 5.361043348908424,
+      "reward_std": 2.4409327004104853,
+      "rewards/concensus_correctness_reward_func": 1.7483749873936176,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.6557028242386878,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.72888717148453,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.078125,
-      "rewards/xmlcount_reward_func": 1.1103437542915344,
+      "rewards/strict_format_reward_func": 0.171875,
+      "rewards/xmlcount_reward_func": 1.0244062431156635,
       "step": 6
     },
     {
-      "completion_length": 279.84375,
-      "epoch": 1.6,
-      "grad_norm": 2.6787109375,
-      "kl": 0.002656653927260777,
+      "completion_length": 264.375,
+      "epoch": 2.0,
+      "grad_norm": 2.265690803527832,
+      "kl": 0.010897997921953598,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 3.922634281218052,
-      "reward_std": 1.499501348240301,
-      "rewards/concensus_correctness_reward_func": 1.0228124931454659,
-      "rewards/consensus_reward_func": 1.0625,
+      "reward": 6.615325629711151,
+      "reward_std": 1.6580882593989372,
+      "rewards/concensus_correctness_reward_func": 2.7746666657427945,
+      "rewards/consensus_reward_func": 1.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.5061030379729345,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.6600756930808226,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.8468437492847443,
+      "rewards/strict_format_reward_func": 0.08333333333333333,
+      "rewards/xmlcount_reward_func": 1.0139166687925656,
       "step": 8
     },
     {
-      "completion_length": 213.53125,
-      "epoch": 2.0,
-      "grad_norm": 3.4874277114868164,
-      "kl": 0.003580357151804492,
+      "completion_length": 252.75,
+      "epoch": 2.571428571428571,
+      "grad_norm": 2.7409284114837646,
+      "kl": 0.016758157726144418,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 4.8787659257650375,
-      "reward_std": 0.9820138990326086,
-      "rewards/concensus_correctness_reward_func": 1.5206249915063381,
-      "rewards/consensus_reward_func": 1.4375,
+      "reward": 4.8129510916769505,
+      "reward_std": 2.2558501958847046,
+      "rewards/concensus_correctness_reward_func": 1.6773124877363443,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6674221679568291,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.5812323654536158,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.09375,
-      "rewards/xmlcount_reward_func": 1.1594687700271606,
+      "rewards/strict_format_reward_func": 0.125,
+      "rewards/xmlcount_reward_func": 0.9919062685221434,
       "step": 10
     },
     {
-      "completion_length": 233.15625,
-      "epoch": 2.4,
-      "grad_norm": 3.6188204288482666,
-      "kl": 0.004368863592389971,
+      "completion_length": 280.9583333333333,
+      "epoch": 3.0,
+      "grad_norm": 1.7997511625289917,
+      "kl": 0.01736647670622915,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 4.069844126701355,
-      "reward_std": 1.5751521023921669,
-      "rewards/concensus_correctness_reward_func": 0.9566249996423721,
-      "rewards/consensus_reward_func": 0.8125,
+      "reward": 5.146967609723409,
+      "reward_std": 1.518940186748902,
+      "rewards/concensus_correctness_reward_func": 1.3999999947845936,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.660312901251018,
+      "rewards/final_correctness_reward_func": 1.0833333333333333,
+      "rewards/question_recreation_reward_func": 0.5750093022361398,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.140625,
-      "rewards/xmlcount_reward_func": 1.1872812695801258,
+      "rewards/strict_format_reward_func": 0.10416666666666667,
+      "rewards/xmlcount_reward_func": 0.9844583372275034,
       "step": 12
     },
     {
-      "completion_length": 222.28125,
-      "epoch": 2.8,
-      "grad_norm": 2.7793495655059814,
-      "kl": 0.004178055489319377,
+      "completion_length": 282.625,
+      "epoch": 3.571428571428571,
+      "grad_norm": 2.471196174621582,
+      "kl": 0.02955857322376687,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 4.857381120324135,
-      "reward_std": 0.6476292543957243,
-      "rewards/concensus_correctness_reward_func": 1.4023125022649765,
-      "rewards/consensus_reward_func": 1.25,
+      "reward": 4.996489986777306,
+      "reward_std": 0.9792956251185387,
+      "rewards/concensus_correctness_reward_func": 1.1874375059269369,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5974122577463277,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.7584899198263884,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 1.1701562851667404,
+      "rewards/strict_format_reward_func": 0.125,
+      "rewards/xmlcount_reward_func": 1.113062508404255,
       "step": 14
     },
     {
-      "completion_length": 236.34375,
-      "epoch": 3.2,
-      "grad_norm": 3.321694850921631,
-      "kl": 0.005652376130456105,
+      "completion_length": 271.3333333333333,
+      "epoch": 4.0,
+      "grad_norm": 2.0855813026428223,
+      "kl": 0.03525549483795961,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 4.658863767981529,
-      "reward_std": 1.2408572635613382,
-      "rewards/concensus_correctness_reward_func": 1.0198749899864197,
-      "rewards/consensus_reward_func": 1.375,
+      "reward": 5.945889592170715,
+      "reward_std": 1.0654800638052013,
+      "rewards/concensus_correctness_reward_func": 1.6479166795810063,
+      "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.6602074466645718,
+      "rewards/final_correctness_reward_func": 1.1666666666666667,
+      "rewards/question_recreation_reward_func": 0.6908895298838615,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.078125,
-      "rewards/xmlcount_reward_func": 1.0881562680006027,
+      "rewards/strict_format_reward_func": 0.14583333333333334,
+      "rewards/xmlcount_reward_func": 1.0445833404858906,
       "step": 16
     },
     {
-      "completion_length": 218.09375,
-      "epoch": 3.6,
-      "grad_norm": 3.550179958343506,
-      "kl": 0.006271327467402443,
+      "completion_length": 264.15625,
+      "epoch": 4.571428571428571,
+      "grad_norm": 2.862443208694458,
+      "kl": 0.022886726481374353,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 4.894970387220383,
-      "reward_std": 1.2736626090481877,
-      "rewards/concensus_correctness_reward_func": 1.3746875002980232,
-      "rewards/consensus_reward_func": 1.375,
+      "reward": 5.546127021312714,
+      "reward_std": 2.9458585816901177,
+      "rewards/concensus_correctness_reward_func": 1.915937508456409,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5793766193091869,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 1.1284062750637531,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.7025957200676203,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.140625,
+      "rewards/xmlcount_reward_func": 1.0838437527418137,
       "step": 18
     },
     {
-      "completion_length": 237.625,
-      "epoch": 4.0,
-      "grad_norm": 2.723349094390869,
-      "kl": 0.004173626010015141,
+      "completion_length": 319.5833333333333,
+      "epoch": 5.0,
+      "grad_norm": 2.0680243968963623,
+      "kl": 0.01664670541261633,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 4.361241207458079,
-      "reward_std": 1.093166567850858,
-      "rewards/concensus_correctness_reward_func": 1.2548750005662441,
-      "rewards/consensus_reward_func": 1.0625,
+      "reward": 4.7121826608975725,
+      "reward_std": 0.9731876458972692,
+      "rewards/concensus_correctness_reward_func": 1.4983333547910054,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.591022448381409,
+      "rewards/final_correctness_reward_func": 0.8333333333333334,
+      "rewards/question_recreation_reward_func": 0.6560991778969765,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 1.0465937703847885,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.8285833323995272,
       "step": 20
     },
     {
-      "epoch": 4.0,
+      "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.685158779058838e-06,
-      "train_runtime": 302.846,
-      "train_samples_per_second": 1.057,
-      "train_steps_per_second": 0.066
+      "train_loss": 1.3907960237702355e-05,
+      "train_runtime": 296.6974,
+      "train_samples_per_second": 1.079,
+      "train_steps_per_second": 0.067
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
+  "num_train_epochs": 7,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {