End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.002904082089662552,
-    "train_runtime": 267.4469,
+    "train_loss": 1.9995495676994324e-06,
+    "train_runtime": 295.6982,
     "train_samples": 28,
-    "train_samples_per_second": 1.196,
-    "train_steps_per_second": 0.075
+    "train_samples_per_second": 1.082,
+    "train_steps_per_second": 0.068
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f266c3df6739323c01a42905c9500363cdaab26c0e6ff4dd7ed661ebec7160de
+oid sha256:b9410acafe0ea79efda62b46323f7a5d516b016a5bb85a02ddbbbb66e97509d0
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.002904082089662552,
-    "train_runtime": 267.4469,
+    "train_loss": 1.9995495676994324e-06,
+    "train_runtime": 295.6982,
     "train_samples": 28,
-    "train_samples_per_second": 1.196,
-    "train_steps_per_second": 0.075
+    "train_samples_per_second": 1.082,
+    "train_steps_per_second": 0.068
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 364.53125,
+      "completion_length": 445.125,
       "epoch": 0.5714285714285714,
-      "grad_norm": 36.14985275268555,
+      "grad_norm": 5.819187164306641,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.376720666885376,
-      "reward_std": 1.1314093207474798,
-      "rewards/concensus_correctness_reward_func": 0.6728749983012676,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.4299878163728863,
+      "reward_std": 0.4049527835159097,
+      "rewards/concensus_correctness_reward_func": 0.01924999989569187,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.38990818394813687,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2543628151761368,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06393750756978989,
+      "rewards/xmlcount_reward_func": 0.15637499606236815,
       "step": 2
     },
     {
-      "completion_length": 338.6666666666667,
+      "completion_length": 446.0,
       "epoch": 1.0,
-      "grad_norm": 4.0901618003845215,
-      "kl": 0.0011947738579086338,
+      "grad_norm": 156.14402770996094,
+      "kl": 0.0013681109024522204,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.882365383207798,
-      "reward_std": 0.8381643230095506,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 1.3144166320562363,
+      "reward_std": 2.482520199380815,
+      "rewards/concensus_correctness_reward_func": 0.8333333333333334,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.453323711330692,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.3682499380471806,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.26237500583132106,
+      "rewards/xmlcount_reward_func": -0.13716666338344416,
       "step": 4
     },
     {
-      "completion_length": 361.8125,
+      "completion_length": 434.8125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 10.009440422058105,
-      "kl": 0.01511065720114857,
+      "grad_norm": 6.273466110229492,
+      "kl": 0.0015552927852695575,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.120104871224612,
-      "reward_std": 1.1055626115048653,
-      "rewards/concensus_correctness_reward_func": 0.0911875031888485,
+      "reward": 1.4866644144058228,
+      "reward_std": 0.9072726890444756,
+      "rewards/concensus_correctness_reward_func": 0.10912499949336052,
       "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.39094860665500164,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.5983831799821928,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3879687525331974,
+      "rewards/xmlcount_reward_func": 0.2791562443599105,
       "step": 6
     },
     {
-      "completion_length": 325.6666666666667,
+      "completion_length": 447.1666666666667,
       "epoch": 2.0,
-      "grad_norm": 69.15676879882812,
-      "kl": 10.71556863428365,
+      "grad_norm": 2.2120509147644043,
+      "kl": 0.001463284672354348,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.008,
-      "reward": 1.1593825854361057,
-      "reward_std": 0.8130506893018415,
-      "rewards/concensus_correctness_reward_func": 0.15424999594688416,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "loss": 0.0,
+      "reward": 1.0152752523620923,
+      "reward_std": 1.0958157057563465,
+      "rewards/concensus_correctness_reward_func": 0.0798333336909612,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4213409485916297,
+      "rewards/question_recreation_reward_func": 0.5489002677301565,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.25045833612481755,
+      "rewards/xmlcount_reward_func": 0.303208331267039,
       "step": 8
     },
     {
-      "completion_length": 349.96875,
+      "completion_length": 353.625,
       "epoch": 2.571428571428571,
-      "grad_norm": 10.420665740966797,
-      "kl": 0.004905327987216879,
+      "grad_norm": 6.458411693572998,
+      "kl": 0.002351721930608619,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.8669011471793056,
-      "reward_std": 1.4712725542485714,
-      "rewards/concensus_correctness_reward_func": 0.7862500250339508,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.1473189276875928,
+      "reward_std": 0.6658630494493991,
+      "rewards/concensus_correctness_reward_func": 0.16381249763071537,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.42768238531425595,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.4298502024030313,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.27796874823980033,
+      "rewards/xmlcount_reward_func": 0.241156249307096,
       "step": 10
     },
     {
-      "completion_length": 435.1666666666667,
+      "completion_length": 430.0833333333333,
       "epoch": 3.0,
-      "grad_norm": 3.3437094688415527,
-      "kl": 0.0076461510713367415,
+      "grad_norm": 3.1426191329956055,
+      "kl": 0.0019996731619661054,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.6248732234040896,
-      "reward_std": 1.62039125066561,
-      "rewards/concensus_correctness_reward_func": 0.1255833332737287,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 1.2683721954623859,
+      "reward_std": 0.6849768863370022,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.504498228430748,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.6212054851154486,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.1718750074505806,
+      "rewards/xmlcount_reward_func": 0.5638333459695181,
       "step": 12
     },
     {
-      "completion_length": 319.96875,
+      "completion_length": 355.78125,
       "epoch": 3.571428571428571,
-      "grad_norm": 75.37753295898438,
-      "kl": 10.455819064613024,
+      "grad_norm": 11.594252586364746,
+      "kl": 0.004082114413904492,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0105,
-      "reward": 0.9599005761556327,
-      "reward_std": 0.5887293635169044,
-      "rewards/concensus_correctness_reward_func": 0.0012499999720603228,
+      "loss": 0.0,
+      "reward": 0.7430537855252624,
+      "reward_std": 0.5826017245417461,
+      "rewards/concensus_correctness_reward_func": 0.012249999679625034,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.4016505675390363,
+      "rewards/question_recreation_reward_func": 0.4506162931211293,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3538750000298023,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.09268749738112092,
       "step": 14
     },
     {
-      "completion_length": 309.9166666666667,
+      "completion_length": 427.0,
       "epoch": 4.0,
-      "grad_norm": 3.6160662174224854,
-      "kl": 1.1093916914154154,
+      "grad_norm": 4.296351432800293,
+      "kl": 0.0034662123459080854,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0008,
-      "reward": 1.3102250987042983,
-      "reward_std": 0.7835694439709187,
-      "rewards/concensus_correctness_reward_func": 0.15491666893164316,
-      "rewards/consensus_reward_func": 0.25,
+      "loss": 0.0,
+      "reward": 0.9003260085980097,
+      "reward_std": 1.165814973724385,
+      "rewards/concensus_correctness_reward_func": 0.03133333226044973,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.4328917254072924,
-      "rewards/soft_format_reward_func": 0.020833333333333332,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.432784349968036,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2849166626110673,
+      "rewards/xmlcount_reward_func": 0.2695416770875454,
       "step": 16
     },
     {
-      "completion_length": 375.5625,
+      "completion_length": 399.96875,
       "epoch": 4.571428571428571,
-      "grad_norm": 39.27841567993164,
-      "kl": 9.685058332230255,
+      "grad_norm": 5.107125282287598,
+      "kl": 0.003304996556835249,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0097,
-      "reward": 0.5228178105317056,
-      "reward_std": 0.7538906417321414,
-      "rewards/concensus_correctness_reward_func": -0.00912499986588955,
-      "rewards/consensus_reward_func": 0.0,
+      "loss": 0.0,
+      "reward": 0.9801066992804408,
+      "reward_std": 0.8676199330948293,
+      "rewards/concensus_correctness_reward_func": 0.025812499225139618,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3308178153820336,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.48079418716952205,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13862499594688416,
+      "rewards/xmlcount_reward_func": 0.2235000031068921,
       "step": 18
     },
     {
-      "completion_length": 434.7916666666667,
+      "completion_length": 383.7083333333333,
       "epoch": 5.0,
-      "grad_norm": 2.2608134746551514,
-      "kl": 0.005378825153457001,
+      "grad_norm": 3.078273296356201,
+      "kl": 0.0034112156621025256,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.1370763749194641,
-      "reward_std": 1.0841339537873864,
-      "rewards/concensus_correctness_reward_func": 0.07991666595141093,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 1.831272717565298,
+      "reward_std": 1.855681760236621,
+      "rewards/concensus_correctness_reward_func": 0.9507499982913336,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.4950347263365984,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.541897700012972,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.31212499861915904,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.06779166931907336,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.002904082089662552,
-      "train_runtime": 267.4469,
-      "train_samples_per_second": 1.196,
-      "train_steps_per_second": 0.075
+      "train_loss": 1.9995495676994324e-06,
+      "train_runtime": 295.6982,
+      "train_samples_per_second": 1.082,
+      "train_steps_per_second": 0.068
     }
   ],
   "logging_steps": 2,