End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.002027319371700287,
-    "train_runtime": 243.1978,
+    "train_loss": 3.66260876035085e-05,
+    "train_runtime": 246.0607,
     "train_samples": 28,
-    "train_samples_per_second": 1.316,
-    "train_steps_per_second": 0.082
+    "train_samples_per_second": 1.3,
+    "train_steps_per_second": 0.081
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:43f4d6a68291b6e540439e102b1a513bb643e089f3f5c2e3fcb81aa36b4299a4
+oid sha256:1060f523ecb6d3c5ac672f76e2e07804dc6e3009436f56a0cae7a2d47a8247b8
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.002027319371700287,
-    "train_runtime": 243.1978,
+    "train_loss": 3.66260876035085e-05,
+    "train_runtime": 246.0607,
     "train_samples": 28,
-    "train_samples_per_second": 1.316,
-    "train_steps_per_second": 0.082
+    "train_samples_per_second": 1.3,
+    "train_steps_per_second": 0.081
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 420.125,
+      "completion_length": 342.96875,
       "epoch": 0.5714285714285714,
-      "grad_norm": 9.172501564025879,
+      "grad_norm": 4.636115550994873,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.8523840941488743,
-      "reward_std": 1.3467477597296238,
-      "rewards/concensus_correctness_reward_func": 0.3463125005364418,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.160812631249428,
+      "reward_std": 0.6587000478175469,
+      "rewards/concensus_correctness_reward_func": 0.4399375021457672,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.48507160134613514,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5692501617595553,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5210000015795231,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.35475000459700823,
       "step": 2
     },
     {
-      "completion_length": 393.7916666666667,
+      "completion_length": 475.5416666666667,
       "epoch": 1.0,
-      "grad_norm": 2.854651927947998,
-      "kl": 0.0015513162773762208,
+      "grad_norm": 2.719712257385254,
+      "kl": 0.0018013427131033193,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.2819704189896584,
-      "reward_std": 1.3122997771327694,
-      "rewards/concensus_correctness_reward_func": 0.38741667071978253,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 3.882448842128118,
+      "reward_std": 2.6914940675099692,
+      "rewards/concensus_correctness_reward_func": 1.1753333335121472,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5407620910555124,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.6292404855291048,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8537916590770086,
+      "rewards/xmlcount_reward_func": 0.9112083464860916,
       "step": 4
     },
     {
-      "completion_length": 282.84375,
+      "completion_length": 292.09375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 7.285346984863281,
-      "kl": 0.005276600772049278,
+      "grad_norm": 8.258121490478516,
+      "kl": 0.029607473421492614,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 2.876493025571108,
-      "reward_std": 1.3032324677624274,
-      "rewards/concensus_correctness_reward_func": 0.42343749664723873,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.336483985185623,
+      "reward_std": 0.8101844638586044,
+      "rewards/concensus_correctness_reward_func": 0.14556249976158142,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.6550554933492094,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5946714862948284,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.985499981790781,
+      "rewards/xmlcount_reward_func": 0.9087500139139593,
       "step": 6
     },
     {
-      "completion_length": 280.5833333333333,
+      "completion_length": 367.3333333333333,
       "epoch": 2.0,
-      "grad_norm": 6.3203630447387695,
-      "kl": 0.009360586351249367,
+      "grad_norm": 2.8354005813598633,
+      "kl": 0.031232848976893973,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 3.3328624963760376,
-      "reward_std": 0.9424603825124601,
-      "rewards/concensus_correctness_reward_func": 0.44849999745686847,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 3.037147363026937,
+      "reward_std": 1.0202810720851023,
+      "rewards/concensus_correctness_reward_func": 0.3407499964038531,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6666666666666666,
-      "rewards/question_recreation_reward_func": 0.750654149800539,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.7943973168730736,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9670416538914045,
+      "rewards/xmlcount_reward_func": 0.9020000025629997,
       "step": 8
     },
     {
-      "completion_length": 255.125,
+      "completion_length": 243.59375,
       "epoch": 2.571428571428571,
-      "grad_norm": 8.027791023254395,
-      "kl": 0.020161927182925865,
+      "grad_norm": 9.608664512634277,
+      "kl": 0.02638791529170703,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 2.9883331805467606,
-      "reward_std": 1.5057067006709985,
-      "rewards/concensus_correctness_reward_func": 0.31987499445676804,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 2.3775548972189426,
+      "reward_std": 1.3022976123647823,
+      "rewards/concensus_correctness_reward_func": 0.3585625011473894,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.7121456605382264,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5948673833627254,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0188124887645245,
+      "rewards/xmlcount_reward_func": 0.924125000834465,
       "step": 10
     },
     {
-      "completion_length": 322.7083333333333,
+      "completion_length": 302.2083333333333,
       "epoch": 3.0,
-      "grad_norm": 4.218914031982422,
-      "kl": 0.09477601277952392,
+      "grad_norm": 5.144018650054932,
+      "kl": 0.01467265534059455,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0001,
-      "reward": 1.9404340485731761,
-      "reward_std": 1.6197142712771893,
-      "rewards/concensus_correctness_reward_func": 0.22758333384990692,
-      "rewards/consensus_reward_func": 0.5,
+      "loss": 0.0,
+      "reward": 3.758754551410675,
+      "reward_std": 1.0187010215595365,
+      "rewards/concensus_correctness_reward_func": 0.6190833350022634,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.6363924071192741,
+      "rewards/final_correctness_reward_func": 0.8333333333333334,
+      "rewards/question_recreation_reward_func": 0.7559212238217393,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4097916707396507,
+      "rewards/xmlcount_reward_func": 0.8837500164906184,
       "step": 12
     },
     {
-      "completion_length": 324.71875,
+      "completion_length": 310.5625,
       "epoch": 3.571428571428571,
-      "grad_norm": 8.915092468261719,
-      "kl": 0.04675563084310852,
+      "grad_norm": 9.093050956726074,
+      "kl": 0.024251463531982154,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.788454730063677,
-      "reward_std": 1.483277335879393,
-      "rewards/concensus_correctness_reward_func": 0.6381875015795231,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 3.3967253491282463,
+      "reward_std": 0.9880957853747532,
+      "rewards/concensus_correctness_reward_func": 0.5252499990165234,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6943297337274998,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.7096316055394709,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7059374991804361,
+      "rewards/xmlcount_reward_func": 0.9743437394499779,
       "step": 14
     },
     {
-      "completion_length": 237.20833333333334,
+      "completion_length": 288.0416666666667,
       "epoch": 4.0,
-      "grad_norm": 7.370723247528076,
-      "kl": 0.022012559231370687,
+      "grad_norm": 4.826526641845703,
+      "kl": 0.05147159545837591,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 3.316575050354004,
-      "reward_std": 2.07821120445927,
-      "rewards/concensus_correctness_reward_func": 0.7534166648983955,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 2.3825169702370963,
+      "reward_std": 0.9278464317370284,
+      "rewards/concensus_correctness_reward_func": 0.3700833370288213,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5365333900166055,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6023503045241038,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9432916790246964,
+      "rewards/xmlcount_reward_func": 0.8267500102519989,
       "step": 16
     },
     {
-      "completion_length": 363.875,
+      "completion_length": 334.5,
       "epoch": 4.571428571428571,
-      "grad_norm": 8.573615074157715,
-      "kl": 0.03373612277209759,
+      "grad_norm": 6.352971076965332,
+      "kl": 0.2050690782489255,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0,
-      "reward": 2.931895741261542,
-      "reward_std": 1.6815357862506062,
-      "rewards/concensus_correctness_reward_func": 0.6655000001192093,
-      "rewards/consensus_reward_func": 0.6875,
+      "loss": 0.0002,
+      "reward": 3.8376564756035805,
+      "reward_std": 2.506588280783035,
+      "rewards/concensus_correctness_reward_func": 1.0865624994039536,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.6954581583850086,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.6192189978901297,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5084375068545341,
+      "rewards/xmlcount_reward_func": 0.9443750027567148,
       "step": 18
     },
     {
-      "completion_length": 209.625,
+      "completion_length": 258.7083333333333,
       "epoch": 5.0,
-      "grad_norm": 3.9853837490081787,
-      "kl": 26.756175404957805,
+      "grad_norm": 3.8929505348205566,
+      "kl": 0.03378994130374243,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0201,
-      "reward": 2.137111852566401,
-      "reward_std": 0.7332117200518647,
-      "rewards/concensus_correctness_reward_func": 0.10125000029802322,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "loss": 0.0,
+      "reward": 2.443352550268173,
+      "reward_std": 0.7956977238257726,
+      "rewards/concensus_correctness_reward_func": 0.3044166713953018,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6128202099353075,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.5871858916555842,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0063749998807907,
+      "rewards/xmlcount_reward_func": 0.9684166759252548,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.002027319371700287,
-      "train_runtime": 243.1978,
-      "train_samples_per_second": 1.316,
-      "train_steps_per_second": 0.082
+      "train_loss": 3.66260876035085e-05,
+      "train_runtime": 246.0607,
+      "train_samples_per_second": 1.3,
+      "train_steps_per_second": 0.081
     }
   ],
   "logging_steps": 2,