End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:56b487afd111e4da52bccf6c9fa8c7197a0add1832cf5fa54bde91f2d4a6075e
+oid sha256:f0db30a336c92b7bad12cff0b9c8b3cd9e169cf50e1b1d0e11e48aefcbbed460
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.061654888369958e-06,
-    "train_runtime": 2276.638,
+    "train_loss": 3.5113219382765236e-06,
+    "train_runtime": 2215.8404,
     "train_samples": 14,
-    "train_samples_per_second": 0.141,
+    "train_samples_per_second": 0.144,
     "train_steps_per_second": 0.009
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.061654888369958e-06,
-    "train_runtime": 2276.638,
+    "train_loss": 3.5113219382765236e-06,
+    "train_runtime": 2215.8404,
     "train_samples": 14,
-    "train_samples_per_second": 0.141,
+    "train_samples_per_second": 0.144,
     "train_steps_per_second": 0.009
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 319.0,
+      "completion_length": 271.53125,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.8265736103057861,
-      "kl": 0.0036303852102719247,
+      "grad_norm": 0.6156684756278992,
+      "kl": 0.003713510144734755,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 3.2230667397379875,
-      "reward_std": 3.523205704987049,
-      "rewards/concensus_correctness_reward_func": 1.508500010939315,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 4.28339558839798,
+      "reward_std": 5.015739507973194,
+      "rewards/concensus_correctness_reward_func": 2.1066250316798687,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.49050420615822077,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.49977052211761475,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2084375023841858,
+      "rewards/xmlcount_reward_func": 0.3644999973475933,
       "step": 2
     },
     {
-      "completion_length": 288.4583333333333,
+      "completion_length": 237.33333333333334,
       "epoch": 1.0,
-      "grad_norm": 0.6778371334075928,
-      "kl": 0.003078966032868872,
+      "grad_norm": 0.3896828293800354,
+      "kl": 0.0036000795274352035,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 6.541472514470418,
-      "reward_std": 7.260249118010203,
-      "rewards/concensus_correctness_reward_func": 4.639250000317891,
-      "rewards/consensus_reward_func": 0.8333333333333334,
+      "reward": 7.742002268632253,
+      "reward_std": 8.532249887784323,
+      "rewards/concensus_correctness_reward_func": 5.1745833332339926,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.2928476979335149,
+      "rewards/final_correctness_reward_func": 1.25,
+      "rewards/question_recreation_reward_func": 0.4115021179119746,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.255208329608043,
+      "rewards/strict_format_reward_func": 0.041666666666666664,
+      "rewards/xmlcount_reward_func": 0.4475833574930827,
       "step": 4
     },
     {
-      "completion_length": 267.40625,
+      "completion_length": 268.1875,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.9330791234970093,
-      "kl": 0.0036824695125687867,
+      "grad_norm": 0.6913865208625793,
+      "kl": 0.003932094754418358,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 3.196286790072918,
-      "reward_std": 2.722376387566328,
-      "rewards/concensus_correctness_reward_func": 1.014437495265156,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 5.461809501051903,
+      "reward_std": 5.302938483655453,
+      "rewards/concensus_correctness_reward_func": 3.2423125002533197,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.39591173361986876,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.43490335159003735,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4734374936670065,
+      "rewards/xmlcount_reward_func": 0.2845937521196902,
       "step": 6
     },
     {
-      "completion_length": 244.20833333333334,
+      "completion_length": 248.33333333333334,
       "epoch": 2.0,
-      "grad_norm": 0.45333999395370483,
-      "kl": 0.003310707824615141,
+      "grad_norm": 0.40261271595954895,
+      "kl": 0.0035152111668139696,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 5.11900262037913,
-      "reward_std": 6.583905945221583,
-      "rewards/concensus_correctness_reward_func": 3.427916668355465,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 6.17376080652078,
+      "reward_std": 5.402185544371605,
+      "rewards/concensus_correctness_reward_func": 3.6611666629711785,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.4223358705639839,
+      "rewards/final_correctness_reward_func": 1.0833333333333333,
+      "rewards/question_recreation_reward_func": 0.3990109084794919,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.581249992052714,
+      "rewards/xmlcount_reward_func": 0.5094166745742162,
       "step": 8
     },
     {
-      "completion_length": 302.6875,
+      "completion_length": 239.0625,
       "epoch": 2.571428571428571,
-      "grad_norm": 0.48581212759017944,
-      "kl": 0.0031960688793333247,
+      "grad_norm": 0.6596890687942505,
+      "kl": 0.004055387194966897,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 4.359011098742485,
-      "reward_std": 5.041795432567596,
-      "rewards/concensus_correctness_reward_func": 2.613250009715557,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 7.403021275997162,
+      "reward_std": 7.769663609564304,
+      "rewards/concensus_correctness_reward_func": 4.745562508702278,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5397296585142612,
+      "rewards/final_correctness_reward_func": 1.125,
+      "rewards/question_recreation_reward_func": 0.42364630103111267,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.4247812554240227,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.46818749513477087,
       "step": 10
     },
     {
-      "completion_length": 264.5416666666667,
+      "completion_length": 272.5,
       "epoch": 3.0,
-      "grad_norm": 0.37057843804359436,
-      "kl": 0.003878361739528676,
+      "grad_norm": 0.3865489363670349,
+      "kl": 0.004031806602142751,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 4.831206654508908,
-      "reward_std": 6.333635171254476,
-      "rewards/concensus_correctness_reward_func": 2.8268333077430725,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 3.3122252325216928,
+      "reward_std": 2.822227214773496,
+      "rewards/concensus_correctness_reward_func": 1.8305832520127296,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6666666666666666,
-      "rewards/question_recreation_reward_func": 0.3961649263898532,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.477475106716156,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3582083247601986,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.3166666701436043,
       "step": 12
     },
     {
-      "completion_length": 304.71875,
+      "completion_length": 245.59375,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.7317449450492859,
-      "kl": 0.003004714220878668,
+      "grad_norm": 0.7244676351547241,
+      "kl": 0.0035437151964288205,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 4.568838551640511,
-      "reward_std": 5.097499243915081,
-      "rewards/concensus_correctness_reward_func": 2.7120625108946115,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 6.540969520807266,
+      "reward_std": 6.686426967382431,
+      "rewards/concensus_correctness_reward_func": 3.60187497548759,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.6519634742289782,
+      "rewards/final_correctness_reward_func": 1.25,
+      "rewards/question_recreation_reward_func": 0.5858760215342045,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3141875136643648,
+      "rewards/xmlcount_reward_func": 0.4625937454402447,
       "step": 14
     },
     {
-      "completion_length": 344.7916666666667,
+      "completion_length": 285.8333333333333,
       "epoch": 4.0,
-      "grad_norm": 0.36377763748168945,
-      "kl": 0.003294540375160674,
+      "grad_norm": 0.4408351480960846,
+      "kl": 0.005074007475438218,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 4.352070172627767,
-      "reward_std": 3.323097010453542,
-      "rewards/concensus_correctness_reward_func": 2.0965833365917206,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 3.404947509368261,
+      "reward_std": 2.91806568702062,
+      "rewards/concensus_correctness_reward_func": 1.8025833343466122,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6666666666666666,
-      "rewards/question_recreation_reward_func": 0.3886534372965495,
+      "rewards/final_correctness_reward_func": 0.5833333333333334,
+      "rewards/question_recreation_reward_func": 0.2744477453331153,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5334999983509382,
+      "rewards/xmlcount_reward_func": 0.4945833434661229,
       "step": 16
     },
     {
-      "completion_length": 281.65625,
+      "completion_length": 245.5625,
       "epoch": 4.571428571428571,
-      "grad_norm": 1.0194026231765747,
-      "kl": 0.0037469247763510793,
+      "grad_norm": 1.1728448867797852,
+      "kl": 0.004925011526211165,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 6.420581094920635,
-      "reward_std": 6.502259276807308,
-      "rewards/concensus_correctness_reward_func": 4.080687532201409,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 8.376463562250137,
+      "reward_std": 6.909352557733655,
+      "rewards/concensus_correctness_reward_func": 5.674437500536442,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.5529873929917812,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.5359011180698872,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.33378125145100057,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.6348750218749046,
       "step": 18
     },
     {
-      "completion_length": 306.0,
+      "completion_length": 228.45833333333334,
       "epoch": 5.0,
-      "grad_norm": 0.5315831899642944,
-      "kl": 0.004204936480770509,
+      "grad_norm": 0.5003474950790405,
+      "kl": 0.003518054281206181,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 4.401223103205363,
-      "reward_std": 4.7618053158124285,
-      "rewards/concensus_correctness_reward_func": 2.6359999974568686,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 2.8317282050848007,
+      "reward_std": 2.5725097159544625,
+      "rewards/concensus_correctness_reward_func": 1.0097499986489613,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.4475146258870761,
+      "rewards/final_correctness_reward_func": 0.8333333333333334,
+      "rewards/question_recreation_reward_func": 0.2900198350350062,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.041666666666666664,
-      "rewards/xmlcount_reward_func": 0.2760416766007741,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.36529167120655376,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.061654888369958e-06,
-      "train_runtime": 2276.638,
-      "train_samples_per_second": 0.141,
+      "train_loss": 3.5113219382765236e-06,
+      "train_runtime": 2215.8404,
+      "train_samples_per_second": 0.144,
       "train_steps_per_second": 0.009
     }
   ],