End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49e7f4b423b70b3f75c43d0ee0042f2ed51bb53084f1c07e599806e1df783b8c
+oid sha256:65ce21fef9d7dca68bcddcd5e892adcd2982504af98c531975fd508708db2202
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.6526825561413716e-06,
-    "train_runtime": 514.3119,
+    "train_loss": 1.760407212714199e-06,
+    "train_runtime": 560.1249,
     "train_samples": 11,
-    "train_samples_per_second": 0.622,
-    "train_steps_per_second": 0.039
+    "train_samples_per_second": 0.571,
+    "train_steps_per_second": 0.036
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.6526825561413716e-06,
-    "train_runtime": 514.3119,
+    "train_loss": 1.760407212714199e-06,
+    "train_runtime": 560.1249,
     "train_samples": 11,
-    "train_samples_per_second": 0.622,
-    "train_steps_per_second": 0.039
+    "train_samples_per_second": 0.571,
+    "train_steps_per_second": 0.036
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 366.3125,
+      "completion_length": 495.34375,
       "epoch": 0.7272727272727273,
-      "grad_norm": 1.4960169792175293,
-      "kl": 0.001837878895457834,
+      "grad_norm": 0.3851245939731598,
+      "kl": 0.0015201627466012724,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.5223995298147202,
-      "reward_std": 0.6293464861810207,
-      "rewards/concensus_correctness_reward_func": 0.1875,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.23531174659729,
+      "reward_std": 1.1557490825653076,
+      "rewards/concensus_correctness_reward_func": 0.0008750000270083547,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5197432637214661,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6263742372393608,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3151562660932541,
+      "rewards/xmlcount_reward_func": 0.1705624982714653,
       "step": 2
     },
     {
-      "completion_length": 397.60714285714283,
+      "completion_length": 362.7142857142857,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.5029628276824951,
-      "kl": 0.0017673390850956952,
+      "grad_norm": 0.46032318472862244,
+      "kl": 0.0023172974719532897,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.9705819306629044,
-      "reward_std": 1.1530599828277315,
-      "rewards/concensus_correctness_reward_func": 0.09707142625536237,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 1.392972162791661,
+      "reward_std": 1.4584640605109078,
+      "rewards/concensus_correctness_reward_func": 0.14257142799241201,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3479033815009253,
+      "rewards/question_recreation_reward_func": 0.3609007064785276,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16846429024423873,
+      "rewards/xmlcount_reward_func": 0.3895000049046108,
       "step": 4
     },
     {
-      "completion_length": 391.2857142857143,
+      "completion_length": 377.7857142857143,
       "epoch": 2.0,
-      "grad_norm": 0.4054848253726959,
-      "kl": 0.00174615705119712,
+      "grad_norm": 3.0854783058166504,
+      "kl": 0.00172044662758708,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.032759734562465,
-      "reward_std": 1.1778842338493891,
-      "rewards/concensus_correctness_reward_func": 0.1072142847946712,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 0.6036367884704045,
+      "reward_std": 0.5871127205235618,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.512688302568027,
+      "rewards/question_recreation_reward_func": 0.5374582058617047,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.18071429857185908,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.0661785677075386,
       "step": 6
     },
     {
-      "completion_length": 412.84375,
+      "completion_length": 432.21875,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.6137565970420837,
-      "kl": 0.001445371744921431,
+      "grad_norm": 0.4442577660083771,
+      "kl": 0.0016866580190253444,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.8012570161372423,
-      "reward_std": 0.7861417420208454,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.3614656776189804,
+      "reward_std": 1.080071672797203,
+      "rewards/concensus_correctness_reward_func": 0.03099999949336052,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5185382533818483,
+      "rewards/question_recreation_reward_func": 0.5839969431981444,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.1420937441289425,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.30896875262260437,
       "step": 8
     },
     {
-      "completion_length": 336.2857142857143,
+      "completion_length": 349.7142857142857,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.536916971206665,
-      "kl": 0.002134649848033275,
+      "grad_norm": 0.5599241852760315,
+      "kl": 0.0020511089491524865,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.4072004471506392,
-      "reward_std": 0.9136639471564975,
-      "rewards/concensus_correctness_reward_func": 0.09392857125827245,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 1.660039212022509,
+      "reward_std": 1.1306958198547363,
+      "rewards/concensus_correctness_reward_func": 0.30899999822889057,
+      "rewards/consensus_reward_func": 0.5714285714285714,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.31530758099896566,
+      "rewards/question_recreation_reward_func": 0.34428915807179045,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7122500155653272,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.39960715600422453,
       "step": 10
     },
     {
-      "completion_length": 460.42857142857144,
+      "completion_length": 495.92857142857144,
       "epoch": 4.0,
-      "grad_norm": 0.3349718153476715,
-      "kl": 0.0014100028035630072,
+      "grad_norm": 0.38454505801200867,
+      "kl": 0.0019243384096106248,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.0341259986162186,
-      "reward_std": 1.216601891177041,
-      "rewards/concensus_correctness_reward_func": 0.05357142857142857,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.0013103996004378,
+      "reward_std": 0.9705695403473718,
+      "rewards/concensus_correctness_reward_func": 0.12135714292526245,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6409116472516742,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5770603716373444,
+      "rewards/soft_format_reward_func": 0.017857142857142856,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1253571462418352,
+      "rewards/xmlcount_reward_func": -0.000678571207182748,
       "step": 12
     },
     {
-      "completion_length": 382.4375,
+      "completion_length": 403.53125,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.5842562317848206,
-      "kl": 0.0018867350881919265,
+      "grad_norm": 0.4447067677974701,
+      "kl": 0.0018273314417456277,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.7885393239557743,
-      "reward_std": 0.8474745936691761,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.136631891131401,
+      "reward_std": 0.8802363304421306,
+      "rewards/concensus_correctness_reward_func": 0.07368750125169754,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.49241431429982185,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.37563188280910254,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10862501058727503,
+      "rewards/xmlcount_reward_func": 0.3123125210404396,
       "step": 14
     },
     {
-      "completion_length": 386.2857142857143,
+      "completion_length": 439.35714285714283,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.5477598905563354,
-      "kl": 0.0016645457030140928,
+      "grad_norm": 0.4888899028301239,
+      "kl": 0.0017850026072535133,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.9445783623627254,
-      "reward_std": 1.0024188586643763,
-      "rewards/concensus_correctness_reward_func": 0.05357142857142857,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 0.7386083092008319,
+      "reward_std": 0.9949612553630557,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5618640652724675,
+      "rewards/question_recreation_reward_func": 0.4658940147076334,
       "rewards/soft_format_reward_func": 0.017857142857142856,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.07914286426135472,
+      "rewards/xmlcount_reward_func": 0.16557144692965917,
       "step": 16
     },
     {
-      "completion_length": 384.5357142857143,
+      "completion_length": 390.10714285714283,
       "epoch": 6.0,
-      "grad_norm": 0.42141640186309814,
-      "kl": 0.002002579309711499,
+      "grad_norm": 0.4372193217277527,
+      "kl": 0.0022854185164240854,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.3217215027127946,
-      "reward_std": 1.270809360912868,
-      "rewards/concensus_correctness_reward_func": 0.16928571249757493,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 1.2345900194985526,
+      "reward_std": 1.5980539066450936,
+      "rewards/concensus_correctness_reward_func": 0.11921428782599312,
+      "rewards/consensus_reward_func": 0.42857142857142855,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5725429398672921,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.18703571068389074,
+      "rewards/question_recreation_reward_func": 0.49841142126492094,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.18839287012815475,
       "step": 18
     },
     {
-      "completion_length": 397.75,
+      "completion_length": 411.34375,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.5499128699302673,
-      "kl": 0.0016771303344285116,
+      "grad_norm": 0.7081403136253357,
+      "kl": 0.0019678706958075054,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.0473634013906121,
-      "reward_std": 0.7728652590885758,
-      "rewards/concensus_correctness_reward_func": 0.10575000196695328,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.8901518955826759,
+      "reward_std": 0.9801672622561455,
+      "rewards/concensus_correctness_reward_func": 0.04268750175833702,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4337696129223332,
+      "rewards/question_recreation_reward_func": 0.4092768579721451,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.17971873981878161,
+      "rewards/xmlcount_reward_func": 0.29756249696947634,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.6526825561413716e-06,
-      "train_runtime": 514.3119,
-      "train_samples_per_second": 0.622,
-      "train_steps_per_second": 0.039
+      "train_loss": 1.760407212714199e-06,
+      "train_runtime": 560.1249,
+      "train_samples_per_second": 0.571,
+      "train_steps_per_second": 0.036
     }
   ],
   "logging_steps": 2,