End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef427109c12e9c134ef312ddff3602d498dc4299f20d5ed75d136d215a012939
+oid sha256:81d59cbbb03553632dba1ae6e1f55f83066c3a0d53b6eac77db813dfa0c94546
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.5715185440967617e-06,
-    "train_runtime": 692.3425,
+    "train_loss": 1.6953708041000937e-06,
+    "train_runtime": 686.112,
     "train_samples": 11,
-    "train_samples_per_second": 0.462,
+    "train_samples_per_second": 0.466,
     "train_steps_per_second": 0.029
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.5715185440967617e-06,
-    "train_runtime": 692.3425,
+    "train_loss": 1.6953708041000937e-06,
+    "train_runtime": 686.112,
     "train_samples": 11,
-    "train_samples_per_second": 0.462,
+    "train_samples_per_second": 0.466,
     "train_steps_per_second": 0.029
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 517.1875,
+      "completion_length": 512.0625,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.6420091390609741,
-      "kl": 0.0019688690153998323,
+      "grad_norm": 0.3976362943649292,
+      "kl": 0.0016056762979133055,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.5067992843687534,
-      "reward_std": 0.986472899094224,
+      "reward": 1.31808702647686,
+      "reward_std": 1.22724399715662,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.4375,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6470805648714304,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.7159307561814785,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.3597187642008066,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.16465624468401074,
       "step": 2
     },
     {
-      "completion_length": 547.2142857142857,
+      "completion_length": 530.0714285714286,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.4010975658893585,
-      "kl": 0.0015678029657075448,
+      "grad_norm": 0.5486065745353699,
+      "kl": 0.001764887128956616,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.2731680827481406,
-      "reward_std": 1.2122740660394942,
-      "rewards/concensus_correctness_reward_func": 0.019857142652784075,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 5.417113099779401,
+      "reward_std": 5.547162072999137,
+      "rewards/concensus_correctness_reward_func": 3.592071396963937,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.6732038089207241,
+      "rewards/final_correctness_reward_func": 0.7142857142857143,
+      "rewards/question_recreation_reward_func": 0.7465773906026568,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03571428571428571,
-      "rewards/xmlcount_reward_func": 0.18725000321865082,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.13582142548901693,
       "step": 4
     },
     {
-      "completion_length": 573.75,
+      "completion_length": 601.6428571428571,
       "epoch": 2.0,
-      "grad_norm": 0.4096458852291107,
-      "kl": 0.0014084425305814616,
+      "grad_norm": 11.482904434204102,
+      "kl": 0.0018090582535868244,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.5097219433103288,
-      "reward_std": 1.2099641816956657,
-      "rewards/concensus_correctness_reward_func": 0.052571428673607964,
-      "rewards/consensus_reward_func": 0.6428571428571429,
+      "reward": 1.0786156143460954,
+      "reward_std": 1.1726488726479667,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6305076128670147,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1837857280458723,
+      "rewards/question_recreation_reward_func": 0.6033298926694053,
+      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.06457142744745527,
       "step": 6
     },
     {
-      "completion_length": 541.46875,
+      "completion_length": 525.21875,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.45404794812202454,
-      "kl": 0.0016080429923022166,
+      "grad_norm": 0.4046158790588379,
+      "kl": 0.0017324830841971561,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.8502027094364166,
-      "reward_std": 1.3040681928396225,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.8011957630515099,
+      "reward_std": 2.2872734293341637,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5663901790976524,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6193519607186317,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.03381251357495785,
+      "rewards/xmlcount_reward_func": -0.00565625075250864,
       "step": 8
     },
     {
-      "completion_length": 601.5357142857143,
+      "completion_length": 622.0714285714286,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.3453723192214966,
-      "kl": 0.001553945301566273,
+      "grad_norm": 0.36950886249542236,
+      "kl": 0.0014689405215904117,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.2584942494119917,
-      "reward_std": 1.4121184519359045,
-      "rewards/concensus_correctness_reward_func": 0.019857142652784075,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 1.1769423612526484,
+      "reward_std": 1.4105787106922694,
+      "rewards/concensus_correctness_reward_func": 0.020642857466425215,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.6241728067398071,
+      "rewards/final_correctness_reward_func": 0.2857142857142857,
+      "rewards/question_recreation_reward_func": 0.7318709662982396,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03571428571428571,
-      "rewards/xmlcount_reward_func": 0.07875001643385206,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": -0.16485714486667088,
       "step": 10
     },
     {
-      "completion_length": 450.0,
+      "completion_length": 587.9642857142857,
       "epoch": 4.0,
-      "grad_norm": 0.46400099992752075,
-      "kl": 0.0015125636543546403,
+      "grad_norm": 0.25444716215133667,
+      "kl": 0.0014601101034453937,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.887412177664893,
-      "reward_std": 0.8893355514322009,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "reward": 0.6373377655233655,
+      "reward_std": 0.9009909970419747,
+      "rewards/concensus_correctness_reward_func": 0.020642857466425215,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.21428571428571427,
-      "rewards/question_recreation_reward_func": 0.5103407331875393,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.5815162871565137,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.01992857881954738,
+      "rewards/xmlcount_reward_func": -0.3219642703022276,
       "step": 12
     },
     {
-      "completion_length": 586.8125,
+      "completion_length": 527.34375,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.28827542066574097,
-      "kl": 0.0011791825891123153,
+      "grad_norm": 0.39833763241767883,
+      "kl": 0.0018258660566061735,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.6554815284907818,
-      "reward_std": 0.9318620897829533,
+      "reward": 1.0284310150891542,
+      "reward_std": 1.0304897874593735,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.125,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5917002856731415,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5533684827387333,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.06121874740347266,
+      "rewards/xmlcount_reward_func": 0.037562498822808266,
       "step": 14
     },
     {
-      "completion_length": 466.39285714285717,
+      "completion_length": 505.10714285714283,
       "epoch": 5.363636363636363,
-      "grad_norm": 1.2276360988616943,
-      "kl": 0.0022477747074195315,
+      "grad_norm": 0.493288516998291,
+      "kl": 0.0022586240832294735,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.2439466140099935,
-      "reward_std": 1.1877283496516091,
-      "rewards/concensus_correctness_reward_func": 0.008928571428571428,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 1.4594463493142809,
+      "reward_std": 2.662624499627522,
+      "rewards/concensus_correctness_reward_func": 0.7142857142857143,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.5682680138519832,
+      "rewards/final_correctness_reward_func": 0.14285714285714285,
+      "rewards/question_recreation_reward_func": 0.45419638071741375,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03571428571428571,
-      "rewards/xmlcount_reward_func": 0.27389287097113474,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": -0.08403570311410087,
       "step": 16
     },
     {
-      "completion_length": 483.9642857142857,
+      "completion_length": 569.5,
       "epoch": 6.0,
-      "grad_norm": 4.336320400238037,
-      "kl": 0.0021773939869100495,
+      "grad_norm": 0.28241637349128723,
+      "kl": 0.0017225055323381508,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.505273767880031,
-      "reward_std": 1.1624436804226466,
-      "rewards/concensus_correctness_reward_func": 0.019857142652784075,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 2.5877433163779124,
+      "reward_std": 3.8677090789590562,
+      "rewards/concensus_correctness_reward_func": 1.4285714285714286,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.7239166242735726,
+      "rewards/final_correctness_reward_func": 0.2857142857142857,
+      "rewards/question_recreation_reward_func": 0.7558861545154026,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.45792857238224577,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.3824285790324211,
       "step": 18
     },
     {
-      "completion_length": 524.40625,
+      "completion_length": 483.0625,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.5881478190422058,
-      "kl": 0.0016475072770845145,
+      "grad_norm": 0.3676411509513855,
+      "kl": 0.0024811434486764483,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.6808819696307182,
-      "reward_std": 1.0567774400115013,
+      "reward": 1.212349385023117,
+      "reward_std": 1.1187334507703781,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.1875,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6210694797337055,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.641536895185709,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.1276874914765358,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.11768750380724669,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.5715185440967617e-06,
-      "train_runtime": 692.3425,
-      "train_samples_per_second": 0.462,
+      "train_loss": 1.6953708041000937e-06,
+      "train_runtime": 686.112,
+      "train_samples_per_second": 0.466,
       "train_steps_per_second": 0.029
     }
   ],