End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19ad4667e479fe18d7ce68b0aec90fbf125c325375f9377c4f30c62aba4e9691
+oid sha256:1d57f57cd1cd2fabf0753ef3438a2408311e3ae411ffee8660441614856db369
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.777217182867389e-07,
-    "train_runtime": 1416.2687,
+    "train_loss": 7.809135922798304e-07,
+    "train_runtime": 1529.8944,
     "train_samples": 11,
-    "train_samples_per_second": 0.226,
-    "train_steps_per_second": 0.014
+    "train_samples_per_second": 0.209,
+    "train_steps_per_second": 0.013
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.777217182867389e-07,
-    "train_runtime": 1416.2687,
+    "train_loss": 7.809135922798304e-07,
+    "train_runtime": 1529.8944,
     "train_samples": 11,
-    "train_samples_per_second": 0.226,
-    "train_steps_per_second": 0.014
+    "train_samples_per_second": 0.209,
+    "train_steps_per_second": 0.013
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 553.875,
+      "completion_length": 483.8125,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.47362053394317627,
-      "kl": 0.0006196982467372436,
+      "grad_norm": 0.4184405207633972,
+      "kl": 0.0007234725198941305,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.4824305549263954,
-      "reward_std": 0.9157494455575943,
-      "rewards/concensus_correctness_reward_func": 0.02500000037252903,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 0.7045045606791973,
+      "reward_std": 0.8394088111817837,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.45093052834272385,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5646920502185822,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.38150000385940075,
+      "rewards/xmlcount_reward_func": -0.1804999839514494,
       "step": 2
     },
     {
-      "completion_length": 354.7857142857143,
+      "completion_length": 504.4642857142857,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.38883236050605774,
-      "kl": 0.0006938135234772094,
+      "grad_norm": 0.5454363822937012,
+      "kl": 0.0007511950097978115,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.720758514744895,
-      "reward_std": 0.988565479006086,
-      "rewards/concensus_correctness_reward_func": 0.0282857141324452,
-      "rewards/consensus_reward_func": 0.7857142857142857,
+      "reward": 1.0028131221021925,
+      "reward_std": 1.126203939318657,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.44272280803748537,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.4283214254038675,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.32374168400253567,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3219285724418504,
       "step": 4
     },
     {
-      "completion_length": 571.75,
+      "completion_length": 526.5,
       "epoch": 2.0,
-      "grad_norm": 0.49806156754493713,
-      "kl": 0.0006804801939454462,
+      "grad_norm": 0.3296413719654083,
+      "kl": 0.0006561284119795476,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.7435984568936485,
-      "reward_std": 1.3163981778281075,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 0.9751809239387512,
+      "reward_std": 1.2371410812650407,
+      "rewards/concensus_correctness_reward_func": -0.004071429371833801,
+      "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5268484950065613,
+      "rewards/final_correctness_reward_func": 0.14285714285714285,
+      "rewards/question_recreation_reward_func": 0.5420380703040532,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.21182143688201904,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.13364285869257791,
       "step": 6
     },
     {
-      "completion_length": 421.5,
+      "completion_length": 464.34375,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.4292489290237427,
-      "kl": 0.0006185105157783255,
+      "grad_norm": 0.4796897768974304,
+      "kl": 0.0021317518876458053,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.316079281270504,
-      "reward_std": 1.1854917779564857,
-      "rewards/concensus_correctness_reward_func": 0.02474999986588955,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.9232642352581024,
+      "reward_std": 1.166212558746338,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.39707926381379366,
+      "rewards/question_recreation_reward_func": 0.3210767302662134,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.550500001758337,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.219375004991889,
       "step": 8
     },
     {
-      "completion_length": 534.8214285714286,
+      "completion_length": 598.6428571428571,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.428897887468338,
-      "kl": 0.000605721421639568,
+      "grad_norm": 0.3463391959667206,
+      "kl": 0.0007349734660238028,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.2259584665298462,
-      "reward_std": 1.177501027073179,
-      "rewards/concensus_correctness_reward_func": 0.0141428570662226,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 1.8327125289610453,
+      "reward_std": 2.5424040130206516,
+      "rewards/concensus_correctness_reward_func": 0.6923571420567376,
+      "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.618815575327192,
+      "rewards/final_correctness_reward_func": 0.42857142857142855,
+      "rewards/question_recreation_reward_func": 0.6093196677310126,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16442857895578658,
+      "rewards/xmlcount_reward_func": -0.04039285225527627,
       "step": 10
     },
     {
-      "completion_length": 521.4285714285714,
+      "completion_length": 501.64285714285717,
       "epoch": 4.0,
-      "grad_norm": 0.2656509578227997,
-      "kl": 0.000656904378307185,
+      "grad_norm": 0.24949033558368683,
+      "kl": 0.0006643205872803394,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.113526205931391,
-      "reward_std": 1.1874884622437614,
-      "rewards/concensus_correctness_reward_func": 0.028571428997176036,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 0.7059421454157148,
+      "reward_std": 0.8951797613075801,
+      "rewards/concensus_correctness_reward_func": 0.017857142857142856,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4133833370038441,
+      "rewards/question_recreation_reward_func": 0.4061564419950758,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.22514286637306213,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.0676428633076804,
       "step": 12
     },
     {
-      "completion_length": 500.78125,
+      "completion_length": 557.3125,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.43524131178855896,
-      "kl": 0.0005758676852565259,
+      "grad_norm": 0.41972818970680237,
+      "kl": 0.0005867193431186024,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.0910731703042984,
-      "reward_std": 1.3809705451130867,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.082656467333436,
+      "reward_std": 1.291143488138914,
+      "rewards/concensus_correctness_reward_func": -0.013250000774860382,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5262919282540679,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.47859397903084755,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.11165625602006912,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.007687479257583618,
       "step": 14
     },
     {
-      "completion_length": 500.5,
+      "completion_length": 460.10714285714283,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.2686573266983032,
-      "kl": 0.0005495383811648935,
+      "grad_norm": 0.4573427736759186,
+      "kl": 0.0008041951306430357,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.9029017346245902,
-      "reward_std": 1.090190895966121,
-      "rewards/concensus_correctness_reward_func": 0.028571428997176036,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 0.9158965008599418,
+      "reward_std": 0.9635738602706364,
+      "rewards/concensus_correctness_reward_func": 0.026785714285714284,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3932946239198957,
+      "rewards/question_recreation_reward_func": 0.5480036437511444,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.12389286820377622,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.03753571292119367,
       "step": 16
     },
     {
-      "completion_length": 483.89285714285717,
+      "completion_length": 586.2857142857143,
       "epoch": 6.0,
-      "grad_norm": 0.2974832355976105,
-      "kl": 0.0006771147558798216,
+      "grad_norm": 0.4108085334300995,
+      "kl": 0.0005893616928785507,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.2937119347708566,
-      "reward_std": 1.1732401209218162,
-      "rewards/concensus_correctness_reward_func": 0.028571428997176036,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 1.8432286190135139,
+      "reward_std": 2.636557706764766,
+      "rewards/concensus_correctness_reward_func": 0.6923571314130511,
+      "rewards/consensus_reward_func": 0.42857142857142855,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5729619477476392,
+      "rewards/final_correctness_reward_func": 0.21428571428571427,
+      "rewards/question_recreation_reward_func": 0.49419292381831575,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3350357115268707,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": -0.004035708095346179,
       "step": 18
     },
     {
-      "completion_length": 478.84375,
+      "completion_length": 548.59375,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.3309420645236969,
-      "kl": 0.0006102465413277969,
+      "grad_norm": 0.36424583196640015,
+      "kl": 0.0007183377165347338,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.4968269914388657,
-      "reward_std": 1.1952479407191277,
-      "rewards/concensus_correctness_reward_func": 0.02500000037252903,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 2.033046916127205,
+      "reward_std": 2.6044920682907104,
+      "rewards/concensus_correctness_reward_func": 0.6058124899864197,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5386707242578268,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.660671953111887,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.4800312491133809,
+      "rewards/xmlcount_reward_func": 0.06343750469386578,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 5.777217182867389e-07,
-      "train_runtime": 1416.2687,
-      "train_samples_per_second": 0.226,
-      "train_steps_per_second": 0.014
+      "train_loss": 7.809135922798304e-07,
+      "train_runtime": 1529.8944,
+      "train_samples_per_second": 0.209,
+      "train_steps_per_second": 0.013
     }
   ],
   "logging_steps": 2,