End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7fb378f4ba903893b3467fc73d9638e12ded461e7d67ff61c062573a6cf22a76
+oid sha256:c76f1a930be4a445293ee9c71ebc1fdf90a228581fc020d755f7ea7288e82e2f
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.621092552450136e-07,
-    "train_runtime": 1630.9123,
+    "train_loss": 7.876865595335403e-07,
+    "train_runtime": 1719.7597,
     "train_samples": 11,
-    "train_samples_per_second": 0.196,
+    "train_samples_per_second": 0.186,
     "train_steps_per_second": 0.012
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.621092552450136e-07,
-    "train_runtime": 1630.9123,
+    "train_loss": 7.876865595335403e-07,
+    "train_runtime": 1719.7597,
     "train_samples": 11,
-    "train_samples_per_second": 0.196,
+    "train_samples_per_second": 0.186,
     "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 430.96875,
+      "completion_length": 493.59375,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.545572817325592,
-      "kl": 0.000742794818506809,
+      "grad_norm": 0.38974353671073914,
+      "kl": 0.0006788387181586586,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.9380801543593407,
-      "reward_std": 1.09766436368227,
-      "rewards/concensus_correctness_reward_func": 0.003874999936670065,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 0.8032935485243797,
+      "reward_std": 0.8788771750405431,
+      "rewards/concensus_correctness_reward_func": 0.01575000025331974,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.466673893854022,
+      "rewards/question_recreation_reward_func": 0.527824804186821,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.09253125265240669,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": -0.005906245671212673,
       "step": 2
     },
     {
-      "completion_length": 417.67857142857144,
+      "completion_length": 408.60714285714283,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.40460318326950073,
-      "kl": 0.0007159239945134946,
+      "grad_norm": 0.48721808195114136,
+      "kl": 0.0007059077033773065,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.2944229756082808,
-      "reward_std": 1.0728516238076347,
-      "rewards/concensus_correctness_reward_func": 0.03157142869063786,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 1.1891927548817225,
+      "reward_std": 0.8131233283451625,
+      "rewards/concensus_correctness_reward_func": 0.009000000144754137,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5914943771702903,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/question_recreation_reward_func": 0.6425141649586814,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2249285727739334,
+      "rewards/xmlcount_reward_func": 0.25196427958352224,
       "step": 4
     },
     {
-      "completion_length": 383.2142857142857,
+      "completion_length": 464.35714285714283,
       "epoch": 2.0,
-      "grad_norm": 0.4155687689781189,
-      "kl": 0.0007597279784801815,
+      "grad_norm": 0.41437041759490967,
+      "kl": 0.0006861174562280732,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.204940242426736,
-      "reward_std": 1.215496735913413,
-      "rewards/concensus_correctness_reward_func": 0.013285713536398751,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 1.5305905533688409,
+      "reward_std": 1.3842910102435522,
+      "rewards/concensus_correctness_reward_func": 0.09757142833301,
+      "rewards/consensus_reward_func": 0.7142857142857143,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.38661879513944897,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4478928723505565,
+      "rewards/question_recreation_reward_func": 0.3316619577152388,
+      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.3513571321964264,
       "step": 6
     },
     {
-      "completion_length": 539.875,
+      "completion_length": 514.28125,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.3224160969257355,
-      "kl": 0.0006412690163415391,
+      "grad_norm": 0.37938645482063293,
+      "kl": 0.0007028070285741705,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.7731022760272026,
-      "reward_std": 0.9987973496317863,
-      "rewards/concensus_correctness_reward_func": 0.003874999936670065,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.1572178862988949,
+      "reward_std": 1.2181185893714428,
+      "rewards/concensus_correctness_reward_func": 0.01575000025331974,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4241334795951843,
+      "rewards/question_recreation_reward_func": 0.43184284679591656,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.03259375039488077,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.1939999982714653,
       "step": 8
     },
     {
-      "completion_length": 411.10714285714283,
+      "completion_length": 458.14285714285717,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.3444700837135315,
-      "kl": 0.0007552724231832794,
+      "grad_norm": 0.5346030592918396,
+      "kl": 0.000904711670175727,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.462347882134574,
-      "reward_std": 1.1783929892948695,
-      "rewards/concensus_correctness_reward_func": 0.0565714282648904,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 0.8834788203239441,
+      "reward_std": 1.2489494255610876,
+      "rewards/concensus_correctness_reward_func": 0.05378571152687073,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6305621351514544,
+      "rewards/question_recreation_reward_func": 0.4776216617652348,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.25735713115760256,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.005071423947811127,
       "step": 10
     },
     {
-      "completion_length": 435.85714285714283,
+      "completion_length": 463.39285714285717,
       "epoch": 4.0,
-      "grad_norm": 0.2986714243888855,
-      "kl": 0.0006829753069074026,
+      "grad_norm": 0.3504575192928314,
+      "kl": 0.0006804205601968404,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.9998348184994289,
-      "reward_std": 0.8780212189470019,
-      "rewards/concensus_correctness_reward_func": 0.03328571521810123,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.053320859159742,
+      "reward_std": 1.0784958175250463,
+      "rewards/concensus_correctness_reward_func": 0.03985714380230222,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5905490645340511,
+      "rewards/question_recreation_reward_func": 0.4849994033575058,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.1438571416905948,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.17132142691739968,
       "step": 12
     },
     {
-      "completion_length": 485.96875,
+      "completion_length": 473.96875,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.3659514784812927,
-      "kl": 0.00071150277290144,
+      "grad_norm": 0.4170607924461365,
+      "kl": 0.0006673134776065126,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.6259073615074158,
-      "reward_std": 1.6027247682213783,
-      "rewards/concensus_correctness_reward_func": 0.02474999986588955,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 1.4295077808201313,
+      "reward_std": 1.0330224707722664,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6519073890522122,
+      "rewards/question_recreation_reward_func": 0.5746952518820763,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.2773750014603138,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.47981252148747444,
       "step": 14
     },
     {
-      "completion_length": 416.25,
+      "completion_length": 524.6071428571429,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.45220518112182617,
-      "kl": 0.0006384395861199924,
+      "grad_norm": 0.40619194507598877,
+      "kl": 0.0006288451591639646,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.3776842228003912,
-      "reward_std": 1.10369587796075,
-      "rewards/concensus_correctness_reward_func": 0.00885714271238872,
-      "rewards/consensus_reward_func": 0.6428571428571429,
+      "reward": 1.4357544311455317,
+      "reward_std": 1.1602395176887512,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6524342170783451,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07353571057319641,
+      "rewards/question_recreation_reward_func": 0.5369686782360077,
+      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/strict_format_reward_func": 0.05357142857142857,
+      "rewards/xmlcount_reward_func": 0.5416428723505565,
       "step": 16
     },
     {
-      "completion_length": 446.64285714285717,
+      "completion_length": 454.57142857142856,
       "epoch": 6.0,
-      "grad_norm": 0.3257089853286743,
-      "kl": 0.0007032139941916934,
+      "grad_norm": 0.3458009660243988,
+      "kl": 0.0007908759538882545,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.0999488064220972,
-      "reward_std": 0.9497513175010681,
-      "rewards/concensus_correctness_reward_func": 0.0282857141324452,
+      "reward": 0.639422527381352,
+      "reward_std": 0.8909217204366412,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5714488093342099,
+      "rewards/question_recreation_reward_func": 0.4681368124272142,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.357357137969562,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.010571436158248357,
       "step": 18
     },
     {
-      "completion_length": 450.65625,
+      "completion_length": 415.8125,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.4257511496543884,
-      "kl": 0.0007408625024254434,
+      "grad_norm": 11.641963005065918,
+      "kl": 0.001962803398782853,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.1619919557124376,
-      "reward_std": 1.1545419916510582,
-      "rewards/concensus_correctness_reward_func": 0.02474999986588955,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.3323246389627457,
+      "reward_std": 1.3373099640011787,
+      "rewards/concensus_correctness_reward_func": 0.08193749841302633,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5668044406920671,
+      "rewards/question_recreation_reward_func": 0.4591683764010668,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.2579374946653843,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.27559376135468483,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 6.621092552450136e-07,
-      "train_runtime": 1630.9123,
-      "train_samples_per_second": 0.196,
+      "train_loss": 7.876865595335403e-07,
+      "train_runtime": 1719.7597,
+      "train_samples_per_second": 0.186,
       "train_steps_per_second": 0.012
     }
   ],