End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c76f1a930be4a445293ee9c71ebc1fdf90a228581fc020d755f7ea7288e82e2f
+oid sha256:ead4f942d938a5c47fa7c195b7d9a45542af88918ad718aa16e38914c753a760
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.876865595335403e-07,
-    "train_runtime": 1719.7597,
+    "train_loss": 6.255729829263146e-07,
+    "train_runtime": 1853.8531,
     "train_samples": 11,
-    "train_samples_per_second": 0.186,
-    "train_steps_per_second": 0.012
+    "train_samples_per_second": 0.173,
+    "train_steps_per_second": 0.011
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.876865595335403e-07,
-    "train_runtime": 1719.7597,
+    "train_loss": 6.255729829263146e-07,
+    "train_runtime": 1853.8531,
     "train_samples": 11,
-    "train_samples_per_second": 0.186,
-    "train_steps_per_second": 0.012
+    "train_samples_per_second": 0.173,
+    "train_steps_per_second": 0.011
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 493.59375,
+      "completion_length": 531.5625,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.38974353671073914,
-      "kl": 0.0006788387181586586,
+      "grad_norm": 0.32100269198417664,
+      "kl": 0.0007212776254164055,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.8032935485243797,
-      "reward_std": 0.8788771750405431,
-      "rewards/concensus_correctness_reward_func": 0.01575000025331974,
+      "reward": 0.8653024211525917,
+      "reward_std": 1.249323770403862,
+      "rewards/concensus_correctness_reward_func": 0.007437500171363354,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.527824804186821,
+      "rewards/question_recreation_reward_func": 0.44517740048468113,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": -0.005906245671212673,
+      "rewards/xmlcount_reward_func": 0.1470624841749668,
       "step": 2
     },
     {
-      "completion_length": 408.60714285714283,
+      "completion_length": 554.6785714285714,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.48721808195114136,
-      "kl": 0.0007059077033773065,
+      "grad_norm": 0.352103054523468,
+      "kl": 0.0006337225744833372,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.1891927548817225,
-      "reward_std": 0.8131233283451625,
-      "rewards/concensus_correctness_reward_func": 0.009000000144754137,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 1.1566207153456551,
+      "reward_std": 0.9632204600742885,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.42857142857142855,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6425141649586814,
+      "rewards/question_recreation_reward_func": 0.6501564085483551,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.25196427958352224,
+      "rewards/xmlcount_reward_func": 0.07789286119597298,
       "step": 4
     },
     {
-      "completion_length": 464.35714285714283,
+      "completion_length": 423.75,
       "epoch": 2.0,
-      "grad_norm": 0.41437041759490967,
-      "kl": 0.0006861174562280732,
+      "grad_norm": 0.7081895470619202,
+      "kl": 0.0007896929829647499,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.5305905533688409,
-      "reward_std": 1.3842910102435522,
-      "rewards/concensus_correctness_reward_func": 0.09757142833301,
-      "rewards/consensus_reward_func": 0.7142857142857143,
+      "reward": 0.7233712758336749,
+      "reward_std": 0.7047450840473175,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3316619577152388,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/question_recreation_reward_func": 0.4309069982596806,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.3513571321964264,
+      "rewards/xmlcount_reward_func": 0.20317856834403106,
       "step": 6
     },
     {
-      "completion_length": 514.28125,
+      "completion_length": 541.75,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.37938645482063293,
-      "kl": 0.0007028070285741705,
+      "grad_norm": 0.3587605059146881,
+      "kl": 0.0005770352618128527,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.1572178862988949,
-      "reward_std": 1.2181185893714428,
-      "rewards/concensus_correctness_reward_func": 0.01575000025331974,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 0.7819546572864056,
+      "reward_std": 0.9537106528878212,
+      "rewards/concensus_correctness_reward_func": 0.014875000342726707,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.43184284679591656,
+      "rewards/question_recreation_reward_func": 0.543517179787159,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.1939999982714653,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.0889375009573996,
       "step": 8
     },
     {
-      "completion_length": 458.14285714285717,
+      "completion_length": 595.3214285714286,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.5346030592918396,
-      "kl": 0.000904711670175727,
+      "grad_norm": 0.2847689092159271,
+      "kl": 0.000647653326658266,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.8834788203239441,
-      "reward_std": 1.2489494255610876,
-      "rewards/concensus_correctness_reward_func": 0.05378571152687073,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 0.8683930380003793,
+      "reward_std": 0.6997007025139672,
+      "rewards/concensus_correctness_reward_func": 0.01792857050895691,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4776216617652348,
+      "rewards/question_recreation_reward_func": 0.5049644847001348,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.005071423947811127,
+      "rewards/xmlcount_reward_func": 0.1312142780848912,
       "step": 10
     },
     {
-      "completion_length": 463.39285714285717,
+      "completion_length": 444.9642857142857,
       "epoch": 4.0,
-      "grad_norm": 0.3504575192928314,
-      "kl": 0.0006804205601968404,
+      "grad_norm": 0.31496578454971313,
+      "kl": 0.0007364009673308049,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.053320859159742,
-      "reward_std": 1.0784958175250463,
-      "rewards/concensus_correctness_reward_func": 0.03985714380230222,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 1.3540408951895577,
+      "reward_std": 1.1649858100073678,
+      "rewards/concensus_correctness_reward_func": 0.008500000195843833,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4849994033575058,
+      "rewards/question_recreation_reward_func": 0.5784337392875126,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17132142691739968,
+      "rewards/strict_format_reward_func": 0.05357142857142857,
+      "rewards/xmlcount_reward_func": 0.42782142971243176,
       "step": 12
     },
     {
-      "completion_length": 473.96875,
+      "completion_length": 576.5,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.4170607924461365,
-      "kl": 0.0006673134776065126,
+      "grad_norm": 0.38832205533981323,
+      "kl": 0.0007369972008746117,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.4295077808201313,
-      "reward_std": 1.0330224707722664,
+      "reward": 0.9104718221351504,
+      "reward_std": 1.0457189995795488,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.375,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5746952518820763,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5795967951416969,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.47981252148747444,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.17462500371038914,
       "step": 14
     },
     {
-      "completion_length": 524.6071428571429,
+      "completion_length": 488.0357142857143,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.40619194507598877,
-      "kl": 0.0006288451591639646,
+      "grad_norm": 0.5469112992286682,
+      "kl": 0.0006665041811564672,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.4357544311455317,
-      "reward_std": 1.1602395176887512,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 1.30920090845653,
+      "reward_std": 1.2249880943979536,
+      "rewards/concensus_correctness_reward_func": 0.008500000195843833,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5369686782360077,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
-      "rewards/strict_format_reward_func": 0.05357142857142857,
-      "rewards/xmlcount_reward_func": 0.5416428723505565,
+      "rewards/question_recreation_reward_func": 0.5316295091594968,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.2512142849820001,
       "step": 16
     },
     {
-      "completion_length": 454.57142857142856,
+      "completion_length": 548.5357142857143,
       "epoch": 6.0,
-      "grad_norm": 0.3458009660243988,
-      "kl": 0.0007908759538882545,
+      "grad_norm": 0.41507840156555176,
+      "kl": 0.000582975144165435,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.639422527381352,
-      "reward_std": 0.8909217204366412,
+      "reward": 0.530327958720071,
+      "reward_std": 0.4705888032913208,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4681368124272142,
+      "rewards/question_recreation_reward_func": 0.37636367763791767,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.010571436158248357,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.15396427576031005,
       "step": 18
     },
     {
-      "completion_length": 415.8125,
+      "completion_length": 539.5625,
       "epoch": 6.7272727272727275,
-      "grad_norm": 11.641963005065918,
-      "kl": 0.001962803398782853,
+      "grad_norm": 0.5632695555686951,
+      "kl": 0.000683278267388232,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.3323246389627457,
-      "reward_std": 1.3373099640011787,
-      "rewards/concensus_correctness_reward_func": 0.08193749841302633,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 0.9871776178479195,
+      "reward_std": 1.048381008207798,
+      "rewards/concensus_correctness_reward_func": 0.02312499936670065,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4591683764010668,
+      "rewards/question_recreation_reward_func": 0.5318338591605425,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.27559376135468483,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.18221875466406345,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.876865595335403e-07,
-      "train_runtime": 1719.7597,
-      "train_samples_per_second": 0.186,
-      "train_steps_per_second": 0.012
+      "train_loss": 6.255729829263146e-07,
+      "train_runtime": 1853.8531,
+      "train_samples_per_second": 0.173,
+      "train_steps_per_second": 0.011
     }
   ],
   "logging_steps": 2,