End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.094016134738922e-06,
-    "train_runtime": 219.6957,
+    "train_loss": 9.198513089359039e-06,
+    "train_runtime": 215.0623,
     "train_samples": 28,
-    "train_samples_per_second": 1.457,
-    "train_steps_per_second": 0.091
+    "train_samples_per_second": 1.488,
+    "train_steps_per_second": 0.093
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72404ef325bc370797abf84fc71a74b94fcac3f316939ae72fa8a470b1bef6c5
+oid sha256:b9ef080e08d6d7b7bb62db78a3ca124283306718c648f0c05a148bc0e8d6f7ae
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.094016134738922e-06,
-    "train_runtime": 219.6957,
+    "train_loss": 9.198513089359039e-06,
+    "train_runtime": 215.0623,
     "train_samples": 28,
-    "train_samples_per_second": 1.457,
-    "train_steps_per_second": 0.091
+    "train_samples_per_second": 1.488,
+    "train_steps_per_second": 0.093
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 344.65625,
+      "completion_length": 298.0625,
       "epoch": 0.5714285714285714,
-      "grad_norm": 5.017642974853516,
+      "grad_norm": 6.717251777648926,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 1.6484445855021477,
-      "reward_std": 0.9143233702052385,
-      "rewards/concensus_correctness_reward_func": 0.15193749591708183,
-      "rewards/consensus_reward_func": 0.3125,
+      "loss": 0.0,
+      "reward": 2.065369065850973,
+      "reward_std": 1.6619373487774283,
+      "rewards/concensus_correctness_reward_func": 0.39149999618530273,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4839133294299245,
+      "rewards/question_recreation_reward_func": 0.45577534078620374,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7000937471166253,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.6868437463417649,
       "step": 2
     },
     {
-      "completion_length": 294.5416666666667,
+      "completion_length": 266.2916666666667,
       "epoch": 1.0,
-      "grad_norm": 3.5389950275421143,
-      "kl": 0.0013586016696838972,
+      "grad_norm": 3.6867785453796387,
+      "kl": 0.001406152257307743,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.2336954176425934,
-      "reward_std": 1.1887832844319444,
-      "rewards/concensus_correctness_reward_func": 0.34041666736205417,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 2.4803318232297897,
+      "reward_std": 1.9100091128299634,
+      "rewards/concensus_correctness_reward_func": 0.5114166711767515,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.638487059623003,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.49745685296754044,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.900625005364418,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.8047916715343794,
       "step": 4
     },
     {
-      "completion_length": 321.46875,
+      "completion_length": 204.0625,
       "epoch": 1.5714285714285714,
-      "grad_norm": 5.161291122436523,
-      "kl": 0.0034227508440380916,
+      "grad_norm": 8.32531452178955,
+      "kl": 0.011501419357955456,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.9783577173948288,
-      "reward_std": 1.2956023588776588,
-      "rewards/concensus_correctness_reward_func": 0.30268749594688416,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.601933002471924,
+      "reward_std": 0.9620060075540096,
+      "rewards/concensus_correctness_reward_func": 0.4566875025629997,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.632795225828886,
+      "rewards/question_recreation_reward_func": 0.5837454923894256,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6678749807178974,
+      "rewards/strict_format_reward_func": 0.09375,
+      "rewards/xmlcount_reward_func": 0.9677500203251839,
       "step": 6
     },
     {
-      "completion_length": 281.6666666666667,
+      "completion_length": 227.25,
       "epoch": 2.0,
-      "grad_norm": 3.7111079692840576,
-      "kl": 0.004361502108319352,
+      "grad_norm": 4.771548271179199,
+      "kl": 0.006304021313553676,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 2.383881355325381,
-      "reward_std": 0.5245453078920642,
-      "rewards/concensus_correctness_reward_func": 0.26100000242392224,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 3.030232419570287,
+      "reward_std": 1.8522576310982306,
+      "rewards/concensus_correctness_reward_func": 0.7245000004768372,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.642589658498764,
+      "rewards/question_recreation_reward_func": 0.6421075016260147,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 1.0011249979337056,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.892791673541069,
       "step": 8
     },
     {
-      "completion_length": 301.5625,
+      "completion_length": 249.625,
       "epoch": 2.571428571428571,
-      "grad_norm": 4.941656589508057,
-      "kl": 0.006099865640862845,
+      "grad_norm": 6.5946149826049805,
+      "kl": 0.011220796084671747,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 2.2924628891050816,
-      "reward_std": 0.7776006847198005,
-      "rewards/concensus_correctness_reward_func": 0.34712499752640724,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 2.330574309453368,
+      "reward_std": 0.9070390482665971,
+      "rewards/concensus_correctness_reward_func": 0.3594375029206276,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5014628782519139,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.42963684926507995,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.9282500073313713,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.9946250058710575,
       "step": 10
     },
     {
-      "completion_length": 199.0,
+      "completion_length": 284.0833333333333,
       "epoch": 3.0,
-      "grad_norm": 4.053720951080322,
-      "kl": 0.007232943249012654,
+      "grad_norm": 8.51677131652832,
+      "kl": 0.01575400565828507,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 2.1971174677213035,
-      "reward_std": 0.7274505247720905,
-      "rewards/concensus_correctness_reward_func": 0.17933333168427149,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 2.885760868589083,
+      "reward_std": 1.9988142798344295,
+      "rewards/concensus_correctness_reward_func": 0.710416667163372,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.736742460479339,
+      "rewards/question_recreation_reward_func": 0.6493442542850971,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9477083285649618,
+      "rewards/strict_format_reward_func": 0.041666666666666664,
+      "rewards/xmlcount_reward_func": 0.7343333313862482,
       "step": 12
     },
     {
-      "completion_length": 264.28125,
+      "completion_length": 287.21875,
       "epoch": 3.571428571428571,
-      "grad_norm": 14.958691596984863,
-      "kl": 0.008564002040657215,
+      "grad_norm": 5.919245719909668,
+      "kl": 0.01331145316362381,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.5280247405171394,
-      "reward_std": 1.2076747512328438,
-      "rewards/concensus_correctness_reward_func": 0.32162499986588955,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.9632712304592133,
+      "reward_std": 1.4405226800590754,
+      "rewards/concensus_correctness_reward_func": 0.756249999627471,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6926184613257647,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6578337242826819,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 1.0919062793254852,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.7835625000298023,
       "step": 14
     },
     {
-      "completion_length": 256.0,
+      "completion_length": 290.2083333333333,
       "epoch": 4.0,
-      "grad_norm": 3.648608446121216,
-      "kl": 0.008650898894605538,
+      "grad_norm": 5.046748161315918,
+      "kl": 0.023930478307496134,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 2.4574286540349326,
-      "reward_std": 0.6175319347530603,
-      "rewards/concensus_correctness_reward_func": 0.32233334084351856,
+      "reward": 2.493785634636879,
+      "reward_std": 0.5378106032779518,
+      "rewards/concensus_correctness_reward_func": 0.34049999713897705,
       "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.544261984527111,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5334939612851789,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.9449999928474426,
+      "rewards/strict_format_reward_func": 0.041666666666666664,
+      "rewards/xmlcount_reward_func": 0.9114583308498064,
       "step": 16
     },
     {
-      "completion_length": 221.4375,
+      "completion_length": 294.375,
       "epoch": 4.571428571428571,
-      "grad_norm": 5.220948696136475,
-      "kl": 0.010831554245669395,
+      "grad_norm": 7.338254928588867,
+      "kl": 0.016075315224952647,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.395220074802637,
-      "reward_std": 0.6563976746983826,
-      "rewards/concensus_correctness_reward_func": 0.3020000010728836,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.63186077773571,
+      "reward_std": 0.7248760172806215,
+      "rewards/concensus_correctness_reward_func": 0.4661874994635582,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6267513148486614,
+      "rewards/question_recreation_reward_func": 0.6226419649028685,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.9664687551558018,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.8711562715470791,
       "step": 18
     },
     {
-      "completion_length": 255.0,
+      "completion_length": 286.5416666666667,
       "epoch": 5.0,
-      "grad_norm": 3.633751630783081,
-      "kl": 0.009951268167545399,
+      "grad_norm": 5.04390811920166,
+      "kl": 0.017868785148796935,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.8240316808223724,
-      "reward_std": 0.2700964094450076,
-      "rewards/concensus_correctness_reward_func": 0.04483333230018616,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 2.4256584346294403,
+      "reward_std": 1.272670385427773,
+      "rewards/concensus_correctness_reward_func": 0.4596666644016902,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5966150037323436,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.5910334512591362,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.041666666666666664,
-      "rewards/xmlcount_reward_func": 0.9742500086625417,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.8749583264191946,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 6.094016134738922e-06,
-      "train_runtime": 219.6957,
-      "train_samples_per_second": 1.457,
-      "train_steps_per_second": 0.091
+      "train_loss": 9.198513089359039e-06,
+      "train_runtime": 215.0623,
+      "train_samples_per_second": 1.488,
+      "train_steps_per_second": 0.093
     }
   ],
   "logging_steps": 2,