End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac5a95f94334726725ac8ad80909555e340a491bc35b69aaca2fa6696994e399
+oid sha256:97b3e471102bd2400f0a64939ce19b042019314950641cfd8916c2b6260015d8
 size 73911112

all_results.json

@@ -1,7 +1,7 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.839546320705267e-06,
-    "train_runtime": 2298.6794,
+    "train_loss": 3.3858226970551185e-06,
+    "train_runtime": 2295.8407,
     "train_samples": 14,
     "train_samples_per_second": 0.139,
     "train_steps_per_second": 0.009

train_results.json

@@ -1,7 +1,7 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.839546320705267e-06,
-    "train_runtime": 2298.6794,
+    "train_loss": 3.3858226970551185e-06,
+    "train_runtime": 2295.8407,
     "train_samples": 14,
     "train_samples_per_second": 0.139,
     "train_steps_per_second": 0.009

trainer_state.json

@@ -10,201 +10,201 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 315.71875,
+      "completion_length": 300.5,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.5491886734962463,
-      "kl": 0.0033342833630740643,
+      "grad_norm": 0.5659001469612122,
+      "kl": 0.004102726932615042,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 3.5814134776592255,
-      "reward_std": 3.557618737220764,
-      "rewards/concensus_correctness_reward_func": 1.9959999984130263,
+      "reward": 3.0786897465586662,
+      "reward_std": 2.879406835883856,
+      "rewards/concensus_correctness_reward_func": 1.1110625080764294,
       "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.45594474393874407,
+      "rewards/final_correctness_reward_func": 0.6875,
+      "rewards/question_recreation_reward_func": 0.4068459216505289,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3169687557965517,
+      "rewards/xmlcount_reward_func": 0.31078125163912773,
       "step": 2
     },
     {
-      "completion_length": 265.7083333333333,
+      "completion_length": 239.5,
       "epoch": 1.0,
-      "grad_norm": 0.45375221967697144,
-      "kl": 0.004827437223866582,
+      "grad_norm": 0.638882040977478,
+      "kl": 0.004074273048900068,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.4515926837921143,
-      "reward_std": 1.1381713052590687,
-      "rewards/concensus_correctness_reward_func": 0.4035833229621251,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 8.177565574645996,
+      "reward_std": 7.039573480685552,
+      "rewards/concensus_correctness_reward_func": 5.386833429336548,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8333333333333334,
-      "rewards/question_recreation_reward_func": 0.5283842980861664,
+      "rewards/final_correctness_reward_func": 1.1666666666666667,
+      "rewards/question_recreation_reward_func": 0.5329404522975286,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6029583464066187,
+      "rewards/xmlcount_reward_func": 0.4244583373268445,
       "step": 4
     },
     {
-      "completion_length": 283.5,
+      "completion_length": 311.9375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.7675709128379822,
-      "kl": 0.004162176148383878,
+      "grad_norm": 0.8447189331054688,
+      "kl": 0.0037213080358924344,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 2.805641271173954,
-      "reward_std": 2.443566210567951,
-      "rewards/concensus_correctness_reward_func": 1.130125051829964,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 3.7713246047496796,
+      "reward_std": 2.976348213851452,
+      "rewards/concensus_correctness_reward_func": 1.3261874932795763,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.46554752066731453,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.6335433274507523,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4599687531590462,
+      "rewards/xmlcount_reward_func": 0.37409374490380287,
       "step": 6
     },
     {
-      "completion_length": 274.2083333333333,
+      "completion_length": 301.375,
       "epoch": 2.0,
-      "grad_norm": 0.36452800035476685,
-      "kl": 0.004451930716944237,
+      "grad_norm": 0.34594935178756714,
+      "kl": 0.003091875657749673,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 3.3771184434493384,
-      "reward_std": 2.740070253610611,
-      "rewards/concensus_correctness_reward_func": 1.3475833336512248,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 6.380806724230449,
+      "reward_std": 5.335273357729117,
+      "rewards/concensus_correctness_reward_func": 3.7923332595576844,
+      "rewards/consensus_reward_func": 0.9166666666666666,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.5558267931143442,
+      "rewards/question_recreation_reward_func": 0.5042233293255171,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3903750069439411,
+      "rewards/xmlcount_reward_func": 0.16758333643277487,
       "step": 8
     },
     {
-      "completion_length": 259.6875,
+      "completion_length": 289.59375,
       "epoch": 2.571428571428571,
-      "grad_norm": 0.5737065672874451,
-      "kl": 0.004188220220385119,
+      "grad_norm": 1.3237749338150024,
+      "kl": 0.0032375632144976407,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 3.42026886343956,
-      "reward_std": 3.293873328715563,
-      "rewards/concensus_correctness_reward_func": 1.4791875034570694,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 8.275667496025562,
+      "reward_std": 5.976843889802694,
+      "rewards/concensus_correctness_reward_func": 5.359187468886375,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.5158939883112907,
+      "rewards/final_correctness_reward_func": 1.3125,
+      "rewards/question_recreation_reward_func": 0.5542614683508873,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.42518749833106995,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.3934687562286854,
       "step": 10
     },
     {
-      "completion_length": 307.75,
+      "completion_length": 298.4583333333333,
       "epoch": 3.0,
-      "grad_norm": 0.3956301510334015,
-      "kl": 0.004619166759463648,
+      "grad_norm": 0.3091566264629364,
+      "kl": 0.003987780287085722,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 3.989995300769806,
-      "reward_std": 3.4375420212745667,
-      "rewards/concensus_correctness_reward_func": 2.046583339571953,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 5.140617907047272,
+      "reward_std": 4.435525173942248,
+      "rewards/concensus_correctness_reward_func": 2.873000076661507,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8333333333333334,
-      "rewards/question_recreation_reward_func": 0.47703704982995987,
+      "rewards/final_correctness_reward_func": 0.5833333333333334,
+      "rewards/question_recreation_reward_func": 0.4436596632003784,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3830416575074196,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.3864583323399226,
       "step": 12
     },
     {
-      "completion_length": 265.3125,
+      "completion_length": 298.75,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.906726598739624,
-      "kl": 0.004731410284875892,
+      "grad_norm": 0.4717908203601837,
+      "kl": 0.0033006055746227503,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.7382691018283367,
-      "reward_std": 2.3467278257012367,
-      "rewards/concensus_correctness_reward_func": 0.9451875127851963,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 7.317512035369873,
+      "reward_std": 7.6424340903759,
+      "rewards/concensus_correctness_reward_func": 4.40049995854497,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.579581655561924,
+      "rewards/final_correctness_reward_func": 1.375,
+      "rewards/question_recreation_reward_func": 0.6019183360040188,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4010000005364418,
+      "rewards/xmlcount_reward_func": 0.3150937668979168,
       "step": 14
     },
     {
-      "completion_length": 298.7083333333333,
+      "completion_length": 297.625,
       "epoch": 4.0,
-      "grad_norm": 0.39787760376930237,
-      "kl": 0.0037641444941982627,
+      "grad_norm": 0.5708794593811035,
+      "kl": 0.004575319200133284,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 4.064295902848244,
-      "reward_std": 4.550448710719745,
-      "rewards/concensus_correctness_reward_func": 2.5,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 3.6336958010991416,
+      "reward_std": 3.153570957481861,
+      "rewards/concensus_correctness_reward_func": 1.876833327114582,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6666666666666666,
-      "rewards/question_recreation_reward_func": 0.3189624299605687,
+      "rewards/final_correctness_reward_func": 0.5833333333333334,
+      "rewards/question_recreation_reward_func": 0.39198747277259827,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.30783333629369736,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.19820834199587503,
       "step": 16
     },
     {
-      "completion_length": 269.625,
+      "completion_length": 254.65625,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.6894464492797852,
-      "kl": 0.005342361866496503,
+      "grad_norm": 5.82739782333374,
+      "kl": 0.0042124263709411025,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.38024440407753,
-      "reward_std": 2.3855159506201744,
-      "rewards/concensus_correctness_reward_func": 0.8445000001229346,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 7.052008777856827,
+      "reward_std": 5.355730663985014,
+      "rewards/concensus_correctness_reward_func": 4.25037491787225,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.4485568329691887,
+      "rewards/final_correctness_reward_func": 1.125,
+      "rewards/question_recreation_reward_func": 0.49616485089063644,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.462187506724149,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.586718762293458,
       "step": 18
     },
     {
-      "completion_length": 235.875,
+      "completion_length": 274.2916666666667,
       "epoch": 5.0,
-      "grad_norm": 0.49525389075279236,
-      "kl": 0.004465930202665429,
+      "grad_norm": 0.7243583798408508,
+      "kl": 0.004538669988202552,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 6.576857527097066,
-      "reward_std": 6.676121334234874,
-      "rewards/concensus_correctness_reward_func": 4.344083309173584,
+      "reward": 4.760532389084498,
+      "reward_std": 5.966201330224673,
+      "rewards/concensus_correctness_reward_func": 2.8519166807333627,
       "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9166666666666666,
-      "rewards/question_recreation_reward_func": 0.39764901002248126,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.3839908093214035,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.41845833758513135,
+      "rewards/xmlcount_reward_func": 0.5246250008543333,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.839546320705267e-06,
-      "train_runtime": 2298.6794,
+      "train_loss": 3.3858226970551185e-06,
+      "train_runtime": 2295.8407,
       "train_samples_per_second": 0.139,
       "train_steps_per_second": 0.009
     }