End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f3a8d3a68e52508b133eb1020a8d479290ed6c51e4057773045ee9b0f6555be
+oid sha256:e2bf2022baf56622c6d8ac7c0b020a4e138e9b27596913fc1a3f797f32fc6ccb
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.004232145391143e-07,
-    "train_runtime": 1497.3635,
+    "train_loss": 6.662672547008697e-07,
+    "train_runtime": 1726.6167,
     "train_samples": 11,
-    "train_samples_per_second": 0.214,
-    "train_steps_per_second": 0.013
+    "train_samples_per_second": 0.185,
+    "train_steps_per_second": 0.012
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.004232145391143e-07,
-    "train_runtime": 1497.3635,
+    "train_loss": 6.662672547008697e-07,
+    "train_runtime": 1726.6167,
     "train_samples": 11,
-    "train_samples_per_second": 0.214,
-    "train_steps_per_second": 0.013
+    "train_samples_per_second": 0.185,
+    "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 444.625,
+      "completion_length": 484.34375,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.3358810544013977,
-      "kl": 0.0007525779074057937,
+      "grad_norm": 0.31932416558265686,
+      "kl": 0.0006641301006311551,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.942502249032259,
-      "reward_std": 1.1949501410126686,
-      "rewards/concensus_correctness_reward_func": 0.03200000151991844,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.8911791052669287,
+      "reward_std": 0.7616452053189278,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5761897154152393,
-      "rewards/soft_format_reward_func": 0.03125,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.037437498569488525,
+      "rewards/question_recreation_reward_func": 0.6187728717923164,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.14740624837577343,
       "step": 2
     },
     {
-      "completion_length": 493.17857142857144,
+      "completion_length": 510.0,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.4376353919506073,
-      "kl": 0.00070392172450998,
+      "grad_norm": 0.320825457572937,
+      "kl": 0.000691832770826295,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.4686146852161204,
-      "reward_std": 1.3468076416424342,
-      "rewards/concensus_correctness_reward_func": 0.06185714474746159,
-      "rewards/consensus_reward_func": 0.5714285714285714,
+      "reward": 0.6444876875196185,
+      "reward_std": 1.1091752818652563,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5830789421285901,
+      "rewards/question_recreation_reward_func": 0.6262734468494143,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2522499965769904,
+      "rewards/xmlcount_reward_func": -0.19607143529823848,
       "step": 4
     },
     {
-      "completion_length": 503.10714285714283,
+      "completion_length": 513.75,
       "epoch": 2.0,
-      "grad_norm": 0.4641832113265991,
-      "kl": 0.0006336624875465142,
+      "grad_norm": 0.25413331389427185,
+      "kl": 0.0009799751902132162,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.1997146138123103,
-      "reward_std": 1.2870798196111406,
-      "rewards/concensus_correctness_reward_func": 0.021571428648063114,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.084031228508268,
+      "reward_std": 1.2072510463850838,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.42067892636571613,
+      "rewards/question_recreation_reward_func": 0.47388836635010584,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.2396071425506047,
+      "rewards/strict_format_reward_func": 0.05357142857142857,
+      "rewards/xmlcount_reward_func": 0.27085714042186737,
       "step": 6
     },
     {
-      "completion_length": 516.28125,
+      "completion_length": 435.15625,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.3577750623226166,
-      "kl": 0.0006857380758447107,
+      "grad_norm": 0.48664921522140503,
+      "kl": 0.0007914057750895154,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.1810798477381468,
-      "reward_std": 1.1422083638608456,
-      "rewards/concensus_correctness_reward_func": 0.06400000303983688,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.1346675045788288,
+      "reward_std": 0.7414112240076065,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5715486072003841,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.2799062551930547,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6151049984619021,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.19143750052899122,
       "step": 8
     },
     {
-      "completion_length": 529.0714285714286,
+      "completion_length": 524.1428571428571,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.34569600224494934,
-      "kl": 0.000677979048175205,
+      "grad_norm": 0.3309638500213623,
+      "kl": 0.0006425218764759068,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.8421074620315007,
-      "reward_std": 1.1149923162800925,
-      "rewards/concensus_correctness_reward_func": 0.012642857219491686,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 0.8435536708150592,
+      "reward_std": 0.9221388867923191,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4952503080878939,
+      "rewards/question_recreation_reward_func": 0.4326251213039671,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11992858882461276,
+      "rewards/xmlcount_reward_func": 0.05378572004181998,
       "step": 10
     },
     {
-      "completion_length": 512.9285714285714,
+      "completion_length": 490.2142857142857,
       "epoch": 4.0,
-      "grad_norm": 0.296815425157547,
-      "kl": 0.0006996494485065341,
+      "grad_norm": 6.519441604614258,
+      "kl": 0.0008424879675398447,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.1889213749340601,
-      "reward_std": 1.010519095829555,
-      "rewards/concensus_correctness_reward_func": 0.008928571428571428,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 1.295002749988011,
+      "reward_std": 1.0125029901308673,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6167427854878562,
+      "rewards/question_recreation_reward_func": 0.7125027477741241,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.11682140401431493,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.33250001072883606,
       "step": 12
     },
     {
-      "completion_length": 449.6875,
+      "completion_length": 425.90625,
       "epoch": 4.7272727272727275,
-      "grad_norm": 6.990741729736328,
-      "kl": 0.0013008478854317218,
+      "grad_norm": 0.4380124807357788,
+      "kl": 0.0006226709811016917,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.026983603835106,
-      "reward_std": 0.7882455755025148,
-      "rewards/concensus_correctness_reward_func": 0.03200000151991844,
+      "reward": 0.8414656817913055,
+      "reward_std": 0.802082123234868,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4787023440003395,
+      "rewards/question_recreation_reward_func": 0.46252819150686264,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.23503126576542854,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.12893750285729766,
       "step": 14
     },
     {
-      "completion_length": 581.3928571428571,
+      "completion_length": 471.2857142857143,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.2926975190639496,
-      "kl": 0.0006121814117899963,
+      "grad_norm": 0.5062863826751709,
+      "kl": 0.0006774923281877168,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.0210010962826865,
-      "reward_std": 0.6594925011907306,
+      "reward": 0.7448562851973942,
+      "reward_std": 1.0423502794333868,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6218225359916687,
+      "rewards/question_recreation_reward_func": 0.5749991663864681,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1134642745767321,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.009142858641488212,
       "step": 16
     },
     {
-      "completion_length": 486.89285714285717,
+      "completion_length": 504.92857142857144,
       "epoch": 6.0,
-      "grad_norm": 0.31777244806289673,
-      "kl": 0.0006165339436847717,
+      "grad_norm": 0.2513563632965088,
+      "kl": 0.0007265556279370296,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.9629412995917457,
-      "reward_std": 1.0609238658632552,
-      "rewards/concensus_correctness_reward_func": 0.049214287527969906,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 0.7107706474406379,
+      "reward_std": 1.2383961762700761,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.42857142857142855,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.5194055991513389,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6080563430275235,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10860715593610491,
+      "rewards/xmlcount_reward_func": -0.32585713480200085,
       "step": 18
     },
     {
-      "completion_length": 490.3125,
+      "completion_length": 437.65625,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.29833632707595825,
-      "kl": 0.0008240973074862268,
+      "grad_norm": 0.3156798481941223,
+      "kl": 0.0006726395040459465,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.7305171675980091,
-      "reward_std": 0.9535290151834488,
+      "reward": 0.8750260164961219,
+      "reward_std": 0.9098680801689625,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.125,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4848296642303467,
+      "rewards/question_recreation_reward_func": 0.3959010145626962,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.12068750895559788,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.21349999587982893,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.004232145391143e-07,
-      "train_runtime": 1497.3635,
-      "train_samples_per_second": 0.214,
-      "train_steps_per_second": 0.013
+      "train_loss": 6.662672547008697e-07,
+      "train_runtime": 1726.6167,
+      "train_samples_per_second": 0.185,
+      "train_steps_per_second": 0.012
     }
   ],
   "logging_steps": 2,