End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2689efbaf112598da3f7bf982cf1d5ef9f9bc802407dc22fba70eaef166fea1e
+oid sha256:20cc4efea2368117d0fb81017ca43a6c687c544d531af17f7d475db7fba2a041
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 9.140628208115231e-07,
-    "train_runtime": 1463.9146,
+    "train_loss": 6.858845210899744e-07,
+    "train_runtime": 1554.7419,
     "train_samples": 11,
-    "train_samples_per_second": 0.219,
-    "train_steps_per_second": 0.014
+    "train_samples_per_second": 0.206,
+    "train_steps_per_second": 0.013
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 9.140628208115231e-07,
-    "train_runtime": 1463.9146,
+    "train_loss": 6.858845210899744e-07,
+    "train_runtime": 1554.7419,
     "train_samples": 11,
-    "train_samples_per_second": 0.219,
-    "train_steps_per_second": 0.014
+    "train_samples_per_second": 0.206,
+    "train_steps_per_second": 0.013
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 617.375,
+      "completion_length": 623.125,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.45765984058380127,
-      "kl": 0.0008022702750167809,
+      "grad_norm": 0.32927563786506653,
+      "kl": 0.000674516981234774,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.4588204845786095,
-      "reward_std": 1.315199837088585,
+      "reward": 1.1585224606096745,
+      "reward_std": 0.9561215024441481,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.4375,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5215704962611198,
+      "rewards/question_recreation_reward_func": 0.5204599518328905,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.4684999957680702,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3099374994635582,
       "step": 2
     },
     {
-      "completion_length": 474.35714285714283,
+      "completion_length": 520.5,
       "epoch": 1.3636363636363638,
-      "grad_norm": 32.118988037109375,
-      "kl": 0.0021380127374348895,
+      "grad_norm": 0.3128676414489746,
+      "kl": 0.0007332045352086425,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.3662904500961304,
-      "reward_std": 1.2328428881508964,
-      "rewards/concensus_correctness_reward_func": 0.048714284385953634,
-      "rewards/consensus_reward_func": 0.7142857142857143,
+      "reward": 1.1351948188883918,
+      "reward_std": 1.0649667382240295,
+      "rewards/concensus_correctness_reward_func": 0.13371428421565465,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5204332854066577,
+      "rewards/question_recreation_reward_func": 0.42790913581848145,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.06500000187328883,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2878571344273431,
       "step": 4
     },
     {
-      "completion_length": 597.75,
+      "completion_length": 471.5357142857143,
       "epoch": 2.0,
-      "grad_norm": 0.27955666184425354,
-      "kl": 0.0009201529568859509,
+      "grad_norm": 0.264969140291214,
+      "kl": 0.0008832713855164391,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.011333272925445,
-      "reward_std": 0.8898934296199253,
-      "rewards/concensus_correctness_reward_func": 0.048714284385953634,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 1.169303366116115,
+      "reward_std": 1.1275735369750433,
+      "rewards/concensus_correctness_reward_func": 0.01257142876940114,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4538689511162894,
+      "rewards/question_recreation_reward_func": 0.45005333849361967,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.20517857585634505,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.31382142433098387,
       "step": 6
     },
     {
-      "completion_length": 547.75,
+      "completion_length": 568.5625,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.3973981440067291,
-      "kl": 0.0009128234341915231,
+      "grad_norm": 0.3894009292125702,
+      "kl": 0.0007739260036032647,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.0256272777915,
-      "reward_std": 0.9381261616945267,
-      "rewards/concensus_correctness_reward_func": 0.04262499883770943,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.2243245132267475,
+      "reward_std": 1.0291738770902157,
+      "rewards/concensus_correctness_reward_func": 0.04325000080280006,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.404314786195755,
+      "rewards/question_recreation_reward_func": 0.6076369620859623,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.20368750765919685,
+      "rewards/xmlcount_reward_func": 0.07343750260770321,
       "step": 8
     },
     {
-      "completion_length": 596.5,
+      "completion_length": 394.0,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.47157469391822815,
-      "kl": 0.0008126678128194596,
+      "grad_norm": 0.38925281167030334,
+      "kl": 0.000897788758655744,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.7186819974865232,
-      "reward_std": 1.2666186456169402,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 1.0290327072143555,
+      "reward_std": 0.7650471691574369,
+      "rewards/concensus_correctness_reward_func": 0.07557142792003495,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.40625341875212534,
+      "rewards/question_recreation_reward_func": 0.3865684462445123,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.04471428905214582,
+      "rewards/xmlcount_reward_func": 0.352607151227338,
       "step": 10
     },
     {
-      "completion_length": 600.0714285714286,
+      "completion_length": 670.1785714285714,
       "epoch": 4.0,
-      "grad_norm": 0.2663576602935791,
-      "kl": 0.0007566870855433601,
+      "grad_norm": 0.22875076532363892,
+      "kl": 0.0007112990549233343,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.9719596760613578,
-      "reward_std": 0.8707286055598941,
-      "rewards/concensus_correctness_reward_func": 0.03571428571428571,
+      "reward": 1.2564320734569006,
+      "reward_std": 1.069840439728328,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5011025326592582,
+      "rewards/question_recreation_reward_func": 0.5129677695887429,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.06014286194528852,
+      "rewards/xmlcount_reward_func": 0.3684642847095217,
       "step": 12
     },
     {
-      "completion_length": 534.3125,
+      "completion_length": 557.75,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.36583825945854187,
-      "kl": 0.0009529158487566747,
+      "grad_norm": 0.4749363660812378,
+      "kl": 0.0007068660997902043,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.9084033668041229,
-      "reward_std": 0.9644215032458305,
+      "reward": 0.6339508574455976,
+      "reward_std": 0.9481349308043718,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.1875,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4169658739119768,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.25706249149516225,
+      "rewards/question_recreation_reward_func": 0.42854462005198,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.14290626347064972,
       "step": 14
     },
     {
-      "completion_length": 559.8571428571429,
+      "completion_length": 553.4642857142857,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.4845457673072815,
-      "kl": 0.0008421636136647846,
+      "grad_norm": 0.41309165954589844,
+      "kl": 0.0006724515198064702,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.653425931930542,
-      "reward_std": 0.9448825589248112,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "reward": 0.9500248410872051,
+      "reward_std": 1.3247052260807581,
+      "rewards/concensus_correctness_reward_func": 0.0455714293888637,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4679616093635559,
+      "rewards/question_recreation_reward_func": 0.5669177089418683,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.024750034723963057,
+      "rewards/xmlcount_reward_func": 0.03396429760115487,
       "step": 16
     },
     {
-      "completion_length": 606.8214285714286,
+      "completion_length": 577.5,
       "epoch": 6.0,
-      "grad_norm": 1.4453972578048706,
-      "kl": 0.000794754596427083,
+      "grad_norm": 0.3813837468624115,
+      "kl": 0.0007483729610352644,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.522873040820871,
-      "reward_std": 1.1045581017221724,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 1.689372181892395,
+      "reward_std": 1.2251285059111459,
+      "rewards/concensus_correctness_reward_func": 0.07892857092831816,
+      "rewards/consensus_reward_func": 0.6428571428571429,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4767658795629229,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6059436116899762,
+      "rewards/soft_format_reward_func": 0.017857142857142856,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.31103571717228207,
+      "rewards/xmlcount_reward_func": 0.34378572021211895,
       "step": 18
     },
     {
-      "completion_length": 554.71875,
+      "completion_length": 532.96875,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.3777633309364319,
-      "kl": 0.0009271802045986988,
+      "grad_norm": 0.3846774399280548,
+      "kl": 0.0007266555148817133,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.8723046258091927,
-      "reward_std": 1.0695999152958393,
-      "rewards/concensus_correctness_reward_func": 0.06812499836087227,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.9871923327445984,
+      "reward_std": 1.0693884268403053,
+      "rewards/concensus_correctness_reward_func": 0.047500000800937414,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3830233681946993,
+      "rewards/question_recreation_reward_func": 0.4166611051186919,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.21803124621510506,
+      "rewards/xmlcount_reward_func": 0.19490626268088818,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 9.140628208115231e-07,
-      "train_runtime": 1463.9146,
-      "train_samples_per_second": 0.219,
-      "train_steps_per_second": 0.014
+      "train_loss": 6.858845210899744e-07,
+      "train_runtime": 1554.7419,
+      "train_samples_per_second": 0.206,
+      "train_steps_per_second": 0.013
     }
   ],
   "logging_steps": 2,