End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b69d8dac1962887399cb09429886a871cce210bf5b30724ec386f549f5d336b
+oid sha256:49e7f4b423b70b3f75c43d0ee0042f2ed51bb53084f1c07e599806e1df783b8c
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.5328842096096195e-06,
-    "train_runtime": 525.9274,
+    "train_loss": 1.6526825561413716e-06,
+    "train_runtime": 514.3119,
     "train_samples": 11,
-    "train_samples_per_second": 0.608,
-    "train_steps_per_second": 0.038
+    "train_samples_per_second": 0.622,
+    "train_steps_per_second": 0.039
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.5328842096096195e-06,
-    "train_runtime": 525.9274,
+    "train_loss": 1.6526825561413716e-06,
+    "train_runtime": 514.3119,
     "train_samples": 11,
-    "train_samples_per_second": 0.608,
-    "train_steps_per_second": 0.038
+    "train_samples_per_second": 0.622,
+    "train_steps_per_second": 0.039
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 381.9375,
+      "completion_length": 366.3125,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.4596123993396759,
-      "kl": 0.001440838270355016,
+      "grad_norm": 1.4960169792175293,
+      "kl": 0.001837878895457834,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.9129732511937618,
-      "reward_std": 1.1515442375093699,
-      "rewards/concensus_correctness_reward_func": 0.01431250013411045,
+      "reward": 1.5223995298147202,
+      "reward_std": 0.6293464861810207,
+      "rewards/concensus_correctness_reward_func": 0.1875,
       "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.512410731986165,
+      "rewards/question_recreation_reward_func": 0.5197432637214661,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.11375000374391675,
+      "rewards/xmlcount_reward_func": 0.3151562660932541,
       "step": 2
     },
     {
-      "completion_length": 462.17857142857144,
+      "completion_length": 397.60714285714283,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.3766481876373291,
-      "kl": 0.001980220849093582,
+      "grad_norm": 0.5029628276824951,
+      "kl": 0.0017673390850956952,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.0409666427544184,
-      "reward_std": 1.0517850214881557,
-      "rewards/concensus_correctness_reward_func": 0.029214284249714444,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.9705819306629044,
+      "reward_std": 1.1530599828277315,
+      "rewards/concensus_correctness_reward_func": 0.09707142625536237,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.3860381011451994,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3479033815009253,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1257142914193017,
+      "rewards/xmlcount_reward_func": 0.16846429024423873,
       "step": 4
     },
     {
-      "completion_length": 387.4642857142857,
+      "completion_length": 391.2857142857143,
       "epoch": 2.0,
-      "grad_norm": 0.34000086784362793,
-      "kl": 0.0014845101395621896,
+      "grad_norm": 0.4054848253726959,
+      "kl": 0.00174615705119712,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.327462715762002,
-      "reward_std": 1.294439171041761,
-      "rewards/concensus_correctness_reward_func": 0.04364285724503653,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 1.032759734562465,
+      "reward_std": 1.1778842338493891,
+      "rewards/concensus_correctness_reward_func": 0.1072142847946712,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.559355554836137,
+      "rewards/question_recreation_reward_func": 0.512688302568027,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.29589285382202696,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.18071429857185908,
       "step": 6
     },
     {
-      "completion_length": 511.46875,
+      "completion_length": 412.84375,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.3722342550754547,
-      "kl": 0.001470490089559462,
+      "grad_norm": 0.6137565970420837,
+      "kl": 0.001445371744921431,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.6776670143008232,
-      "reward_std": 1.140218161046505,
+      "reward": 0.8012570161372423,
+      "reward_std": 0.7861417420208454,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.25,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6425107698887587,
+      "rewards/question_recreation_reward_func": 0.5185382533818483,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.21484376676380634,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.1420937441289425,
       "step": 8
     },
     {
-      "completion_length": 317.67857142857144,
+      "completion_length": 336.2857142857143,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.5521669387817383,
-      "kl": 0.0018845175259879657,
+      "grad_norm": 0.536916971206665,
+      "kl": 0.002134649848033275,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.34071518267904,
-      "reward_std": 1.0933335423469543,
-      "rewards/concensus_correctness_reward_func": 0.12164285566125597,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.4072004471506392,
+      "reward_std": 0.9136639471564975,
+      "rewards/concensus_correctness_reward_func": 0.09392857125827245,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.21428571428571427,
-      "rewards/question_recreation_reward_func": 0.3257508873939514,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.31530758099896566,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.44689284903662546,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.7122500155653272,
       "step": 10
     },
     {
-      "completion_length": 413.7142857142857,
+      "completion_length": 460.42857142857144,
       "epoch": 4.0,
-      "grad_norm": 0.6070003509521484,
-      "kl": 0.0016493868814515217,
+      "grad_norm": 0.3349718153476715,
+      "kl": 0.0014100028035630072,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.3134214558771677,
-      "reward_std": 1.2431852221488953,
-      "rewards/concensus_correctness_reward_func": 0.06857142703873771,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 1.0341259986162186,
+      "reward_std": 1.216601891177041,
+      "rewards/concensus_correctness_reward_func": 0.05357142857142857,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2857142857142857,
-      "rewards/question_recreation_reward_func": 0.6075999992234367,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6409116472516742,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06582143264157432,
+      "rewards/xmlcount_reward_func": 0.1253571462418352,
       "step": 12
     },
     {
-      "completion_length": 363.75,
+      "completion_length": 382.4375,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.5632243156433105,
-      "kl": 0.0016079658380476758,
+      "grad_norm": 0.5842562317848206,
+      "kl": 0.0018867350881919265,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.1775269322097301,
-      "reward_std": 1.1245576068758965,
-      "rewards/concensus_correctness_reward_func": 0.009812500327825546,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 0.7885393239557743,
+      "reward_std": 0.8474745936691761,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.39168317802250385,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.49241431429982185,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.33853124268352985,
+      "rewards/xmlcount_reward_func": 0.10862501058727503,
       "step": 14
     },
     {
-      "completion_length": 391.5357142857143,
+      "completion_length": 386.2857142857143,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.6763136386871338,
-      "kl": 0.0016269180424777524,
+      "grad_norm": 0.5477598905563354,
+      "kl": 0.0016645457030140928,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.8577623793057032,
-      "reward_std": 2.8034206799098422,
-      "rewards/concensus_correctness_reward_func": 0.75,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 0.9445783623627254,
+      "reward_std": 1.0024188586643763,
+      "rewards/concensus_correctness_reward_func": 0.05357142857142857,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2857142857142857,
-      "rewards/question_recreation_reward_func": 0.4767981693148613,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.011892854103020259,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5618640652724675,
+      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.07914286426135472,
       "step": 16
     },
     {
-      "completion_length": 379.42857142857144,
+      "completion_length": 384.5357142857143,
       "epoch": 6.0,
-      "grad_norm": 0.3384060859680176,
-      "kl": 0.0018665683123150042,
+      "grad_norm": 0.42141640186309814,
+      "kl": 0.002002579309711499,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.8319548964500427,
-      "reward_std": 2.2698605997221812,
-      "rewards/concensus_correctness_reward_func": 0.7650000027247837,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "reward": 1.3217215027127946,
+      "reward_std": 1.270809360912868,
+      "rewards/concensus_correctness_reward_func": 0.16928571249757493,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2857142857142857,
-      "rewards/question_recreation_reward_func": 0.4369191506079265,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.20146428900105612,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5725429398672921,
+      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.18703571068389074,
       "step": 18
     },
     {
-      "completion_length": 403.71875,
+      "completion_length": 397.75,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.4062493145465851,
-      "kl": 0.0016297149268211797,
+      "grad_norm": 0.5499128699302673,
+      "kl": 0.0016771303344285116,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.052502359263599,
-      "reward_std": 1.1235482916235924,
-      "rewards/concensus_correctness_reward_func": 0.03818750008940697,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.0473634013906121,
+      "reward_std": 0.7728652590885758,
+      "rewards/concensus_correctness_reward_func": 0.10575000196695328,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4699398223310709,
+      "rewards/question_recreation_reward_func": 0.4337696129223332,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16937501076608896,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.17971873981878161,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.5328842096096195e-06,
-      "train_runtime": 525.9274,
-      "train_samples_per_second": 0.608,
-      "train_steps_per_second": 0.038
+      "train_loss": 1.6526825561413716e-06,
+      "train_runtime": 514.3119,
+      "train_samples_per_second": 0.622,
+      "train_steps_per_second": 0.039
     }
   ],
   "logging_steps": 2,