End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2df2af059f1fa85583af8337e8d0c3d14017bfb3501fbc559b13d650a92d1456
+oid sha256:b6ff199f447e2453c6bae789ee115e27b9caea5f324a0534d1d59ddd2f29a918
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.414163181034382e-07,
-    "train_runtime": 1813.6186,
+    "train_loss": 6.86128595361879e-07,
+    "train_runtime": 1616.1217,
     "train_samples": 11,
-    "train_samples_per_second": 0.176,
-    "train_steps_per_second": 0.011
+    "train_samples_per_second": 0.198,
+    "train_steps_per_second": 0.012
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.414163181034382e-07,
-    "train_runtime": 1813.6186,
+    "train_loss": 6.86128595361879e-07,
+    "train_runtime": 1616.1217,
     "train_samples": 11,
-    "train_samples_per_second": 0.176,
-    "train_steps_per_second": 0.011
+    "train_samples_per_second": 0.198,
+    "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 493.8125,
+      "completion_length": 448.34375,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.3305293619632721,
-      "kl": 0.000681818990415195,
+      "grad_norm": 0.9550420641899109,
+      "kl": 0.0007673161817365326,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.663579361513257,
-      "reward_std": 0.9879630729556084,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.6378161907196045,
+      "reward_std": 0.7000563866458833,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4023918565362692,
+      "rewards/question_recreation_reward_func": 0.3441286841407418,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13618750125169754,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.2077499981969595,
       "step": 2
     },
     {
-      "completion_length": 545.2857142857143,
+      "completion_length": 559.7142857142857,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.6514630317687988,
-      "kl": 0.0005856585963296571,
+      "grad_norm": 0.3691655397415161,
+      "kl": 0.0006211901283157724,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.1346928136689323,
-      "reward_std": 0.8935785293579102,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 1.157321231705802,
+      "reward_std": 1.319876023701259,
+      "rewards/concensus_correctness_reward_func": 0.0353571426655565,
       "rewards/consensus_reward_func": 0.42857142857142855,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4740499486880643,
-      "rewards/soft_format_reward_func": 0.03571428571428571,
+      "rewards/question_recreation_reward_func": 0.5186783479792731,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.17849999879087722,
+      "rewards/xmlcount_reward_func": 0.15685713929789408,
       "step": 4
     },
     {
-      "completion_length": 475.89285714285717,
+      "completion_length": 410.32142857142856,
       "epoch": 2.0,
-      "grad_norm": 0.3919878304004669,
-      "kl": 0.0006864020584284195,
+      "grad_norm": 0.3856706917285919,
+      "kl": 0.0007643453011821423,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.8100964426994324,
-      "reward_std": 0.5613211989402771,
+      "reward": 0.7809734291263989,
+      "reward_std": 0.7767886732305799,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3268821526850973,
+      "rewards/question_recreation_reward_func": 0.29372341611555647,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.3939285682780402,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.3801071345806122,
       "step": 6
     },
     {
-      "completion_length": 545.53125,
+      "completion_length": 398.9375,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.3520386517047882,
-      "kl": 0.0006004377246426884,
+      "grad_norm": 0.4694315195083618,
+      "kl": 0.0007544287436758168,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.8596796654164791,
-      "reward_std": 1.0479932762682438,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.7728503812104464,
+      "reward_std": 0.7368343286216259,
+      "rewards/concensus_correctness_reward_func": 0.027374999597668648,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5163671942427754,
+      "rewards/question_recreation_reward_func": 0.4214128777384758,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.030812502838671207,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.23031250946223736,
       "step": 8
     },
     {
-      "completion_length": 469.89285714285717,
+      "completion_length": 412.32142857142856,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.42280104756355286,
-      "kl": 0.0006955595038432096,
+      "grad_norm": 0.6740966439247131,
+      "kl": 0.0007240102443444942,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.528952385698046,
-      "reward_std": 0.9152748286724091,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 0.33227934156145367,
+      "reward_std": 0.5858121908136776,
+      "rewards/concensus_correctness_reward_func": 0.008928571428571428,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2419881054333278,
+      "rewards/question_recreation_reward_func": 0.2764579105590071,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07267856278589793,
+      "rewards/xmlcount_reward_func": -0.02453570972595896,
       "step": 10
     },
     {
-      "completion_length": 416.60714285714283,
+      "completion_length": 458.7857142857143,
       "epoch": 4.0,
-      "grad_norm": 1.5434492826461792,
-      "kl": 0.0008391083060165069,
+      "grad_norm": 0.5076102018356323,
+      "kl": 0.0007106905063015542,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.9031282152448382,
-      "reward_std": 0.9117257467338017,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.0340332176004137,
+      "reward_std": 0.9801140172140939,
+      "rewards/concensus_correctness_reward_func": 0.008928571428571428,
+      "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.38680679883275715,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/question_recreation_reward_func": 0.46624752453395296,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.03571428571428571,
-      "rewards/xmlcount_reward_func": 0.2484642948423113,
+      "rewards/xmlcount_reward_func": 0.38028571967567715,
       "step": 12
     },
     {
-      "completion_length": 474.125,
+      "completion_length": 449.09375,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.4103376865386963,
-      "kl": 0.0006365402732626535,
+      "grad_norm": 1.6110213994979858,
+      "kl": 0.0007361223615589552,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.9181887079030275,
-      "reward_std": 0.8712512701749802,
-      "rewards/concensus_correctness_reward_func": -0.011187500320374966,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.0659161694347858,
+      "reward_std": 0.9395686965435743,
+      "rewards/concensus_correctness_reward_func": 0.0037499999161809683,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5364074558019638,
+      "rewards/question_recreation_reward_func": 0.40613494254648685,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.18984375521540642,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4060312546789646,
       "step": 14
     },
     {
-      "completion_length": 500.42857142857144,
+      "completion_length": 387.7857142857143,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.5999810695648193,
-      "kl": 0.0007516410883649119,
+      "grad_norm": 0.3922109007835388,
+      "kl": 0.0008256308252124914,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.6752335769789559,
-      "reward_std": 1.169873799596514,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 0.8362700939178467,
+      "reward_std": 0.9171016386577061,
+      "rewards/concensus_correctness_reward_func": 0.008928571428571428,
+      "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3122692938361849,
+      "rewards/question_recreation_reward_func": 0.20062722345548017,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.13082141535622732,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.4481428542307445,
       "step": 16
     },
     {
-      "completion_length": 591.2142857142857,
+      "completion_length": 411.75,
       "epoch": 6.0,
-      "grad_norm": 0.29178813099861145,
-      "kl": 0.0006257130298763514,
+      "grad_norm": 0.3162197470664978,
+      "kl": 0.00065167569222727,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.6936301929610116,
-      "reward_std": 0.916986380304609,
+      "reward": 0.9783175247056144,
+      "reward_std": 0.8466850008283343,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.400630207998412,
+      "rewards/question_recreation_reward_func": 0.5067460983991623,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03571428571428571,
-      "rewards/xmlcount_reward_func": 0.11442857554980687,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.3108571491071156,
       "step": 18
     },
     {
-      "completion_length": 461.53125,
+      "completion_length": 476.1875,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.48311421275138855,
-      "kl": 0.00061785375510226,
+      "grad_norm": 0.44239941239356995,
+      "kl": 0.0006094321288401261,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.6026521008461714,
-      "reward_std": 0.782015647739172,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.7325360793620348,
+      "reward_std": 1.0415890887379646,
+      "rewards/concensus_correctness_reward_func": 0.0037499999161809683,
       "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3484645844437182,
+      "rewards/question_recreation_reward_func": 0.4430048484355211,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06668751128017902,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.06703124567866325,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 6.414163181034382e-07,
-      "train_runtime": 1813.6186,
-      "train_samples_per_second": 0.176,
-      "train_steps_per_second": 0.011
+      "train_loss": 6.86128595361879e-07,
+      "train_runtime": 1616.1217,
+      "train_samples_per_second": 0.198,
+      "train_steps_per_second": 0.012
     }
   ],
   "logging_steps": 2,