End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d400e22463117128c5687405661f6c0c1de57bf15b029d0d86b95dbeef93166
+oid sha256:1ee8a6119dc23bf21d6ea563438fa9a3568bbaab2b0d3ac0da87cf92e400bd97
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.993006249762403e-07,
-    "train_runtime": 660.6931,
+    "train_loss": 6.453263267758303e-07,
+    "train_runtime": 694.7411,
     "train_samples": 11,
-    "train_samples_per_second": 0.484,
-    "train_steps_per_second": 0.03
+    "train_samples_per_second": 0.461,
+    "train_steps_per_second": 0.029
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.993006249762403e-07,
-    "train_runtime": 660.6931,
+    "train_loss": 6.453263267758303e-07,
+    "train_runtime": 694.7411,
     "train_samples": 11,
-    "train_samples_per_second": 0.484,
-    "train_steps_per_second": 0.03
+    "train_samples_per_second": 0.461,
+    "train_steps_per_second": 0.029
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 518.5,
+      "completion_length": 563.34375,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.555176317691803,
-      "kl": 0.0007004447579674888,
+      "grad_norm": 0.3412521779537201,
+      "kl": 0.0006879963693791069,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.8274347987025976,
-      "reward_std": 1.1000676304101944,
-      "rewards/concensus_correctness_reward_func": 0.015625,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.73992083594203,
+      "reward_std": 1.143914290703833,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.49177856743335724,
+      "rewards/question_recreation_reward_func": 0.47585833724588156,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.007531255949288607,
+      "rewards/xmlcount_reward_func": -0.11093749850988388,
       "step": 2
     },
     {
-      "completion_length": 380.92857142857144,
+      "completion_length": 535.3214285714286,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.43665412068367004,
-      "kl": 0.0008776169935507434,
+      "grad_norm": 0.30657586455345154,
+      "kl": 0.0006268782391478973,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.7335536139351981,
-      "reward_std": 0.7739071760858808,
+      "reward": 0.7660549112728664,
+      "reward_std": 1.128754803112575,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.07142857142857142,
+      "rewards/consensus_reward_func": 0.42857142857142855,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.38558934203216005,
+      "rewards/question_recreation_reward_func": 0.30587634657110485,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.2586785948702267,
+      "rewards/xmlcount_reward_func": 0.013750001788139343,
       "step": 4
     },
     {
-      "completion_length": 544.2142857142857,
+      "completion_length": 531.1428571428571,
       "epoch": 2.0,
-      "grad_norm": 0.31843292713165283,
-      "kl": 0.0007293318132204669,
+      "grad_norm": 0.3350563645362854,
+      "kl": 0.0007434671528504363,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.8059655938829694,
-      "reward_std": 1.1535072326660156,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 0.7349748356001717,
+      "reward_std": 1.101830163172313,
+      "rewards/concensus_correctness_reward_func": 0.006642857300383704,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5153227469750813,
+      "rewards/question_recreation_reward_func": 0.36926053890160154,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.058500008923666816,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.0019285551139286586,
       "step": 6
     },
     {
-      "completion_length": 562.4375,
+      "completion_length": 594.0625,
       "epoch": 2.7272727272727275,
-      "grad_norm": 4.0187530517578125,
-      "kl": 0.0014460544553003274,
+      "grad_norm": 0.9346657395362854,
+      "kl": 0.0006868069322081283,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.044090647250414,
-      "reward_std": 0.9590163063257933,
+      "reward": 0.7243713960051537,
+      "reward_std": 1.3020212054252625,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.5625,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.44065316766500473,
+      "rewards/question_recreation_reward_func": 0.38334012776613235,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.04093749704770744,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": -0.002718733623623848,
       "step": 8
     },
     {
-      "completion_length": 495.75,
+      "completion_length": 623.7857142857143,
       "epoch": 3.3636363636363638,
-      "grad_norm": 3.4272704124450684,
-      "kl": 0.000784852820548362,
+      "grad_norm": 0.29732203483581543,
+      "kl": 0.0006290328788704105,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.6675052131925311,
-      "reward_std": 1.1034608014992304,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 1.1950739026069641,
+      "reward_std": 1.3936849406787328,
+      "rewards/concensus_correctness_reward_func": 0.006642857300383704,
+      "rewards/consensus_reward_func": 0.42857142857142855,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.42725521538938793,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/question_recreation_reward_func": 0.47925250232219696,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": -0.15260715250458037,
+      "rewards/xmlcount_reward_func": 0.2627500146627426,
       "step": 10
     },
     {
-      "completion_length": 512.6428571428571,
+      "completion_length": 546.3571428571429,
       "epoch": 4.0,
-      "grad_norm": 0.3071759045124054,
-      "kl": 0.0008274220578771617,
+      "grad_norm": 0.3038821220397949,
+      "kl": 0.0007142754205103431,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.309668013027736,
-      "reward_std": 1.1936028088842119,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.8773872426577977,
+      "reward_std": 1.5168753096035548,
+      "rewards/concensus_correctness_reward_func": 0.013285714600767409,
       "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5594894502844129,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/question_recreation_reward_func": 0.42256578377314974,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.21446428767272405,
+      "rewards/xmlcount_reward_func": -0.07632141986063548,
       "step": 12
     },
     {
-      "completion_length": 550.09375,
+      "completion_length": 562.21875,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.3173958361148834,
-      "kl": 0.0008341810971614905,
+      "grad_norm": 0.2944260239601135,
+      "kl": 0.000732177410100121,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.7159233158454299,
-      "reward_std": 1.0219199433922768,
+      "reward": 1.0529435947537422,
+      "reward_std": 1.2766156494617462,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.125,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4055483331903815,
+      "rewards/question_recreation_reward_func": 0.49947487749159336,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.16975000500679016,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.17846874706447124,
       "step": 14
     },
     {
-      "completion_length": 479.0357142857143,
+      "completion_length": 508.92857142857144,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.4549387991428375,
-      "kl": 0.0008137404719101531,
+      "grad_norm": 0.5001223087310791,
+      "kl": 0.0007892606330902449,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.1487190595694952,
-      "reward_std": 0.8686993377549308,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 0.882321685552597,
+      "reward_std": 1.0122005002839225,
+      "rewards/concensus_correctness_reward_func": 0.013285714600767409,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5325047671794891,
+      "rewards/question_recreation_reward_func": 0.5161074038062777,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.2412142881325313,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.06721428781747818,
       "step": 16
     },
     {
-      "completion_length": 450.35714285714283,
+      "completion_length": 665.0,
       "epoch": 6.0,
-      "grad_norm": 0.9689341187477112,
-      "kl": 0.0009170615979071174,
+      "grad_norm": 0.29188090562820435,
+      "kl": 0.0005866806064399757,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.4135749212333134,
-      "reward_std": 1.0793337864535195,
-      "rewards/concensus_correctness_reward_func": 0.014714285731315613,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 1.2284236805779594,
+      "reward_std": 0.8941658990723746,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3206820615700313,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": -0.2432499953678676,
+      "rewards/question_recreation_reward_func": 0.4088879461799349,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3195357152393886,
       "step": 18
     },
     {
-      "completion_length": 572.3125,
+      "completion_length": 514.6875,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.3049357235431671,
-      "kl": 0.0007123384966689628,
+      "grad_norm": 0.47368043661117554,
+      "kl": 0.000769777390814852,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.6959485113620758,
-      "reward_std": 1.0689484477043152,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.8404022082686424,
+      "reward_std": 1.426487885415554,
+      "rewards/concensus_correctness_reward_func": 0.005812500137835741,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.49332351610064507,
+      "rewards/question_recreation_reward_func": 0.406339680776,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07762499898672104,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.03762500360608101,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.993006249762403e-07,
-      "train_runtime": 660.6931,
-      "train_samples_per_second": 0.484,
-      "train_steps_per_second": 0.03
+      "train_loss": 6.453263267758303e-07,
+      "train_runtime": 694.7411,
+      "train_samples_per_second": 0.461,
+      "train_steps_per_second": 0.029
     }
   ],
   "logging_steps": 2,