End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1ee8a6119dc23bf21d6ea563438fa9a3568bbaab2b0d3ac0da87cf92e400bd97
+oid sha256:06bfb7a8f0f26f4bbb23752893ba522b711b503433d1d877cecfd3beb3bb6329
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.453263267758303e-07,
-    "train_runtime": 694.7411,
+    "train_loss": 8.002021615993727e-07,
+    "train_runtime": 706.8151,
     "train_samples": 11,
-    "train_samples_per_second": 0.461,
-    "train_steps_per_second": 0.029
+    "train_samples_per_second": 0.453,
+    "train_steps_per_second": 0.028
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.453263267758303e-07,
-    "train_runtime": 694.7411,
+    "train_loss": 8.002021615993727e-07,
+    "train_runtime": 706.8151,
     "train_samples": 11,
-    "train_samples_per_second": 0.461,
-    "train_steps_per_second": 0.029
+    "train_samples_per_second": 0.453,
+    "train_steps_per_second": 0.028
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 563.34375,
+      "completion_length": 590.0,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.3412521779537201,
-      "kl": 0.0006879963693791069,
+      "grad_norm": 0.4663006365299225,
+      "kl": 0.0007288729466381483,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.73992083594203,
-      "reward_std": 1.143914290703833,
+      "reward": 0.7590441051870584,
+      "reward_std": 1.1202260591089725,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.375,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.47585833724588156,
+      "rewards/question_recreation_reward_func": 0.4478878565132618,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.11093749850988388,
+      "rewards/xmlcount_reward_func": 0.3111562505364418,
       "step": 2
     },
     {
-      "completion_length": 535.3214285714286,
+      "completion_length": 612.1785714285714,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.30657586455345154,
-      "kl": 0.0006268782391478973,
+      "grad_norm": 0.6310429573059082,
+      "kl": 0.0007378177126936082,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.7660549112728664,
-      "reward_std": 1.128754803112575,
+      "reward": 0.798093746815409,
+      "reward_std": 1.0958153435162135,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.30587634657110485,
+      "rewards/question_recreation_reward_func": 0.2663080633750984,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.013750001788139343,
+      "rewards/xmlcount_reward_func": 0.2282142894608634,
       "step": 4
     },
     {
-      "completion_length": 531.1428571428571,
+      "completion_length": 558.3571428571429,
       "epoch": 2.0,
-      "grad_norm": 0.3350563645362854,
-      "kl": 0.0007434671528504363,
+      "grad_norm": 20.305686950683594,
+      "kl": 0.002111146758709635,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.7349748356001717,
-      "reward_std": 1.101830163172313,
-      "rewards/concensus_correctness_reward_func": 0.006642857300383704,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 0.7155568769999913,
+      "reward_std": 0.7241366165024894,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.36926053890160154,
+      "rewards/question_recreation_reward_func": 0.4653426195893969,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0019285551139286586,
+      "rewards/xmlcount_reward_func": 0.2502142935991287,
       "step": 6
     },
     {
-      "completion_length": 594.0625,
+      "completion_length": 569.0625,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.9346657395362854,
-      "kl": 0.0006868069322081283,
+      "grad_norm": 0.2997376620769501,
+      "kl": 0.0006852330989204347,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.7243713960051537,
-      "reward_std": 1.3020212054252625,
+      "reward": 0.5706977061927319,
+      "reward_std": 0.9442779440432787,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.3125,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.38334012776613235,
+      "rewards/question_recreation_reward_func": 0.44401017762720585,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": -0.002718733623623848,
+      "rewards/xmlcount_reward_func": 0.032937511801719666,
       "step": 8
     },
     {
-      "completion_length": 623.7857142857143,
+      "completion_length": 570.9642857142857,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.29732203483581543,
-      "kl": 0.0006290328788704105,
+      "grad_norm": 0.5129194855690002,
+      "kl": 0.0007547670975327492,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.1950739026069641,
-      "reward_std": 1.3936849406787328,
-      "rewards/concensus_correctness_reward_func": 0.006642857300383704,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 0.7262742732252393,
+      "reward_std": 1.1136714539357595,
+      "rewards/concensus_correctness_reward_func": 0.03571428571428571,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.47925250232219696,
+      "rewards/question_recreation_reward_func": 0.3148814226899828,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.2627500146627426,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.018535726836749485,
       "step": 10
     },
     {
-      "completion_length": 546.3571428571429,
+      "completion_length": 565.7857142857143,
       "epoch": 4.0,
-      "grad_norm": 0.3038821220397949,
-      "kl": 0.0007142754205103431,
+      "grad_norm": 0.24238479137420654,
+      "kl": 0.0007750183743025575,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.8773872426577977,
-      "reward_std": 1.5168753096035548,
-      "rewards/concensus_correctness_reward_func": 0.013285714600767409,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 0.853008508682251,
+      "reward_std": 1.244115846497672,
+      "rewards/concensus_correctness_reward_func": 0.014785714447498322,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.42256578377314974,
+      "rewards/question_recreation_reward_func": 0.48872279269354685,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": -0.07632141986063548,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.007642852940729686,
       "step": 12
     },
     {
-      "completion_length": 562.21875,
+      "completion_length": 523.875,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.2944260239601135,
-      "kl": 0.000732177410100121,
+      "grad_norm": 0.30150485038757324,
+      "kl": 0.0007246694731293246,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.0529435947537422,
-      "reward_std": 1.2766156494617462,
+      "reward": 0.5891091991215944,
+      "reward_std": 1.0257326923310757,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.375,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.49947487749159336,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.3265154678374529,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17846874706447124,
+      "rewards/xmlcount_reward_func": 0.07509374804794788,
       "step": 14
     },
     {
-      "completion_length": 508.92857142857144,
+      "completion_length": 563.6071428571429,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.5001223087310791,
-      "kl": 0.0007892606330902449,
+      "grad_norm": 0.42829781770706177,
+      "kl": 0.0008008114403180246,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.882321685552597,
-      "reward_std": 1.0122005002839225,
-      "rewards/concensus_correctness_reward_func": 0.013285714600767409,
+      "reward": 0.6023855464799064,
+      "reward_std": 1.1734420231410436,
+      "rewards/concensus_correctness_reward_func": 0.012642857219491686,
       "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5161074038062777,
+      "rewards/question_recreation_reward_func": 0.4298141087804522,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06721428781747818,
+      "rewards/xmlcount_reward_func": -0.12578569991247995,
       "step": 16
     },
     {
-      "completion_length": 665.0,
+      "completion_length": 493.5357142857143,
       "epoch": 6.0,
-      "grad_norm": 0.29188090562820435,
-      "kl": 0.0005866806064399757,
+      "grad_norm": 0.33995261788368225,
+      "kl": 0.0007940116338431835,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.2284236805779594,
-      "reward_std": 0.8941658990723746,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.0086630071912492,
+      "reward_std": 0.7423376888036728,
+      "rewards/concensus_correctness_reward_func": 0.014785714447498322,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4088879461799349,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2943415769508907,
+      "rewards/soft_format_reward_func": 0.017857142857142856,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3195357152393886,
+      "rewards/xmlcount_reward_func": 0.3959642882858004,
       "step": 18
     },
     {
-      "completion_length": 514.6875,
+      "completion_length": 567.84375,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.47368043661117554,
-      "kl": 0.000769777390814852,
+      "grad_norm": 0.38296636939048767,
+      "kl": 0.0006954705531825311,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.8404022082686424,
-      "reward_std": 1.426487885415554,
-      "rewards/concensus_correctness_reward_func": 0.005812500137835741,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 0.9041110388934612,
+      "reward_std": 1.277186619117856,
+      "rewards/concensus_correctness_reward_func": 0.024000000208616257,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.406339680776,
+      "rewards/question_recreation_reward_func": 0.477923521772027,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.03762500360608101,
+      "rewards/xmlcount_reward_func": 0.0740625150501728,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 6.453263267758303e-07,
-      "train_runtime": 694.7411,
-      "train_samples_per_second": 0.461,
-      "train_steps_per_second": 0.029
+      "train_loss": 8.002021615993727e-07,
+      "train_runtime": 706.8151,
+      "train_samples_per_second": 0.453,
+      "train_steps_per_second": 0.028
     }
   ],
   "logging_steps": 2,