End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:319d4b1f518bd420e0ac536761fac9fe88abe6a005537fcfc687d0a7abf56313
+oid sha256:06297c81f85c777ce5474785196cf01778d44aab7c10e0597e1e3bbd96534d64
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.6533955204067753e-06,
-    "train_runtime": 1303.4828,
+    "train_loss": 1.6401619859607309e-06,
+    "train_runtime": 1223.4771,
     "train_samples": 11,
-    "train_samples_per_second": 0.245,
-    "train_steps_per_second": 0.015
+    "train_samples_per_second": 0.262,
+    "train_steps_per_second": 0.016
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.6533955204067753e-06,
-    "train_runtime": 1303.4828,
+    "train_loss": 1.6401619859607309e-06,
+    "train_runtime": 1223.4771,
     "train_samples": 11,
-    "train_samples_per_second": 0.245,
-    "train_steps_per_second": 0.015
+    "train_samples_per_second": 0.262,
+    "train_steps_per_second": 0.016
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 451.28125,
+      "completion_length": 473.125,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.44182097911834717,
-      "kl": 0.0019356507327756844,
+      "grad_norm": 0.5780012011528015,
+      "kl": 0.0018195659504272044,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.2238102182745934,
-      "reward_std": 1.2390022426843643,
-      "rewards/concensus_correctness_reward_func": 0.015687499195337296,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 0.7852539010345936,
+      "reward_std": 0.6700366213917732,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.468091506510973,
+      "rewards/question_recreation_reward_func": 0.6348789054900408,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.3337812479585409,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.15037500904873013,
       "step": 2
     },
     {
-      "completion_length": 457.35714285714283,
+      "completion_length": 461.32142857142856,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.45675912499427795,
-      "kl": 0.0015680542564950883,
+      "grad_norm": 0.43347495794296265,
+      "kl": 0.0015060585823708347,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.9336767899138587,
-      "reward_std": 0.890420743397304,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "reward": 1.1546246686152049,
+      "reward_std": 0.9197601931435722,
+      "rewards/concensus_correctness_reward_func": 0.03864285775593349,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4334625133446285,
+      "rewards/question_recreation_reward_func": 0.41933891496488024,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.05357142857142857,
-      "rewards/xmlcount_reward_func": 0.30378569875444683,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3395000100135803,
       "step": 4
     },
     {
-      "completion_length": 440.82142857142856,
+      "completion_length": 553.6785714285714,
       "epoch": 2.0,
-      "grad_norm": 0.4970490634441376,
-      "kl": 0.0022127327475962894,
+      "grad_norm": 0.24650119245052338,
+      "kl": 0.001656832073682121,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.0376974301678794,
-      "reward_std": 1.0912910274096899,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 1.446252771786281,
+      "reward_std": 1.6744095257350378,
+      "rewards/concensus_correctness_reward_func": 0.19264285692146846,
+      "rewards/consensus_reward_func": 0.7857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.40855455504996435,
+      "rewards/question_recreation_reward_func": 0.5666455977729389,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.27200000413826536,
+      "rewards/xmlcount_reward_func": -0.09874998139483589,
       "step": 6
     },
     {
-      "completion_length": 470.96875,
+      "completion_length": 489.90625,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.42499014735221863,
-      "kl": 0.0016785570624051616,
+      "grad_norm": 0.36353105306625366,
+      "kl": 0.0016505489984410815,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.272936761379242,
-      "reward_std": 1.2292308993637562,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.2562414929270744,
+      "reward_std": 1.0721860602498055,
+      "rewards/concensus_correctness_reward_func": 0.05324999988079071,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.621280487626791,
+      "rewards/question_recreation_reward_func": 0.5846789591014385,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.19853125419467688,
+      "rewards/xmlcount_reward_func": 0.2276875041425228,
       "step": 8
     },
     {
-      "completion_length": 418.64285714285717,
+      "completion_length": 500.7142857142857,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.4545181691646576,
-      "kl": 0.0017308940878137946,
+      "grad_norm": 0.5545067191123962,
+      "kl": 0.0017322281741404108,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.7651273161172867,
-      "reward_std": 0.7860261691468102,
-      "rewards/concensus_correctness_reward_func": 0.03585714101791382,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.230509843145098,
+      "reward_std": 1.23474839755467,
+      "rewards/concensus_correctness_reward_func": 0.0247142847095217,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2769844537334783,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.3760812836033957,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.2201428519827979,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.25828570978982107,
       "step": 10
     },
     {
-      "completion_length": 450.64285714285717,
+      "completion_length": 540.7142857142857,
       "epoch": 4.0,
-      "grad_norm": 0.46858593821525574,
-      "kl": 0.0018417536879756621,
+      "grad_norm": 0.2851618528366089,
+      "kl": 0.0018666927935555577,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.1109825521707535,
-      "reward_std": 1.060609472649438,
-      "rewards/concensus_correctness_reward_func": 0.01792857050895691,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 0.9000063027654376,
+      "reward_std": 0.8522530368396214,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4803754304136549,
+      "rewards/question_recreation_reward_func": 0.5019705896931035,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.18410714449627058,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.2373214289546013,
       "step": 12
     },
     {
-      "completion_length": 506.125,
+      "completion_length": 490.59375,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.4882889986038208,
-      "kl": 0.0016498971672262996,
+      "grad_norm": 0.43547067046165466,
+      "kl": 0.0016540642027393915,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.722807576879859,
-      "reward_std": 0.9430956915020943,
-      "rewards/concensus_correctness_reward_func": 0.03137499839067459,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.7657471336424351,
+      "reward_std": 0.8933027181774378,
+      "rewards/concensus_correctness_reward_func": 0.02162499912083149,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.44049508310854435,
+      "rewards/question_recreation_reward_func": 0.44059090316295624,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0009375154040753841,
+      "rewards/xmlcount_reward_func": 0.1160312567371875,
       "step": 14
     },
     {
-      "completion_length": 438.7857142857143,
+      "completion_length": 456.0,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.4207870066165924,
-      "kl": 0.0016474326673362935,
+      "grad_norm": 0.427194744348526,
+      "kl": 0.0021815421558650477,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.5417963096073696,
-      "reward_std": 0.9399098455905914,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.1721123542104448,
+      "reward_std": 0.9155192162309375,
+      "rewards/concensus_correctness_reward_func": 0.0684285717351096,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.45022486788885935,
+      "rewards/question_recreation_reward_func": 0.43671950165714535,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.12271428321089063,
+      "rewards/xmlcount_reward_func": 0.38124998978206087,
       "step": 16
     },
     {
-      "completion_length": 491.92857142857144,
+      "completion_length": 458.92857142857144,
       "epoch": 6.0,
-      "grad_norm": 0.38668152689933777,
-      "kl": 0.0017830188485926815,
+      "grad_norm": 0.5137476325035095,
+      "kl": 0.00196937275385218,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.8857221049921853,
-      "reward_std": 0.9831528876508985,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 1.2816025529588972,
+      "reward_std": 1.0332180857658386,
+      "rewards/concensus_correctness_reward_func": 0.031071428741727556,
       "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4772578179836273,
+      "rewards/question_recreation_reward_func": 0.5437096644725118,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.03346426784992218,
+      "rewards/xmlcount_reward_func": 0.3318214310067041,
       "step": 18
     },
     {
-      "completion_length": 494.6875,
+      "completion_length": 501.9375,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.3912516236305237,
-      "kl": 0.0017925204883795232,
+      "grad_norm": 0.6245306730270386,
+      "kl": 0.0016426051588496193,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.1969642359763384,
-      "reward_std": 1.1354937702417374,
-      "rewards/concensus_correctness_reward_func": 0.015687499195337296,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 0.8476437460631132,
+      "reward_std": 0.9873409513384104,
+      "rewards/concensus_correctness_reward_func": 0.0521249994635582,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.49490175396203995,
+      "rewards/question_recreation_reward_func": 0.5099562611430883,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.10824999772012234,
+      "rewards/xmlcount_reward_func": 0.0824374994263053,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.6533955204067753e-06,
-      "train_runtime": 1303.4828,
-      "train_samples_per_second": 0.245,
-      "train_steps_per_second": 0.015
+      "train_loss": 1.6401619859607309e-06,
+      "train_runtime": 1223.4771,
+      "train_samples_per_second": 0.262,
+      "train_steps_per_second": 0.016
     }
   ],
   "logging_steps": 2,