End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e72e5d62d0ccb2fc2532de462edff86bdcddee71a72b56ef0683a624256092a8
+oid sha256:56b487afd111e4da52bccf6c9fa8c7197a0add1832cf5fa54bde91f2d4a6075e
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.94423991565418e-06,
-    "train_runtime": 2208.7808,
+    "train_loss": 3.061654888369958e-06,
+    "train_runtime": 2276.638,
     "train_samples": 14,
-    "train_samples_per_second": 0.145,
+    "train_samples_per_second": 0.141,
     "train_steps_per_second": 0.009
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.94423991565418e-06,
-    "train_runtime": 2208.7808,
+    "train_loss": 3.061654888369958e-06,
+    "train_runtime": 2276.638,
     "train_samples": 14,
-    "train_samples_per_second": 0.145,
+    "train_samples_per_second": 0.141,
     "train_steps_per_second": 0.009
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 295.84375,
+      "completion_length": 319.0,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.8563258647918701,
-      "kl": 0.003536480959155597,
+      "grad_norm": 0.8265736103057861,
+      "kl": 0.0036303852102719247,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.5090076625347137,
-      "reward_std": 2.025131955742836,
-      "rewards/concensus_correctness_reward_func": 0.27750000450760126,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 3.2230667397379875,
+      "reward_std": 3.523205704987049,
+      "rewards/concensus_correctness_reward_func": 1.508500010939315,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.3784138932824135,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.49050420615822077,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16559374425560236,
+      "rewards/xmlcount_reward_func": 0.2084375023841858,
       "step": 2
     },
     {
-      "completion_length": 229.20833333333334,
+      "completion_length": 288.4583333333333,
       "epoch": 1.0,
-      "grad_norm": 0.9148445129394531,
-      "kl": 0.006021915857369701,
+      "grad_norm": 0.6778371334075928,
+      "kl": 0.003078966032868872,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 6.8821214238802595,
-      "reward_std": 1.12631660203139,
-      "rewards/concensus_correctness_reward_func": 3.9894166390101113,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 6.541472514470418,
+      "reward_std": 7.260249118010203,
+      "rewards/concensus_correctness_reward_func": 4.639250000317891,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.4860379497210185,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.2928476979335149,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.49000000456968945,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.255208329608043,
       "step": 4
     },
     {
-      "completion_length": 266.78125,
+      "completion_length": 267.40625,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.7374868988990784,
-      "kl": 0.003570650515030138,
+      "grad_norm": 0.9330791234970093,
+      "kl": 0.0036824695125687867,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 6.189605325460434,
-      "reward_std": 4.721047542989254,
-      "rewards/concensus_correctness_reward_func": 4.111750059761107,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 3.196286790072918,
+      "reward_std": 2.722376387566328,
+      "rewards/concensus_correctness_reward_func": 1.014437495265156,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.42754288017749786,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.39591173361986876,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3846874926239252,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4734374936670065,
       "step": 6
     },
     {
-      "completion_length": 260.5,
+      "completion_length": 244.20833333333334,
       "epoch": 2.0,
-      "grad_norm": 0.5508885979652405,
-      "kl": 0.004868219878214101,
+      "grad_norm": 0.45333999395370483,
+      "kl": 0.003310707824615141,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 6.274152000745137,
-      "reward_std": 4.543557991584142,
-      "rewards/concensus_correctness_reward_func": 3.6819167161981263,
-      "rewards/consensus_reward_func": 0.75,
+      "reward": 5.11900262037913,
+      "reward_std": 6.583905945221583,
+      "rewards/concensus_correctness_reward_func": 3.427916668355465,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0833333333333333,
-      "rewards/question_recreation_reward_func": 0.38919370373090106,
-      "rewards/soft_format_reward_func": 0.020833333333333332,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3488749985893567,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.4223358705639839,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.581249992052714,
       "step": 8
     },
     {
-      "completion_length": 254.25,
+      "completion_length": 302.6875,
       "epoch": 2.571428571428571,
-      "grad_norm": 0.6594353318214417,
-      "kl": 0.003944829441024922,
+      "grad_norm": 0.48581212759017944,
+      "kl": 0.0031960688793333247,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 3.2260841988027096,
-      "reward_std": 2.6478710919618607,
-      "rewards/concensus_correctness_reward_func": 1.4044374972581863,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 4.359011098742485,
+      "reward_std": 5.041795432567596,
+      "rewards/concensus_correctness_reward_func": 2.613250009715557,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.875,
-      "rewards/question_recreation_reward_func": 0.4373656213283539,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5397296585142612,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3217812580987811,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4247812554240227,
       "step": 10
     },
     {
-      "completion_length": 233.79166666666666,
+      "completion_length": 264.5416666666667,
       "epoch": 3.0,
-      "grad_norm": 0.7712763547897339,
-      "kl": 0.006244191473039488,
+      "grad_norm": 0.37057843804359436,
+      "kl": 0.003878361739528676,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 3.1116245090961456,
-      "reward_std": 4.354177655031283,
-      "rewards/concensus_correctness_reward_func": 1.7080833328266938,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 4.831206654508908,
+      "reward_std": 6.333635171254476,
+      "rewards/concensus_correctness_reward_func": 2.8268333077430725,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.4336661696434021,
+      "rewards/final_correctness_reward_func": 0.6666666666666666,
+      "rewards/question_recreation_reward_func": 0.3961649263898532,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3865416534245014,
+      "rewards/xmlcount_reward_func": 0.3582083247601986,
       "step": 12
     },
     {
-      "completion_length": 294.34375,
+      "completion_length": 304.71875,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.8130471706390381,
-      "kl": 0.003477307560387999,
+      "grad_norm": 0.7317449450492859,
+      "kl": 0.003004714220878668,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.4697456657886505,
-      "reward_std": 2.4868993014097214,
-      "rewards/concensus_correctness_reward_func": 0.8234375007450581,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 4.568838551640511,
+      "reward_std": 5.097499243915081,
+      "rewards/concensus_correctness_reward_func": 2.7120625108946115,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5231832601130009,
+      "rewards/question_recreation_reward_func": 0.6519634742289782,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3731249952688813,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3141875136643648,
       "step": 14
     },
     {
-      "completion_length": 274.5833333333333,
+      "completion_length": 344.7916666666667,
       "epoch": 4.0,
-      "grad_norm": 0.34250977635383606,
-      "kl": 0.00459623197093606,
+      "grad_norm": 0.36377763748168945,
+      "kl": 0.003294540375160674,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 4.730737209320068,
-      "reward_std": 5.185523370901744,
-      "rewards/concensus_correctness_reward_func": 2.949750006198883,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 4.352070172627767,
+      "reward_std": 3.323097010453542,
+      "rewards/concensus_correctness_reward_func": 2.0965833365917206,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5833333333333334,
-      "rewards/question_recreation_reward_func": 0.33107048521439236,
+      "rewards/final_correctness_reward_func": 0.6666666666666666,
+      "rewards/question_recreation_reward_func": 0.3886534372965495,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.44991667692859966,
+      "rewards/xmlcount_reward_func": 0.5334999983509382,
       "step": 16
     },
     {
-      "completion_length": 269.78125,
+      "completion_length": 281.65625,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.5748184323310852,
-      "kl": 0.0047870511771179736,
+      "grad_norm": 1.0194026231765747,
+      "kl": 0.0037469247763510793,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 5.355020493268967,
-      "reward_std": 3.6786953806877136,
-      "rewards/concensus_correctness_reward_func": 3.2864374984055758,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 6.420581094920635,
+      "reward_std": 6.502259276807308,
+      "rewards/concensus_correctness_reward_func": 4.080687532201409,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.491989191621542,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.5529873929917812,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4515937529504299,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.33378125145100057,
       "step": 18
     },
     {
-      "completion_length": 221.66666666666666,
+      "completion_length": 306.0,
       "epoch": 5.0,
-      "grad_norm": 0.5979350805282593,
-      "kl": 0.004887227822716038,
+      "grad_norm": 0.5315831899642944,
+      "kl": 0.004204936480770509,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 6.617766787608464,
-      "reward_std": 5.993567039569219,
-      "rewards/concensus_correctness_reward_func": 4.354166651765506,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 4.401223103205363,
+      "reward_std": 4.7618053158124285,
+      "rewards/concensus_correctness_reward_func": 2.6359999974568686,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9166666666666666,
-      "rewards/question_recreation_reward_func": 0.40672506640354794,
+      "rewards/final_correctness_reward_func": 0.4166666666666667,
+      "rewards/question_recreation_reward_func": 0.4475146258870761,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4402083406845729,
+      "rewards/strict_format_reward_func": 0.041666666666666664,
+      "rewards/xmlcount_reward_func": 0.2760416766007741,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.94423991565418e-06,
-      "train_runtime": 2208.7808,
-      "train_samples_per_second": 0.145,
+      "train_loss": 3.061654888369958e-06,
+      "train_runtime": 2276.638,
+      "train_samples_per_second": 0.141,
       "train_steps_per_second": 0.009
     }
   ],