End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e7cc6260c4d27307d3c6f23331f0397f9fb0a1553c268c9152cf66ee8c43959f
+oid sha256:c73cf49765fd3138999355598bf85f20a8a937900e0f7ff9d189cb9c883215e9
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.136558790283743e-07,
-    "train_runtime": 1684.1044,
+    "train_loss": 7.369539957835514e-07,
+    "train_runtime": 1589.5578,
     "train_samples": 14,
-    "train_samples_per_second": 0.19,
-    "train_steps_per_second": 0.012
+    "train_samples_per_second": 0.201,
+    "train_steps_per_second": 0.013
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.136558790283743e-07,
-    "train_runtime": 1684.1044,
+    "train_loss": 7.369539957835514e-07,
+    "train_runtime": 1589.5578,
     "train_samples": 14,
-    "train_samples_per_second": 0.19,
-    "train_steps_per_second": 0.012
+    "train_samples_per_second": 0.201,
+    "train_steps_per_second": 0.013
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 324.75,
+      "completion_length": 308.375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 7.494189739227295,
-      "kl": 0.0015203254515654407,
+      "grad_norm": 0.4618178904056549,
+      "kl": 0.0008210769738070667,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.6173278782516718,
-      "reward_std": 2.0453468784689903,
-      "rewards/concensus_correctness_reward_func": 0.7067499980330467,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 3.7530966848134995,
+      "reward_std": 3.5663677155971527,
+      "rewards/concensus_correctness_reward_func": 1.902500000782311,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.4556090999394655,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.42531548626720905,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07996875047683716,
+      "rewards/xmlcount_reward_func": 0.5502812545746565,
       "step": 2
     },
     {
-      "completion_length": 346.3333333333333,
+      "completion_length": 294.125,
       "epoch": 1.0,
-      "grad_norm": 0.27051088213920593,
-      "kl": 0.0007224793662317097,
+      "grad_norm": 0.3711313009262085,
+      "kl": 0.0007352615842440476,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 4.278629879156749,
-      "reward_std": 4.529092192649841,
-      "rewards/concensus_correctness_reward_func": 2.4939166667560735,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 2.8463947971661887,
+      "reward_std": 2.6672874043385186,
+      "rewards/concensus_correctness_reward_func": 0.8670000011722246,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5833333333333334,
-      "rewards/question_recreation_reward_func": 0.5247964213291804,
+      "rewards/final_correctness_reward_func": 0.6666666666666666,
+      "rewards/question_recreation_reward_func": 0.571061501900355,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.34325000022848445,
+      "rewards/xmlcount_reward_func": 0.49166666467984516,
       "step": 4
     },
     {
-      "completion_length": 330.78125,
+      "completion_length": 331.03125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.3755961060523987,
-      "kl": 0.0007677657122258097,
+      "grad_norm": 0.5452682375907898,
+      "kl": 0.0008879703818820417,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.6671322477050126,
-      "reward_std": 2.2411595210433006,
-      "rewards/concensus_correctness_reward_func": 0.6502499990165234,
+      "reward": 2.6651667281985283,
+      "reward_std": 2.2832678612321615,
+      "rewards/concensus_correctness_reward_func": 1.2805000003427267,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5165697513148189,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.3908854443579912,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.18781251867767423,
+      "rewards/xmlcount_reward_func": 0.30628125090152025,
       "step": 6
     },
     {
-      "completion_length": 367.7916666666667,
+      "completion_length": 315.4583333333333,
       "epoch": 2.0,
-      "grad_norm": 0.3104553520679474,
-      "kl": 0.0007298217872933795,
+      "grad_norm": 0.3351956009864807,
+      "kl": 0.0007843189911606411,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 4.552031417687734,
-      "reward_std": 4.407095680634181,
-      "rewards/concensus_correctness_reward_func": 2.5124999998758235,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 2.160457650820414,
+      "reward_std": 1.2612790241837502,
+      "rewards/concensus_correctness_reward_func": 0.03366666783889135,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5833333333333334,
-      "rewards/question_recreation_reward_func": 0.5381147364775339,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.583832656343778,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5014166831970215,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.43879168232282,
       "step": 8
     },
     {
-      "completion_length": 321.875,
+      "completion_length": 297.53125,
       "epoch": 2.571428571428571,
-      "grad_norm": 0.42915189266204834,
-      "kl": 0.00089692858455237,
+      "grad_norm": 0.5597347617149353,
+      "kl": 0.000914437769097276,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 2.966095581650734,
-      "reward_std": 3.4431328922510147,
-      "rewards/concensus_correctness_reward_func": 1.2461875001899898,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 3.6466805040836334,
+      "reward_std": 3.303773660212755,
+      "rewards/concensus_correctness_reward_func": 1.8898750003427267,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5967517923563719,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.4443992068991065,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.49815625976771116,
+      "rewards/xmlcount_reward_func": 0.3749062614515424,
       "step": 10
     },
     {
-      "completion_length": 275.4166666666667,
+      "completion_length": 345.0833333333333,
       "epoch": 3.0,
-      "grad_norm": 0.37654978036880493,
-      "kl": 0.0009604951289171974,
+      "grad_norm": 0.2649274170398712,
+      "kl": 0.0008190043930274745,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 2.896782418092092,
-      "reward_std": 2.705845886220535,
-      "rewards/concensus_correctness_reward_func": 1.7107499986886978,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.2134440243244171,
+      "reward_std": 1.232163891196251,
+      "rewards/concensus_correctness_reward_func": 0.041666666666666664,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.5344907194375992,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.36773564169804257,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.23487500528184077,
+      "rewards/xmlcount_reward_func": 0.1373750095566114,
       "step": 12
     },
     {
-      "completion_length": 360.46875,
+      "completion_length": 296.5,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.4417196810245514,
-      "kl": 0.0008304499169753399,
+      "grad_norm": 0.5115249752998352,
+      "kl": 0.0007272411894518882,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 4.035018794238567,
-      "reward_std": 3.590598814189434,
-      "rewards/concensus_correctness_reward_func": 1.9950624704360962,
+      "reward": 4.0324734300374985,
+      "reward_std": 4.162970960140228,
+      "rewards/concensus_correctness_reward_func": 1.9586250027641654,
       "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.628081239759922,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.6743796616792679,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.41187499929219484,
+      "rewards/xmlcount_reward_func": 0.46196874510496855,
       "step": 14
     },
     {
-      "completion_length": 316.375,
+      "completion_length": 318.375,
       "epoch": 4.0,
-      "grad_norm": 0.3043082356452942,
-      "kl": 0.0009063526037304351,
+      "grad_norm": 0.41379380226135254,
+      "kl": 0.0008508136185506979,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.0959248145421345,
-      "reward_std": 1.0131016621987026,
-      "rewards/concensus_correctness_reward_func": 0.09658333100378513,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 2.4556567668914795,
+      "reward_std": 2.669686953226725,
+      "rewards/concensus_correctness_reward_func": 1.0971667543053627,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3403414661685626,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.31807343785961467,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.492333322763443,
+      "rewards/xmlcount_reward_func": 0.45708333949248,
       "step": 16
     },
     {
-      "completion_length": 268.46875,
+      "completion_length": 315.90625,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.4458009898662567,
-      "kl": 0.0009120181421167217,
+      "grad_norm": 0.497284859418869,
+      "kl": 0.0008221051321015693,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 3.6639642491936684,
-      "reward_std": 3.6108987629413605,
-      "rewards/concensus_correctness_reward_func": 1.9879999980330467,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 3.725215867161751,
+      "reward_std": 3.5660266540944576,
+      "rewards/concensus_correctness_reward_func": 1.875,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.4907767567783594,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.6019969191402197,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.419562506955117,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.49821875616908073,
       "step": 18
     },
     {
-      "completion_length": 340.3333333333333,
+      "completion_length": 254.33333333333334,
       "epoch": 5.0,
-      "grad_norm": 0.40639910101890564,
-      "kl": 0.0008975758876961967,
+      "grad_norm": 0.3269297778606415,
+      "kl": 0.0010246408152549218,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.0511793394883473,
-      "reward_std": 0.7933718264102936,
-      "rewards/concensus_correctness_reward_func": 0.008416666959722837,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 2.541357174515724,
+      "reward_std": 2.4800077378749847,
+      "rewards/concensus_correctness_reward_func": 0.9776667108138403,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.4137210374077161,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2957083272437255,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.4136489120622476,
+      "rewards/soft_format_reward_func": 0.020833333333333332,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.4417083350320657,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 8.136558790283743e-07,
-      "train_runtime": 1684.1044,
-      "train_samples_per_second": 0.19,
-      "train_steps_per_second": 0.012
+      "train_loss": 7.369539957835514e-07,
+      "train_runtime": 1589.5578,
+      "train_samples_per_second": 0.201,
+      "train_steps_per_second": 0.013
     }
   ],
   "logging_steps": 2,