End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:33cf78606cd36cdf106cd865918c4a1c4f3ae02fd2d158818f1a20be65dbda28
+oid sha256:e7cc6260c4d27307d3c6f23331f0397f9fb0a1553c268c9152cf66ee8c43959f
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.307810619749944e-07,
-    "train_runtime": 1695.0641,
+    "train_loss": 8.136558790283743e-07,
+    "train_runtime": 1684.1044,
     "train_samples": 14,
-    "train_samples_per_second": 0.189,
+    "train_samples_per_second": 0.19,
     "train_steps_per_second": 0.012
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.307810619749944e-07,
-    "train_runtime": 1695.0641,
+    "train_loss": 8.136558790283743e-07,
+    "train_runtime": 1684.1044,
     "train_samples": 14,
-    "train_samples_per_second": 0.189,
+    "train_samples_per_second": 0.19,
     "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 286.6875,
+      "completion_length": 324.75,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.4535891115665436,
-      "kl": 0.000859841635246994,
+      "grad_norm": 7.494189739227295,
+      "kl": 0.0015203254515654407,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 2.995594423264265,
-      "reward_std": 2.5770161375403404,
-      "rewards/concensus_correctness_reward_func": 1.2781875003129244,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 1.6173278782516718,
+      "reward_std": 2.0453468784689903,
+      "rewards/concensus_correctness_reward_func": 0.7067499980330467,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.45025062188506126,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4556090999394655,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.37653125170618296,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.07996875047683716,
       "step": 2
     },
     {
-      "completion_length": 283.4166666666667,
+      "completion_length": 346.3333333333333,
       "epoch": 1.0,
-      "grad_norm": 0.47619107365608215,
-      "kl": 0.0008487212374651184,
+      "grad_norm": 0.27051088213920593,
+      "kl": 0.0007224793662317097,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 5.357995266715686,
-      "reward_std": 5.9385765840609865,
-      "rewards/concensus_correctness_reward_func": 2.870083341995875,
-      "rewards/consensus_reward_func": 0.75,
+      "reward": 4.278629879156749,
+      "reward_std": 4.529092192649841,
+      "rewards/concensus_correctness_reward_func": 2.4939166667560735,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.6812870502471924,
+      "rewards/final_correctness_reward_func": 0.5833333333333334,
+      "rewards/question_recreation_reward_func": 0.5247964213291804,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30662500113248825,
+      "rewards/xmlcount_reward_func": 0.34325000022848445,
       "step": 4
     },
     {
-      "completion_length": 288.03125,
+      "completion_length": 330.78125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.48158586025238037,
-      "kl": 0.0008262748160632327,
+      "grad_norm": 0.3755961060523987,
+      "kl": 0.0007677657122258097,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 3.6037064269185066,
-      "reward_std": 3.6371278911828995,
-      "rewards/concensus_correctness_reward_func": 2.0355000011622906,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 1.6671322477050126,
+      "reward_std": 2.2411595210433006,
+      "rewards/concensus_correctness_reward_func": 0.6502499990165234,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.47433141618967056,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5165697513148189,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.3126249937340617,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.18781251867767423,
       "step": 6
     },
     {
-      "completion_length": 269.7083333333333,
+      "completion_length": 367.7916666666667,
       "epoch": 2.0,
-      "grad_norm": 0.29907792806625366,
-      "kl": 0.0008629298342081407,
+      "grad_norm": 0.3104553520679474,
+      "kl": 0.0007298217872933795,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 2.6299867729345956,
-      "reward_std": 2.6098424941301346,
-      "rewards/concensus_correctness_reward_func": 1.0914999743302662,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 4.552031417687734,
+      "reward_std": 4.407095680634181,
+      "rewards/concensus_correctness_reward_func": 2.5124999998758235,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.5365285724401474,
+      "rewards/final_correctness_reward_func": 0.5833333333333334,
+      "rewards/question_recreation_reward_func": 0.5381147364775339,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.25195833543936413,
+      "rewards/xmlcount_reward_func": 0.5014166831970215,
       "step": 8
     },
     {
-      "completion_length": 285.875,
+      "completion_length": 321.875,
       "epoch": 2.571428571428571,
-      "grad_norm": 0.5044719576835632,
-      "kl": 0.0008350330826942809,
+      "grad_norm": 0.42915189266204834,
+      "kl": 0.00089692858455237,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.9113747850060463,
-      "reward_std": 1.2056577801704407,
-      "rewards/concensus_correctness_reward_func": 0.19181249663233757,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.966095581650734,
+      "reward_std": 3.4431328922510147,
+      "rewards/concensus_correctness_reward_func": 1.2461875001899898,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5010622590780258,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5967517923563719,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.45287501998245716,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.49815625976771116,
       "step": 10
     },
     {
-      "completion_length": 298.2083333333333,
+      "completion_length": 275.4166666666667,
       "epoch": 3.0,
-      "grad_norm": 0.33939021825790405,
-      "kl": 0.0007859368730957309,
+      "grad_norm": 0.37654978036880493,
+      "kl": 0.0009604951289171974,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 4.694664716720581,
-      "reward_std": 4.910978585481644,
-      "rewards/concensus_correctness_reward_func": 2.4921666669348874,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 2.896782418092092,
+      "reward_std": 2.705845886220535,
+      "rewards/concensus_correctness_reward_func": 1.7107499986886978,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6666666666666666,
-      "rewards/question_recreation_reward_func": 0.4820397098859151,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.5344907194375992,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.38712501401702565,
+      "rewards/xmlcount_reward_func": 0.23487500528184077,
       "step": 12
     },
     {
-      "completion_length": 292.40625,
+      "completion_length": 360.46875,
       "epoch": 3.571428571428571,
-      "grad_norm": 1.0123093128204346,
-      "kl": 0.0007496148900827393,
+      "grad_norm": 0.4417196810245514,
+      "kl": 0.0008304499169753399,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.1247988790273666,
-      "reward_std": 1.2916134409606457,
-      "rewards/concensus_correctness_reward_func": 0.1236874982714653,
+      "reward": 4.035018794238567,
+      "reward_std": 3.590598814189434,
+      "rewards/concensus_correctness_reward_func": 1.9950624704360962,
       "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5985488146543503,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.628081239759922,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.44943750090897083,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.41187499929219484,
       "step": 14
     },
     {
-      "completion_length": 281.625,
+      "completion_length": 316.375,
       "epoch": 4.0,
-      "grad_norm": 0.3682559132575989,
-      "kl": 0.0007413495234989872,
+      "grad_norm": 0.3043082356452942,
+      "kl": 0.0009063526037304351,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 3.6117839366197586,
-      "reward_std": 4.430135985215505,
-      "rewards/concensus_correctness_reward_func": 1.876833366851012,
-      "rewards/consensus_reward_func": 0.75,
+      "reward": 1.0959248145421345,
+      "reward_std": 1.0131016621987026,
+      "rewards/concensus_correctness_reward_func": 0.09658333100378513,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.30882569402456284,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3403414661685626,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3427916696916024,
+      "rewards/xmlcount_reward_func": 0.492333322763443,
       "step": 16
     },
     {
-      "completion_length": 329.46875,
+      "completion_length": 268.46875,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.5471922159194946,
-      "kl": 0.0009025832550833002,
+      "grad_norm": 0.4458009898662567,
+      "kl": 0.0009120181421167217,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 4.727515637874603,
-      "reward_std": 3.2995193153619766,
-      "rewards/concensus_correctness_reward_func": 2.6106249820441008,
+      "reward": 3.6639642491936684,
+      "reward_std": 3.6108987629413605,
+      "rewards/concensus_correctness_reward_func": 1.9879999980330467,
       "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.5705154612660408,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4907767567783594,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2963750045746565,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.419562506955117,
       "step": 18
     },
     {
-      "completion_length": 320.6666666666667,
+      "completion_length": 340.3333333333333,
       "epoch": 5.0,
-      "grad_norm": 0.26856714487075806,
-      "kl": 0.0008216216325915108,
+      "grad_norm": 0.40639910101890564,
+      "kl": 0.0008975758876961967,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 4.267755409081777,
-      "reward_std": 3.1953567837675414,
-      "rewards/concensus_correctness_reward_func": 2.6530833318829536,
-      "rewards/consensus_reward_func": 0.75,
+      "reward": 1.0511793394883473,
+      "reward_std": 0.7933718264102936,
+      "rewards/concensus_correctness_reward_func": 0.008416666959722837,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.6007552792628607,
+      "rewards/question_recreation_reward_func": 0.4137210374077161,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.0764166625837485,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2957083272437255,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.307810619749944e-07,
-      "train_runtime": 1695.0641,
-      "train_samples_per_second": 0.189,
+      "train_loss": 8.136558790283743e-07,
+      "train_runtime": 1684.1044,
+      "train_samples_per_second": 0.19,
       "train_steps_per_second": 0.012
     }
   ],