End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c80cdc031285aa3c15f2520026e25de7fa65e8fdde5c8c5893ea9f65ba45882c
+oid sha256:231da430b99290f5102d961fb27c9b52da0408e378d07a7619acc81d21c406c0
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.7048853123924347e-06,
-    "train_runtime": 344.2094,
+    "train_loss": 1.8071034446620615e-06,
+    "train_runtime": 348.0669,
     "train_samples": 14,
-    "train_samples_per_second": 0.93,
-    "train_steps_per_second": 0.058
+    "train_samples_per_second": 0.919,
+    "train_steps_per_second": 0.057
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.7048853123924347e-06,
-    "train_runtime": 344.2094,
+    "train_loss": 1.8071034446620615e-06,
+    "train_runtime": 348.0669,
     "train_samples": 14,
-    "train_samples_per_second": 0.93,
-    "train_steps_per_second": 0.058
+    "train_samples_per_second": 0.919,
+    "train_steps_per_second": 0.057
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 295.15625,
+      "completion_length": 306.375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.4980431795120239,
-      "kl": 0.001723670880892314,
+      "grad_norm": 0.5308642387390137,
+      "kl": 0.0021895101526752114,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 5.167072042822838,
-      "reward_std": 5.309514049440622,
-      "rewards/concensus_correctness_reward_func": 3.190250001847744,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.5153492391109467,
+      "reward_std": 2.6090498343110085,
+      "rewards/concensus_correctness_reward_func": 0.7049374994821846,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.44085316359996796,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.3991617187857628,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.535968747921288,
+      "rewards/xmlcount_reward_func": 0.4737500064074993,
       "step": 2
     },
     {
-      "completion_length": 281.0416666666667,
+      "completion_length": 274.2916666666667,
       "epoch": 1.0,
-      "grad_norm": 0.4395172894001007,
-      "kl": 0.002263232871579627,
+      "grad_norm": 0.372209370136261,
+      "kl": 0.002308575358862678,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 6.546656807263692,
-      "reward_std": 7.525084922711055,
-      "rewards/concensus_correctness_reward_func": 3.587750043720007,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 6.7491885622342425,
+      "reward_std": 7.915084143479665,
+      "rewards/concensus_correctness_reward_func": 4.166666666666667,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.6386151264111201,
+      "rewards/final_correctness_reward_func": 1.0833333333333333,
+      "rewards/question_recreation_reward_func": 0.532355268796285,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4036250151693821,
+      "rewards/xmlcount_reward_func": 0.46683332324028015,
       "step": 4
     },
     {
-      "completion_length": 300.6875,
+      "completion_length": 322.75,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.662812352180481,
-      "kl": 0.0016874009161256254,
+      "grad_norm": 0.43669286370277405,
+      "kl": 0.001961336594831664,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 4.936138093471527,
-      "reward_std": 5.001859599724412,
-      "rewards/concensus_correctness_reward_func": 3.066062521189451,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 2.2698651757091284,
+      "reward_std": 2.0269790440797806,
+      "rewards/concensus_correctness_reward_func": 0.676749998703599,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.45535678416490555,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.4571463596075773,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2897187480702996,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.24534375802613795,
       "step": 6
     },
     {
-      "completion_length": 248.20833333333334,
+      "completion_length": 277.2083333333333,
       "epoch": 2.0,
-      "grad_norm": 0.42877623438835144,
-      "kl": 0.002104680344928056,
+      "grad_norm": 0.30822211503982544,
+      "kl": 0.00212874977538983,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 4.107728699843089,
-      "reward_std": 4.388790463407834,
-      "rewards/concensus_correctness_reward_func": 1.7869999992350738,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 7.554377923409144,
+      "reward_std": 6.587712789575259,
+      "rewards/concensus_correctness_reward_func": 4.650249878565471,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9166666666666666,
-      "rewards/question_recreation_reward_func": 0.45460371673107147,
+      "rewards/final_correctness_reward_func": 1.25,
+      "rewards/question_recreation_reward_func": 0.44750309487183887,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.532791664203008,
+      "rewards/xmlcount_reward_func": 0.45662500460942584,
       "step": 8
     },
     {
-      "completion_length": 281.90625,
+      "completion_length": 312.1875,
       "epoch": 2.571428571428571,
-      "grad_norm": 3.1693649291992188,
-      "kl": 0.0021985476050758734,
+      "grad_norm": 0.5058867931365967,
+      "kl": 0.002013914636336267,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 5.133505783975124,
-      "reward_std": 3.6785213071852922,
-      "rewards/concensus_correctness_reward_func": 3.1693749986588955,
+      "reward": 5.197073700837791,
+      "reward_std": 6.339561741799116,
+      "rewards/concensus_correctness_reward_func": 3.262062467634678,
       "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5224119387567043,
+      "rewards/final_correctness_reward_func": 0.6875,
+      "rewards/question_recreation_reward_func": 0.5408236430957913,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5042187524959445,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.253562499769032,
       "step": 10
     },
     {
-      "completion_length": 314.7916666666667,
+      "completion_length": 297.7916666666667,
       "epoch": 3.0,
-      "grad_norm": 0.2839249074459076,
-      "kl": 0.001704779511783272,
+      "grad_norm": 0.30220890045166016,
+      "kl": 0.0019135521530794601,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 3.554308513800303,
-      "reward_std": 3.1588053504625955,
-      "rewards/concensus_correctness_reward_func": 1.3915833197534084,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 4.277362247308095,
+      "reward_std": 3.7730555484692254,
+      "rewards/concensus_correctness_reward_func": 1.9397499933838844,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1666666666666667,
-      "rewards/question_recreation_reward_func": 0.4168501968185107,
+      "rewards/final_correctness_reward_func": 1.0833333333333333,
+      "rewards/question_recreation_reward_func": 0.43482065945863724,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.32920832683642703,
+      "rewards/xmlcount_reward_func": 0.4861249973376592,
       "step": 12
     },
     {
-      "completion_length": 271.03125,
+      "completion_length": 303.90625,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.6410653591156006,
-      "kl": 0.0021672843577107415,
+      "grad_norm": 0.4894811511039734,
+      "kl": 0.0018768926383927464,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 7.146000355482101,
-      "reward_std": 6.1719986870884895,
-      "rewards/concensus_correctness_reward_func": 4.559312526136637,
+      "reward": 5.925179444253445,
+      "reward_std": 6.7717363722622395,
+      "rewards/concensus_correctness_reward_func": 3.75,
       "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.6316566355526447,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.621023166924715,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5175312533974648,
+      "rewards/xmlcount_reward_func": 0.30415625125169754,
       "step": 14
     },
     {
-      "completion_length": 268.875,
+      "completion_length": 278.125,
       "epoch": 4.0,
-      "grad_norm": 0.3645438253879547,
-      "kl": 0.0018159543784956138,
+      "grad_norm": 0.392229825258255,
+      "kl": 0.002562401001341641,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 4.66881267229716,
-      "reward_std": 4.626638899246852,
-      "rewards/concensus_correctness_reward_func": 2.5479166644314923,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 1.5568751394748688,
+      "reward_std": 1.468297431866328,
+      "rewards/concensus_correctness_reward_func": 0.15199999511241913,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5833333333333334,
-      "rewards/question_recreation_reward_func": 0.41914614538351697,
+      "rewards/final_correctness_reward_func": 0.6666666666666666,
+      "rewards/question_recreation_reward_func": 0.1795418101052443,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4517500003178914,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.28783332866926986,
       "step": 16
     },
     {
-      "completion_length": 336.6875,
+      "completion_length": 331.4375,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.47637107968330383,
-      "kl": 0.0015734998160041869,
+      "grad_norm": 0.6803017258644104,
+      "kl": 0.0018350362952332944,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 5.934840962290764,
-      "reward_std": 6.618309460580349,
-      "rewards/concensus_correctness_reward_func": 3.75,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 2.4337521493434906,
+      "reward_std": 2.3893414810299873,
+      "rewards/concensus_correctness_reward_func": 0.6932499990798533,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.5804660655558109,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.5723147038370371,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4168750122189522,
+      "rewards/xmlcount_reward_func": 0.3556875018402934,
       "step": 18
     },
     {
-      "completion_length": 253.20833333333334,
+      "completion_length": 268.5,
       "epoch": 5.0,
-      "grad_norm": 0.864767849445343,
-      "kl": 0.0024529556976631284,
+      "grad_norm": 0.3107430040836334,
+      "kl": 0.0019618809456005692,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 2.0046025762955346,
-      "reward_std": 1.5288229087988536,
-      "rewards/concensus_correctness_reward_func": 0.45499999945362407,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 4.82015057404836,
+      "reward_std": 2.9356290424863496,
+      "rewards/concensus_correctness_reward_func": 2.6206666628519693,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.33506091435750324,
+      "rewards/final_correctness_reward_func": 0.8333333333333334,
+      "rewards/question_recreation_reward_func": 0.43869226922591525,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2978750020265579,
+      "rewards/xmlcount_reward_func": 0.3441249926884969,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.7048853123924347e-06,
-      "train_runtime": 344.2094,
-      "train_samples_per_second": 0.93,
-      "train_steps_per_second": 0.058
+      "train_loss": 1.8071034446620615e-06,
+      "train_runtime": 348.0669,
+      "train_samples_per_second": 0.919,
+      "train_steps_per_second": 0.057
     }
   ],
   "logging_steps": 2,