End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0d8fbe7f1c37cd2d48078cd78cb70dad8811f6682ab2d6f06bf6c9537b8ddc1
+oid sha256:bad0579d87073d4a49dff57cbacea8c12a5b289ca5d39f5cbc20d7f430c3da58
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.4959041345719016e-05,
-    "train_runtime": 502.6524,
+    "train_loss": 1.8507867025618907e-05,
+    "train_runtime": 516.6128,
     "train_samples": 14,
-    "train_samples_per_second": 0.637,
-    "train_steps_per_second": 0.04
+    "train_samples_per_second": 0.619,
+    "train_steps_per_second": 0.039
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.4959041345719016e-05,
-    "train_runtime": 502.6524,
+    "train_loss": 1.8507867025618907e-05,
+    "train_runtime": 516.6128,
     "train_samples": 14,
-    "train_samples_per_second": 0.637,
-    "train_steps_per_second": 0.04
+    "train_samples_per_second": 0.619,
+    "train_steps_per_second": 0.039
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 300.6875,
+      "completion_length": 244.125,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.5476604104042053,
-      "kl": 0.017130549531430006,
+      "grad_norm": 0.6719356775283813,
+      "kl": 0.021937294979579747,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 7.829952538013458,
-      "reward_std": 5.701882891356945,
-      "rewards/concensus_correctness_reward_func": 5.542687471956015,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 4.525261968374252,
+      "reward_std": 3.8179611265659332,
+      "rewards/concensus_correctness_reward_func": 2.364500004798174,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.48470262438058853,
+      "rewards/final_correctness_reward_func": 0.6875,
+      "rewards/question_recreation_reward_func": 0.40348084457218647,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.4744374956935644,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.632281250320375,
       "step": 2
     },
     {
-      "completion_length": 247.16666666666666,
+      "completion_length": 231.83333333333334,
       "epoch": 1.0,
-      "grad_norm": 0.4445508122444153,
-      "kl": 0.01814927641923229,
+      "grad_norm": 0.43023428320884705,
+      "kl": 0.023038096881161135,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 8.992417554060617,
-      "reward_std": 9.302019089460373,
-      "rewards/concensus_correctness_reward_func": 6.101666688919067,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 5.652425408363342,
+      "reward_std": 5.419331391652425,
+      "rewards/concensus_correctness_reward_func": 3.1954166491826377,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1666666666666667,
-      "rewards/question_recreation_reward_func": 0.4685428738594055,
+      "rewards/final_correctness_reward_func": 0.9166666666666666,
+      "rewards/question_recreation_reward_func": 0.6258837779362997,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.041666666666666664,
-      "rewards/xmlcount_reward_func": 0.547208326558272,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5811249911785126,
       "step": 4
     },
     {
-      "completion_length": 235.9375,
+      "completion_length": 243.125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.8429892063140869,
-      "kl": 0.021250308491289616,
+      "grad_norm": 1.7448420524597168,
+      "kl": 0.020654373452998698,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 6.9223052859306335,
-      "reward_std": 6.0704952627420425,
-      "rewards/concensus_correctness_reward_func": 4.337062530219555,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 4.262659437954426,
+      "reward_std": 4.423336744308472,
+      "rewards/concensus_correctness_reward_func": 2.6919375024735928,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9375,
-      "rewards/question_recreation_reward_func": 0.49346147291362286,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.37369068525731564,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5292812464758754,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3689062474295497,
       "step": 6
     },
     {
-      "completion_length": 261.2083333333333,
+      "completion_length": 233.66666666666666,
       "epoch": 2.0,
-      "grad_norm": 0.40513259172439575,
-      "kl": 0.016181573426971834,
+      "grad_norm": 0.5539745688438416,
+      "kl": 0.019762453467895586,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 7.106939196586609,
-      "reward_std": 6.75446496407191,
-      "rewards/concensus_correctness_reward_func": 4.253333330154419,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 7.194277405738831,
+      "reward_std": 6.900168001651764,
+      "rewards/concensus_correctness_reward_func": 3.990916669368744,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.47435584167639416,
+      "rewards/final_correctness_reward_func": 1.25,
+      "rewards/question_recreation_reward_func": 0.590402327477932,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5459166740377744,
+      "rewards/strict_format_reward_func": 0.041666666666666664,
+      "rewards/xmlcount_reward_func": 0.48795833190282184,
       "step": 8
     },
     {
-      "completion_length": 250.65625,
+      "completion_length": 237.8125,
       "epoch": 2.571428571428571,
-      "grad_norm": 1.1486411094665527,
-      "kl": 0.016840377589687705,
+      "grad_norm": 0.7340793013572693,
+      "kl": 0.022954123560339212,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 8.095225438475609,
-      "reward_std": 5.941859779879451,
-      "rewards/concensus_correctness_reward_func": 5.472312554717064,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 7.7875232100486755,
+      "reward_std": 6.256930936127901,
+      "rewards/concensus_correctness_reward_func": 4.968125019222498,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.4965067356824875,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.4076562486588955,
+      "rewards/final_correctness_reward_func": 0.9375,
+      "rewards/question_recreation_reward_func": 0.6027106828987598,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.513562511652708,
       "step": 10
     },
     {
-      "completion_length": 263.9583333333333,
+      "completion_length": 232.20833333333334,
       "epoch": 3.0,
-      "grad_norm": 0.33428069949150085,
-      "kl": 0.014431457811345657,
+      "grad_norm": 0.7022180557250977,
+      "kl": 0.01902877725660801,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 5.579716891050339,
-      "reward_std": 6.354978660742442,
-      "rewards/concensus_correctness_reward_func": 3.7206667264302573,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 2.9621255894502005,
+      "reward_std": 2.618810474872589,
+      "rewards/concensus_correctness_reward_func": 1.2934166491031647,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8333333333333334,
-      "rewards/question_recreation_reward_func": 0.4373418341080348,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.5026255945364634,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2550416737794876,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.5619166642427444,
       "step": 12
     },
     {
-      "completion_length": 247.71875,
+      "completion_length": 249.78125,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.8113343715667725,
-      "kl": 0.018531960784457624,
+      "grad_norm": 0.6851270794868469,
+      "kl": 0.021324723376892507,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 6.202991157770157,
-      "reward_std": 6.211837485432625,
-      "rewards/concensus_correctness_reward_func": 3.422125019133091,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 7.44910529255867,
+      "reward_std": 7.687609039247036,
+      "rewards/concensus_correctness_reward_func": 4.81756254285574,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.6378974206745625,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.596093749627471,
+      "rewards/final_correctness_reward_func": 0.9375,
+      "rewards/question_recreation_reward_func": 0.5702925845980644,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4674999946728349,
       "step": 14
     },
     {
-      "completion_length": 282.875,
+      "completion_length": 240.375,
       "epoch": 4.0,
-      "grad_norm": 0.6455506086349487,
-      "kl": 0.015719143596167367,
+      "grad_norm": 0.42777204513549805,
+      "kl": 0.020246231307586033,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 8.652406414349874,
-      "reward_std": 7.971785386403401,
-      "rewards/concensus_correctness_reward_func": 6.0429166952768965,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 5.361208260059357,
+      "reward_std": 4.523568580547969,
+      "rewards/concensus_correctness_reward_func": 3.5072499910990396,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.2613231750826041,
+      "rewards/final_correctness_reward_func": 0.6666666666666666,
+      "rewards/question_recreation_reward_func": 0.319458469748497,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.43150002509355545,
+      "rewards/xmlcount_reward_func": 0.45116666704416275,
       "step": 16
     },
     {
-      "completion_length": 276.90625,
+      "completion_length": 242.875,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.6044275164604187,
-      "kl": 0.01645567046944052,
+      "grad_norm": 0.807776689529419,
+      "kl": 0.02085455262567848,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 7.950043350458145,
-      "reward_std": 8.045561142265797,
-      "rewards/concensus_correctness_reward_func": 5.522000052034855,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 3.4024015069007874,
+      "reward_std": 3.9184409603476524,
+      "rewards/concensus_correctness_reward_func": 1.618999995291233,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.5236059874296188,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5032764822244644,
       "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.4981875102967024,
+      "rewards/xmlcount_reward_func": 0.49887499399483204,
       "step": 18
     },
     {
-      "completion_length": 295.5416666666667,
+      "completion_length": 257.7916666666667,
       "epoch": 5.0,
-      "grad_norm": 0.2983975112438202,
-      "kl": 0.014737597511460384,
+      "grad_norm": 0.5556287169456482,
+      "kl": 0.021133776288479567,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 5.004824082056682,
-      "reward_std": 4.7791677216688795,
-      "rewards/concensus_correctness_reward_func": 2.7082499563694,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 8.280698696772257,
+      "reward_std": 6.51280556122462,
+      "rewards/concensus_correctness_reward_func": 5.379750000933806,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9166666666666666,
-      "rewards/question_recreation_reward_func": 0.3839908813436826,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.48974035680294037,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.3917500053842862,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.577874998251597,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.4959041345719016e-05,
-      "train_runtime": 502.6524,
-      "train_samples_per_second": 0.637,
-      "train_steps_per_second": 0.04
+      "train_loss": 1.8507867025618907e-05,
+      "train_runtime": 516.6128,
+      "train_samples_per_second": 0.619,
+      "train_steps_per_second": 0.039
     }
   ],
   "logging_steps": 2,