End of training

README.md

@@ -40,8 +40,8 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 
 - TRL: 0.15.2
 - Transformers: 4.51.3
-- Pytorch: 2.6.0
-- Datasets: 3.5.1
+- Pytorch: 2.6.0+cu124
+- Datasets: 3.6.0
 - Tokenizers: 0.21.1
 
 ## Citations

adapter_config.json

@@ -26,11 +26,11 @@
   "target_modules": [
     "v_proj",
     "q_proj",
-    "k_proj",
+    "up_proj",
     "o_proj",
-    "down_proj",
     "gate_proj",
-    "up_proj"
+    "k_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10d39f8baec25c01e9de2e548489447c54a6bc762389c0af3f473a1733b43d57
+oid sha256:b9c682c34dde4db4dd710344c3e73befda53f0b532a7948cff6f7bb072f1ee96
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.2314932791923639e-05,
-    "train_runtime": 998.884,
-    "train_samples": 16,
-    "train_samples_per_second": 0.32,
-    "train_steps_per_second": 0.02
+    "train_loss": 7.289324287285126e-07,
+    "train_runtime": 1571.8166,
+    "train_samples": 14,
+    "train_samples_per_second": 0.204,
+    "train_steps_per_second": 0.013
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.2314932791923639e-05,
-    "train_runtime": 998.884,
-    "train_samples": 16,
-    "train_samples_per_second": 0.32,
-    "train_steps_per_second": 0.02
+    "train_loss": 7.289324287285126e-07,
+    "train_runtime": 1571.8166,
+    "train_samples": 14,
+    "train_samples_per_second": 0.204,
+    "train_steps_per_second": 0.013
 }

trainer_state.json

@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 284.84375,
-      "epoch": 0.5,
-      "grad_norm": 0.6431036591529846,
-      "kl": 0.011925784056074917,
+      "completion_length": 327.65625,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 0.5158918499946594,
+      "kl": 0.0008259698588517494,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 4.0141816437244415,
-      "reward_std": 2.4115020632743835,
-      "rewards/concensus_correctness_reward_func": 0.8819999992847443,
-      "rewards/consensus_reward_func": 0.8125,
+      "reward": 2.894564166665077,
+      "reward_std": 3.509520131163299,
+      "rewards/concensus_correctness_reward_func": 1.25,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.5264317132532597,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5704704113304615,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4182500038295984,
+      "rewards/xmlcount_reward_func": 0.5115937506780028,
       "step": 2
     },
     {
-      "completion_length": 291.375,
+      "completion_length": 376.4166666666667,
       "epoch": 1.0,
-      "grad_norm": 0.6375234723091125,
-      "kl": 0.010492167028132826,
+      "grad_norm": 0.33813101053237915,
+      "kl": 0.0007738670295414826,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 4.376498609781265,
-      "reward_std": 3.10993455350399,
-      "rewards/concensus_correctness_reward_func": 1.4150624573230743,
+      "reward": 5.040141661961873,
+      "reward_std": 6.29193310936292,
+      "rewards/concensus_correctness_reward_func": 3.542833333214124,
       "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.6194675415754318,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5513499031464258,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.46696874871850014,
+      "rewards/xmlcount_reward_func": 0.1959583361943563,
       "step": 4
     },
     {
-      "completion_length": 259.6875,
-      "epoch": 1.5,
-      "grad_norm": 1.1588741540908813,
-      "kl": 0.01240836747456342,
+      "completion_length": 333.6875,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 0.3675234019756317,
+      "kl": 0.0006849843921372667,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 5.587010830640793,
-      "reward_std": 3.3741167187690735,
-      "rewards/concensus_correctness_reward_func": 2.038812465965748,
-      "rewards/consensus_reward_func": 0.8125,
+      "reward": 4.553593710064888,
+      "reward_std": 3.967165894806385,
+      "rewards/concensus_correctness_reward_func": 2.583875000476837,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.5824484638869762,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.5825937986373901,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.622000003233552,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.44962499663233757,
       "step": 6
     },
     {
-      "completion_length": 238.625,
+      "completion_length": 340.5,
       "epoch": 2.0,
-      "grad_norm": 1.5165284872055054,
-      "kl": 0.01328942901454866,
+      "grad_norm": 0.445524126291275,
+      "kl": 0.0007936773666491111,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 3.591675192117691,
-      "reward_std": 1.914122048765421,
-      "rewards/concensus_correctness_reward_func": 0.753250002861023,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 5.078198462724686,
+      "reward_std": 4.9849435687065125,
+      "rewards/concensus_correctness_reward_func": 3.3385833334177732,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0625,
-      "rewards/question_recreation_reward_func": 0.5360814481973648,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.567906683931748,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.6148437410593033,
+      "rewards/xmlcount_reward_func": 0.2758749971787135,
       "step": 8
     },
     {
-      "completion_length": 295.875,
-      "epoch": 2.5,
-      "grad_norm": 0.6974892020225525,
-      "kl": 0.00982225313782692,
+      "completion_length": 377.40625,
+      "epoch": 2.571428571428571,
+      "grad_norm": 0.3984737694263458,
+      "kl": 0.0008149275454343297,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 5.198153331875801,
-      "reward_std": 4.107647001743317,
-      "rewards/concensus_correctness_reward_func": 2.116812489926815,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.5888163596391678,
+      "reward_std": 2.401412595063448,
+      "rewards/concensus_correctness_reward_func": 0.6479375008493662,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.6258407421410084,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5396913513541222,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.6117500104010105,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.13556250906549394,
       "step": 10
     },
     {
-      "completion_length": 267.46875,
+      "completion_length": 320.7916666666667,
       "epoch": 3.0,
-      "grad_norm": 0.6824747323989868,
-      "kl": 0.013338925957214087,
+      "grad_norm": 0.8845546841621399,
+      "kl": 0.0008283675512454162,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 3.235099136829376,
-      "reward_std": 2.496397987008095,
-      "rewards/concensus_correctness_reward_func": 0.7566250115633011,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 3.094367762406667,
+      "reward_std": 4.639908462762833,
+      "rewards/concensus_correctness_reward_func": 1.789000004529953,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.875,
-      "rewards/question_recreation_reward_func": 0.461099105887115,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.41028442482153576,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.43925001844763756,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.311749999721845,
       "step": 12
     },
     {
-      "completion_length": 259.3125,
-      "epoch": 3.5,
-      "grad_norm": 0.5737769603729248,
-      "kl": 0.010779978358186781,
+      "completion_length": 289.40625,
+      "epoch": 3.571428571428571,
+      "grad_norm": 0.5609108805656433,
+      "kl": 0.000864443281898275,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 4.8104297965765,
-      "reward_std": 4.084704261273146,
-      "rewards/concensus_correctness_reward_func": 2.5522499680519104,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 4.097056046128273,
+      "reward_std": 3.7959868870675564,
+      "rewards/concensus_correctness_reward_func": 1.9438125044107437,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.875,
-      "rewards/question_recreation_reward_func": 0.4921174459159374,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.6310247220098972,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.5004375018179417,
+      "rewards/xmlcount_reward_func": 0.5065937452018261,
       "step": 14
     },
     {
-      "completion_length": 233.8125,
+      "completion_length": 301.5416666666667,
       "epoch": 4.0,
-      "grad_norm": 0.6655871868133545,
-      "kl": 0.014783572289161384,
+      "grad_norm": 0.39758506417274475,
+      "kl": 0.0007808880084970345,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 2.6022290140390396,
-      "reward_std": 1.702585056424141,
-      "rewards/concensus_correctness_reward_func": 0.28581250458955765,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.6380778153737388,
+      "reward_std": 2.545287271340688,
+      "rewards/concensus_correctness_reward_func": 1.0830000340938568,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.40194769483059645,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.319161057472229,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.586343752220273,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.40258333335320157,
       "step": 16
     },
     {
-      "completion_length": 258.78125,
-      "epoch": 4.5,
-      "grad_norm": 0.6985666155815125,
-      "kl": 0.011472002835944295,
+      "completion_length": 354.0625,
+      "epoch": 4.571428571428571,
+      "grad_norm": 0.5217888951301575,
+      "kl": 0.0009133577696047723,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 3.076596900820732,
-      "reward_std": 1.970288049429655,
-      "rewards/concensus_correctness_reward_func": 0.4724375084042549,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 3.523820221424103,
+      "reward_std": 4.024020619690418,
+      "rewards/concensus_correctness_reward_func": 1.6346250772476196,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.375,
-      "rewards/question_recreation_reward_func": 0.6544406935572624,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5721640214323997,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.44971875101327896,
+      "rewards/xmlcount_reward_func": 0.3014062512665987,
       "step": 18
     },
     {
-      "completion_length": 237.71875,
+      "completion_length": 288.5416666666667,
       "epoch": 5.0,
-      "grad_norm": 1.5129072666168213,
-      "kl": 0.014775739575270563,
+      "grad_norm": 0.3385997414588928,
+      "kl": 0.0008402868891910961,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 2.9806275367736816,
-      "reward_std": 1.9444493725895882,
-      "rewards/concensus_correctness_reward_func": 0.5696874931454659,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.1194004565477371,
+      "reward_std": 0.9371432662010193,
+      "rewards/concensus_correctness_reward_func": 0.030583334465821583,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9375,
-      "rewards/question_recreation_reward_func": 0.42878376692533493,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4427338168025017,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.49778125435113907,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.47941667834917706,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.2314932791923639e-05,
-      "train_runtime": 998.884,
-      "train_samples_per_second": 0.32,
-      "train_steps_per_second": 0.02
+      "train_loss": 7.289324287285126e-07,
+      "train_runtime": 1571.8166,
+      "train_samples_per_second": 0.204,
+      "train_steps_per_second": 0.013
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 7,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79dc523b85b0d3d7fb6b02c7070367d92c3d50aa0a50d515f13bb82d88b2cb4d
+oid sha256:9035c212864973378c8f2ca58c568d67c200a53e47c79dca8c5d11aa7ca802eb
 size 6008