End of training

README.md

@@ -40,7 +40,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 
 - TRL: 0.15.2
 - Transformers: 4.51.2
-- Pytorch: 2.6.0
+- Pytorch: 2.5.1
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1
 

adapter_config.json

@@ -24,13 +24,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
-    "q_proj",
+    "up_proj",
+    "k_proj",
     "o_proj",
-    "v_proj",
     "gate_proj",
-    "up_proj",
-    "k_proj"
+    "v_proj",
+    "down_proj",
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5955105b5be57f6d5fc03cfab43e55e20cc66c2b47261c5296dc6e408257f2c8
+oid sha256:ac0a211e3df4e26a62134de9dd4a4e5c50fbcee8a2f8d7db1f464d2e2d271ccb
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.2624624287127516e-05,
-    "train_runtime": 318.7505,
+    "train_loss": 7.469066019893944e-07,
+    "train_runtime": 313.1331,
     "train_samples": 14,
-    "train_samples_per_second": 1.004,
-    "train_steps_per_second": 0.063
+    "train_samples_per_second": 1.022,
+    "train_steps_per_second": 0.064
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.2624624287127516e-05,
-    "train_runtime": 318.7505,
+    "train_loss": 7.469066019893944e-07,
+    "train_runtime": 313.1331,
     "train_samples": 14,
-    "train_samples_per_second": 1.004,
-    "train_steps_per_second": 0.063
+    "train_samples_per_second": 1.022,
+    "train_steps_per_second": 0.064
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 286.71875,
+      "completion_length": 327.78125,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.6248641610145569,
-      "kl": 0.013412793457973748,
+      "grad_norm": 0.45513784885406494,
+      "kl": 0.0007744804534013383,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 4.181982383131981,
-      "reward_std": 2.771310269832611,
-      "rewards/concensus_correctness_reward_func": 2.1404375042766333,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.2494101841002703,
+      "reward_std": 2.1352708376944065,
+      "rewards/concensus_correctness_reward_func": 0.721000000834465,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.5507635921239853,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.5667225942015648,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.28765624295920134,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.14918749872595072,
       "step": 2
     },
     {
-      "completion_length": 270.9583333333333,
+      "completion_length": 271.7083333333333,
       "epoch": 1.0,
-      "grad_norm": 0.3707655370235443,
-      "kl": 0.014095600694417953,
+      "grad_norm": 0.42274269461631775,
+      "kl": 0.0010220681821616988,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 6.016905357440312,
-      "reward_std": 6.030558327833812,
-      "rewards/concensus_correctness_reward_func": 3.81974991162618,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 4.305304487546285,
+      "reward_std": 2.481597055991491,
+      "rewards/concensus_correctness_reward_func": 2.66349999854962,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8333333333333334,
-      "rewards/question_recreation_reward_func": 0.5182803173859915,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.599512904882431,
+      "rewards/soft_format_reward_func": 0.020833333333333332,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.512208333859841,
+      "rewards/xmlcount_reward_func": 0.27145832777023315,
       "step": 4
     },
     {
-      "completion_length": 288.65625,
+      "completion_length": 307.03125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.5696437954902649,
-      "kl": 0.012311801430769265,
+      "grad_norm": 13.239145278930664,
+      "kl": 0.0010518985181988683,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 3.8420243561267853,
-      "reward_std": 3.859384536743164,
-      "rewards/concensus_correctness_reward_func": 1.6384375132620335,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 3.155073456466198,
+      "reward_std": 3.6922449842095375,
+      "rewards/concensus_correctness_reward_func": 1.3747499901801348,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.875,
-      "rewards/question_recreation_reward_func": 0.4874306097626686,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.42832337878644466,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3255312517285347,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.3832500111311674,
       "step": 6
     },
     {
-      "completion_length": 270.875,
+      "completion_length": 293.75,
       "epoch": 2.0,
-      "grad_norm": 0.6221694946289062,
-      "kl": 0.015311841387301683,
+      "grad_norm": 0.41131001710891724,
+      "kl": 0.0008157214033417404,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 5.322434385617574,
-      "reward_std": 6.316161473592122,
-      "rewards/concensus_correctness_reward_func": 2.648166666428248,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 4.246236711740494,
+      "reward_std": 4.285125603278478,
+      "rewards/concensus_correctness_reward_func": 2.533833332359791,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1666666666666667,
-      "rewards/question_recreation_reward_func": 0.5574344595273336,
+      "rewards/final_correctness_reward_func": 0.5833333333333334,
+      "rewards/question_recreation_reward_func": 0.5278615256150564,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.512666662534078,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.35120833789308864,
       "step": 8
     },
     {
-      "completion_length": 251.59375,
+      "completion_length": 284.09375,
       "epoch": 2.571428571428571,
-      "grad_norm": 0.6904033422470093,
-      "kl": 0.016136467922478914,
+      "grad_norm": 0.5300907492637634,
+      "kl": 0.0007643571079825051,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 5.954379126429558,
-      "reward_std": 2.2650047317147255,
-      "rewards/concensus_correctness_reward_func": 3.6618750169873238,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 3.6151050850749016,
+      "reward_std": 3.5139888897538185,
+      "rewards/concensus_correctness_reward_func": 1.8828750001266599,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.49144152272492647,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.45229253359138966,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6135625038295984,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3893125085160136,
       "step": 10
     },
     {
-      "completion_length": 240.375,
+      "completion_length": 307.625,
       "epoch": 3.0,
-      "grad_norm": 2.257899284362793,
-      "kl": 0.01891724842910965,
+      "grad_norm": 0.2573208808898926,
+      "kl": 0.0008810065822520604,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 5.2814386785030365,
-      "reward_std": 4.464167500535647,
-      "rewards/concensus_correctness_reward_func": 2.9076666409770646,
+      "reward": 1.8000794450441997,
+      "reward_std": 1.0749290486176808,
+      "rewards/concensus_correctness_reward_func": 0.06416666756073634,
       "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0833333333333333,
-      "rewards/question_recreation_reward_func": 0.348563772936662,
+      "rewards/final_correctness_reward_func": 0.4166666666666667,
+      "rewards/question_recreation_reward_func": 0.5521627912918726,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.041666666666666664,
-      "rewards/xmlcount_reward_func": 0.5668750032782555,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.4129166677594185,
       "step": 12
     },
     {
-      "completion_length": 283.375,
+      "completion_length": 303.6875,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.7673466205596924,
-      "kl": 0.013287136680446565,
+      "grad_norm": 0.6238363981246948,
+      "kl": 0.0008335267993970774,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 6.9643110781908035,
-      "reward_std": 6.077938843518496,
-      "rewards/concensus_correctness_reward_func": 4.241875037550926,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 2.697949156165123,
+      "reward_std": 3.5988308005034924,
+      "rewards/concensus_correctness_reward_func": 1.2578125,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.875,
-      "rewards/question_recreation_reward_func": 0.5903111323714256,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.6246991790831089,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.5539999920874834,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.19043751433491707,
       "step": 14
     },
     {
-      "completion_length": 243.95833333333334,
+      "completion_length": 313.4583333333333,
       "epoch": 4.0,
-      "grad_norm": 0.8013021349906921,
-      "kl": 0.016221244353801012,
+      "grad_norm": 0.34566551446914673,
+      "kl": 0.0008214676102700954,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 4.895285129547119,
-      "reward_std": 4.644445935885112,
-      "rewards/concensus_correctness_reward_func": 2.9503334388136864,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 3.4665555357933044,
+      "reward_std": 2.7648710012435913,
+      "rewards/concensus_correctness_reward_func": 1.8328333298365276,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6666666666666666,
-      "rewards/question_recreation_reward_func": 0.261535182595253,
+      "rewards/final_correctness_reward_func": 0.5833333333333334,
+      "rewards/question_recreation_reward_func": 0.39530544976393384,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.4959166720509529,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.32175001005331677,
       "step": 16
     },
     {
-      "completion_length": 270.1875,
+      "completion_length": 325.65625,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.665776252746582,
-      "kl": 0.013211678306106478,
+      "grad_norm": 0.5649901628494263,
+      "kl": 0.0007845223735785112,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 5.104047998785973,
-      "reward_std": 4.970302836969495,
-      "rewards/concensus_correctness_reward_func": 2.938875000923872,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 3.5771792829036713,
+      "reward_std": 3.5350502906367183,
+      "rewards/concensus_correctness_reward_func": 1.9469999969005585,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.5428603421896696,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.5195855796337128,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.4191875113174319,
+      "rewards/xmlcount_reward_func": 0.15746875060722232,
       "step": 18
     },
     {
-      "completion_length": 292.75,
+      "completion_length": 285.375,
       "epoch": 5.0,
-      "grad_norm": 0.38175004720687866,
-      "kl": 0.012505050360535582,
+      "grad_norm": 0.33395493030548096,
+      "kl": 0.000825791716730843,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 3.267457067966461,
-      "reward_std": 2.8946244418621063,
-      "rewards/concensus_correctness_reward_func": 1.2395833532015483,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 2.3533659875392914,
+      "reward_std": 2.8434973508119583,
+      "rewards/concensus_correctness_reward_func": 0.8655000006159147,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.39274873584508896,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.4751576582590739,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5517916778723398,
+      "rewards/strict_format_reward_func": 0.041666666666666664,
+      "rewards/xmlcount_reward_func": 0.30437499781449634,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.2624624287127516e-05,
-      "train_runtime": 318.7505,
-      "train_samples_per_second": 1.004,
-      "train_steps_per_second": 0.063
+      "train_loss": 7.469066019893944e-07,
+      "train_runtime": 313.1331,
+      "train_samples_per_second": 1.022,
+      "train_steps_per_second": 0.064
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:311a31c61551d364640b7ba0b6f2e0e174d037cb0808262a99ed2d71a81d25ba
+oid sha256:fbe8d4b7c01fbb3024ed6e881f4d5e554022dbace47ce1539b45aa48e20bf06b
 size 6008