End of training

README.md

@@ -1,5 +1,5 @@
 ---
-base_model: Gensyn/Qwen2.5-0.5B-Instruct
+base_model: Gensyn/Qwen2.5-1.5B-Instruct
 library_name: transformers
 model_name: gensyn-checkpoints-masked_vocal_salmon
 tags:
@@ -8,13 +8,14 @@ tags:
 - grpo
 - gensyn
 - I am masked vocal salmon
+- unsloth
 - trl
 licence: license
 ---
 
 # Model Card for gensyn-checkpoints-masked_vocal_salmon
 
-This model is a fine-tuned version of [Gensyn/Qwen2.5-0.5B-Instruct](https://huggingface.co/Gensyn/Qwen2.5-0.5B-Instruct).
+This model is a fine-tuned version of [Gensyn/Qwen2.5-1.5B-Instruct](https://huggingface.co/Gensyn/Qwen2.5-1.5B-Instruct).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -39,7 +40,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 
 - TRL: 0.15.2
 - Transformers: 4.51.3
-- Pytorch: 2.5.1
+- Pytorch: 2.6.0
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1
 

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Gensyn/Qwen2.5-1.5B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "gate_proj",
+    "down_proj",
+    "q_proj",
+    "o_proj",
+    "v_proj",
+    "up_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4aaa02a548ef17ca89d161df6a85af7e0e4203ad9b67593b1e00d234582b821
+size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0014741608130862005,
-    "train_runtime": 143.5727,
-    "train_samples": 28,
-    "train_samples_per_second": 2.229,
-    "train_steps_per_second": 0.139
+    "train_loss": 6.654366700331593e-07,
+    "train_runtime": 1175.1261,
+    "train_samples": 11,
+    "train_samples_per_second": 0.272,
+    "train_steps_per_second": 0.017
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0014741608130862005,
-    "train_runtime": 143.5727,
-    "train_samples": 28,
-    "train_samples_per_second": 2.229,
-    "train_steps_per_second": 0.139
+    "train_loss": 6.654366700331593e-07,
+    "train_runtime": 1175.1261,
+    "train_samples": 11,
+    "train_samples_per_second": 0.272,
+    "train_steps_per_second": 0.017
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 6.7272727272727275,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 221.9375,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 9.694320678710938,
-      "kl": 0.0,
+      "completion_length": 472.15625,
+      "epoch": 0.7272727272727273,
+      "grad_norm": 0.6036210656166077,
+      "kl": 0.0008934065590437967,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 2.8578853271901608,
-      "reward_std": 1.0625475882552564,
-      "rewards/concensus_correctness_reward_func": 0.5570000037550926,
-      "rewards/consensus_reward_func": 0.6875,
+      "loss": 0.0,
+      "reward": 1.4180201515555382,
+      "reward_std": 1.395761176943779,
+      "rewards/concensus_correctness_reward_func": 0.018624999560415745,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6873852936550975,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.423113907687366,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.109375,
-      "rewards/xmlcount_reward_func": 0.7541250037029386,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4450312480330467,
       "step": 2
     },
     {
-      "completion_length": 136.83333333333334,
-      "epoch": 1.0,
-      "grad_norm": 5.852896690368652,
-      "kl": 0.0063058519347881275,
+      "completion_length": 419.2142857142857,
+      "epoch": 1.3636363636363638,
+      "grad_norm": 0.5538027882575989,
+      "kl": 0.0007605135199680392,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 3.7386062343915305,
-      "reward_std": 1.4028123530248802,
-      "rewards/concensus_correctness_reward_func": 0.70066666106383,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 0.9143300311905997,
+      "reward_std": 0.8392358294555119,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.6852311789989471,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.30008004180022646,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3333333333333333,
-      "rewards/xmlcount_reward_func": 1.1860416730244954,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.5071071471486773,
       "step": 4
     },
     {
-      "completion_length": 146.78125,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 10.775361061096191,
-      "kl": 0.022688565848511644,
+      "completion_length": 542.4642857142857,
+      "epoch": 2.0,
+      "grad_norm": 0.39062559604644775,
+      "kl": 0.0006340955400706402,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 4.566311419010162,
-      "reward_std": 1.5661333670868771,
-      "rewards/concensus_correctness_reward_func": 1.4004999957978725,
-      "rewards/consensus_reward_func": 0.9375,
+      "reward": 0.6439937661801066,
+      "reward_std": 0.8819772984300341,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.7594364183023572,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5107437585081372,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.1719999946653843,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.08103571246777262,
       "step": 6
     },
     {
-      "completion_length": 142.91666666666666,
-      "epoch": 2.0,
-      "grad_norm": 9.037089347839355,
-      "kl": 0.31922435003798455,
+      "completion_length": 452.40625,
+      "epoch": 2.7272727272727275,
+      "grad_norm": 0.484588086605072,
+      "kl": 0.0007340605188801419,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0002,
-      "reward": 4.1081428329149885,
-      "reward_std": 1.029661375455665,
-      "rewards/concensus_correctness_reward_func": 0.7650833452741305,
-      "rewards/consensus_reward_func": 0.9166666666666666,
+      "loss": 0.0,
+      "reward": 1.2139809615910053,
+      "reward_std": 1.1542998775839806,
+      "rewards/concensus_correctness_reward_func": 0.013812499586492777,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6767677310854197,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4430122319608927,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1871250073115032,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.16340623889118433,
       "step": 8
     },
     {
-      "completion_length": 143.09375,
-      "epoch": 2.571428571428571,
-      "grad_norm": 11.110583305358887,
-      "kl": 0.5744109276565723,
+      "completion_length": 580.5714285714286,
+      "epoch": 3.3636363636363638,
+      "grad_norm": 0.3272243142127991,
+      "kl": 0.0006291726936719247,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0006,
-      "reward": 3.682307220995426,
-      "reward_std": 1.1703917261911556,
-      "rewards/concensus_correctness_reward_func": 0.7889375016093254,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.0,
+      "reward": 0.9775400928088597,
+      "reward_std": 0.9005033576062748,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.6156822107732296,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.49911151826381683,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.1058125048875809,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.19271428563765117,
       "step": 10
     },
     {
-      "completion_length": 136.41666666666666,
-      "epoch": 3.0,
-      "grad_norm": 5.076348304748535,
-      "kl": 2.3429599769879133,
+      "completion_length": 479.92857142857144,
+      "epoch": 4.0,
+      "grad_norm": 0.37380141019821167,
+      "kl": 0.00070970255183056,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0018,
-      "reward": 4.40448385477066,
-      "reward_std": 1.6560175865888596,
-      "rewards/concensus_correctness_reward_func": 0.8267500052849451,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "loss": 0.0,
+      "reward": 1.1728975645133428,
+      "reward_std": 1.135693175452096,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.8022754968454441,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5700761207512447,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.2129583358764648,
+      "rewards/strict_format_reward_func": 0.05357142857142857,
+      "rewards/xmlcount_reward_func": 0.19210714208228247,
       "step": 12
     },
     {
-      "completion_length": 169.6875,
-      "epoch": 3.571428571428571,
-      "grad_norm": 8.082115173339844,
-      "kl": 7.364466468396131,
+      "completion_length": 569.4375,
+      "epoch": 4.7272727272727275,
+      "grad_norm": 0.31213265657424927,
+      "kl": 0.0007356460555456579,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0074,
-      "reward": 4.209167793393135,
-      "reward_std": 1.530743383220397,
-      "rewards/concensus_correctness_reward_func": 0.805249996483326,
-      "rewards/consensus_reward_func": 1.0625,
+      "loss": 0.0,
+      "reward": 0.5268249623477459,
+      "reward_std": 0.9648920074105263,
+      "rewards/concensus_correctness_reward_func": 0.004812499973922968,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.6893240767531097,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4122624695301056,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.1677187494933605,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.20275000482797623,
       "step": 14
     },
     {
-      "completion_length": 151.83333333333334,
-      "epoch": 4.0,
-      "grad_norm": 23.623441696166992,
-      "kl": 5.0384774438571185,
+      "completion_length": 550.0714285714286,
+      "epoch": 5.363636363636363,
+      "grad_norm": 0.3187633454799652,
+      "kl": 0.0006976349395699799,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0038,
-      "reward": 3.1199777324994407,
-      "reward_std": 1.2431689854711294,
-      "rewards/concensus_correctness_reward_func": 0.32775000234444934,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "loss": 0.0,
+      "reward": 1.2211733332702093,
+      "reward_std": 1.0074801189558846,
+      "rewards/concensus_correctness_reward_func": 0.010285713842936925,
+      "rewards/consensus_reward_func": 0.5714285714285714,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.714102791932722,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5048876021589551,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1822916666666667,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.11671427318028041,
       "step": 16
     },
     {
-      "completion_length": 202.25,
-      "epoch": 4.571428571428571,
-      "grad_norm": 9.800145149230957,
-      "kl": 0.6018333853571676,
+      "completion_length": 499.0,
+      "epoch": 6.0,
+      "grad_norm": 0.48127031326293945,
+      "kl": 0.0007189161011150905,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0006,
-      "reward": 3.698561042547226,
-      "reward_std": 1.2764807561179623,
-      "rewards/concensus_correctness_reward_func": 0.6848750058561563,
-      "rewards/consensus_reward_func": 0.9375,
+      "loss": 0.0,
+      "reward": 0.8536344426018851,
+      "reward_std": 1.009829261473247,
+      "rewards/concensus_correctness_reward_func": 0.010285713842936925,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.7144985762424767,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4199201501905918,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 1.096062496304512,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.0662857059921537,
       "step": 18
     },
     {
-      "completion_length": 162.91666666666666,
-      "epoch": 5.0,
-      "grad_norm": 7.49417781829834,
-      "kl": 0.5289312032206605,
+      "completion_length": 409.71875,
+      "epoch": 6.7272727272727275,
+      "grad_norm": 3.820303440093994,
+      "kl": 0.0007408617093460634,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0004,
-      "reward": 4.249745080868403,
-      "reward_std": 1.1143968345907826,
-      "rewards/concensus_correctness_reward_func": 0.8489166696866354,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0,
+      "reward": 1.0658142920583487,
+      "reward_std": 1.05014331266284,
+      "rewards/concensus_correctness_reward_func": 0.004812499973922968,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.7353284160296122,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.46540803741663694,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2916666666666667,
-      "rewards/xmlcount_reward_func": 1.2071666717529297,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.22059374302625656,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.0014741608130862005,
-      "train_runtime": 143.5727,
-      "train_samples_per_second": 2.229,
-      "train_steps_per_second": 0.139
+      "train_loss": 6.654366700331593e-07,
+      "train_runtime": 1175.1261,
+      "train_samples_per_second": 0.272,
+      "train_steps_per_second": 0.017
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 10,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -227,7 +227,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 2,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:123588bd2f338759d72e8685564678580a958041438fa52b6cbc5acd86f969e5
-size 5880
+oid sha256:c2728207bad49d3f20aa37952e67ccc6d3233f5e53095ed17a152bcaff255850
+size 5944