End of training

README.md

@@ -1,5 +1,5 @@
 ---
-base_model: Gensyn/Qwen2.5-0.5B-Instruct
+base_model: Gensyn/Qwen2.5-1.5B-Instruct
 library_name: transformers
 model_name: gensyn-checkpoints-peaceful_shrewd_marmot
 tags:
@@ -8,13 +8,14 @@ tags:
 - grpo
 - gensyn
 - I am peaceful shrewd marmot
+- unsloth
 - trl
 licence: license
 ---
 
 # Model Card for gensyn-checkpoints-peaceful_shrewd_marmot
 
-This model is a fine-tuned version of [Gensyn/Qwen2.5-0.5B-Instruct](https://huggingface.co/Gensyn/Qwen2.5-0.5B-Instruct).
+This model is a fine-tuned version of [Gensyn/Qwen2.5-1.5B-Instruct](https://huggingface.co/Gensyn/Qwen2.5-1.5B-Instruct).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -39,7 +40,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 
 - TRL: 0.15.2
 - Transformers: 4.51.3
-- Pytorch: 2.5.1
+- Pytorch: 2.6.0
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1
 

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Gensyn/Qwen2.5-1.5B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "q_proj",
+    "gate_proj",
+    "k_proj",
+    "v_proj",
+    "up_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:814d4700311c61edd4973358003830fb4e8379a9c389ee291251860f9b54c9d2
+size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0003779519018280553,
-    "train_runtime": 451.5947,
-    "train_samples": 28,
-    "train_samples_per_second": 0.709,
-    "train_steps_per_second": 0.044
+    "train_loss": 8.5623666450374e-07,
+    "train_runtime": 1083.5601,
+    "train_samples": 20,
+    "train_samples_per_second": 0.295,
+    "train_steps_per_second": 0.018
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0003779519018280553,
-    "train_runtime": 451.5947,
-    "train_samples": 28,
-    "train_samples_per_second": 0.709,
-    "train_steps_per_second": 0.044
+    "train_loss": 8.5623666450374e-07,
+    "train_runtime": 1083.5601,
+    "train_samples": 20,
+    "train_samples_per_second": 0.295,
+    "train_steps_per_second": 0.018
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 4.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 344.625,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 10.772970199584961,
-      "kl": 0.0,
+      "completion_length": 395.875,
+      "epoch": 0.4,
+      "grad_norm": 0.5111533999443054,
+      "kl": 0.0006723058941133786,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 0.24003540235571563,
-      "reward_std": 0.3615740498935338,
+      "loss": 0.0,
+      "reward": 1.8240740671753883,
+      "reward_std": 1.3498586490750313,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.25100415269844234,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5756990369409323,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.010968748014420271,
+      "rewards/xmlcount_reward_func": 0.37337500508874655,
       "step": 2
     },
     {
-      "completion_length": 361.7083333333333,
-      "epoch": 1.0,
-      "grad_norm": 42.27848434448242,
-      "kl": 0.0015548948867945,
+      "completion_length": 461.90625,
+      "epoch": 0.8,
+      "grad_norm": 0.580380380153656,
+      "kl": 0.0007322138044401072,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.4717333244625479,
-      "reward_std": 0.5787315891357139,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.451542153954506,
+      "reward_std": 1.2028871849179268,
+      "rewards/concensus_correctness_reward_func": 0.07575000077486038,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.2576499928254634,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5127608813345432,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13075000412451723,
+      "rewards/xmlcount_reward_func": 0.488031261600554,
       "step": 4
     },
     {
-      "completion_length": 383.4375,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 6.3906779289245605,
-      "kl": 0.003872946799674537,
+      "completion_length": 387.71875,
+      "epoch": 1.2,
+      "grad_norm": 0.5032566785812378,
+      "kl": 0.000788287041359581,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.4794421146507375,
-      "reward_std": 0.27617324620950967,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.6773777455091476,
+      "reward_std": 1.304907813668251,
+      "rewards/concensus_correctness_reward_func": 0.07562499865889549,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.30297336814692244,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.4745964966714382,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11396875674836338,
+      "rewards/xmlcount_reward_func": 0.36153125762939453,
       "step": 6
     },
     {
-      "completion_length": 388.7916666666667,
-      "epoch": 2.0,
-      "grad_norm": 10.911130905151367,
-      "kl": 0.0028913425194332376,
+      "completion_length": 382.71875,
+      "epoch": 1.6,
+      "grad_norm": 0.4953606128692627,
+      "kl": 0.0007655509398318827,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.46779738351081807,
-      "reward_std": 0.56250952805082,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.7268302105367184,
+      "reward_std": 1.188648235052824,
+      "rewards/concensus_correctness_reward_func": 0.2141249980777502,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.2721307026222348,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.6044552035164088,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11233333125710487,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.45512500032782555,
       "step": 8
     },
     {
-      "completion_length": 355.5,
-      "epoch": 2.571428571428571,
-      "grad_norm": 9.093505859375,
-      "kl": 0.00975232083874289,
+      "completion_length": 466.375,
+      "epoch": 2.0,
+      "grad_norm": 0.3901965618133545,
+      "kl": 0.0007470509153790772,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.4602516919840127,
-      "reward_std": 0.4189709427018897,
+      "reward": 1.0460964776575565,
+      "reward_std": 0.9714201502501965,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.2312829417642206,
+      "rewards/question_recreation_reward_func": 0.47784643806517124,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16646874835714698,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.1151250135153532,
       "step": 10
     },
     {
-      "completion_length": 291.4166666666667,
-      "epoch": 3.0,
-      "grad_norm": 66.55801391601562,
-      "kl": 0.18349976455404735,
+      "completion_length": 382.71875,
+      "epoch": 2.4,
+      "grad_norm": 0.5561316013336182,
+      "kl": 0.0008165190956788138,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0001,
-      "reward": 0.32576117835318047,
-      "reward_std": 0.2934178128295268,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "loss": 0.0,
+      "reward": 1.866930216550827,
+      "reward_std": 1.6471154689788818,
+      "rewards/concensus_correctness_reward_func": 0.12025000154972076,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.24851117640112838,
-      "rewards/soft_format_reward_func": 0.020833333333333332,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.05641666303078333,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.7318051867187023,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.17112500546500087,
       "step": 12
     },
     {
-      "completion_length": 343.09375,
-      "epoch": 3.571428571428571,
-      "grad_norm": 17.57956886291504,
-      "kl": 0.0372060123263509,
+      "completion_length": 412.90625,
+      "epoch": 2.8,
+      "grad_norm": 0.6610909104347229,
+      "kl": 0.0011565974200493656,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.22849576943553984,
-      "reward_std": 0.7314633782953024,
-      "rewards/concensus_correctness_reward_func": 0.0078125,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.5754297897219658,
+      "reward_std": 1.5814464390277863,
+      "rewards/concensus_correctness_reward_func": 0.078125,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.22037077555432916,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4452110081911087,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.06218750216066837,
+      "rewards/xmlcount_reward_func": 0.3645937405526638,
       "step": 14
     },
     {
-      "completion_length": 383.7083333333333,
-      "epoch": 4.0,
-      "grad_norm": 19.69514274597168,
-      "kl": 3.7607118334659995,
+      "completion_length": 473.53125,
+      "epoch": 3.2,
+      "grad_norm": 1.5192562341690063,
+      "kl": 0.0009948557199095376,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0028,
-      "reward": 0.08723691183452804,
-      "reward_std": 0.3238961525882284,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "loss": 0.0,
+      "reward": 1.4332580585032701,
+      "reward_std": 1.424522940069437,
+      "rewards/concensus_correctness_reward_func": 0.09793750010430813,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.15461191271121302,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.4286955539137125,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.15070833389957747,
+      "rewards/xmlcount_reward_func": 0.15662500262260437,
       "step": 16
     },
     {
-      "completion_length": 402.25,
-      "epoch": 4.571428571428571,
-      "grad_norm": 15.42269515991211,
-      "kl": 0.0024876317584130447,
+      "completion_length": 392.96875,
+      "epoch": 3.6,
+      "grad_norm": 0.756603479385376,
+      "kl": 0.0009181452624034137,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.46647889574524015,
-      "reward_std": 0.3916395944033866,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.9255880266427994,
+      "reward_std": 1.5188024193048477,
+      "rewards/concensus_correctness_reward_func": 0.15568749699741602,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3051351552712731,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.4566505029797554,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.09884374821558595,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.42262499406933784,
       "step": 18
     },
     {
-      "completion_length": 278.6666666666667,
-      "epoch": 5.0,
-      "grad_norm": 5.755941867828369,
-      "kl": 1.0229452109100141,
+      "completion_length": 329.6875,
+      "epoch": 4.0,
+      "grad_norm": 0.5636370778083801,
+      "kl": 0.0008640880332677625,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0008,
-      "reward": 0.49779408425092697,
-      "reward_std": 0.30871247686445713,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "loss": 0.0,
+      "reward": 1.6222331374883652,
+      "reward_std": 0.9177208282053471,
+      "rewards/concensus_correctness_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3205857429032524,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5897956416010857,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17720833172400793,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.5011875098571181,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 4.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.0003779519018280553,
-      "train_runtime": 451.5947,
-      "train_samples_per_second": 0.709,
-      "train_steps_per_second": 0.044
+      "train_loss": 8.5623666450374e-07,
+      "train_runtime": 1083.5601,
+      "train_samples_per_second": 0.295,
+      "train_steps_per_second": 0.018
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 4,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -227,7 +227,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 2,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8a740c114889c9cfd9d6df66d8762679a5440a165be9704bbcc18da98ff08fd
-size 5880
+oid sha256:e94c82ec35a1fffc0809f8c8354d8a30c35bcba25e8732c0f87807667bc4b1e7
+size 5944