End of training

adapter_config.json

@@ -24,13 +24,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
+    "o_proj",
+    "v_proj",
     "q_proj",
     "up_proj",
+    "down_proj",
     "gate_proj",
-    "k_proj",
-    "v_proj",
-    "o_proj"
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1120a3d01d8d014643d02d821b91f7970f58603171986d25224cf43d410df86c
+oid sha256:90ee50811893e4c12d868d63ef58a769cbe0257752abe3e2326a7ae9e9c844db
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.552342190640047e-06,
-    "train_runtime": 1068.5785,
-    "train_samples": 14,
-    "train_samples_per_second": 0.299,
-    "train_steps_per_second": 0.019
+    "train_loss": 5.938015531725682e-07,
+    "train_runtime": 1203.9652,
+    "train_samples": 9,
+    "train_samples_per_second": 0.266,
+    "train_steps_per_second": 0.017
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.552342190640047e-06,
-    "train_runtime": 1068.5785,
-    "train_samples": 14,
-    "train_samples_per_second": 0.299,
-    "train_steps_per_second": 0.019
+    "train_loss": 5.938015531725682e-07,
+    "train_runtime": 1203.9652,
+    "train_samples": 9,
+    "train_samples_per_second": 0.266,
+    "train_steps_per_second": 0.017
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 6.888888888888889,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 293.75,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 0.665027379989624,
-      "kl": 0.0053783293114975095,
+      "completion_length": 346.71875,
+      "epoch": 0.8888888888888888,
+      "grad_norm": 0.49128618836402893,
+      "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 2.289050579071045,
-      "reward_std": 0.7755527645349503,
-      "rewards/concensus_correctness_reward_func": 0.09375,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.4949689898639917,
+      "reward_std": 1.1950396299362183,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0625,
-      "rewards/question_recreation_reward_func": 0.4583318280056119,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.40200022887438536,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.549468757584691,
+      "rewards/xmlcount_reward_func": 0.40546875447034836,
       "step": 2
     },
     {
-      "completion_length": 285.1666666666667,
-      "epoch": 1.0,
-      "grad_norm": 0.530790388584137,
-      "kl": 0.006566934597988923,
+      "completion_length": 372.45,
+      "epoch": 1.4444444444444444,
+      "grad_norm": 7.855459213256836,
+      "kl": 0.0011587344692088664,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 4.521254261334737,
-      "reward_std": 4.512782156467438,
-      "rewards/concensus_correctness_reward_func": 1.8588333477576573,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 2.4613747358322144,
+      "reward_std": 1.3496225833892823,
+      "rewards/concensus_correctness_reward_func": 0.175,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.6164624790350596,
-      "rewards/soft_format_reward_func": 0.020833333333333332,
+      "rewards/final_correctness_reward_func": 0.9,
+      "rewards/question_recreation_reward_func": 0.6027747333049774,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.35845833520094555,
+      "rewards/xmlcount_reward_func": 0.7835999846458435,
       "step": 4
     },
     {
-      "completion_length": 273.09375,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 0.7246836423873901,
-      "kl": 0.0067243746016174555,
+      "completion_length": 353.45,
+      "epoch": 2.0,
+      "grad_norm": 0.3375403583049774,
+      "kl": 0.000796216621529311,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 2.9700495898723602,
-      "reward_std": 2.3836634270846844,
-      "rewards/concensus_correctness_reward_func": 0.671875,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.6292999267578125,
+      "reward_std": 0.9524844944477081,
+      "rewards/concensus_correctness_reward_func": 0.0875,
+      "rewards/consensus_reward_func": 0.2,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.4809871045872569,
+      "rewards/final_correctness_reward_func": 0.2,
+      "rewards/question_recreation_reward_func": 0.5393498949706554,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.6765625048428774,
+      "rewards/strict_format_reward_func": 0.025,
+      "rewards/xmlcount_reward_func": 0.5774500086903572,
       "step": 6
     },
     {
-      "completion_length": 296.375,
-      "epoch": 2.0,
-      "grad_norm": 0.5065819621086121,
-      "kl": 0.006312550472406049,
+      "completion_length": 409.53125,
+      "epoch": 2.888888888888889,
+      "grad_norm": 0.4388130009174347,
+      "kl": 0.0008436714124400169,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 3.7771472533543906,
-      "reward_std": 4.439713845650355,
-      "rewards/concensus_correctness_reward_func": 1.6666666666666667,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 1.6039557978510857,
+      "reward_std": 1.5356585681438446,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.5523556098341942,
-      "rewards/soft_format_reward_func": 0.020833333333333332,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.48914328031241894,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.37062498927116394,
+      "rewards/xmlcount_reward_func": 0.4273125003091991,
       "step": 8
     },
     {
-      "completion_length": 275.875,
-      "epoch": 2.571428571428571,
-      "grad_norm": 0.6104355454444885,
-      "kl": 0.007143966417061165,
+      "completion_length": 391.2,
+      "epoch": 3.4444444444444446,
+      "grad_norm": 0.539831280708313,
+      "kl": 0.0007707841461524368,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 4.970172703266144,
-      "reward_std": 5.592903956770897,
-      "rewards/concensus_correctness_reward_func": 2.8150624968111515,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.4420040369033813,
+      "reward_std": 1.1969814956188203,
+      "rewards/concensus_correctness_reward_func": 0.1625,
+      "rewards/consensus_reward_func": 0.4,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9375,
-      "rewards/question_recreation_reward_func": 0.4048290243372321,
+      "rewards/final_correctness_reward_func": 0.3,
+      "rewards/question_recreation_reward_func": 0.36160409450531006,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.375281248241663,
+      "rewards/xmlcount_reward_func": 0.21790000498294831,
       "step": 10
     },
     {
-      "completion_length": 304.0833333333333,
-      "epoch": 3.0,
-      "grad_norm": 0.4668538272380829,
-      "kl": 0.005717102981482943,
+      "completion_length": 339.3,
+      "epoch": 4.0,
+      "grad_norm": 0.191449835896492,
+      "kl": 0.0007514836092013865,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 4.4208576281865435,
-      "reward_std": 4.536543635030587,
-      "rewards/concensus_correctness_reward_func": 2.0118333250284195,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 2.318739914894104,
+      "reward_std": 1.7537718057632445,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.9166666666666666,
-      "rewards/question_recreation_reward_func": 0.4727743864059448,
+      "rewards/final_correctness_reward_func": 1.1,
+      "rewards/question_recreation_reward_func": 0.4956398606300354,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3529166827599208,
+      "rewards/xmlcount_reward_func": 0.6231000006198884,
       "step": 12
     },
     {
-      "completion_length": 305.1875,
-      "epoch": 3.571428571428571,
-      "grad_norm": 0.47888341546058655,
-      "kl": 0.006005744566209614,
+      "completion_length": 359.09375,
+      "epoch": 4.888888888888889,
+      "grad_norm": 1.54889976978302,
+      "kl": 0.0008484305944875814,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 3.988325133919716,
-      "reward_std": 4.117011643946171,
-      "rewards/concensus_correctness_reward_func": 1.5698749981820583,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.9775982201099396,
+      "reward_std": 1.2569940611720085,
+      "rewards/concensus_correctness_reward_func": 0.21875,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.875,
-      "rewards/question_recreation_reward_func": 0.629481378942728,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.42125441413372755,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.50771873164922,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.650093749165535,
       "step": 14
     },
     {
-      "completion_length": 276.4583333333333,
-      "epoch": 4.0,
-      "grad_norm": 0.33002349734306335,
-      "kl": 0.006965941439072291,
+      "completion_length": 407.5,
+      "epoch": 5.444444444444445,
+      "grad_norm": 0.41609179973602295,
+      "kl": 0.0008388939779251814,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 4.395283937454224,
-      "reward_std": 4.75975959499677,
-      "rewards/concensus_correctness_reward_func": 2.5625,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.9900086879730225,
+      "reward_std": 1.1237601399421693,
+      "rewards/concensus_correctness_reward_func": 0.1625,
+      "rewards/consensus_reward_func": 0.2,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8333333333333334,
-      "rewards/question_recreation_reward_func": 0.2941172122955322,
+      "rewards/final_correctness_reward_func": 0.8,
+      "rewards/question_recreation_reward_func": 0.41860867887735365,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.45533334898451966,
+      "rewards/xmlcount_reward_func": 0.40889999866485593,
       "step": 16
     },
     {
-      "completion_length": 273.09375,
-      "epoch": 4.571428571428571,
-      "grad_norm": 0.5967588424682617,
-      "kl": 0.006533314939588308,
+      "completion_length": 381.25,
+      "epoch": 6.0,
+      "grad_norm": 0.2035149782896042,
+      "kl": 0.0008319188258610666,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 4.299996465444565,
-      "reward_std": 5.0971772111952305,
-      "rewards/concensus_correctness_reward_func": 1.956750001758337,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.4723462462425232,
+      "reward_std": 1.298855483531952,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.3,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.5635589696466923,
+      "rewards/final_correctness_reward_func": 0.3,
+      "rewards/question_recreation_reward_func": 0.45944626331329347,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.6078125014901161,
+      "rewards/strict_format_reward_func": 0.025,
+      "rewards/xmlcount_reward_func": 0.38790000379085543,
       "step": 18
     },
     {
-      "completion_length": 286.8333333333333,
-      "epoch": 5.0,
-      "grad_norm": 0.5166720151901245,
-      "kl": 0.00690275647987922,
+      "completion_length": 419.59375,
+      "epoch": 6.888888888888889,
+      "grad_norm": 0.42345717549324036,
+      "kl": 0.0009374981746077538,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 7.515388667583466,
-      "reward_std": 7.997843941052754,
-      "rewards/concensus_correctness_reward_func": 5.3416666984558105,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 1.7654030844569206,
+      "reward_std": 1.5419669449329376,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.3129719669620196,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.4614968504756689,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4440833354989688,
+      "rewards/xmlcount_reward_func": 0.5539062768220901,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 6.888888888888889,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 5.552342190640047e-06,
-      "train_runtime": 1068.5785,
-      "train_samples_per_second": 0.299,
-      "train_steps_per_second": 0.019
+      "train_loss": 5.938015531725682e-07,
+      "train_runtime": 1203.9652,
+      "train_samples_per_second": 0.266,
+      "train_steps_per_second": 0.017
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 10,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e958f1c974cd9d9bd06cbc144cbaf8fc44ab5bd16503f651c70ffce75b813b17
+oid sha256:c52b8e45c2d0c1096950d6bb82823e0caed4ef18ee8a79a988bfb6845acd84aa
 size 6008