End of training

adapter_config.json

@@ -24,13 +24,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "q_proj",
+    "k_proj",
     "up_proj",
     "o_proj",
     "gate_proj",
-    "v_proj",
-    "k_proj",
-    "q_proj",
-    "down_proj"
+    "down_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:414342088961dfd74c63cbe89b34866ba3d5bdd815b18a419c4d403811724420
+oid sha256:35b1d89bbdbbe5e1d73bed96f09434e6e67fa496cdae5af9db1c1762e9e234f9
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.672815686670219e-07,
-    "train_runtime": 1038.255,
-    "train_samples": 11,
-    "train_samples_per_second": 0.308,
-    "train_steps_per_second": 0.019
+    "train_loss": 6.671937740065914e-07,
+    "train_runtime": 1430.3514,
+    "train_samples": 9,
+    "train_samples_per_second": 0.224,
+    "train_steps_per_second": 0.014
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.672815686670219e-07,
-    "train_runtime": 1038.255,
-    "train_samples": 11,
-    "train_samples_per_second": 0.308,
-    "train_steps_per_second": 0.019
+    "train_loss": 6.671937740065914e-07,
+    "train_runtime": 1430.3514,
+    "train_samples": 9,
+    "train_samples_per_second": 0.224,
+    "train_steps_per_second": 0.014
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 6.7272727272727275,
+  "epoch": 6.888888888888889,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 385.53125,
-      "epoch": 0.7272727272727273,
-      "grad_norm": 0.5399614572525024,
-      "kl": 0.0007184698333730921,
+      "completion_length": 356.03125,
+      "epoch": 0.8888888888888888,
+      "grad_norm": 0.6314395666122437,
+      "kl": 0.0007967295678099617,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.100014790892601,
-      "reward_std": 0.9939730558544397,
-      "rewards/concensus_correctness_reward_func": 0.046937499195337296,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.4575330335646868,
+      "reward_std": 0.9215883761644363,
+      "rewards/concensus_correctness_reward_func": 0.02812500111758709,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5981710441410542,
-      "rewards/soft_format_reward_func": 0.03125,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.22053126245737076,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.55325180105865,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.501156248152256,
       "step": 2
     },
     {
-      "completion_length": 373.7857142857143,
-      "epoch": 1.3636363636363638,
-      "grad_norm": 1.7267061471939087,
-      "kl": 0.0009554715486176844,
+      "completion_length": 477.1,
+      "epoch": 1.4444444444444444,
+      "grad_norm": 0.4168018698692322,
+      "kl": 0.0006883293273858726,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.0411313814776284,
-      "reward_std": 0.9137314856052399,
-      "rewards/concensus_correctness_reward_func": 0.05357142857142857,
-      "rewards/consensus_reward_func": 0.07142857142857142,
+      "reward": 1.215496826171875,
+      "reward_std": 0.7959818780422211,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.43952426314353943,
+      "rewards/question_recreation_reward_func": 0.7412468314170837,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.45875001166548046,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3742500126361847,
       "step": 4
     },
     {
-      "completion_length": 453.60714285714283,
+      "completion_length": 330.75,
       "epoch": 2.0,
-      "grad_norm": 0.5066748261451721,
-      "kl": 0.000570978453781988,
+      "grad_norm": 0.20514653623104095,
+      "kl": 0.0009159827372059226,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.0355133158820016,
-      "reward_std": 1.0863181459052222,
-      "rewards/concensus_correctness_reward_func": 0.10728571244648524,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "reward": 1.698195791244507,
+      "reward_std": 1.2764226973056794,
+      "rewards/concensus_correctness_reward_func": 0.10770000070333481,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5739419098411288,
+      "rewards/final_correctness_reward_func": 0.1,
+      "rewards/question_recreation_reward_func": 0.47604577243328094,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.19357144300426757,
+      "rewards/strict_format_reward_func": 0.025,
+      "rewards/xmlcount_reward_func": 0.4894500017166138,
       "step": 6
     },
     {
-      "completion_length": 436.375,
-      "epoch": 2.7272727272727275,
-      "grad_norm": 0.34225988388061523,
-      "kl": 0.0006407255314115901,
+      "completion_length": 335.8125,
+      "epoch": 2.888888888888889,
+      "grad_norm": 0.7058514952659607,
+      "kl": 0.0008629741714685224,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.8332138750702143,
-      "reward_std": 1.266435667872429,
-      "rewards/concensus_correctness_reward_func": 0.12299999874085188,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.3411639034748077,
+      "reward_std": 1.0308659747242928,
+      "rewards/concensus_correctness_reward_func": 0.029249999672174454,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.44002638664096594,
+      "rewards/question_recreation_reward_func": 0.5329451244324446,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.10481250332668424,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.45084376633167267,
       "step": 8
     },
     {
-      "completion_length": 404.57142857142856,
-      "epoch": 3.3636363636363638,
-      "grad_norm": 0.3635714054107666,
-      "kl": 0.0007320014244344618,
+      "completion_length": 315.1,
+      "epoch": 3.4444444444444446,
+      "grad_norm": 0.5046074986457825,
+      "kl": 0.0007503833738155663,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.3576448474611555,
-      "reward_std": 1.0595165938138962,
-      "rewards/concensus_correctness_reward_func": 0.13399999908038548,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 1.2790536284446716,
+      "reward_std": 1.0574151515960692,
+      "rewards/concensus_correctness_reward_func": 0.03510000109672547,
+      "rewards/consensus_reward_func": 0.2,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4522162995168141,
+      "rewards/final_correctness_reward_func": 0.1,
+      "rewards/question_recreation_reward_func": 0.7279036343097687,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.485714287097965,
+      "rewards/xmlcount_reward_func": 0.21605000048875808,
       "step": 10
     },
     {
-      "completion_length": 477.10714285714283,
+      "completion_length": 363.2,
       "epoch": 4.0,
-      "grad_norm": 0.32631227374076843,
-      "kl": 0.0006598751559587461,
+      "grad_norm": 0.20469354093074799,
+      "kl": 0.0006909701623953879,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.8694216098104205,
-      "reward_std": 0.7272259593009949,
-      "rewards/concensus_correctness_reward_func": 0.009000000144754137,
-      "rewards/consensus_reward_func": 0.07142857142857142,
+      "reward": 1.0307862520217896,
+      "reward_std": 0.6254790127277374,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5253501683473587,
+      "rewards/question_recreation_reward_func": 0.5521362721920013,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2636428624391556,
+      "rewards/strict_format_reward_func": 0.025,
+      "rewards/xmlcount_reward_func": 0.35364999286830423,
       "step": 12
     },
     {
-      "completion_length": 423.21875,
-      "epoch": 4.7272727272727275,
-      "grad_norm": 0.3718009889125824,
-      "kl": 0.0006742304467479698,
+      "completion_length": 333.40625,
+      "epoch": 4.888888888888889,
+      "grad_norm": 1.0354853868484497,
+      "kl": 0.0014274672030296642,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.8267503269016743,
-      "reward_std": 0.8771254681050777,
-      "rewards/concensus_correctness_reward_func": 0.0374374995008111,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.4639385491609573,
+      "reward_std": 1.0350679904222488,
+      "rewards/concensus_correctness_reward_func": 0.021937500685453415,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5107190869748592,
+      "rewards/question_recreation_reward_func": 0.5511260535567999,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.09109376277774572,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4846249930560589,
       "step": 14
     },
     {
-      "completion_length": 433.4642857142857,
-      "epoch": 5.363636363636363,
-      "grad_norm": 0.5318070650100708,
-      "kl": 0.0006712128025745707,
+      "completion_length": 373.45,
+      "epoch": 5.444444444444445,
+      "grad_norm": 1.1001330614089966,
+      "kl": 0.0006557022570632398,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.2695725475038802,
-      "reward_std": 1.3976846294743674,
-      "rewards/concensus_correctness_reward_func": 0.21014285619769776,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 1.1861554026603698,
+      "reward_std": 1.0914301306009293,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.2,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4444296487740108,
+      "rewards/question_recreation_reward_func": 0.7789053797721863,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2578571449433054,
+      "rewards/strict_format_reward_func": 0.05,
+      "rewards/xmlcount_reward_func": 0.15724998712539673,
       "step": 16
     },
     {
-      "completion_length": 394.14285714285717,
+      "completion_length": 404.95,
       "epoch": 6.0,
-      "grad_norm": 0.9077908992767334,
-      "kl": 0.0008383357780985534,
+      "grad_norm": 0.12595321238040924,
+      "kl": 0.0008602307643741369,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.9978287645748684,
-      "reward_std": 0.9985724900450025,
-      "rewards/concensus_correctness_reward_func": 0.05357142857142857,
-      "rewards/consensus_reward_func": 0.07142857142857142,
+      "reward": 1.3635105848312379,
+      "reward_std": 0.8709330588579178,
+      "rewards/concensus_correctness_reward_func": 0.02250000089406967,
+      "rewards/consensus_reward_func": 0.2,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5058645009994507,
+      "rewards/final_correctness_reward_func": 0.2,
+      "rewards/question_recreation_reward_func": 0.42216054201126096,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.36696428060531616,
+      "rewards/xmlcount_reward_func": 0.5188500076532364,
       "step": 18
     },
     {
-      "completion_length": 375.53125,
-      "epoch": 6.7272727272727275,
-      "grad_norm": 1.2648330926895142,
-      "kl": 0.0008303213326144032,
+      "completion_length": 379.375,
+      "epoch": 6.888888888888889,
+      "grad_norm": 0.3285180628299713,
+      "kl": 0.0007949093233037274,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.055504847317934,
-      "reward_std": 0.8788291048258543,
-      "rewards/concensus_correctness_reward_func": 0.05274999886751175,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.661986481398344,
+      "reward_std": 0.6507729813456535,
+      "rewards/concensus_correctness_reward_func": -0.009937499649822712,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.44391110353171825,
+      "rewards/question_recreation_reward_func": 0.4172677341848612,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.37134375516325235,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.17653125058859587,
       "step": 20
     },
     {
-      "epoch": 6.7272727272727275,
+      "epoch": 6.888888888888889,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 6.672815686670219e-07,
-      "train_runtime": 1038.255,
-      "train_samples_per_second": 0.308,
-      "train_steps_per_second": 0.019
+      "train_loss": 6.671937740065914e-07,
+      "train_runtime": 1430.3514,
+      "train_samples_per_second": 0.224,
+      "train_steps_per_second": 0.014
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3434c1d28515481066b888484d628bc1ffa55a17494013d670f14e2c27418239
+oid sha256:f8550070dcbf31eb2b1a5f94205e0051e1f7447528d586fac90e792ff1b94288
 size 5944