End of training

adapter_config.json

@@ -24,13 +24,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
-    "k_proj",
+    "v_proj",
     "q_proj",
+    "gate_proj",
     "o_proj",
-    "v_proj",
     "up_proj",
-    "down_proj"
+    "down_proj",
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eda520eb2553c1945b78d40e82a0bfd317e5e2c82e2c946a10a43c288d5555c6
+oid sha256:ad0765a9fe64ca40d6ce9bf26684d1cea0ce43df6b24ac55d782eec5ed54b060
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.1318876374843966e-06,
-    "train_runtime": 545.1088,
-    "train_samples": 14,
-    "train_samples_per_second": 0.587,
-    "train_steps_per_second": 0.037
+    "train_loss": 8.158007614333939e-07,
+    "train_runtime": 1460.3551,
+    "train_samples": 12,
+    "train_samples_per_second": 0.219,
+    "train_steps_per_second": 0.014
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.1318876374843966e-06,
-    "train_runtime": 545.1088,
-    "train_samples": 14,
-    "train_samples_per_second": 0.587,
-    "train_steps_per_second": 0.037
+    "train_loss": 8.158007614333939e-07,
+    "train_runtime": 1460.3551,
+    "train_samples": 12,
+    "train_samples_per_second": 0.219,
+    "train_steps_per_second": 0.014
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 6.666666666666667,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 282.125,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 0.7033820748329163,
-      "kl": 0.0014059239474590868,
+      "completion_length": 305.375,
+      "epoch": 0.6666666666666666,
+      "grad_norm": 0.8305250406265259,
+      "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 3.856389135122299,
-      "reward_std": 3.8554979264736176,
-      "rewards/concensus_correctness_reward_func": 1.921875,
-      "rewards/consensus_reward_func": 0.5625,
+      "loss": -0.0,
+      "reward": 1.539582073688507,
+      "reward_std": 0.923794575035572,
+      "rewards/concensus_correctness_reward_func": 0.046875,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5537954121828079,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4275937583297491,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.45251956954598427,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.6495624948292971,
       "step": 2
     },
     {
-      "completion_length": 266.4166666666667,
-      "epoch": 1.0,
-      "grad_norm": 0.38228628039360046,
-      "kl": 0.0011829413124360144,
+      "completion_length": 324.90625,
+      "epoch": 1.3333333333333333,
+      "grad_norm": 0.6652752757072449,
+      "kl": 0.0008809074424789287,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.485657254854838,
-      "reward_std": 2.8838004966576896,
-      "rewards/concensus_correctness_reward_func": 0.9249166660010815,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 0.9363901242613792,
+      "reward_std": 0.9168383367359638,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.549990639090538,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.4294838737696409,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.34408334270119667,
+      "rewards/xmlcount_reward_func": 0.19440625130664557,
       "step": 4
     },
     {
-      "completion_length": 294.5625,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 0.39524662494659424,
-      "kl": 0.001252303110959474,
+      "completion_length": 335.90625,
+      "epoch": 2.0,
+      "grad_norm": 0.8124482035636902,
+      "kl": 0.0008842248062137514,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 2.970046356320381,
-      "reward_std": 3.5414111353456974,
-      "rewards/concensus_correctness_reward_func": 1.3329375013709068,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.0841968432068825,
+      "reward_std": 1.182814422994852,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.5125776026397943,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.47269680351018906,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3120312597602606,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.1583749931305647,
       "step": 6
     },
     {
-      "completion_length": 252.95833333333334,
-      "epoch": 2.0,
-      "grad_norm": 0.36599600315093994,
-      "kl": 0.0012746660019426297,
+      "completion_length": 332.3125,
+      "epoch": 2.6666666666666665,
+      "grad_norm": 1.0054805278778076,
+      "kl": 0.0008862593604135327,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.857189121345679,
-      "reward_std": 1.3523634672164917,
-      "rewards/concensus_correctness_reward_func": 0.06349999954303105,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 0.8604639675468206,
+      "reward_std": 1.0349821876734495,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5466473984221617,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.48099521454423666,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.33037501325209934,
+      "rewards/xmlcount_reward_func": 0.0669687557965517,
       "step": 8
     },
     {
-      "completion_length": 276.25,
-      "epoch": 2.571428571428571,
-      "grad_norm": 0.7125920057296753,
-      "kl": 0.0014186603511916474,
+      "completion_length": 343.25,
+      "epoch": 3.3333333333333335,
+      "grad_norm": 0.5403996706008911,
+      "kl": 0.0008731155321584083,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 3.0660413801670074,
-      "reward_std": 2.600810881704092,
-      "rewards/concensus_correctness_reward_func": 1.3980624992400408,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 0.8438624609261751,
+      "reward_std": 0.9167536506429315,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5118539500981569,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.32392496382817626,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.21862499741837382,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.25431250408291817,
       "step": 10
     },
     {
-      "completion_length": 310.875,
-      "epoch": 3.0,
-      "grad_norm": 0.35926124453544617,
-      "kl": 0.0011175758360574644,
+      "completion_length": 338.71875,
+      "epoch": 4.0,
+      "grad_norm": 0.5702744126319885,
+      "kl": 0.0008382313608308323,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 2.6451631089051566,
-      "reward_std": 2.777275949716568,
-      "rewards/concensus_correctness_reward_func": 0.9941666759550571,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 1.2192110121250153,
+      "reward_std": 0.9378089085221291,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.5017463713884354,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.6342110075056553,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.378416674832503,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3818749971687794,
       "step": 12
     },
     {
-      "completion_length": 247.1875,
-      "epoch": 3.571428571428571,
-      "grad_norm": 0.8195107579231262,
-      "kl": 0.001302031785598956,
+      "completion_length": 313.65625,
+      "epoch": 4.666666666666667,
+      "grad_norm": 0.5005449652671814,
+      "kl": 0.0008908945746952668,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.3996790796518326,
-      "reward_std": 2.415383957326412,
-      "rewards/concensus_correctness_reward_func": 0.7305624997243285,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.1570591032505035,
+      "reward_std": 0.641424436122179,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.4978354126214981,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.45824659150093794,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.48378124041482806,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.5269374959170818,
       "step": 14
     },
     {
-      "completion_length": 283.75,
-      "epoch": 4.0,
-      "grad_norm": 0.7575254440307617,
-      "kl": 0.0012653738958761096,
+      "completion_length": 364.90625,
+      "epoch": 5.333333333333333,
+      "grad_norm": 0.5262055993080139,
+      "kl": 0.0009737010987009853,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 4.887010504802068,
-      "reward_std": 4.870325267314911,
-      "rewards/concensus_correctness_reward_func": 3.4498333471516767,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 0.9238902628421783,
+      "reward_std": 1.0804461874067783,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.43109386414289474,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.42392150964587927,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2560833248620232,
+      "rewards/xmlcount_reward_func": 0.2499687452800572,
       "step": 16
     },
     {
-      "completion_length": 292.4375,
-      "epoch": 4.571428571428571,
-      "grad_norm": 0.6278160810470581,
-      "kl": 0.0012706017587333918,
+      "completion_length": 271.8125,
+      "epoch": 6.0,
+      "grad_norm": 0.5890949368476868,
+      "kl": 0.0009729154990054667,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.916475921869278,
-      "reward_std": 1.562115877866745,
-      "rewards/concensus_correctness_reward_func": 0.140625,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.2119700014591217,
+      "reward_std": 0.8509260080754757,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.4949758695438504,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4545950014144182,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.21837499644607306,
+      "rewards/xmlcount_reward_func": 0.44487499445676804,
       "step": 18
     },
     {
-      "completion_length": 264.375,
-      "epoch": 5.0,
-      "grad_norm": 0.388331800699234,
-      "kl": 0.0012651081973065932,
+      "completion_length": 283.25,
+      "epoch": 6.666666666666667,
+      "grad_norm": 0.6624741554260254,
+      "kl": 0.0009808850372792222,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 2.642484724521637,
-      "reward_std": 3.0436351001262665,
-      "rewards/concensus_correctness_reward_func": 1.0363333486020565,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 1.0398481525480747,
+      "reward_std": 1.0907393302768469,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.43598459909359616,
-      "rewards/soft_format_reward_func": 0.020833333333333332,
+      "rewards/question_recreation_reward_func": 0.29278566036373377,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.316000006472071,
+      "rewards/xmlcount_reward_func": 0.3095625005662441,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 6.666666666666667,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.1318876374843966e-06,
-      "train_runtime": 545.1088,
-      "train_samples_per_second": 0.587,
-      "train_steps_per_second": 0.037
+      "train_loss": 8.158007614333939e-07,
+      "train_runtime": 1460.3551,
+      "train_samples_per_second": 0.219,
+      "train_steps_per_second": 0.014
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47321d91f4ce4b5bf9496232930f0f95d3c90cf7cdfb4b32ac8f7346e8b557e5
+oid sha256:9d80ebc5aba346914cf3b05dad48e6224e1bccd8ec5e38c0fec66d2a637f8574
 size 6008