End of training

adapter_config.json

@@ -24,13 +24,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
+    "q_proj",
+    "o_proj",
+    "k_proj",
     "v_proj",
-    "up_proj",
+    "down_proj",
     "gate_proj",
-    "k_proj",
-    "o_proj",
-    "q_proj"
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:868fd0c4e03516d0ccda8a10431a0abb250070154933879081c5a7d377c7c5dc
+oid sha256:c161787132baa5b42839a4ffb608635bbe84e49a24bd0508d07e03c4a8bfab9c
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.639158449516495e-07,
-    "train_runtime": 1717.2834,
-    "train_samples": 21,
-    "train_samples_per_second": 0.28,
-    "train_steps_per_second": 0.017
+    "train_loss": 7.550418615664967e-07,
+    "train_runtime": 594.0398,
+    "train_samples": 14,
+    "train_samples_per_second": 0.539,
+    "train_steps_per_second": 0.034
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.639158449516495e-07,
-    "train_runtime": 1717.2834,
-    "train_samples": 21,
-    "train_samples_per_second": 0.28,
-    "train_steps_per_second": 0.017
+    "train_loss": 7.550418615664967e-07,
+    "train_runtime": 594.0398,
+    "train_samples": 14,
+    "train_samples_per_second": 0.539,
+    "train_steps_per_second": 0.034
 }

trainer_state.json

@@ -4,310 +4,215 @@
   "best_model_checkpoint": null,
   "epoch": 5.0,
   "eval_steps": 500,
-  "global_step": 30,
+  "global_step": 20,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 304.625,
-      "epoch": 0.38095238095238093,
-      "grad_norm": 0.5284512639045715,
-      "kl": 0.0007773352263029665,
+      "completion_length": 335.625,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 0.5254331827163696,
+      "kl": 0.0007864277868065983,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 4.023562505841255,
-      "reward_std": 3.1125364899635315,
-      "rewards/concensus_correctness_reward_func": 1.3129999861121178,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 1.940154641866684,
+      "reward_std": 2.1765673831105232,
+      "rewards/concensus_correctness_reward_func": 0.640625,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3125,
-      "rewards/question_recreation_reward_func": 0.5383437238633633,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5108109563589096,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.2815937480190769,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.35121875163167715,
       "step": 2
     },
     {
-      "completion_length": 239.09375,
-      "epoch": 0.7619047619047619,
-      "grad_norm": 0.5153515934944153,
-      "kl": 0.0008589375793235376,
-      "learning_rate": 4.941551389275217e-07,
+      "completion_length": 209.54166666666666,
+      "epoch": 1.0,
+      "grad_norm": 1.1438218355178833,
+      "kl": 0.001036973485800748,
+      "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.7685961797833443,
-      "reward_std": 2.092758573591709,
-      "rewards/concensus_correctness_reward_func": 0.703125,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.632412592569987,
+      "reward_std": 1.6522801121075947,
+      "rewards/concensus_correctness_reward_func": 0.25066666801770526,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.5785962119698524,
+      "rewards/final_correctness_reward_func": 0.9166666666666666,
+      "rewards/question_recreation_reward_func": 0.5156625856955847,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.580624982714653,
+      "rewards/strict_format_reward_func": 0.041666666666666664,
+      "rewards/xmlcount_reward_func": 0.4077500080068906,
       "step": 4
     },
     {
-      "completion_length": 286.25,
-      "epoch": 1.0,
-      "grad_norm": 0.19600075483322144,
-      "kl": 0.0009585698484443128,
-      "learning_rate": 4.768938549177392e-07,
+      "completion_length": 295.3125,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 0.6330137252807617,
+      "kl": 0.0009554048301652074,
+      "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 2.937186098098755,
-      "reward_std": 3.6447759747505186,
-      "rewards/concensus_correctness_reward_func": 1.206700000166893,
-      "rewards/consensus_reward_func": 0.3,
+      "reward": 2.1943196058273315,
+      "reward_std": 2.59543714299798,
+      "rewards/concensus_correctness_reward_func": 0.7225000001490116,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8,
-      "rewards/question_recreation_reward_func": 0.46698602437973025,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.4514446333050728,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.025,
-      "rewards/xmlcount_reward_func": 0.13849999755620956,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.19224999798461795,
       "step": 6
     },
     {
-      "completion_length": 265.15625,
-      "epoch": 1.380952380952381,
-      "grad_norm": 0.5939684510231018,
-      "kl": 0.0009940789168467745,
-      "learning_rate": 4.490232664264109e-07,
+      "completion_length": 258.9166666666667,
+      "epoch": 2.0,
+      "grad_norm": 0.41280338168144226,
+      "kl": 0.0008769648945114265,
+      "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 3.3382604867219925,
-      "reward_std": 3.5747185684740543,
-      "rewards/concensus_correctness_reward_func": 1.5023749880492687,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 2.6730779806772866,
+      "reward_std": 2.167528917392095,
+      "rewards/concensus_correctness_reward_func": 0.2864166634778182,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.5651666708290577,
+      "rewards/final_correctness_reward_func": 0.9166666666666666,
+      "rewards/question_recreation_reward_func": 0.5900363326072693,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.23946874495595694,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.3174583340684573,
       "step": 8
     },
     {
-      "completion_length": 323.8125,
-      "epoch": 1.7619047619047619,
-      "grad_norm": 0.42236611247062683,
-      "kl": 0.0007506760448450223,
-      "learning_rate": 4.118465711954569e-07,
+      "completion_length": 295.125,
+      "epoch": 2.571428571428571,
+      "grad_norm": 0.645525336265564,
+      "kl": 0.0008544060474378057,
+      "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 3.1966800540685654,
-      "reward_std": 2.871971268206835,
-      "rewards/concensus_correctness_reward_func": 1.0033124908804893,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 3.4242160469293594,
+      "reward_std": 3.0434161946177483,
+      "rewards/concensus_correctness_reward_func": 1.416874997317791,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0625,
-      "rewards/question_recreation_reward_func": 0.64896135404706,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.5722471997141838,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16940625081770122,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.37259376933798194,
       "step": 10
     },
     {
-      "completion_length": 296.65,
-      "epoch": 2.0,
-      "grad_norm": 0.29138192534446716,
-      "kl": 0.0008530282881110907,
-      "learning_rate": 3.6710211017494754e-07,
+      "completion_length": 285.7916666666667,
+      "epoch": 3.0,
+      "grad_norm": 0.3467922806739807,
+      "kl": 0.0007908445356103281,
+      "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.7486515998840333,
-      "reward_std": 1.3477203607559205,
-      "rewards/concensus_correctness_reward_func": 0.03899999856948853,
-      "rewards/consensus_reward_func": 0.2,
+      "reward": 2.24852288266023,
+      "reward_std": 1.6140415569146473,
+      "rewards/concensus_correctness_reward_func": 0.18200000002980232,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8,
-      "rewards/question_recreation_reward_func": 0.36615161299705506,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.4489395481844743,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.34350000619888305,
+      "rewards/xmlcount_reward_func": 0.20091667026281357,
       "step": 12
     },
     {
-      "completion_length": 234.8125,
-      "epoch": 2.380952380952381,
-      "grad_norm": 0.6359131932258606,
-      "kl": 0.0009479617510805838,
-      "learning_rate": 3.168820846323053e-07,
+      "completion_length": 299.0,
+      "epoch": 3.571428571428571,
+      "grad_norm": 0.5615944266319275,
+      "kl": 0.0007969704092829488,
+      "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 3.1933727264404297,
-      "reward_std": 2.891321759670973,
-      "rewards/concensus_correctness_reward_func": 0.9353749975562096,
+      "reward": 2.481991246342659,
+      "reward_std": 1.5491515547037125,
+      "rewards/concensus_correctness_reward_func": 0.2594375014305115,
       "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.45012272521853447,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.5661787614226341,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.4641250092536211,
+      "rewards/xmlcount_reward_func": 0.5626250058412552,
       "step": 14
     },
     {
-      "completion_length": 237.53125,
-      "epoch": 2.761904761904762,
-      "grad_norm": 0.7391633987426758,
-      "kl": 0.0008452322508674115,
-      "learning_rate": 2.635347271463544e-07,
+      "completion_length": 304.0,
+      "epoch": 4.0,
+      "grad_norm": 0.40277090668678284,
+      "kl": 0.000872418963505576,
+      "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 4.267088107764721,
-      "reward_std": 4.962071567773819,
-      "rewards/concensus_correctness_reward_func": 2.008500002324581,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.444976548353831,
+      "reward_std": 1.702726533015569,
+      "rewards/concensus_correctness_reward_func": 0.34324999898672104,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0625,
-      "rewards/question_recreation_reward_func": 0.4506507311016321,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.39860157140841085,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.4173124972730875,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.18229166294137636,
       "step": 16
     },
     {
-      "completion_length": 275.1,
-      "epoch": 3.0,
-      "grad_norm": 0.14013737440109253,
-      "kl": 0.0009094005450606346,
-      "learning_rate": 2.0955450086180881e-07,
+      "completion_length": 308.4375,
+      "epoch": 4.571428571428571,
+      "grad_norm": 0.7489370107650757,
+      "kl": 0.0008767009203438647,
+      "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 7.2557789325714115,
-      "reward_std": 7.94908595085144,
-      "rewards/concensus_correctness_reward_func": 4.360799989104271,
-      "rewards/consensus_reward_func": 0.7,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3,
-      "rewards/question_recreation_reward_func": 0.5974290490150451,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.29755001068115233,
-      "step": 18
-    },
-    {
-      "completion_length": 304.375,
-      "epoch": 3.380952380952381,
-      "grad_norm": 0.5780467987060547,
-      "kl": 0.0008160869110724889,
-      "learning_rate": 1.5746546116502139e-07,
-      "loss": 0.0,
-      "reward": 5.3449102863669395,
-      "reward_std": 4.34605273604393,
-      "rewards/concensus_correctness_reward_func": 2.842437505722046,
-      "rewards/consensus_reward_func": 0.375,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.5805664826184511,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2969062477350235,
-      "step": 20
-    },
-    {
-      "completion_length": 256.875,
-      "epoch": 3.761904761904762,
-      "grad_norm": 0.6656950116157532,
-      "kl": 0.0009350914479000494,
-      "learning_rate": 1.0970323365940443e-07,
-      "loss": 0.0,
-      "reward": 2.6813122555613518,
-      "reward_std": 2.048238165676594,
-      "rewards/concensus_correctness_reward_func": 0.5099374949932098,
+      "reward": 3.167308270931244,
+      "reward_std": 2.8198581263422966,
+      "rewards/concensus_correctness_reward_func": 1.25,
       "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.6501871980726719,
+      "rewards/question_recreation_reward_func": 0.5250894874334335,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3805624935775995,
-      "step": 22
-    },
-    {
-      "completion_length": 278.4,
-      "epoch": 4.0,
-      "grad_norm": 0.2863069772720337,
-      "kl": 0.0008415899705141783,
-      "learning_rate": 6.850112701921735e-08,
-      "loss": 0.0,
-      "reward": 2.3153826713562013,
-      "reward_std": 2.1163893222808836,
-      "rewards/concensus_correctness_reward_func": 0.3267999976873398,
-      "rewards/consensus_reward_func": 0.4,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.7,
-      "rewards/question_recreation_reward_func": 0.6259327292442322,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.025,
-      "rewards/xmlcount_reward_func": 0.23765000849962234,
-      "step": 24
-    },
-    {
-      "completion_length": 228.1875,
-      "epoch": 4.380952380952381,
-      "grad_norm": 0.583082377910614,
-      "kl": 0.0008953025680966675,
-      "learning_rate": 3.578570595810274e-08,
-      "loss": 0.0,
-      "reward": 3.213442385196686,
-      "reward_std": 1.8543748632073402,
-      "rewards/concensus_correctness_reward_func": 0.5673749968409538,
-      "rewards/consensus_reward_func": 0.25,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.5275673819705844,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30600000359117985,
-      "step": 26
-    },
-    {
-      "completion_length": 263.78125,
-      "epoch": 4.761904761904762,
-      "grad_norm": 0.9703101515769958,
-      "kl": 0.0008642729299026541,
-      "learning_rate": 1.3086707204299413e-08,
-      "loss": 0.0,
-      "reward": 2.9460783302783966,
-      "reward_std": 3.104323089122772,
-      "rewards/concensus_correctness_reward_func": 0.932937502861023,
-      "rewards/consensus_reward_func": 0.4375,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.40417213924229145,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.468343760818243,
-      "step": 28
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.2359687564894557,
+      "step": 18
     },
     {
-      "completion_length": 266.5,
+      "completion_length": 320.875,
       "epoch": 5.0,
-      "grad_norm": 0.2689303755760193,
-      "kl": 0.000776640942785889,
-      "learning_rate": 1.4655107114101007e-09,
+      "grad_norm": 0.37519264221191406,
+      "kl": 0.000752140263405939,
+      "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 2.5046738386154175,
-      "reward_std": 2.042145240306854,
-      "rewards/concensus_correctness_reward_func": 0.4296000003814697,
-      "rewards/consensus_reward_func": 0.6,
+      "reward": 1.4269973983367283,
+      "reward_std": 1.24178742369016,
+      "rewards/concensus_correctness_reward_func": 0.10074999804298083,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6,
-      "rewards/question_recreation_reward_func": 0.5618738174438477,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.511705718934536,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.025,
-      "rewards/xmlcount_reward_func": 0.2882000029087067,
-      "step": 30
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.48120833685000736,
+      "step": 20
     },
     {
       "epoch": 5.0,
-      "step": 30,
+      "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.639158449516495e-07,
-      "train_runtime": 1717.2834,
-      "train_samples_per_second": 0.28,
-      "train_steps_per_second": 0.017
+      "train_loss": 7.550418615664967e-07,
+      "train_runtime": 594.0398,
+      "train_samples_per_second": 0.539,
+      "train_steps_per_second": 0.034
     }
   ],
   "logging_steps": 2,
-  "max_steps": 30,
+  "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 6,
+  "num_train_epochs": 7,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7274ec51f6f7deced9ec5c781dff7c45081be2110b6ef4d077cb8afd1286e62e
+oid sha256:4b574dc18fd82e1cd241fd23368fcabd617873fe1081e9e5efe22c29eda7e943
 size 6008