Training in progress, step 50, checkpoint

last-checkpoint/adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "k_proj",
     "gate_proj",
-    "v_proj",
+    "q_proj",
     "o_proj",
+    "k_proj",
     "down_proj",
-    "q_proj"
+    "v_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d5a1b97e55ae79b41655d684852d0e162ccb63a232710e9e79c091bce0352a93
+oid sha256:e07433ce7979fcdc05e88071e39fc5597ad1cef8904bec0ffb98e12fd587669f
 size 859942080

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6df64e2aacaf6c467309406fb7b12f86de671f49adeeb980c0dd831b77a2615c
+oid sha256:ad865a4f06f01f8a7c70a731954c3a338fa017215435f6b8285f9f6f4997ec3a
 size 671466706

last-checkpoint/trainer_state.json

@@ -1,5 +1,5 @@
 {
-  "best_metric": 2.4684722423553467,
+  "best_metric": 2.4652342796325684,
   "best_model_checkpoint": "miner_id_24/checkpoint-50",
   "epoch": 0.62402496099844,
   "eval_steps": 25,
@@ -10,376 +10,376 @@
   "log_history": [
     {
       "epoch": 0.0124804992199688,
-      "grad_norm": 241.28233337402344,
+      "grad_norm": 48.33766555786133,
       "learning_rate": 5e-05,
-      "loss": 47.5625,
+      "loss": 41.9453,
       "step": 1
     },
     {
       "epoch": 0.0124804992199688,
-      "eval_loss": 3.496701240539551,
-      "eval_runtime": 16.2306,
-      "eval_samples_per_second": 33.27,
-      "eval_steps_per_second": 4.19,
+      "eval_loss": 3.2026607990264893,
+      "eval_runtime": 16.3037,
+      "eval_samples_per_second": 33.121,
+      "eval_steps_per_second": 4.171,
       "step": 1
     },
     {
       "epoch": 0.0249609984399376,
-      "grad_norm": 225.81707763671875,
+      "grad_norm": 63.55783462524414,
       "learning_rate": 0.0001,
-      "loss": 48.6521,
+      "loss": 43.5994,
       "step": 2
     },
     {
       "epoch": 0.0374414976599064,
-      "grad_norm": 54.283016204833984,
+      "grad_norm": 33.458194732666016,
       "learning_rate": 9.989294616193017e-05,
-      "loss": 45.3746,
+      "loss": 44.1308,
       "step": 3
     },
     {
       "epoch": 0.0499219968798752,
-      "grad_norm": 30.326000213623047,
+      "grad_norm": 28.39899253845215,
       "learning_rate": 9.957224306869053e-05,
-      "loss": 42.3161,
+      "loss": 41.3101,
       "step": 4
     },
     {
       "epoch": 0.062402496099843996,
-      "grad_norm": 30.251224517822266,
+      "grad_norm": 25.842370986938477,
       "learning_rate": 9.903926402016153e-05,
-      "loss": 43.854,
+      "loss": 43.278,
       "step": 5
     },
     {
       "epoch": 0.0748829953198128,
-      "grad_norm": 36.94489288330078,
+      "grad_norm": 27.863216400146484,
       "learning_rate": 9.829629131445342e-05,
-      "loss": 43.8111,
+      "loss": 43.1905,
       "step": 6
     },
     {
       "epoch": 0.0873634945397816,
-      "grad_norm": 23.006725311279297,
+      "grad_norm": 20.345382690429688,
       "learning_rate": 9.73465064747553e-05,
-      "loss": 42.6537,
+      "loss": 42.1296,
       "step": 7
     },
     {
       "epoch": 0.0998439937597504,
-      "grad_norm": 28.94269371032715,
+      "grad_norm": 28.974214553833008,
       "learning_rate": 9.619397662556435e-05,
-      "loss": 43.799,
+      "loss": 43.5732,
       "step": 8
     },
     {
       "epoch": 0.11232449297971919,
-      "grad_norm": 26.714143753051758,
+      "grad_norm": 26.12350845336914,
       "learning_rate": 9.484363707663442e-05,
-      "loss": 43.9843,
+      "loss": 43.796,
       "step": 9
     },
     {
       "epoch": 0.12480499219968799,
-      "grad_norm": 26.85364532470703,
+      "grad_norm": 26.77032470703125,
       "learning_rate": 9.330127018922194e-05,
-      "loss": 44.0964,
+      "loss": 43.8661,
       "step": 10
     },
     {
       "epoch": 0.1372854914196568,
-      "grad_norm": 35.250606536865234,
+      "grad_norm": 28.507535934448242,
       "learning_rate": 9.157348061512727e-05,
-      "loss": 43.5598,
+      "loss": 43.4069,
       "step": 11
     },
     {
       "epoch": 0.1497659906396256,
-      "grad_norm": 30.30172348022461,
+      "grad_norm": 31.786388397216797,
       "learning_rate": 8.966766701456177e-05,
-      "loss": 43.4464,
+      "loss": 43.3076,
       "step": 12
     },
     {
       "epoch": 0.1622464898595944,
-      "grad_norm": 33.46178436279297,
+      "grad_norm": 27.120195388793945,
       "learning_rate": 8.759199037394887e-05,
-      "loss": 41.5899,
+      "loss": 41.2234,
       "step": 13
     },
     {
       "epoch": 0.1747269890795632,
-      "grad_norm": 27.445087432861328,
+      "grad_norm": 21.42611312866211,
       "learning_rate": 8.535533905932738e-05,
-      "loss": 40.1862,
+      "loss": 39.9329,
       "step": 14
     },
     {
       "epoch": 0.187207488299532,
-      "grad_norm": 18.624109268188477,
+      "grad_norm": 16.399066925048828,
       "learning_rate": 8.296729075500344e-05,
-      "loss": 39.7267,
+      "loss": 39.5397,
       "step": 15
     },
     {
       "epoch": 0.1996879875195008,
-      "grad_norm": 17.867002487182617,
+      "grad_norm": 16.056814193725586,
       "learning_rate": 8.043807145043604e-05,
-      "loss": 39.4295,
+      "loss": 39.251,
       "step": 16
     },
     {
       "epoch": 0.2121684867394696,
-      "grad_norm": 20.40807342529297,
+      "grad_norm": 18.577831268310547,
       "learning_rate": 7.777851165098012e-05,
-      "loss": 42.3176,
+      "loss": 42.0921,
       "step": 17
     },
     {
       "epoch": 0.22464898595943839,
-      "grad_norm": 18.3370418548584,
+      "grad_norm": 16.399547576904297,
       "learning_rate": 7.500000000000001e-05,
-      "loss": 40.4421,
+      "loss": 40.1743,
       "step": 18
     },
     {
       "epoch": 0.23712948517940718,
-      "grad_norm": 19.78119659423828,
+      "grad_norm": 18.57469367980957,
       "learning_rate": 7.211443451095007e-05,
-      "loss": 40.9037,
+      "loss": 40.791,
       "step": 19
     },
     {
       "epoch": 0.24960998439937598,
-      "grad_norm": 19.116260528564453,
+      "grad_norm": 18.809473037719727,
       "learning_rate": 6.91341716182545e-05,
-      "loss": 40.9953,
+      "loss": 40.9133,
       "step": 20
     },
     {
       "epoch": 0.2620904836193448,
-      "grad_norm": 19.80707359313965,
+      "grad_norm": 19.56980323791504,
       "learning_rate": 6.607197326515808e-05,
-      "loss": 40.5555,
+      "loss": 40.4617,
       "step": 21
     },
     {
       "epoch": 0.2745709828393136,
-      "grad_norm": 25.69529151916504,
+      "grad_norm": 22.437828063964844,
       "learning_rate": 6.294095225512603e-05,
-      "loss": 40.7941,
+      "loss": 40.6184,
       "step": 22
     },
     {
       "epoch": 0.2870514820592824,
-      "grad_norm": 26.47637367248535,
+      "grad_norm": 24.266427993774414,
       "learning_rate": 5.9754516100806423e-05,
-      "loss": 41.6646,
+      "loss": 41.6315,
       "step": 23
     },
     {
       "epoch": 0.2995319812792512,
-      "grad_norm": 27.254323959350586,
+      "grad_norm": 26.298166275024414,
       "learning_rate": 5.6526309611002594e-05,
-      "loss": 43.7913,
+      "loss": 43.6817,
       "step": 24
     },
     {
       "epoch": 0.31201248049922,
-      "grad_norm": 37.66843795776367,
+      "grad_norm": 34.5001335144043,
       "learning_rate": 5.327015646150716e-05,
-      "loss": 46.5396,
+      "loss": 46.5044,
       "step": 25
     },
     {
       "epoch": 0.31201248049922,
-      "eval_loss": 2.5203487873077393,
-      "eval_runtime": 16.2045,
-      "eval_samples_per_second": 33.324,
-      "eval_steps_per_second": 4.196,
+      "eval_loss": 2.5227160453796387,
+      "eval_runtime": 16.2938,
+      "eval_samples_per_second": 33.142,
+      "eval_steps_per_second": 4.173,
       "step": 25
     },
     {
       "epoch": 0.3244929797191888,
-      "grad_norm": 13.104182243347168,
+      "grad_norm": 14.59637451171875,
       "learning_rate": 5e-05,
-      "loss": 38.2475,
+      "loss": 38.2747,
       "step": 26
     },
     {
       "epoch": 0.3369734789391576,
-      "grad_norm": 15.340250015258789,
+      "grad_norm": 15.654855728149414,
       "learning_rate": 4.6729843538492847e-05,
-      "loss": 37.9144,
+      "loss": 37.8957,
       "step": 27
     },
     {
       "epoch": 0.3494539781591264,
-      "grad_norm": 13.768589973449707,
+      "grad_norm": 13.646095275878906,
       "learning_rate": 4.347369038899744e-05,
-      "loss": 38.5212,
+      "loss": 38.4908,
       "step": 28
     },
     {
       "epoch": 0.3619344773790952,
-      "grad_norm": 14.8656005859375,
+      "grad_norm": 14.295589447021484,
       "learning_rate": 4.0245483899193595e-05,
-      "loss": 40.1984,
+      "loss": 40.0716,
       "step": 29
     },
     {
       "epoch": 0.374414976599064,
-      "grad_norm": 17.970172882080078,
+      "grad_norm": 16.36422348022461,
       "learning_rate": 3.705904774487396e-05,
-      "loss": 39.7075,
+      "loss": 39.6821,
       "step": 30
     },
     {
       "epoch": 0.3868954758190328,
-      "grad_norm": 16.23076629638672,
+      "grad_norm": 15.698250770568848,
       "learning_rate": 3.392802673484193e-05,
-      "loss": 39.7191,
+      "loss": 39.6605,
       "step": 31
     },
     {
       "epoch": 0.3993759750390016,
-      "grad_norm": 17.51127052307129,
+      "grad_norm": 17.636642456054688,
       "learning_rate": 3.086582838174551e-05,
-      "loss": 39.8791,
+      "loss": 39.8463,
       "step": 32
     },
     {
       "epoch": 0.4118564742589704,
-      "grad_norm": 19.608243942260742,
+      "grad_norm": 19.906291961669922,
       "learning_rate": 2.7885565489049946e-05,
-      "loss": 40.0612,
+      "loss": 40.0594,
       "step": 33
     },
     {
       "epoch": 0.4243369734789392,
-      "grad_norm": 19.153104782104492,
+      "grad_norm": 19.436504364013672,
       "learning_rate": 2.500000000000001e-05,
-      "loss": 41.4167,
+      "loss": 41.4247,
       "step": 34
     },
     {
       "epoch": 0.43681747269890797,
-      "grad_norm": 20.9382266998291,
+      "grad_norm": 21.34272003173828,
       "learning_rate": 2.2221488349019903e-05,
-      "loss": 41.2812,
+      "loss": 41.3794,
       "step": 35
     },
     {
       "epoch": 0.44929797191887677,
-      "grad_norm": 24.44215965270996,
+      "grad_norm": 25.100749969482422,
       "learning_rate": 1.9561928549563968e-05,
-      "loss": 41.4316,
+      "loss": 41.5146,
       "step": 36
     },
     {
       "epoch": 0.46177847113884557,
-      "grad_norm": 28.488000869750977,
+      "grad_norm": 30.39947509765625,
       "learning_rate": 1.703270924499656e-05,
-      "loss": 41.4623,
+      "loss": 41.5509,
       "step": 37
     },
     {
       "epoch": 0.47425897035881437,
-      "grad_norm": 25.207191467285156,
+      "grad_norm": 26.53461456298828,
       "learning_rate": 1.4644660940672627e-05,
-      "loss": 40.3727,
+      "loss": 40.5865,
       "step": 38
     },
     {
       "epoch": 0.48673946957878317,
-      "grad_norm": 12.194273948669434,
+      "grad_norm": 10.599092483520508,
       "learning_rate": 1.2408009626051137e-05,
-      "loss": 37.1192,
+      "loss": 37.0101,
       "step": 39
     },
     {
       "epoch": 0.49921996879875197,
-      "grad_norm": 14.081475257873535,
+      "grad_norm": 12.43578052520752,
       "learning_rate": 1.0332332985438248e-05,
-      "loss": 37.3612,
+      "loss": 37.2016,
       "step": 40
     },
     {
       "epoch": 0.5117004680187207,
-      "grad_norm": 16.696203231811523,
+      "grad_norm": 15.518513679504395,
       "learning_rate": 8.426519384872733e-06,
-      "loss": 38.8394,
+      "loss": 38.7602,
       "step": 41
     },
     {
       "epoch": 0.5241809672386896,
-      "grad_norm": 16.718196868896484,
+      "grad_norm": 15.59644603729248,
       "learning_rate": 6.698729810778065e-06,
-      "loss": 39.069,
+      "loss": 38.9373,
       "step": 42
     },
     {
       "epoch": 0.5366614664586583,
-      "grad_norm": 16.142709732055664,
+      "grad_norm": 15.444360733032227,
       "learning_rate": 5.156362923365588e-06,
-      "loss": 38.2544,
+      "loss": 38.1155,
       "step": 43
     },
     {
       "epoch": 0.5491419656786272,
-      "grad_norm": 16.090139389038086,
+      "grad_norm": 15.52137279510498,
       "learning_rate": 3.8060233744356633e-06,
-      "loss": 39.326,
+      "loss": 39.2679,
       "step": 44
     },
     {
       "epoch": 0.5616224648985959,
-      "grad_norm": 16.413667678833008,
+      "grad_norm": 16.007932662963867,
       "learning_rate": 2.653493525244721e-06,
-      "loss": 38.9714,
+      "loss": 38.9434,
       "step": 45
     },
     {
       "epoch": 0.5741029641185648,
-      "grad_norm": 20.610652923583984,
+      "grad_norm": 19.981130599975586,
       "learning_rate": 1.70370868554659e-06,
-      "loss": 41.5153,
+      "loss": 41.4802,
       "step": 46
     },
     {
       "epoch": 0.5865834633385335,
-      "grad_norm": 18.844303131103516,
+      "grad_norm": 19.083805084228516,
       "learning_rate": 9.607359798384785e-07,
-      "loss": 40.6101,
+      "loss": 40.5354,
       "step": 47
     },
     {
       "epoch": 0.5990639625585024,
-      "grad_norm": 22.228759765625,
+      "grad_norm": 21.28178596496582,
       "learning_rate": 4.277569313094809e-07,
-      "loss": 42.249,
+      "loss": 42.1493,
       "step": 48
     },
     {
       "epoch": 0.6115444617784711,
-      "grad_norm": 25.627452850341797,
+      "grad_norm": 25.080459594726562,
       "learning_rate": 1.0705383806982606e-07,
-      "loss": 43.0037,
+      "loss": 42.958,
       "step": 49
     },
     {
       "epoch": 0.62402496099844,
-      "grad_norm": 38.99866485595703,
+      "grad_norm": 37.190128326416016,
       "learning_rate": 0.0,
-      "loss": 45.5983,
+      "loss": 45.5318,
       "step": 50
     },
     {
       "epoch": 0.62402496099844,
-      "eval_loss": 2.4684722423553467,
-      "eval_runtime": 16.2579,
-      "eval_samples_per_second": 33.215,
-      "eval_steps_per_second": 4.183,
+      "eval_loss": 2.4652342796325684,
+      "eval_runtime": 16.3474,
+      "eval_samples_per_second": 33.033,
+      "eval_steps_per_second": 4.16,
       "step": 50
     }
   ],

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:273a226a1af468f917cbd7fed30a2f1736dd864b4080c3f6d58049a483765b92
+oid sha256:3fa7e702f3900d7dba4ca57c72cfa41df314e30949a396810e2b8e1a82ccf370
 size 6776