Model save

adapter_config.json

@@ -20,10 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
     "q_proj",
+    "k_proj",
     "o_proj",
-    "k_proj"
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c6ab52bff14f59b499586ef6e8faf038c25da4ed1b4d2bd15f5c2c64707cd64
-size 1085473720
+oid sha256:b7e1351a67db68dd417cbfbeca9e67f9a39f8701d1f2c67035233497cc66b6e0
+size 1148400880

trainer_state.json

@@ -10,68 +10,68 @@
   "log_history": [
     {
       "epoch": 0.4854368932038835,
-      "grad_norm": 0.43196743726730347,
+      "grad_norm": 0.4750896096229553,
       "learning_rate": 8.794946550048592e-05,
-      "loss": 1.0964,
+      "loss": 0.854,
       "step": 250
     },
     {
       "epoch": 0.970873786407767,
-      "grad_norm": 0.544486403465271,
+      "grad_norm": 0.3962755501270294,
       "learning_rate": 7.580174927113704e-05,
-      "loss": 0.6417,
+      "loss": 0.4107,
       "step": 500
     },
     {
       "epoch": 1.4563106796116505,
-      "grad_norm": 0.5797905921936035,
+      "grad_norm": 0.4067479968070984,
       "learning_rate": 6.365403304178815e-05,
-      "loss": 0.5176,
+      "loss": 0.3693,
       "step": 750
     },
     {
       "epoch": 1.941747572815534,
-      "grad_norm": 0.5782710313796997,
+      "grad_norm": 0.38044601678848267,
       "learning_rate": 5.150631681243926e-05,
-      "loss": 0.4687,
+      "loss": 0.3561,
       "step": 1000
     },
     {
       "epoch": 2.4271844660194173,
-      "grad_norm": 0.5743305087089539,
+      "grad_norm": 0.3728051483631134,
       "learning_rate": 3.9358600583090386e-05,
-      "loss": 0.4397,
+      "loss": 0.3473,
       "step": 1250
     },
     {
       "epoch": 2.912621359223301,
-      "grad_norm": 0.5558674931526184,
+      "grad_norm": 0.40094345808029175,
       "learning_rate": 2.72108843537415e-05,
-      "loss": 0.4245,
+      "loss": 0.3414,
       "step": 1500
     },
     {
       "epoch": 3.3980582524271843,
-      "grad_norm": 0.49328354001045227,
+      "grad_norm": 0.3950284719467163,
       "learning_rate": 1.5063168124392615e-05,
-      "loss": 0.4121,
+      "loss": 0.3321,
       "step": 1750
     },
     {
       "epoch": 3.883495145631068,
-      "grad_norm": 0.4594404697418213,
+      "grad_norm": 0.3837919235229492,
       "learning_rate": 2.915451895043732e-06,
-      "loss": 0.4077,
+      "loss": 0.3271,
       "step": 2000
     },
     {
       "epoch": 4.0,
       "step": 2060,
-      "total_flos": 5.002863068338053e+17,
-      "train_loss": 0.5468143407580922,
-      "train_runtime": 6426.3106,
-      "train_samples_per_second": 82.061,
-      "train_steps_per_second": 0.321
+      "total_flos": 5.083160308163666e+17,
+      "train_loss": 0.4145523006476245,
+      "train_runtime": 7540.7405,
+      "train_samples_per_second": 69.933,
+      "train_steps_per_second": 0.273
     }
   ],
   "logging_steps": 250,
@@ -91,7 +91,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 5.002863068338053e+17,
+  "total_flos": 5.083160308163666e+17,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c76eb7885bf87316287c24725e8ba0d384615fdd1b26e46f4e8e7f140e9689ad
+oid sha256:37060c46874a7281744ce6e4a73c75ab26f52036c0376a488d3598e1446fba39
 size 5240