Upload folder using huggingface_hub

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
     "v_proj",
-    "up_proj",
-    "k_proj",
     "gate_proj",
     "down_proj",
-    "o_proj"
+    "q_proj",
+    "o_proj",
+    "k_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d1f2f5cfc5826a2b2fef3a62b0c9435c2c082d599ec3d97712eb11bec1ab2af
+oid sha256:edfb040750ea22cc194598c6f5a6c2e7f554d203f21d0541477961960926719e
 size 35669232

image_projector.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67ace4e904ebf5e0524ca78eeb7d9899e1d5f6a4d84135469950b27c791e10a9
+oid sha256:2ebab6acad7c435a7f377ae18e8625635aba4122c811632caba17a9a0439fba0
 size 29379352

lora_weights.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05190f07b7d3f2c882001373a6156ca05d7ee7a5324beddc57959449720c41e6
+oid sha256:1a34b2ae4068c112b000ec9c23d72c44e2b0119e44037d1ad8dfc3f42f7e429f
 size 35697862

trainer_state.json

@@ -1,181 +1,141 @@
 {
-  "epoch": 0.16309412861137,
+  "epoch": 0.03262034577566522,
   "global_step": 350,
-  "max_steps": 2146,
-  "logging_steps": 25,
+  "max_steps": 3000,
+  "logging_steps": 50,
   "eval_steps": 50,
-  "save_steps": 25,
+  "save_steps": 50,
   "train_batch_size": 8,
   "num_train_epochs": 1,
   "num_input_tokens_seen": 0,
-  "total_flos": 1.6047778948703846e+17,
+  "total_flos": 1.6004947053699072e+17,
   "log_history": [
     {
-      "loss": 3.9545,
-      "grad_norm": 20.241615295410156,
-      "learning_rate": 5.8139534883720933e-05,
-      "epoch": 0.011649580615097856,
-      "step": 25
-    },
-    {
-      "loss": 0.6469,
-      "grad_norm": 0.6640440821647644,
-      "learning_rate": 0.00011627906976744187,
-      "epoch": 0.023299161230195712,
+      "loss": 11.5092,
+      "grad_norm": 27.41529655456543,
+      "learning_rate": 4.666666666666667e-05,
+      "epoch": 0.004660049396523603,
       "step": 50
     },
     {
-      "eval_loss": 0.332077294588089,
-      "eval_runtime": 1616.9669,
-      "eval_samples_per_second": 0.558,
-      "eval_steps_per_second": 0.14,
-      "epoch": 0.023299161230195712,
+      "eval_loss": 6.5413079261779785,
+      "eval_runtime": 484.9209,
+      "eval_samples_per_second": 1.862,
+      "eval_steps_per_second": 0.466,
+      "epoch": 0.004660049396523603,
       "step": 50
     },
     {
-      "loss": 0.1711,
-      "grad_norm": 0.0800417810678482,
-      "learning_rate": 0.0001744186046511628,
-      "epoch": 0.03494874184529357,
-      "step": 75
-    },
-    {
-      "loss": 0.4039,
-      "grad_norm": 0.19737310707569122,
-      "learning_rate": 0.00023255813953488373,
-      "epoch": 0.046598322460391424,
+      "loss": 0.907,
+      "grad_norm": 0.3267304003238678,
+      "learning_rate": 9.999988344964554e-05,
+      "epoch": 0.009320098793047207,
       "step": 100
     },
     {
-      "eval_loss": 0.6975870728492737,
-      "eval_runtime": 1615.7283,
-      "eval_samples_per_second": 0.559,
-      "eval_steps_per_second": 0.14,
-      "epoch": 0.046598322460391424,
+      "eval_loss": 0.2963584065437317,
+      "eval_runtime": 482.3744,
+      "eval_samples_per_second": 1.872,
+      "eval_steps_per_second": 0.469,
+      "epoch": 0.009320098793047207,
       "step": 100
     },
     {
-      "loss": 0.2002,
-      "grad_norm": 0.0352032296359539,
-      "learning_rate": 0.00029069767441860465,
-      "epoch": 0.058247903075489285,
-      "step": 125
-    },
-    {
-      "loss": 0.3939,
-      "grad_norm": 0.17062194645404816,
-      "learning_rate": 0.0003488372093023256,
-      "epoch": 0.06989748369058714,
+      "loss": 0.2887,
+      "grad_norm": 0.23999030888080597,
+      "learning_rate": 9.992123261946325e-05,
+      "epoch": 0.01398014818957081,
       "step": 150
     },
     {
-      "eval_loss": 0.2825596034526825,
-      "eval_runtime": 1614.1197,
-      "eval_samples_per_second": 0.559,
-      "eval_steps_per_second": 0.14,
-      "epoch": 0.06989748369058714,
+      "eval_loss": 0.26408717036247253,
+      "eval_runtime": 483.0929,
+      "eval_samples_per_second": 1.869,
+      "eval_steps_per_second": 0.468,
+      "epoch": 0.01398014818957081,
       "step": 150
     },
     {
-      "loss": 0.1607,
-      "grad_norm": 0.08992636203765869,
-      "learning_rate": 0.00040697674418604653,
-      "epoch": 0.081547064305685,
-      "step": 175
-    },
-    {
-      "loss": 0.3829,
-      "grad_norm": 0.960233747959137,
-      "learning_rate": 0.00046511627906976747,
-      "epoch": 0.09319664492078285,
+      "loss": 0.273,
+      "grad_norm": 0.22597914934158325,
+      "learning_rate": 9.96971586146684e-05,
+      "epoch": 0.018640197586094413,
       "step": 200
     },
     {
-      "eval_loss": 0.2506251931190491,
-      "eval_runtime": 1617.641,
-      "eval_samples_per_second": 0.558,
-      "eval_steps_per_second": 0.14,
-      "epoch": 0.09319664492078285,
+      "eval_loss": 0.2530948221683502,
+      "eval_runtime": 483.695,
+      "eval_samples_per_second": 1.867,
+      "eval_steps_per_second": 0.467,
+      "epoch": 0.018640197586094413,
       "step": 200
     },
     {
-      "loss": 0.1605,
-      "grad_norm": 0.04875573888421059,
-      "learning_rate": 0.0004974106680476438,
-      "epoch": 0.1048462255358807,
-      "step": 225
-    },
-    {
-      "loss": 0.3913,
-      "grad_norm": 0.13655845820903778,
-      "learning_rate": 0.0004909373381667531,
-      "epoch": 0.11649580615097857,
+      "loss": 0.2655,
+      "grad_norm": 0.2108173966407776,
+      "learning_rate": 9.932831417461484e-05,
+      "epoch": 0.023300246982618015,
       "step": 250
     },
     {
-      "eval_loss": 0.43965786695480347,
-      "eval_runtime": 1617.1306,
-      "eval_samples_per_second": 0.558,
-      "eval_steps_per_second": 0.14,
-      "epoch": 0.11649580615097857,
+      "eval_loss": 0.2521112263202667,
+      "eval_runtime": 484.1571,
+      "eval_samples_per_second": 1.865,
+      "eval_steps_per_second": 0.467,
+      "epoch": 0.023300246982618015,
       "step": 250
     },
     {
-      "loss": 0.1632,
-      "grad_norm": 0.019430797547101974,
-      "learning_rate": 0.0004844640082858622,
-      "epoch": 0.12814538676607642,
-      "step": 275
-    },
-    {
-      "loss": 0.3724,
-      "grad_norm": 0.2898576855659485,
-      "learning_rate": 0.0004779906784049715,
-      "epoch": 0.13979496738117428,
+      "loss": 0.2635,
+      "grad_norm": 0.16866298019886017,
+      "learning_rate": 9.881577376254393e-05,
+      "epoch": 0.02796029637914162,
       "step": 300
     },
     {
-      "eval_loss": 0.9187588095664978,
-      "eval_runtime": 1617.9145,
-      "eval_samples_per_second": 0.558,
-      "eval_steps_per_second": 0.14,
-      "epoch": 0.13979496738117428,
+      "eval_loss": 0.24954503774642944,
+      "eval_runtime": 482.9845,
+      "eval_samples_per_second": 1.87,
+      "eval_steps_per_second": 0.468,
+      "epoch": 0.02796029637914162,
       "step": 300
     },
     {
-      "loss": 0.2034,
-      "grad_norm": 0.02103794552385807,
-      "learning_rate": 0.0004715173485240808,
-      "epoch": 0.15144454799627213,
-      "step": 325
-    },
-    {
-      "loss": 0.3585,
-      "grad_norm": 0.12554290890693665,
-      "learning_rate": 0.0004650440186431901,
-      "epoch": 0.16309412861137,
+      "loss": 0.2622,
+      "grad_norm": 0.19963641464710236,
+      "learning_rate": 9.816103043561648e-05,
+      "epoch": 0.03262034577566522,
       "step": 350
     },
     {
-      "eval_loss": 0.24577070772647858,
-      "eval_runtime": 1617.816,
-      "eval_samples_per_second": 0.558,
-      "eval_steps_per_second": 0.14,
-      "epoch": 0.16309412861137,
+      "eval_loss": 0.24790766835212708,
+      "eval_runtime": 482.7586,
+      "eval_samples_per_second": 1.87,
+      "eval_steps_per_second": 0.468,
+      "epoch": 0.03262034577566522,
       "step": 350
     }
   ],
-  "best_metric": null,
-  "best_model_checkpoint": null,
+  "best_metric": 0.24790766835212708,
+  "best_model_checkpoint": "./multimodal-phi3_5-mini-instruct-llava_adapter/checkpoint-350",
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "is_hyper_param_search": false,
   "trial_name": null,
   "trial_params": null,
   "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 3,
+        "early_stopping_threshold": 0.01
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 3
+      }
+    },
     "TrainerControl": {
       "args": {
-        "should_training_stop": false,
+        "should_training_stop": true,
         "should_epoch_stop": false,
         "should_save": true,
         "should_evaluate": false,