Fine-tune nsi319/legal-led-base-16384 on FiscalNote/billsum subset

README.md

@@ -16,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [nsi319/legal-led-base-16384](https://huggingface.co/nsi319/legal-led-base-16384) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 6.6293
+- Loss: 0.8303
 
 ## Model description
 
@@ -49,10 +49,10 @@ The following hyperparameters were used during training:
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss |
-|:-------------:|:-----:|:----:|:---------------:|
-| No log        | 1.0   | 1    | 7.4076          |
-| No log        | 2.0   | 2    | 6.6293          |
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.4957        | 1.0    | 297  | 0.8795          |
+| 1.0639        | 1.9963 | 592  | 0.8303          |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.0,
-    "total_flos": 371962445211648.0,
-    "train_loss": 5.818258762359619,
-    "train_runtime": 153.9001,
-    "train_samples_per_second": 0.832,
-    "train_steps_per_second": 0.013
+    "epoch": 1.9963060686015832,
+    "total_flos": 1.0987423797624422e+17,
+    "train_loss": 1.5217120325243152,
+    "train_runtime": 38602.7875,
+    "train_samples_per_second": 0.982,
+    "train_steps_per_second": 0.015
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84d2882b56e85593d829ce7086a07cea8af76d46d1052d16ae2fec3ce9110f5a
+oid sha256:06850b8387bd2729725fe7c42e6f792b8165afbc27ec3fc855fb9278c3434696
 size 647614116

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.0,
-    "total_flos": 371962445211648.0,
-    "train_loss": 5.818258762359619,
-    "train_runtime": 153.9001,
-    "train_samples_per_second": 0.832,
-    "train_steps_per_second": 0.013
+    "epoch": 1.9963060686015832,
+    "total_flos": 1.0987423797624422e+17,
+    "train_loss": 1.5217120325243152,
+    "train_runtime": 38602.7875,
+    "train_samples_per_second": 0.982,
+    "train_steps_per_second": 0.015
 }

trainer_state.json

@@ -1,42 +1,77 @@
 {
-  "best_global_step": 2,
-  "best_metric": 6.629301071166992,
-  "best_model_checkpoint": "./results_billsum/checkpoint-2",
-  "epoch": 2.0,
+  "best_global_step": 592,
+  "best_metric": 0.830290675163269,
+  "best_model_checkpoint": "./results_billsum/checkpoint-592",
+  "epoch": 1.9963060686015832,
   "eval_steps": 500,
-  "global_step": 2,
+  "global_step": 592,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
+    {
+      "epoch": 0.33773087071240104,
+      "grad_norm": 838200.5,
+      "learning_rate": 1.7544483985765128e-05,
+      "loss": 3.203,
+      "step": 100
+    },
+    {
+      "epoch": 0.6754617414248021,
+      "grad_norm": 158957.71875,
+      "learning_rate": 1.3985765124555162e-05,
+      "loss": 1.4957,
+      "step": 200
+    },
     {
       "epoch": 1.0,
-      "eval_loss": 7.40756368637085,
-      "eval_runtime": 12.5285,
-      "eval_samples_per_second": 2.554,
-      "eval_steps_per_second": 0.639,
-      "step": 1
+      "eval_loss": 0.8794804811477661,
+      "eval_runtime": 1277.7764,
+      "eval_samples_per_second": 2.558,
+      "eval_steps_per_second": 0.64,
+      "step": 297
+    },
+    {
+      "epoch": 1.010131926121372,
+      "grad_norm": 73460.7890625,
+      "learning_rate": 1.0427046263345197e-05,
+      "loss": 1.1737,
+      "step": 300
+    },
+    {
+      "epoch": 1.347862796833773,
+      "grad_norm": 66100.03125,
+      "learning_rate": 6.868327402135232e-06,
+      "loss": 1.0938,
+      "step": 400
     },
     {
-      "epoch": 2.0,
-      "eval_loss": 6.629301071166992,
-      "eval_runtime": 12.9385,
-      "eval_samples_per_second": 2.473,
-      "eval_steps_per_second": 0.618,
-      "step": 2
+      "epoch": 1.6855936675461742,
+      "grad_norm": 63527.66796875,
+      "learning_rate": 3.309608540925267e-06,
+      "loss": 1.0639,
+      "step": 500
+    },
+    {
+      "epoch": 1.9963060686015832,
+      "eval_loss": 0.830290675163269,
+      "eval_runtime": 1279.4932,
+      "eval_samples_per_second": 2.555,
+      "eval_steps_per_second": 0.639,
+      "step": 592
     },
     {
-      "epoch": 2.0,
-      "step": 2,
-      "total_flos": 371962445211648.0,
-      "train_loss": 5.818258762359619,
-      "train_runtime": 153.9001,
-      "train_samples_per_second": 0.832,
-      "train_steps_per_second": 0.013
+      "epoch": 1.9963060686015832,
+      "step": 592,
+      "total_flos": 1.0987423797624422e+17,
+      "train_loss": 1.5217120325243152,
+      "train_runtime": 38602.7875,
+      "train_samples_per_second": 0.982,
+      "train_steps_per_second": 0.015
     }
   ],
   "logging_steps": 100,
-  "max_steps": 2,
+  "max_steps": 592,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 500,
@@ -61,7 +96,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 371962445211648.0,
+  "total_flos": 1.0987423797624422e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null