Fine-tune nsi319/legal-led-base-16384 on FiscalNote/billsum subset

README.md

@@ -16,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [nsi319/legal-led-base-16384](https://huggingface.co/nsi319/legal-led-base-16384) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 6.4175
+- Loss: 6.6293
 
 ## Model description
 
@@ -44,16 +44,15 @@ The following hyperparameters were used during training:
 - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_ratio: 0.05
-- num_epochs: 3
+- num_epochs: 2
 - mixed_precision_training: Native AMP
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| No log        | 1.0   | 1    | 7.4017          |
-| No log        | 2.0   | 2    | 6.6088          |
-| No log        | 3.0   | 3    | 6.4175          |
+| No log        | 1.0   | 1    | 7.4076          |
+| No log        | 2.0   | 2    | 6.6293          |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 3.0,
-    "total_flos": 579817568360448.0,
-    "train_loss": 5.7508284250895185,
-    "train_runtime": 238.1241,
-    "train_samples_per_second": 0.806,
+    "epoch": 2.0,
+    "total_flos": 371962445211648.0,
+    "train_loss": 5.818258762359619,
+    "train_runtime": 153.9001,
+    "train_samples_per_second": 0.832,
     "train_steps_per_second": 0.013
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c7639c0059e8fbcc5f4dacac685cbe09f8c5bf529888d9d8bbdfa85709f9a948
+oid sha256:84d2882b56e85593d829ce7086a07cea8af76d46d1052d16ae2fec3ce9110f5a
 size 647614116

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 512,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 3.0,
-    "total_flos": 579817568360448.0,
-    "train_loss": 5.7508284250895185,
-    "train_runtime": 238.1241,
-    "train_samples_per_second": 0.806,
+    "epoch": 2.0,
+    "total_flos": 371962445211648.0,
+    "train_loss": 5.818258762359619,
+    "train_runtime": 153.9001,
+    "train_samples_per_second": 0.832,
     "train_steps_per_second": 0.013
 }

trainer_state.json

@@ -1,52 +1,44 @@
 {
-  "best_global_step": 3,
-  "best_metric": 6.417472839355469,
-  "best_model_checkpoint": "./results_billsum/checkpoint-3",
-  "epoch": 3.0,
+  "best_global_step": 2,
+  "best_metric": 6.629301071166992,
+  "best_model_checkpoint": "./results_billsum/checkpoint-2",
+  "epoch": 2.0,
   "eval_steps": 500,
-  "global_step": 3,
+  "global_step": 2,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 1.0,
-      "eval_loss": 7.4016804695129395,
-      "eval_runtime": 13.2975,
-      "eval_samples_per_second": 2.406,
-      "eval_steps_per_second": 0.602,
+      "eval_loss": 7.40756368637085,
+      "eval_runtime": 12.5285,
+      "eval_samples_per_second": 2.554,
+      "eval_steps_per_second": 0.639,
       "step": 1
     },
     {
       "epoch": 2.0,
-      "eval_loss": 6.608830451965332,
-      "eval_runtime": 13.15,
-      "eval_samples_per_second": 2.433,
-      "eval_steps_per_second": 0.608,
+      "eval_loss": 6.629301071166992,
+      "eval_runtime": 12.9385,
+      "eval_samples_per_second": 2.473,
+      "eval_steps_per_second": 0.618,
       "step": 2
     },
     {
-      "epoch": 3.0,
-      "eval_loss": 6.417472839355469,
-      "eval_runtime": 13.1507,
-      "eval_samples_per_second": 2.433,
-      "eval_steps_per_second": 0.608,
-      "step": 3
-    },
-    {
-      "epoch": 3.0,
-      "step": 3,
-      "total_flos": 579817568360448.0,
-      "train_loss": 5.7508284250895185,
-      "train_runtime": 238.1241,
-      "train_samples_per_second": 0.806,
+      "epoch": 2.0,
+      "step": 2,
+      "total_flos": 371962445211648.0,
+      "train_loss": 5.818258762359619,
+      "train_runtime": 153.9001,
+      "train_samples_per_second": 0.832,
       "train_steps_per_second": 0.013
     }
   ],
   "logging_steps": 100,
-  "max_steps": 3,
+  "max_steps": 2,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 2,
   "save_steps": 500,
   "stateful_callbacks": {
     "EarlyStoppingCallback": {
@@ -69,7 +61,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 579817568360448.0,
+  "total_flos": 371962445211648.0,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63aaa7c8720fbc7e6af2821aa660c1124ea6176e78437270954587f667dc7361
+oid sha256:21a31db65a62ac1ef8468a49a03cfea852c52cec8c76738eaf6570aa7e535472
 size 5496