rank experiment

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/args.json

@@ -0,0 +1,35 @@
+{
+    "task_name": "cola",
+    "train_file": null,
+    "validation_file": null,
+    "max_length": 256,
+    "pad_to_max_length": false,
+    "model_name_or_path": "google-bert/bert-base-uncased",
+    "use_slow_tokenizer": false,
+    "per_device_train_batch_size": 32,
+    "per_device_eval_batch_size": 32,
+    "learning_rate": 5e-05,
+    "max_grad_norm": 0.5,
+    "weight_decay": 0.0,
+    "num_train_epochs": 5,
+    "max_train_steps": null,
+    "gradient_accumulation_steps": 1,
+    "lr_scheduler_type": "linear",
+    "num_warmup_steps": 0,
+    "output_dir": "./outputs/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345",
+    "seed": 12345,
+    "push_to_hub": false,
+    "hub_model_id": null,
+    "hub_token": null,
+    "checkpointing_steps": null,
+    "resume_from_checkpoint": null,
+    "with_tracking": false,
+    "report_to": "all",
+    "ignore_mismatched_sizes": true,
+    "save_train_results": false,
+    "lora_r": 16,
+    "lora_alpha": 16,
+    "lora_dropout": 0.1,
+    "testing_set": "train_val",
+    "lm_head": true
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/logfile.log

@@ -0,0 +1,137 @@
+04/30/2024 20:34:21 - INFO - __main__ -  Number of labels detected = 2
+04/30/2024 20:34:22 - INFO - __main__ - None
+04/30/2024 20:34:22 - INFO - __main__ - Sample 3412 of the training set: {'input_ids': [101, 1045, 12781, 1996, 7427, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1], 'labels': 1}.
+04/30/2024 20:34:22 - INFO - __main__ - Sample 6002 of the training set: {'input_ids': [101, 1045, 2442, 2064, 4521, 22088, 2015, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': 0}.
+04/30/2024 20:34:22 - INFO - __main__ - Sample 83 of the training set: {'input_ids': [101, 1996, 7764, 22257, 2993, 2000, 1996, 2598, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': 0}.
+04/30/2024 20:34:22 - INFO - __main__ -  Max training steps before recalculation = None
+04/30/2024 20:34:22 - INFO - __main__ -  num_update_steps_per_epoch initial = 214
+04/30/2024 20:34:22 - INFO - __main__ -  num training epochs initial = 5
+04/30/2024 20:34:22 - INFO - __main__ - Adjusted num_train_epochs based on max_train_steps: 5
+04/30/2024 20:34:22 - INFO - __main__ - PeftModelForSequenceClassification(
+  (base_model): LoraModel(
+    (model): BertForSequenceClassification(
+      (bert): BertModel(
+        (embeddings): BertEmbeddings(
+          (word_embeddings): Embedding(30522, 768, padding_idx=0)
+          (position_embeddings): Embedding(512, 768)
+          (token_type_embeddings): Embedding(2, 768)
+          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+        )
+        (encoder): BertEncoder(
+          (layer): ModuleList(
+            (0-11): 12 x BertLayer(
+              (attention): BertAttention(
+                (self): BertSelfAttention(
+                  (query): lora.Linear(
+                    (base_layer): Linear(in_features=768, out_features=768, bias=True)
+                    (lora_dropout): ModuleDict(
+                      (default): Dropout(p=0.1, inplace=False)
+                    )
+                    (lora_A): ModuleDict(
+                      (default): Linear(in_features=768, out_features=16, bias=False)
+                    )
+                    (lora_B): ModuleDict(
+                      (default): Linear(in_features=16, out_features=768, bias=False)
+                    )
+                    (lora_embedding_A): ParameterDict()
+                    (lora_embedding_B): ParameterDict()
+                  )
+                  (key): Linear(in_features=768, out_features=768, bias=True)
+                  (value): lora.Linear(
+                    (base_layer): Linear(in_features=768, out_features=768, bias=True)
+                    (lora_dropout): ModuleDict(
+                      (default): Dropout(p=0.1, inplace=False)
+                    )
+                    (lora_A): ModuleDict(
+                      (default): Linear(in_features=768, out_features=16, bias=False)
+                    )
+                    (lora_B): ModuleDict(
+                      (default): Linear(in_features=16, out_features=768, bias=False)
+                    )
+                    (lora_embedding_A): ParameterDict()
+                    (lora_embedding_B): ParameterDict()
+                  )
+                  (dropout): Dropout(p=0.1, inplace=False)
+                )
+                (output): BertSelfOutput(
+                  (dense): Linear(in_features=768, out_features=768, bias=True)
+                  (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
+                  (dropout): Dropout(p=0.1, inplace=False)
+                )
+              )
+              (intermediate): BertIntermediate(
+                (dense): Linear(in_features=768, out_features=3072, bias=True)
+                (intermediate_act_fn): GELUActivation()
+              )
+              (output): BertOutput(
+                (dense): Linear(in_features=3072, out_features=768, bias=True)
+                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
+                (dropout): Dropout(p=0.1, inplace=False)
+              )
+            )
+          )
+        )
+        (pooler): BertPooler(
+          (dense): Linear(in_features=768, out_features=768, bias=True)
+          (activation): Tanh()
+        )
+      )
+      (dropout): Dropout(p=0.1, inplace=False)
+      (classifier): ModulesToSaveWrapper(
+        (original_module): lora.Linear(
+          (base_layer): Linear(in_features=768, out_features=2, bias=True)
+          (lora_dropout): ModuleDict(
+            (default): Dropout(p=0.1, inplace=False)
+          )
+          (lora_A): ModuleDict(
+            (default): Linear(in_features=768, out_features=16, bias=False)
+          )
+          (lora_B): ModuleDict(
+            (default): Linear(in_features=16, out_features=2, bias=False)
+          )
+          (lora_embedding_A): ParameterDict()
+          (lora_embedding_B): ParameterDict()
+        )
+        (modules_to_save): ModuleDict(
+          (default): lora.Linear(
+            (base_layer): Linear(in_features=768, out_features=2, bias=True)
+            (lora_dropout): ModuleDict(
+              (default): Dropout(p=0.1, inplace=False)
+            )
+            (lora_A): ModuleDict(
+              (default): Linear(in_features=768, out_features=16, bias=False)
+            )
+            (lora_B): ModuleDict(
+              (default): Linear(in_features=16, out_features=2, bias=False)
+            )
+            (lora_embedding_A): ParameterDict()
+            (lora_embedding_B): ParameterDict()
+          )
+        )
+      )
+    )
+  )
+)
+04/30/2024 20:34:23 - INFO - __main__ -  num_update_steps_per_epoch before recalculation = 214
+04/30/2024 20:34:23 - INFO - __main__ -  num_update_steps_per_epoch after recalculation = 214
+04/30/2024 20:34:23 - INFO - __main__ -  num training epochs before recalculation = 5
+04/30/2024 20:34:23 - INFO - __main__ - ***** Running training *****
+04/30/2024 20:34:23 - INFO - __main__ -   Num examples = 6840
+04/30/2024 20:34:23 - INFO - __main__ -   Num Epochs = 5
+04/30/2024 20:34:23 - INFO - __main__ -   Instantaneous batch size per device = 32
+04/30/2024 20:34:23 - INFO - __main__ -   Total train batch size (w. parallel, distributed & accumulation) = 32
+04/30/2024 20:34:23 - INFO - __main__ -   Gradient Accumulation steps = 1
+04/30/2024 20:34:23 - INFO - __main__ -   Total optimization steps = 1070
+04/30/2024 20:34:25 - INFO - __main__ - epoch 0: {'matthews_correlation': -0.008044856322926524}
+04/30/2024 20:34:28 - INFO - __main__ - epoch 0: {'matthews_correlation': -0.008842368995538273}
+04/30/2024 20:34:48 - INFO - __main__ - epoch 0: {'matthews_correlation': 0.018148342420931135}
+04/30/2024 20:34:51 - INFO - __main__ - epoch 0: {'matthews_correlation': 0.0}
+04/30/2024 20:35:11 - INFO - __main__ - epoch 1: {'matthews_correlation': 0.360510315590827}
+04/30/2024 20:35:14 - INFO - __main__ - epoch 1: {'matthews_correlation': 0.39632675374632054}
+04/30/2024 20:35:35 - INFO - __main__ - epoch 2: {'matthews_correlation': 0.400329623173134}
+04/30/2024 20:35:37 - INFO - __main__ - epoch 2: {'matthews_correlation': 0.4154373562708837}
+04/30/2024 20:35:58 - INFO - __main__ - epoch 3: {'matthews_correlation': 0.40629511747030483}
+04/30/2024 20:36:00 - INFO - __main__ - epoch 3: {'matthews_correlation': 0.4242069800395323}
+04/30/2024 20:36:22 - INFO - __main__ - epoch 4: {'matthews_correlation': 0.40928849875350365}
+04/30/2024 20:36:24 - INFO - __main__ - epoch 4: {'matthews_correlation': 0.42308532319611225}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_0/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: google-bert/bert-base-uncased
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.10.0

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_0/adapter_config.json

@@ -0,0 +1,30 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google-bert/bert-base-uncased",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "query",
+    "classifier",
+    "value"
+  ],
+  "task_type": "SEQ_CLS",
+  "use_dora": false,
+  "use_rslora": false
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_0/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:999f3ac3b8eea554339512988be8e3a801cdc8709ad520c86ecc3bf8888078ee
+size 2520992

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_0/all_results.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": -0.008044856322926524}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_0/all_results_val.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": -0.008842368995538273}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_0/gpu_stats.json

@@ -0,0 +1,127 @@
+{
+    "memory_allocated": 450086912,
+    "max_memory_allocated": 488288256,
+    "memory_reserved": 557842432,
+    "max_memory_reserved": 557842432,
+    "memory_stats": {
+        "active.all.allocated": 47793,
+        "active.all.current": 269,
+        "active.all.freed": 47524,
+        "active.all.peak": 283,
+        "active.large_pool.allocated": 23288,
+        "active.large_pool.current": 76,
+        "active.large_pool.freed": 23212,
+        "active.large_pool.peak": 85,
+        "active.small_pool.allocated": 24505,
+        "active.small_pool.current": 193,
+        "active.small_pool.freed": 24312,
+        "active.small_pool.peak": 207,
+        "active_bytes.all.allocated": 72149238272,
+        "active_bytes.all.current": 450086912,
+        "active_bytes.all.freed": 71699151360,
+        "active_bytes.all.peak": 488288256,
+        "active_bytes.large_pool.allocated": 63002771456,
+        "active_bytes.large_pool.current": 447086592,
+        "active_bytes.large_pool.freed": 62555684864,
+        "active_bytes.large_pool.peak": 485261312,
+        "active_bytes.small_pool.allocated": 9146466816,
+        "active_bytes.small_pool.current": 3000320,
+        "active_bytes.small_pool.freed": 9143466496,
+        "active_bytes.small_pool.peak": 8622592,
+        "allocated_bytes.all.allocated": 72149238272,
+        "allocated_bytes.all.current": 450086912,
+        "allocated_bytes.all.freed": 71699151360,
+        "allocated_bytes.all.peak": 488288256,
+        "allocated_bytes.large_pool.allocated": 63002771456,
+        "allocated_bytes.large_pool.current": 447086592,
+        "allocated_bytes.large_pool.freed": 62555684864,
+        "allocated_bytes.large_pool.peak": 485261312,
+        "allocated_bytes.small_pool.allocated": 9146466816,
+        "allocated_bytes.small_pool.current": 3000320,
+        "allocated_bytes.small_pool.freed": 9143466496,
+        "allocated_bytes.small_pool.peak": 8622592,
+        "allocation.all.allocated": 47793,
+        "allocation.all.current": 269,
+        "allocation.all.freed": 47524,
+        "allocation.all.peak": 283,
+        "allocation.large_pool.allocated": 23288,
+        "allocation.large_pool.current": 76,
+        "allocation.large_pool.freed": 23212,
+        "allocation.large_pool.peak": 85,
+        "allocation.small_pool.allocated": 24505,
+        "allocation.small_pool.current": 193,
+        "allocation.small_pool.freed": 24312,
+        "allocation.small_pool.peak": 207,
+        "inactive_split.all.allocated": 29498,
+        "inactive_split.all.current": 21,
+        "inactive_split.all.freed": 29477,
+        "inactive_split.all.peak": 30,
+        "inactive_split.large_pool.allocated": 20697,
+        "inactive_split.large_pool.current": 18,
+        "inactive_split.large_pool.freed": 20679,
+        "inactive_split.large_pool.peak": 24,
+        "inactive_split.small_pool.allocated": 8801,
+        "inactive_split.small_pool.current": 3,
+        "inactive_split.small_pool.freed": 8798,
+        "inactive_split.small_pool.peak": 10,
+        "inactive_split_bytes.all.allocated": 75156058624,
+        "inactive_split_bytes.all.current": 46938112,
+        "inactive_split_bytes.all.freed": 75109120512,
+        "inactive_split_bytes.all.peak": 92003840,
+        "inactive_split_bytes.large_pool.allocated": 65641709568,
+        "inactive_split_bytes.large_pool.current": 45744128,
+        "inactive_split_bytes.large_pool.freed": 65595965440,
+        "inactive_split_bytes.large_pool.peak": 90832896,
+        "inactive_split_bytes.small_pool.allocated": 9514349056,
+        "inactive_split_bytes.small_pool.current": 1193984,
+        "inactive_split_bytes.small_pool.freed": 9513155072,
+        "inactive_split_bytes.small_pool.peak": 5207552,
+        "max_split_size": -1,
+        "num_alloc_retries": 0,
+        "num_ooms": 0,
+        "oversize_allocations.allocated": 0,
+        "oversize_allocations.current": 0,
+        "oversize_allocations.freed": 0,
+        "oversize_allocations.peak": 0,
+        "oversize_segments.allocated": 0,
+        "oversize_segments.current": 0,
+        "oversize_segments.freed": 0,
+        "oversize_segments.peak": 0,
+        "requested_bytes.all.allocated": 65374491376,
+        "requested_bytes.all.current": 448949908,
+        "requested_bytes.all.freed": 64925541468,
+        "requested_bytes.all.peak": 486823572,
+        "requested_bytes.large_pool.allocated": 56231950336,
+        "requested_bytes.large_pool.current": 445954048,
+        "requested_bytes.large_pool.freed": 55785996288,
+        "requested_bytes.large_pool.peak": 483801088,
+        "requested_bytes.small_pool.allocated": 9142541040,
+        "requested_bytes.small_pool.current": 2995860,
+        "requested_bytes.small_pool.freed": 9139545180,
+        "requested_bytes.small_pool.peak": 8617680,
+        "reserved_bytes.all.allocated": 557842432,
+        "reserved_bytes.all.current": 557842432,
+        "reserved_bytes.all.freed": 0,
+        "reserved_bytes.all.peak": 557842432,
+        "reserved_bytes.large_pool.allocated": 547356672,
+        "reserved_bytes.large_pool.current": 547356672,
+        "reserved_bytes.large_pool.freed": 0,
+        "reserved_bytes.large_pool.peak": 547356672,
+        "reserved_bytes.small_pool.allocated": 10485760,
+        "reserved_bytes.small_pool.current": 10485760,
+        "reserved_bytes.small_pool.freed": 0,
+        "reserved_bytes.small_pool.peak": 10485760,
+        "segment.all.allocated": 28,
+        "segment.all.current": 28,
+        "segment.all.freed": 0,
+        "segment.all.peak": 28,
+        "segment.large_pool.allocated": 23,
+        "segment.large_pool.current": 23,
+        "segment.large_pool.freed": 0,
+        "segment.large_pool.peak": 23,
+        "segment.small_pool.allocated": 5,
+        "segment.small_pool.current": 5,
+        "segment.small_pool.freed": 0,
+        "segment.small_pool.peak": 5
+    }
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_0/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_0/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "padding_side": "left",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_1069/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: google-bert/bert-base-uncased
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.10.0

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_1069/adapter_config.json

@@ -0,0 +1,30 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google-bert/bert-base-uncased",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "query",
+    "classifier",
+    "value"
+  ],
+  "task_type": "SEQ_CLS",
+  "use_dora": false,
+  "use_rslora": false
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_1069/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c299a879c64ae5fdaf6bf7a8d33c8226bceb06e9b710bfd83d186559aeff05d
+size 2520992

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_1069/all_results.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.40928849875350365}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_1069/all_results_val.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.42308532319611225}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_1069/gpu_stats.json

@@ -0,0 +1,127 @@
+{
+    "memory_allocated": 463434752,
+    "max_memory_allocated": 1283469824,
+    "memory_reserved": 1409286144,
+    "max_memory_reserved": 1409286144,
+    "memory_stats": {
+        "active.all.allocated": 1456108,
+        "active.all.current": 374,
+        "active.all.freed": 1455734,
+        "active.all.peak": 637,
+        "active.large_pool.allocated": 804416,
+        "active.large_pool.current": 77,
+        "active.large_pool.freed": 804339,
+        "active.large_pool.peak": 248,
+        "active.small_pool.allocated": 651692,
+        "active.small_pool.current": 297,
+        "active.small_pool.freed": 651395,
+        "active.small_pool.peak": 465,
+        "active_bytes.all.allocated": 2580269732352,
+        "active_bytes.all.current": 463434752,
+        "active_bytes.all.freed": 2579806297600,
+        "active_bytes.all.peak": 1283469824,
+        "active_bytes.large_pool.allocated": 2323193320960,
+        "active_bytes.large_pool.current": 455606272,
+        "active_bytes.large_pool.freed": 2322737714688,
+        "active_bytes.large_pool.peak": 1262828032,
+        "active_bytes.small_pool.allocated": 257076411392,
+        "active_bytes.small_pool.current": 7828480,
+        "active_bytes.small_pool.freed": 257068582912,
+        "active_bytes.small_pool.peak": 67981312,
+        "allocated_bytes.all.allocated": 2580269732352,
+        "allocated_bytes.all.current": 463434752,
+        "allocated_bytes.all.freed": 2579806297600,
+        "allocated_bytes.all.peak": 1283469824,
+        "allocated_bytes.large_pool.allocated": 2323193320960,
+        "allocated_bytes.large_pool.current": 455606272,
+        "allocated_bytes.large_pool.freed": 2322737714688,
+        "allocated_bytes.large_pool.peak": 1262828032,
+        "allocated_bytes.small_pool.allocated": 257076411392,
+        "allocated_bytes.small_pool.current": 7828480,
+        "allocated_bytes.small_pool.freed": 257068582912,
+        "allocated_bytes.small_pool.peak": 67981312,
+        "allocation.all.allocated": 1456108,
+        "allocation.all.current": 374,
+        "allocation.all.freed": 1455734,
+        "allocation.all.peak": 637,
+        "allocation.large_pool.allocated": 804416,
+        "allocation.large_pool.current": 77,
+        "allocation.large_pool.freed": 804339,
+        "allocation.large_pool.peak": 248,
+        "allocation.small_pool.allocated": 651692,
+        "allocation.small_pool.current": 297,
+        "allocation.small_pool.freed": 651395,
+        "allocation.small_pool.peak": 465,
+        "inactive_split.all.allocated": 857459,
+        "inactive_split.all.current": 36,
+        "inactive_split.all.freed": 857423,
+        "inactive_split.all.peak": 93,
+        "inactive_split.large_pool.allocated": 635683,
+        "inactive_split.large_pool.current": 19,
+        "inactive_split.large_pool.freed": 635664,
+        "inactive_split.large_pool.peak": 43,
+        "inactive_split.small_pool.allocated": 221776,
+        "inactive_split.small_pool.current": 17,
+        "inactive_split.small_pool.freed": 221759,
+        "inactive_split.small_pool.peak": 61,
+        "inactive_split_bytes.all.allocated": 2708686925824,
+        "inactive_split_bytes.all.current": 67144704,
+        "inactive_split_bytes.all.freed": 2708619781120,
+        "inactive_split_bytes.all.peak": 132381696,
+        "inactive_split_bytes.large_pool.allocated": 2427621849600,
+        "inactive_split_bytes.large_pool.current": 58195968,
+        "inactive_split_bytes.large_pool.freed": 2427563653632,
+        "inactive_split_bytes.large_pool.peak": 124715008,
+        "inactive_split_bytes.small_pool.allocated": 281065076224,
+        "inactive_split_bytes.small_pool.current": 8948736,
+        "inactive_split_bytes.small_pool.freed": 281056127488,
+        "inactive_split_bytes.small_pool.peak": 36578816,
+        "max_split_size": -1,
+        "num_alloc_retries": 0,
+        "num_ooms": 0,
+        "oversize_allocations.allocated": 0,
+        "oversize_allocations.current": 0,
+        "oversize_allocations.freed": 0,
+        "oversize_allocations.peak": 0,
+        "oversize_segments.allocated": 0,
+        "oversize_segments.current": 0,
+        "oversize_segments.freed": 0,
+        "oversize_segments.peak": 0,
+        "requested_bytes.all.allocated": 2466322347332,
+        "requested_bytes.all.current": 462295332,
+        "requested_bytes.all.freed": 2465860052000,
+        "requested_bytes.all.peak": 1256508208,
+        "requested_bytes.large_pool.allocated": 2209290269696,
+        "requested_bytes.large_pool.current": 454473728,
+        "requested_bytes.large_pool.freed": 2208835795968,
+        "requested_bytes.large_pool.peak": 1235879936,
+        "requested_bytes.small_pool.allocated": 257032077636,
+        "requested_bytes.small_pool.current": 7821604,
+        "requested_bytes.small_pool.freed": 257024256032,
+        "requested_bytes.small_pool.peak": 67963824,
+        "reserved_bytes.all.allocated": 1409286144,
+        "reserved_bytes.all.current": 1409286144,
+        "reserved_bytes.all.freed": 0,
+        "reserved_bytes.all.peak": 1409286144,
+        "reserved_bytes.large_pool.allocated": 1333788672,
+        "reserved_bytes.large_pool.current": 1333788672,
+        "reserved_bytes.large_pool.freed": 0,
+        "reserved_bytes.large_pool.peak": 1333788672,
+        "reserved_bytes.small_pool.allocated": 75497472,
+        "reserved_bytes.small_pool.current": 75497472,
+        "reserved_bytes.small_pool.freed": 0,
+        "reserved_bytes.small_pool.peak": 75497472,
+        "segment.all.allocated": 101,
+        "segment.all.current": 101,
+        "segment.all.freed": 0,
+        "segment.all.peak": 101,
+        "segment.large_pool.allocated": 65,
+        "segment.large_pool.current": 65,
+        "segment.large_pool.freed": 0,
+        "segment.large_pool.peak": 65,
+        "segment.small_pool.allocated": 36,
+        "segment.small_pool.current": 36,
+        "segment.small_pool.freed": 0,
+        "segment.small_pool.peak": 36
+    }
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_1069/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_1069/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "padding_side": "left",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_213/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: google-bert/bert-base-uncased
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.10.0

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_213/adapter_config.json

@@ -0,0 +1,30 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google-bert/bert-base-uncased",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "query",
+    "classifier",
+    "value"
+  ],
+  "task_type": "SEQ_CLS",
+  "use_dora": false,
+  "use_rslora": false
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_213/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c4750e899af6ed37526deda08b18d6cbe728a714eefa8c1fc03574734c314b1
+size 2520992

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_213/all_results.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.018148342420931135}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_213/all_results_val.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.0}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_213/gpu_stats.json

@@ -0,0 +1,127 @@
+{
+    "memory_allocated": 463434752,
+    "max_memory_allocated": 1282286592,
+    "memory_reserved": 1409286144,
+    "max_memory_reserved": 1409286144,
+    "memory_stats": {
+        "active.all.allocated": 328664,
+        "active.all.current": 374,
+        "active.all.freed": 328290,
+        "active.all.peak": 637,
+        "active.large_pool.allocated": 178844,
+        "active.large_pool.current": 77,
+        "active.large_pool.freed": 178767,
+        "active.large_pool.peak": 248,
+        "active.small_pool.allocated": 149820,
+        "active.small_pool.current": 297,
+        "active.small_pool.freed": 149523,
+        "active.small_pool.peak": 465,
+        "active_bytes.all.allocated": 571512007680,
+        "active_bytes.all.current": 463434752,
+        "active_bytes.all.freed": 571048572928,
+        "active_bytes.all.peak": 1282286592,
+        "active_bytes.large_pool.allocated": 512859335680,
+        "active_bytes.large_pool.current": 455606272,
+        "active_bytes.large_pool.freed": 512403729408,
+        "active_bytes.large_pool.peak": 1261644800,
+        "active_bytes.small_pool.allocated": 58652672000,
+        "active_bytes.small_pool.current": 7828480,
+        "active_bytes.small_pool.freed": 58644843520,
+        "active_bytes.small_pool.peak": 67981312,
+        "allocated_bytes.all.allocated": 571512007680,
+        "allocated_bytes.all.current": 463434752,
+        "allocated_bytes.all.freed": 571048572928,
+        "allocated_bytes.all.peak": 1282286592,
+        "allocated_bytes.large_pool.allocated": 512859335680,
+        "allocated_bytes.large_pool.current": 455606272,
+        "allocated_bytes.large_pool.freed": 512403729408,
+        "allocated_bytes.large_pool.peak": 1261644800,
+        "allocated_bytes.small_pool.allocated": 58652672000,
+        "allocated_bytes.small_pool.current": 7828480,
+        "allocated_bytes.small_pool.freed": 58644843520,
+        "allocated_bytes.small_pool.peak": 67981312,
+        "allocation.all.allocated": 328664,
+        "allocation.all.current": 374,
+        "allocation.all.freed": 328290,
+        "allocation.all.peak": 637,
+        "allocation.large_pool.allocated": 178844,
+        "allocation.large_pool.current": 77,
+        "allocation.large_pool.freed": 178767,
+        "allocation.large_pool.peak": 248,
+        "allocation.small_pool.allocated": 149820,
+        "allocation.small_pool.current": 297,
+        "allocation.small_pool.freed": 149523,
+        "allocation.small_pool.peak": 465,
+        "inactive_split.all.allocated": 194129,
+        "inactive_split.all.current": 35,
+        "inactive_split.all.freed": 194094,
+        "inactive_split.all.peak": 90,
+        "inactive_split.large_pool.allocated": 143003,
+        "inactive_split.large_pool.current": 19,
+        "inactive_split.large_pool.freed": 142984,
+        "inactive_split.large_pool.peak": 43,
+        "inactive_split.small_pool.allocated": 51126,
+        "inactive_split.small_pool.current": 16,
+        "inactive_split.small_pool.freed": 51110,
+        "inactive_split.small_pool.peak": 60,
+        "inactive_split_bytes.all.allocated": 602413305856,
+        "inactive_split_bytes.all.current": 67144704,
+        "inactive_split_bytes.all.freed": 602346161152,
+        "inactive_split_bytes.all.peak": 132381696,
+        "inactive_split_bytes.large_pool.allocated": 538671667200,
+        "inactive_split_bytes.large_pool.current": 58195968,
+        "inactive_split_bytes.large_pool.freed": 538613471232,
+        "inactive_split_bytes.large_pool.peak": 124715008,
+        "inactive_split_bytes.small_pool.allocated": 63741638656,
+        "inactive_split_bytes.small_pool.current": 8948736,
+        "inactive_split_bytes.small_pool.freed": 63732689920,
+        "inactive_split_bytes.small_pool.peak": 36578816,
+        "max_split_size": -1,
+        "num_alloc_retries": 0,
+        "num_ooms": 0,
+        "oversize_allocations.allocated": 0,
+        "oversize_allocations.current": 0,
+        "oversize_allocations.freed": 0,
+        "oversize_allocations.peak": 0,
+        "oversize_segments.allocated": 0,
+        "oversize_segments.current": 0,
+        "oversize_segments.freed": 0,
+        "oversize_segments.peak": 0,
+        "requested_bytes.all.allocated": 543352862308,
+        "requested_bytes.all.current": 462295908,
+        "requested_bytes.all.freed": 542890566400,
+        "requested_bytes.all.peak": 1256508208,
+        "requested_bytes.large_pool.allocated": 484712155136,
+        "requested_bytes.large_pool.current": 454473728,
+        "requested_bytes.large_pool.freed": 484257681408,
+        "requested_bytes.large_pool.peak": 1235879936,
+        "requested_bytes.small_pool.allocated": 58640707172,
+        "requested_bytes.small_pool.current": 7822180,
+        "requested_bytes.small_pool.freed": 58632884992,
+        "requested_bytes.small_pool.peak": 67963824,
+        "reserved_bytes.all.allocated": 1409286144,
+        "reserved_bytes.all.current": 1409286144,
+        "reserved_bytes.all.freed": 0,
+        "reserved_bytes.all.peak": 1409286144,
+        "reserved_bytes.large_pool.allocated": 1333788672,
+        "reserved_bytes.large_pool.current": 1333788672,
+        "reserved_bytes.large_pool.freed": 0,
+        "reserved_bytes.large_pool.peak": 1333788672,
+        "reserved_bytes.small_pool.allocated": 75497472,
+        "reserved_bytes.small_pool.current": 75497472,
+        "reserved_bytes.small_pool.freed": 0,
+        "reserved_bytes.small_pool.peak": 75497472,
+        "segment.all.allocated": 101,
+        "segment.all.current": 101,
+        "segment.all.freed": 0,
+        "segment.all.peak": 101,
+        "segment.large_pool.allocated": 65,
+        "segment.large_pool.current": 65,
+        "segment.large_pool.freed": 0,
+        "segment.large_pool.peak": 65,
+        "segment.small_pool.allocated": 36,
+        "segment.small_pool.current": 36,
+        "segment.small_pool.freed": 0,
+        "segment.small_pool.peak": 36
+    }
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_213/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_213/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "padding_side": "left",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_427/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: google-bert/bert-base-uncased
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.10.0

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_427/adapter_config.json

@@ -0,0 +1,30 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google-bert/bert-base-uncased",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "query",
+    "classifier",
+    "value"
+  ],
+  "task_type": "SEQ_CLS",
+  "use_dora": false,
+  "use_rslora": false
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_427/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27de2aa2a7bb5323af4dbbb050d515a5d390b99fe6318c8dcd0cc21628ce72c8
+size 2520992

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_427/all_results.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.360510315590827}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_427/all_results_val.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.39632675374632054}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_427/gpu_stats.json

@@ -0,0 +1,127 @@
+{
+    "memory_allocated": 463436288,
+    "max_memory_allocated": 1283469824,
+    "memory_reserved": 1409286144,
+    "max_memory_reserved": 1409286144,
+    "memory_stats": {
+        "active.all.allocated": 610525,
+        "active.all.current": 374,
+        "active.all.freed": 610151,
+        "active.all.peak": 637,
+        "active.large_pool.allocated": 335117,
+        "active.large_pool.current": 77,
+        "active.large_pool.freed": 335040,
+        "active.large_pool.peak": 248,
+        "active.small_pool.allocated": 275408,
+        "active.small_pool.current": 297,
+        "active.small_pool.freed": 275111,
+        "active.small_pool.peak": 465,
+        "active_bytes.all.allocated": 1072609452544,
+        "active_bytes.all.current": 463436288,
+        "active_bytes.all.freed": 1072146016256,
+        "active_bytes.all.peak": 1283469824,
+        "active_bytes.large_pool.allocated": 964333470208,
+        "active_bytes.large_pool.current": 455606272,
+        "active_bytes.large_pool.freed": 963877863936,
+        "active_bytes.large_pool.peak": 1262828032,
+        "active_bytes.small_pool.allocated": 108275982336,
+        "active_bytes.small_pool.current": 7830016,
+        "active_bytes.small_pool.freed": 108268152320,
+        "active_bytes.small_pool.peak": 67981312,
+        "allocated_bytes.all.allocated": 1072609452544,
+        "allocated_bytes.all.current": 463436288,
+        "allocated_bytes.all.freed": 1072146016256,
+        "allocated_bytes.all.peak": 1283469824,
+        "allocated_bytes.large_pool.allocated": 964333470208,
+        "allocated_bytes.large_pool.current": 455606272,
+        "allocated_bytes.large_pool.freed": 963877863936,
+        "allocated_bytes.large_pool.peak": 1262828032,
+        "allocated_bytes.small_pool.allocated": 108275982336,
+        "allocated_bytes.small_pool.current": 7830016,
+        "allocated_bytes.small_pool.freed": 108268152320,
+        "allocated_bytes.small_pool.peak": 67981312,
+        "allocation.all.allocated": 610525,
+        "allocation.all.current": 374,
+        "allocation.all.freed": 610151,
+        "allocation.all.peak": 637,
+        "allocation.large_pool.allocated": 335117,
+        "allocation.large_pool.current": 77,
+        "allocation.large_pool.freed": 335040,
+        "allocation.large_pool.peak": 248,
+        "allocation.small_pool.allocated": 275408,
+        "allocation.small_pool.current": 297,
+        "allocation.small_pool.freed": 275111,
+        "allocation.small_pool.peak": 465,
+        "inactive_split.all.allocated": 359105,
+        "inactive_split.all.current": 35,
+        "inactive_split.all.freed": 359070,
+        "inactive_split.all.peak": 91,
+        "inactive_split.large_pool.allocated": 265879,
+        "inactive_split.large_pool.current": 19,
+        "inactive_split.large_pool.freed": 265860,
+        "inactive_split.large_pool.peak": 43,
+        "inactive_split.small_pool.allocated": 93226,
+        "inactive_split.small_pool.current": 16,
+        "inactive_split.small_pool.freed": 93210,
+        "inactive_split.small_pool.peak": 61,
+        "inactive_split_bytes.all.allocated": 1128239007232,
+        "inactive_split_bytes.all.current": 67143168,
+        "inactive_split_bytes.all.freed": 1128171864064,
+        "inactive_split_bytes.all.peak": 132381696,
+        "inactive_split_bytes.large_pool.allocated": 1010142608384,
+        "inactive_split_bytes.large_pool.current": 58195968,
+        "inactive_split_bytes.large_pool.freed": 1010084412416,
+        "inactive_split_bytes.large_pool.peak": 124715008,
+        "inactive_split_bytes.small_pool.allocated": 118096398848,
+        "inactive_split_bytes.small_pool.current": 8947200,
+        "inactive_split_bytes.small_pool.freed": 118087451648,
+        "inactive_split_bytes.small_pool.peak": 36578816,
+        "max_split_size": -1,
+        "num_alloc_retries": 0,
+        "num_ooms": 0,
+        "oversize_allocations.allocated": 0,
+        "oversize_allocations.current": 0,
+        "oversize_allocations.freed": 0,
+        "oversize_allocations.peak": 0,
+        "oversize_segments.allocated": 0,
+        "oversize_segments.current": 0,
+        "oversize_segments.freed": 0,
+        "oversize_segments.peak": 0,
+        "requested_bytes.all.allocated": 1022857096956,
+        "requested_bytes.all.current": 462296484,
+        "requested_bytes.all.freed": 1022394800472,
+        "requested_bytes.all.peak": 1256508208,
+        "requested_bytes.large_pool.allocated": 914601113600,
+        "requested_bytes.large_pool.current": 454473728,
+        "requested_bytes.large_pool.freed": 914146639872,
+        "requested_bytes.large_pool.peak": 1235879936,
+        "requested_bytes.small_pool.allocated": 108255983356,
+        "requested_bytes.small_pool.current": 7822756,
+        "requested_bytes.small_pool.freed": 108248160600,
+        "requested_bytes.small_pool.peak": 67963824,
+        "reserved_bytes.all.allocated": 1409286144,
+        "reserved_bytes.all.current": 1409286144,
+        "reserved_bytes.all.freed": 0,
+        "reserved_bytes.all.peak": 1409286144,
+        "reserved_bytes.large_pool.allocated": 1333788672,
+        "reserved_bytes.large_pool.current": 1333788672,
+        "reserved_bytes.large_pool.freed": 0,
+        "reserved_bytes.large_pool.peak": 1333788672,
+        "reserved_bytes.small_pool.allocated": 75497472,
+        "reserved_bytes.small_pool.current": 75497472,
+        "reserved_bytes.small_pool.freed": 0,
+        "reserved_bytes.small_pool.peak": 75497472,
+        "segment.all.allocated": 101,
+        "segment.all.current": 101,
+        "segment.all.freed": 0,
+        "segment.all.peak": 101,
+        "segment.large_pool.allocated": 65,
+        "segment.large_pool.current": 65,
+        "segment.large_pool.freed": 0,
+        "segment.large_pool.peak": 65,
+        "segment.small_pool.allocated": 36,
+        "segment.small_pool.current": 36,
+        "segment.small_pool.freed": 0,
+        "segment.small_pool.peak": 36
+    }
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_427/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

Rank_LoRA/cola/google-bert/bert-base-uncased_lora_lmheadtrain_val_16_16_0.1_5e-05_12345/step_427/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "padding_side": "left",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}