2 runs

outputs/args.json

@@ -0,0 +1,40 @@
+{
+    "task_name": "mrpc",
+    "train_file": null,
+    "validation_file": null,
+    "max_length": 300,
+    "pad_to_max_length": false,
+    "model_name_or_path": "roberta-base",
+    "use_slow_tokenizer": false,
+    "per_device_train_batch_size": 8,
+    "per_device_eval_batch_size": 8,
+    "learning_rate": 0.0001,
+    "weight_decay": 0.0,
+    "num_train_epochs": 3,
+    "max_train_steps": 10000,
+    "gradient_accumulation_steps": 1,
+    "lr_scheduler_type": "linear",
+    "num_warmup_steps": 0,
+    "output_dir": "./outputs",
+    "peft_method": null,
+    "seed": 42,
+    "push_to_hub": false,
+    "hub_model_id": null,
+    "hub_token": null,
+    "checkpointing_steps": "1000",
+    "resume_from_checkpoint": null,
+    "with_tracking": false,
+    "report_to": "all",
+    "ignore_mismatched_sizes": true,
+    "save": false,
+    "load_step": 999,
+    "laplace_hessian": "kron",
+    "laplace_sub": "all",
+    "laplace_prior": "homo",
+    "laplace_optim_step": 1000,
+    "testing_set": "train_val",
+    "laplace_predict": "mc_corr",
+    "lm_head": false,
+    "cache_dir": "/content/cache/huggingface/metrics/glue",
+    "step_list": []
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/args.json

@@ -0,0 +1,33 @@
+{
+    "task_name": "cola",
+    "train_file": null,
+    "validation_file": null,
+    "max_length": 300,
+    "pad_to_max_length": false,
+    "model_name_or_path": "bert-base-uncased",
+    "use_slow_tokenizer": false,
+    "per_device_train_batch_size": 8,
+    "per_device_eval_batch_size": 8,
+    "learning_rate": 0.0001,
+    "max_grad_norm": 0.5,
+    "weight_decay": 0.0,
+    "num_train_epochs": 3,
+    "max_train_steps": 10000,
+    "gradient_accumulation_steps": 1,
+    "lr_scheduler_type": "linear",
+    "num_warmup_steps": 0,
+    "output_dir": "./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000",
+    "seed": 12345,
+    "push_to_hub": false,
+    "hub_model_id": null,
+    "hub_token": null,
+    "checkpointing_steps": "2000",
+    "resume_from_checkpoint": null,
+    "with_tracking": false,
+    "report_to": "all",
+    "ignore_mismatched_sizes": true,
+    "save_train_results": false,
+    "testing_set": "train_val",
+    "lm_head": true,
+    "leave_out": null
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/logfile.log

@@ -0,0 +1,379 @@
+06/01/2024 17:48:32 - INFO - __main__ -  Number of labels detected = 2
+06/01/2024 17:48:40 - INFO - adapters.heads.model_mixin - Adding head 'default' with config {'head_type': 'masked_lm', 'vocab_size': 30522, 'embedding_size': 768, 'layers': 2, 'activation_function': 'gelu', 'layer_norm': True, 'bias': True, 'shift_labels': False, 'label2id': None}.
+06/01/2024 17:48:41 - INFO - __main__ -  Number of labels detected = 2
+06/01/2024 17:48:41 - INFO - adapters.heads.model_mixin - Adding head 'cola' with config {'head_type': 'classification', 'num_labels': 2, 'layers': 2, 'activation_function': 'tanh', 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'use_pooler': False, 'bias': True, 'dropout_prob': None}.
+06/01/2024 17:48:41 - INFO - adapters.configuration.model_adapters_config - Adding adapter 'cola'.
+06/01/2024 17:48:41 - INFO - __main__ - ================================================================================
+Name                     Architecture         #Param      %Param  Active   Train
+--------------------------------------------------------------------------------
+cola                     bottleneck        1,789,056       1.634       1       1
+--------------------------------------------------------------------------------
+Full model                               109,482,240     100.000               0
+================================================================================
+06/01/2024 17:48:41 - INFO - __main__ - printing model
+06/01/2024 17:48:41 - INFO - __main__ - BertAdapterModel(
+  (bert): BertModel(
+    (embeddings): BertEmbeddings(
+      (word_embeddings): Embedding(30522, 768, padding_idx=0)
+      (position_embeddings): Embedding(512, 768)
+      (token_type_embeddings): Embedding(2, 768)
+      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
+      (dropout): Dropout(p=0.1, inplace=False)
+    )
+    (encoder): BertEncoder(
+      (layer): ModuleList(
+        (0-11): 12 x BertLayer(
+          (attention): BertAttention(
+            (self): BertSelfAttentionWithAdapters(
+              (query): LoRALinearTorch(
+                in_features=768, out_features=768, bias=True
+                (loras): ModuleDict()
+              )
+              (key): LoRALinearTorch(
+                in_features=768, out_features=768, bias=True
+                (loras): ModuleDict()
+              )
+              (value): LoRALinearTorch(
+                in_features=768, out_features=768, bias=True
+                (loras): ModuleDict()
+              )
+              (dropout): Dropout(p=0.1, inplace=False)
+              (prefix_tuning): PrefixTuningLayer(
+                (prefix_gates): ModuleDict()
+                (pool): PrefixTuningPool(
+                  (prefix_tunings): ModuleDict()
+                )
+              )
+            )
+            (output): BertSelfOutputWithAdapters(
+              (dense): Linear(in_features=768, out_features=768, bias=True)
+              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
+              (dropout): Dropout(p=0.1, inplace=False)
+              (adapters): ModuleDict(
+                (cola): Adapter(
+                  (non_linearity): Activation_Function_Class(
+                    (f): SiLU()
+                  )
+                  (adapter_down): Sequential(
+                    (0): Linear(in_features=768, out_features=48, bias=True)
+                    (1): Activation_Function_Class(
+                      (f): SiLU()
+                    )
+                  )
+                  (adapter_up): Linear(in_features=48, out_features=768, bias=True)
+                  (dropout): Dropout(p=0.0, inplace=False)
+                )
+              )
+              (adapter_fusion_layer): ModuleDict()
+            )
+          )
+          (intermediate): BertIntermediate(
+            (dense): LoRALinearTorch(
+              in_features=768, out_features=3072, bias=True
+              (loras): ModuleDict()
+            )
+            (intermediate_act_fn): GELUActivation()
+          )
+          (output): BertOutputWithAdapters(
+            (dense): LoRALinearTorch(
+              in_features=3072, out_features=768, bias=True
+              (loras): ModuleDict()
+            )
+            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
+            (dropout): Dropout(p=0.1, inplace=False)
+            (adapters): ModuleDict(
+              (cola): Adapter(
+                (non_linearity): Activation_Function_Class(
+                  (f): SiLU()
+                )
+                (adapter_down): Sequential(
+                  (0): Linear(in_features=768, out_features=48, bias=True)
+                  (1): Activation_Function_Class(
+                    (f): SiLU()
+                  )
+                )
+                (adapter_up): Linear(in_features=48, out_features=768, bias=True)
+                (dropout): Dropout(p=0.0, inplace=False)
+              )
+            )
+            (adapter_fusion_layer): ModuleDict()
+          )
+        )
+      )
+    )
+    (pooler): BertPooler(
+      (dense): Linear(in_features=768, out_features=768, bias=True)
+      (activation): Tanh()
+    )
+    (invertible_adapters): ModuleDict()
+    (shared_parameters): ModuleDict()
+    (prefix_tuning): PrefixTuningPool(
+      (prefix_tunings): ModuleDict()
+    )
+    (prompt_tuning): PromptTuningLayer(
+      (base_model_embeddings): Embedding(30522, 768, padding_idx=0)
+      (prompt_tunings): ModuleDict()
+    )
+  )
+  (heads): ModuleDict(
+    (default): BertStyleMaskedLMHead(
+      (0): Linear(in_features=768, out_features=768, bias=True)
+      (1): Activation_Function_Class(
+        (f): GELUActivation()
+      )
+      (2): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
+      (3): Linear(in_features=768, out_features=30522, bias=True)
+    )
+    (cola): ClassificationHead(
+      (0): Dropout(p=0.1, inplace=False)
+      (1): Linear(in_features=768, out_features=768, bias=True)
+      (2): Activation_Function_Class(
+        (f): Tanh()
+      )
+      (3): Dropout(p=0.1, inplace=False)
+      (4): Linear(in_features=768, out_features=2, bias=True)
+    )
+  )
+)
+06/01/2024 17:48:42 - INFO - __main__ - Sample 3412 of the training set: {'input_ids': [101, 1045, 12781, 1996, 7427, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1], 'labels': 1}.
+06/01/2024 17:48:42 - INFO - __main__ - Sample 6002 of the training set: {'input_ids': [101, 1045, 2442, 2064, 4521, 22088, 2015, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': 0}.
+06/01/2024 17:48:42 - INFO - __main__ - Sample 83 of the training set: {'input_ids': [101, 1996, 7764, 22257, 2993, 2000, 1996, 2598, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': 0}.
+06/01/2024 17:48:42 - INFO - __main__ -  Max training steps before recalculation = 10000
+06/01/2024 17:48:42 - INFO - __main__ -  num_update_steps_per_epoch initial = 855
+06/01/2024 17:48:42 - INFO - __main__ -  num training epochs initial = 3
+06/01/2024 17:48:42 - INFO - __main__ - Adjusted num_train_epochs based on max_train_steps: 3
+06/01/2024 17:48:42 - INFO - __main__ -  num_update_steps_per_epoch before recalculation = 855
+06/01/2024 17:48:42 - INFO - __main__ -  num_update_steps_per_epoch after recalculation = 855
+06/01/2024 17:48:42 - INFO - __main__ -  num training epochs before recalculation = 12
+06/01/2024 17:48:43 - INFO - __main__ - ***** Running training *****
+06/01/2024 17:48:43 - INFO - __main__ -   Num examples = 6840
+06/01/2024 17:48:43 - INFO - __main__ -   Num Epochs = 12
+06/01/2024 17:48:43 - INFO - __main__ -   Instantaneous batch size per device = 8
+06/01/2024 17:48:43 - INFO - __main__ -   Total train batch size (w. parallel, distributed & accumulation) = 8
+06/01/2024 17:48:43 - INFO - __main__ -   Gradient Accumulation steps = 1
+06/01/2024 17:48:43 - INFO - __main__ -   Total optimization steps = 10000
+06/01/2024 17:48:49 - INFO - __main__ - epoch 0: {'matthews_correlation': -0.02929206145132745}
+06/01/2024 17:48:49 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/adapter_config.json
+06/01/2024 17:48:49 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/pytorch_adapter.bin
+06/01/2024 17:48:49 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/head_config.json
+06/01/2024 17:48:49 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/pytorch_model_head.bin
+06/01/2024 17:48:54 - INFO - __main__ - epoch 0: {'matthews_correlation': 0.0}
+06/01/2024 17:48:54 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/adapter_config.json
+06/01/2024 17:48:54 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/pytorch_adapter.bin
+06/01/2024 17:48:54 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/head_config.json
+06/01/2024 17:48:54 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/pytorch_model_head.bin
+06/01/2024 17:51:04 - INFO - __main__ -  Number of labels detected = 2
+06/01/2024 17:51:05 - INFO - adapters.heads.model_mixin - Adding head 'default' with config {'head_type': 'masked_lm', 'vocab_size': 30522, 'embedding_size': 768, 'layers': 2, 'activation_function': 'gelu', 'layer_norm': True, 'bias': True, 'shift_labels': False, 'label2id': None}.
+06/01/2024 17:51:06 - INFO - __main__ -  Number of labels detected = 2
+06/01/2024 17:51:07 - INFO - adapters.heads.model_mixin - Adding head 'cola' with config {'head_type': 'classification', 'num_labels': 2, 'layers': 2, 'activation_function': 'tanh', 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'use_pooler': False, 'bias': True, 'dropout_prob': None}.
+06/01/2024 17:51:07 - INFO - adapters.configuration.model_adapters_config - Adding adapter 'cola'.
+06/01/2024 17:51:07 - INFO - __main__ - ================================================================================
+Name                     Architecture         #Param      %Param  Active   Train
+--------------------------------------------------------------------------------
+cola                     bottleneck        1,789,056       1.634       1       1
+--------------------------------------------------------------------------------
+Full model                               109,482,240     100.000               0
+================================================================================
+06/01/2024 17:51:07 - INFO - __main__ - printing model
+06/01/2024 17:51:07 - INFO - __main__ - BertAdapterModel(
+  (bert): BertModel(
+    (embeddings): BertEmbeddings(
+      (word_embeddings): Embedding(30522, 768, padding_idx=0)
+      (position_embeddings): Embedding(512, 768)
+      (token_type_embeddings): Embedding(2, 768)
+      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
+      (dropout): Dropout(p=0.1, inplace=False)
+    )
+    (encoder): BertEncoder(
+      (layer): ModuleList(
+        (0-11): 12 x BertLayer(
+          (attention): BertAttention(
+            (self): BertSelfAttentionWithAdapters(
+              (query): LoRALinearTorch(
+                in_features=768, out_features=768, bias=True
+                (loras): ModuleDict()
+              )
+              (key): LoRALinearTorch(
+                in_features=768, out_features=768, bias=True
+                (loras): ModuleDict()
+              )
+              (value): LoRALinearTorch(
+                in_features=768, out_features=768, bias=True
+                (loras): ModuleDict()
+              )
+              (dropout): Dropout(p=0.1, inplace=False)
+              (prefix_tuning): PrefixTuningLayer(
+                (prefix_gates): ModuleDict()
+                (pool): PrefixTuningPool(
+                  (prefix_tunings): ModuleDict()
+                )
+              )
+            )
+            (output): BertSelfOutputWithAdapters(
+              (dense): Linear(in_features=768, out_features=768, bias=True)
+              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
+              (dropout): Dropout(p=0.1, inplace=False)
+              (adapters): ModuleDict(
+                (cola): Adapter(
+                  (non_linearity): Activation_Function_Class(
+                    (f): SiLU()
+                  )
+                  (adapter_down): Sequential(
+                    (0): Linear(in_features=768, out_features=48, bias=True)
+                    (1): Activation_Function_Class(
+                      (f): SiLU()
+                    )
+                  )
+                  (adapter_up): Linear(in_features=48, out_features=768, bias=True)
+                  (dropout): Dropout(p=0.0, inplace=False)
+                )
+              )
+              (adapter_fusion_layer): ModuleDict()
+            )
+          )
+          (intermediate): BertIntermediate(
+            (dense): LoRALinearTorch(
+              in_features=768, out_features=3072, bias=True
+              (loras): ModuleDict()
+            )
+            (intermediate_act_fn): GELUActivation()
+          )
+          (output): BertOutputWithAdapters(
+            (dense): LoRALinearTorch(
+              in_features=3072, out_features=768, bias=True
+              (loras): ModuleDict()
+            )
+            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
+            (dropout): Dropout(p=0.1, inplace=False)
+            (adapters): ModuleDict(
+              (cola): Adapter(
+                (non_linearity): Activation_Function_Class(
+                  (f): SiLU()
+                )
+                (adapter_down): Sequential(
+                  (0): Linear(in_features=768, out_features=48, bias=True)
+                  (1): Activation_Function_Class(
+                    (f): SiLU()
+                  )
+                )
+                (adapter_up): Linear(in_features=48, out_features=768, bias=True)
+                (dropout): Dropout(p=0.0, inplace=False)
+              )
+            )
+            (adapter_fusion_layer): ModuleDict()
+          )
+        )
+      )
+    )
+    (pooler): BertPooler(
+      (dense): Linear(in_features=768, out_features=768, bias=True)
+      (activation): Tanh()
+    )
+    (invertible_adapters): ModuleDict()
+    (shared_parameters): ModuleDict()
+    (prefix_tuning): PrefixTuningPool(
+      (prefix_tunings): ModuleDict()
+    )
+    (prompt_tuning): PromptTuningLayer(
+      (base_model_embeddings): Embedding(30522, 768, padding_idx=0)
+      (prompt_tunings): ModuleDict()
+    )
+  )
+  (heads): ModuleDict(
+    (default): BertStyleMaskedLMHead(
+      (0): Linear(in_features=768, out_features=768, bias=True)
+      (1): Activation_Function_Class(
+        (f): GELUActivation()
+      )
+      (2): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
+      (3): Linear(in_features=768, out_features=30522, bias=True)
+    )
+    (cola): ClassificationHead(
+      (0): Dropout(p=0.1, inplace=False)
+      (1): Linear(in_features=768, out_features=768, bias=True)
+      (2): Activation_Function_Class(
+        (f): Tanh()
+      )
+      (3): Dropout(p=0.1, inplace=False)
+      (4): Linear(in_features=768, out_features=2, bias=True)
+    )
+  )
+)
+06/01/2024 17:51:07 - INFO - __main__ - Sample 3412 of the training set: {'input_ids': [101, 1045, 12781, 1996, 7427, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1], 'labels': 1}.
+06/01/2024 17:51:07 - INFO - __main__ - Sample 6002 of the training set: {'input_ids': [101, 1045, 2442, 2064, 4521, 22088, 2015, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': 0}.
+06/01/2024 17:51:07 - INFO - __main__ - Sample 83 of the training set: {'input_ids': [101, 1996, 7764, 22257, 2993, 2000, 1996, 2598, 1012, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': 0}.
+06/01/2024 17:51:07 - INFO - __main__ -  Max training steps before recalculation = 10000
+06/01/2024 17:51:07 - INFO - __main__ -  num_update_steps_per_epoch initial = 855
+06/01/2024 17:51:07 - INFO - __main__ -  num training epochs initial = 3
+06/01/2024 17:51:07 - INFO - __main__ - Adjusted num_train_epochs based on max_train_steps: 3
+06/01/2024 17:51:08 - INFO - __main__ -  num_update_steps_per_epoch before recalculation = 855
+06/01/2024 17:51:08 - INFO - __main__ -  num_update_steps_per_epoch after recalculation = 855
+06/01/2024 17:51:08 - INFO - __main__ -  num training epochs before recalculation = 12
+06/01/2024 17:51:09 - INFO - __main__ - ***** Running training *****
+06/01/2024 17:51:09 - INFO - __main__ -   Num examples = 6840
+06/01/2024 17:51:09 - INFO - __main__ -   Num Epochs = 12
+06/01/2024 17:51:09 - INFO - __main__ -   Instantaneous batch size per device = 8
+06/01/2024 17:51:09 - INFO - __main__ -   Total train batch size (w. parallel, distributed & accumulation) = 8
+06/01/2024 17:51:09 - INFO - __main__ -   Gradient Accumulation steps = 1
+06/01/2024 17:51:09 - INFO - __main__ -   Total optimization steps = 10000
+06/01/2024 17:51:16 - INFO - __main__ - epoch 0: {'matthews_correlation': -0.02929206145132745}
+06/01/2024 17:51:16 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/adapter_config.json
+06/01/2024 17:51:16 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/pytorch_adapter.bin
+06/01/2024 17:51:16 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/head_config.json
+06/01/2024 17:51:16 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/pytorch_model_head.bin
+06/01/2024 17:51:24 - INFO - __main__ - epoch 0: {'matthews_correlation': 0.0}
+06/01/2024 17:51:24 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/adapter_config.json
+06/01/2024 17:51:24 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/pytorch_adapter.bin
+06/01/2024 17:51:24 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/head_config.json
+06/01/2024 17:51:24 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/pytorch_model_head.bin
+06/01/2024 17:53:22 - INFO - __main__ - epoch 2: {'matthews_correlation': 0.4747045393662238}
+06/01/2024 17:53:22 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/adapter_config.json
+06/01/2024 17:53:22 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/pytorch_adapter.bin
+06/01/2024 17:53:22 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/head_config.json
+06/01/2024 17:53:22 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/pytorch_model_head.bin
+06/01/2024 17:53:28 - INFO - __main__ - epoch 2: {'matthews_correlation': 0.5174977706442678}
+06/01/2024 17:53:28 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/adapter_config.json
+06/01/2024 17:53:29 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/pytorch_adapter.bin
+06/01/2024 17:53:29 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/head_config.json
+06/01/2024 17:53:29 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/pytorch_model_head.bin
+06/01/2024 17:55:23 - INFO - __main__ - epoch 4: {'matthews_correlation': 0.542244787638552}
+06/01/2024 17:55:23 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/adapter_config.json
+06/01/2024 17:55:23 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/pytorch_adapter.bin
+06/01/2024 17:55:23 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/head_config.json
+06/01/2024 17:55:23 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/pytorch_model_head.bin
+06/01/2024 17:55:29 - INFO - __main__ - epoch 4: {'matthews_correlation': 0.5568270392858016}
+06/01/2024 17:55:29 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/adapter_config.json
+06/01/2024 17:55:29 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/pytorch_adapter.bin
+06/01/2024 17:55:29 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/head_config.json
+06/01/2024 17:55:29 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/pytorch_model_head.bin
+06/01/2024 17:57:25 - INFO - __main__ - epoch 7: {'matthews_correlation': 0.5073664747016221}
+06/01/2024 17:57:25 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/adapter_config.json
+06/01/2024 17:57:25 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/pytorch_adapter.bin
+06/01/2024 17:57:25 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/head_config.json
+06/01/2024 17:57:25 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/pytorch_model_head.bin
+06/01/2024 17:57:31 - INFO - __main__ - epoch 7: {'matthews_correlation': 0.552840881500124}
+06/01/2024 17:57:31 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/adapter_config.json
+06/01/2024 17:57:31 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/pytorch_adapter.bin
+06/01/2024 17:57:31 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/head_config.json
+06/01/2024 17:57:31 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/pytorch_model_head.bin
+06/01/2024 17:59:27 - INFO - __main__ - epoch 9: {'matthews_correlation': 0.5353925809123671}
+06/01/2024 17:59:27 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_7999/adapter_config.json
+06/01/2024 17:59:27 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_7999/pytorch_adapter.bin
+06/01/2024 17:59:27 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_7999/head_config.json
+06/01/2024 17:59:27 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_7999/pytorch_model_head.bin
+06/01/2024 17:59:33 - INFO - __main__ - epoch 9: {'matthews_correlation': 0.5495593400575438}
+06/01/2024 17:59:33 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_7999/adapter_config.json
+06/01/2024 17:59:33 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_7999/pytorch_adapter.bin
+06/01/2024 17:59:33 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_7999/head_config.json
+06/01/2024 17:59:33 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_7999/pytorch_model_head.bin
+06/01/2024 18:01:29 - INFO - __main__ - epoch 11: {'matthews_correlation': 0.5226700639354173}
+06/01/2024 18:01:29 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_9999/adapter_config.json
+06/01/2024 18:01:29 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_9999/pytorch_adapter.bin
+06/01/2024 18:01:29 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_9999/head_config.json
+06/01/2024 18:01:29 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_9999/pytorch_model_head.bin
+06/01/2024 18:01:35 - INFO - __main__ - epoch 11: {'matthews_correlation': 0.540076124607824}
+06/01/2024 18:01:35 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_9999/adapter_config.json
+06/01/2024 18:01:35 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_9999/pytorch_adapter.bin
+06/01/2024 18:01:35 - INFO - adapters.loading - Configuration saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_9999/head_config.json
+06/01/2024 18:01:35 - INFO - adapters.loading - Module weights saved in ./outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_9999/pytorch_model_head.bin
+06/01/2024 18:01:35 - INFO - __main__ - ***** Completed training *****

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "config": {
+    "adapter_residual_before_ln": false,
+    "cross_adapter": false,
+    "dropout": 0.0,
+    "factorized_phm_W": true,
+    "factorized_phm_rule": false,
+    "hypercomplex_nonlinearity": "glorot-uniform",
+    "init_weights": "bert",
+    "inv_adapter": null,
+    "inv_adapter_reduction_factor": null,
+    "is_parallel": false,
+    "learn_phm": true,
+    "leave_out": [],
+    "ln_after": false,
+    "ln_before": false,
+    "mh_adapter": true,
+    "non_linearity": "swish",
+    "original_ln_after": true,
+    "original_ln_before": false,
+    "output_adapter": true,
+    "phm_bias": true,
+    "phm_c_init": "normal",
+    "phm_dim": 4,
+    "phm_init_range": 0.0001,
+    "phm_layer": false,
+    "phm_rank": 1,
+    "reduction_factor": 16,
+    "residual_before_ln": true,
+    "scaling": 1.0,
+    "shared_W_phm": false,
+    "shared_phm_rule": true,
+    "use_gating": false
+  },
+  "hidden_size": 768,
+  "model_class": "BertAdapterModel",
+  "model_name": "bert-base-uncased",
+  "model_type": "bert",
+  "name": "cola",
+  "version": "0.2.1"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/all_results.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": -0.02929206145132745}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/all_results_val.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.0}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/gpu_stats.json

@@ -0,0 +1,130 @@
+{
+    "memory_allocated": 459623936,
+    "max_memory_allocated": 471682048,
+    "memory_reserved": 532676608,
+    "max_memory_reserved": 532676608,
+    "memory_stats": {
+        "active.all.allocated": 229718,
+        "active.all.current": 319,
+        "active.all.freed": 229399,
+        "active.all.peak": 337,
+        "active.large_pool.allocated": 7614,
+        "active.large_pool.current": 78,
+        "active.large_pool.freed": 7536,
+        "active.large_pool.peak": 80,
+        "active.small_pool.allocated": 222104,
+        "active.small_pool.current": 241,
+        "active.small_pool.freed": 221863,
+        "active.small_pool.peak": 259,
+        "active_bytes.all.allocated": 104486025728,
+        "active_bytes.all.current": 459623936,
+        "active_bytes.all.freed": 104026401792,
+        "active_bytes.all.peak": 471682048,
+        "active_bytes.large_pool.allocated": 16832397312,
+        "active_bytes.large_pool.current": 451805184,
+        "active_bytes.large_pool.freed": 16380592128,
+        "active_bytes.large_pool.peak": 458686464,
+        "active_bytes.small_pool.allocated": 87653628416,
+        "active_bytes.small_pool.current": 7818752,
+        "active_bytes.small_pool.freed": 87645809664,
+        "active_bytes.small_pool.peak": 16436224,
+        "allocated_bytes.all.allocated": 104486025728,
+        "allocated_bytes.all.current": 459623936,
+        "allocated_bytes.all.freed": 104026401792,
+        "allocated_bytes.all.peak": 471682048,
+        "allocated_bytes.large_pool.allocated": 16832397312,
+        "allocated_bytes.large_pool.current": 451805184,
+        "allocated_bytes.large_pool.freed": 16380592128,
+        "allocated_bytes.large_pool.peak": 458686464,
+        "allocated_bytes.small_pool.allocated": 87653628416,
+        "allocated_bytes.small_pool.current": 7818752,
+        "allocated_bytes.small_pool.freed": 87645809664,
+        "allocated_bytes.small_pool.peak": 16436224,
+        "allocation.all.allocated": 229718,
+        "allocation.all.current": 319,
+        "allocation.all.freed": 229399,
+        "allocation.all.peak": 337,
+        "allocation.large_pool.allocated": 7614,
+        "allocation.large_pool.current": 78,
+        "allocation.large_pool.freed": 7536,
+        "allocation.large_pool.peak": 80,
+        "allocation.small_pool.allocated": 222104,
+        "allocation.small_pool.current": 241,
+        "allocation.small_pool.freed": 221863,
+        "allocation.small_pool.peak": 259,
+        "inactive_split.all.allocated": 110877,
+        "inactive_split.all.current": 23,
+        "inactive_split.all.freed": 110854,
+        "inactive_split.all.peak": 34,
+        "inactive_split.large_pool.allocated": 7159,
+        "inactive_split.large_pool.current": 18,
+        "inactive_split.large_pool.freed": 7141,
+        "inactive_split.large_pool.peak": 19,
+        "inactive_split.small_pool.allocated": 103718,
+        "inactive_split.small_pool.current": 5,
+        "inactive_split.small_pool.freed": 103713,
+        "inactive_split.small_pool.peak": 15,
+        "inactive_split_bytes.all.allocated": 111369324032,
+        "inactive_split_bytes.all.current": 43692544,
+        "inactive_split_bytes.all.freed": 111325631488,
+        "inactive_split_bytes.all.peak": 67346432,
+        "inactive_split_bytes.large_pool.allocated": 18599641088,
+        "inactive_split_bytes.large_pool.current": 41025536,
+        "inactive_split_bytes.large_pool.freed": 18558615552,
+        "inactive_split_bytes.large_pool.peak": 59146240,
+        "inactive_split_bytes.small_pool.allocated": 92769682944,
+        "inactive_split_bytes.small_pool.current": 2667008,
+        "inactive_split_bytes.small_pool.freed": 92767015936,
+        "inactive_split_bytes.small_pool.peak": 8200192,
+        "max_split_size": -1,
+        "num_alloc_retries": 0,
+        "num_device_alloc": 30,
+        "num_device_free": 0,
+        "num_ooms": 0,
+        "num_sync_all_streams": 0,
+        "oversize_allocations.allocated": 0,
+        "oversize_allocations.current": 0,
+        "oversize_allocations.freed": 0,
+        "oversize_allocations.peak": 0,
+        "oversize_segments.allocated": 0,
+        "oversize_segments.current": 0,
+        "oversize_segments.freed": 0,
+        "oversize_segments.peak": 0,
+        "requested_bytes.all.allocated": 100127988300,
+        "requested_bytes.all.current": 458479208,
+        "requested_bytes.all.freed": 99669509092,
+        "requested_bytes.all.peak": 470535000,
+        "requested_bytes.large_pool.allocated": 12486326272,
+        "requested_bytes.large_pool.current": 450672640,
+        "requested_bytes.large_pool.freed": 12035653632,
+        "requested_bytes.large_pool.peak": 457553920,
+        "requested_bytes.small_pool.allocated": 87641662028,
+        "requested_bytes.small_pool.current": 7806568,
+        "requested_bytes.small_pool.freed": 87633855460,
+        "requested_bytes.small_pool.peak": 16421720,
+        "reserved_bytes.all.allocated": 532676608,
+        "reserved_bytes.all.current": 532676608,
+        "reserved_bytes.all.freed": 0,
+        "reserved_bytes.all.peak": 532676608,
+        "reserved_bytes.large_pool.allocated": 513802240,
+        "reserved_bytes.large_pool.current": 513802240,
+        "reserved_bytes.large_pool.freed": 0,
+        "reserved_bytes.large_pool.peak": 513802240,
+        "reserved_bytes.small_pool.allocated": 18874368,
+        "reserved_bytes.small_pool.current": 18874368,
+        "reserved_bytes.small_pool.freed": 0,
+        "reserved_bytes.small_pool.peak": 18874368,
+        "segment.all.allocated": 30,
+        "segment.all.current": 30,
+        "segment.all.freed": 0,
+        "segment.all.peak": 30,
+        "segment.large_pool.allocated": 21,
+        "segment.large_pool.current": 21,
+        "segment.large_pool.freed": 0,
+        "segment.large_pool.peak": 21,
+        "segment.small_pool.allocated": 9,
+        "segment.small_pool.current": 9,
+        "segment.small_pool.freed": 0,
+        "segment.small_pool.peak": 9
+    }
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/head_config.json

@@ -0,0 +1,21 @@
+{
+  "config": {
+    "activation_function": "tanh",
+    "bias": true,
+    "dropout_prob": null,
+    "head_type": "classification",
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layers": 2,
+    "num_labels": 2,
+    "use_pooler": false
+  },
+  "hidden_size": 768,
+  "model_class": "BertAdapterModel",
+  "model_name": "bert-base-uncased",
+  "model_type": "bert",
+  "name": "cola",
+  "version": "0.2.1"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/pytorch_adapter.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29b06f7b9617fec3c5bc63794ca9c89f4238d46db9edb1de32e4dd4e5086fae0
+size 7191062

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/pytorch_model_head.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:244a549f85f7dcaa9f5082c6ca9f115dff60bbcbbd8534f0a78f92ef9b70720d
+size 2370664

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_0/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "padding_side": "left",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "config": {
+    "adapter_residual_before_ln": false,
+    "cross_adapter": false,
+    "dropout": 0.0,
+    "factorized_phm_W": true,
+    "factorized_phm_rule": false,
+    "hypercomplex_nonlinearity": "glorot-uniform",
+    "init_weights": "bert",
+    "inv_adapter": null,
+    "inv_adapter_reduction_factor": null,
+    "is_parallel": false,
+    "learn_phm": true,
+    "leave_out": [],
+    "ln_after": false,
+    "ln_before": false,
+    "mh_adapter": true,
+    "non_linearity": "swish",
+    "original_ln_after": true,
+    "original_ln_before": false,
+    "output_adapter": true,
+    "phm_bias": true,
+    "phm_c_init": "normal",
+    "phm_dim": 4,
+    "phm_init_range": 0.0001,
+    "phm_layer": false,
+    "phm_rank": 1,
+    "reduction_factor": 16,
+    "residual_before_ln": true,
+    "scaling": 1.0,
+    "shared_W_phm": false,
+    "shared_phm_rule": true,
+    "use_gating": false
+  },
+  "hidden_size": 768,
+  "model_class": "BertAdapterModel",
+  "model_name": "bert-base-uncased",
+  "model_type": "bert",
+  "name": "cola",
+  "version": "0.2.1"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/all_results.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.4747045393662238}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/all_results_val.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.5174977706442678}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/gpu_stats.json

@@ -0,0 +1,130 @@
+{
+    "memory_allocated": 487209472,
+    "max_memory_allocated": 699914240,
+    "memory_reserved": 805306368,
+    "max_memory_reserved": 805306368,
+    "memory_stats": {
+        "active.all.allocated": 3076017,
+        "active.all.current": 520,
+        "active.all.freed": 3075497,
+        "active.all.peak": 777,
+        "active.large_pool.allocated": 116558,
+        "active.large_pool.current": 81,
+        "active.large_pool.freed": 116477,
+        "active.large_pool.peak": 183,
+        "active.small_pool.allocated": 2959459,
+        "active.small_pool.current": 439,
+        "active.small_pool.freed": 2959020,
+        "active.small_pool.peak": 696,
+        "active_bytes.all.allocated": 1290817708544,
+        "active_bytes.all.current": 487209472,
+        "active_bytes.all.freed": 1290330499072,
+        "active_bytes.all.peak": 699914240,
+        "active_bytes.large_pool.allocated": 257307901952,
+        "active_bytes.large_pool.current": 465043456,
+        "active_bytes.large_pool.freed": 256842858496,
+        "active_bytes.large_pool.peak": 646561792,
+        "active_bytes.small_pool.allocated": 1033509806592,
+        "active_bytes.small_pool.current": 22166016,
+        "active_bytes.small_pool.freed": 1033487640576,
+        "active_bytes.small_pool.peak": 133540864,
+        "allocated_bytes.all.allocated": 1290817708544,
+        "allocated_bytes.all.current": 487209472,
+        "allocated_bytes.all.freed": 1290330499072,
+        "allocated_bytes.all.peak": 699914240,
+        "allocated_bytes.large_pool.allocated": 257307901952,
+        "allocated_bytes.large_pool.current": 465043456,
+        "allocated_bytes.large_pool.freed": 256842858496,
+        "allocated_bytes.large_pool.peak": 646561792,
+        "allocated_bytes.small_pool.allocated": 1033509806592,
+        "allocated_bytes.small_pool.current": 22166016,
+        "allocated_bytes.small_pool.freed": 1033487640576,
+        "allocated_bytes.small_pool.peak": 133540864,
+        "allocation.all.allocated": 3076017,
+        "allocation.all.current": 520,
+        "allocation.all.freed": 3075497,
+        "allocation.all.peak": 777,
+        "allocation.large_pool.allocated": 116558,
+        "allocation.large_pool.current": 81,
+        "allocation.large_pool.freed": 116477,
+        "allocation.large_pool.peak": 183,
+        "allocation.small_pool.allocated": 2959459,
+        "allocation.small_pool.current": 439,
+        "allocation.small_pool.freed": 2959020,
+        "allocation.small_pool.peak": 696,
+        "inactive_split.all.allocated": 1669133,
+        "inactive_split.all.current": 45,
+        "inactive_split.all.freed": 1669088,
+        "inactive_split.all.peak": 119,
+        "inactive_split.large_pool.allocated": 99599,
+        "inactive_split.large_pool.current": 19,
+        "inactive_split.large_pool.freed": 99580,
+        "inactive_split.large_pool.peak": 23,
+        "inactive_split.small_pool.allocated": 1569534,
+        "inactive_split.small_pool.current": 26,
+        "inactive_split.small_pool.freed": 1569508,
+        "inactive_split.small_pool.peak": 99,
+        "inactive_split_bytes.all.allocated": 1381495543808,
+        "inactive_split_bytes.all.current": 60147200,
+        "inactive_split_bytes.all.freed": 1381435396608,
+        "inactive_split_bytes.all.peak": 149930496,
+        "inactive_split_bytes.large_pool.allocated": 291350265856,
+        "inactive_split_bytes.large_pool.current": 48758784,
+        "inactive_split_bytes.large_pool.freed": 291301507072,
+        "inactive_split_bytes.large_pool.peak": 75759616,
+        "inactive_split_bytes.small_pool.allocated": 1090145277952,
+        "inactive_split_bytes.small_pool.current": 11388416,
+        "inactive_split_bytes.small_pool.freed": 1090133889536,
+        "inactive_split_bytes.small_pool.peak": 82985472,
+        "max_split_size": -1,
+        "num_alloc_retries": 0,
+        "num_device_alloc": 97,
+        "num_device_free": 0,
+        "num_ooms": 0,
+        "num_sync_all_streams": 0,
+        "oversize_allocations.allocated": 0,
+        "oversize_allocations.current": 0,
+        "oversize_allocations.freed": 0,
+        "oversize_allocations.peak": 0,
+        "oversize_segments.allocated": 0,
+        "oversize_segments.current": 0,
+        "oversize_segments.freed": 0,
+        "oversize_segments.peak": 0,
+        "requested_bytes.all.allocated": 1243709959972,
+        "requested_bytes.all.current": 486047800,
+        "requested_bytes.all.freed": 1243223912172,
+        "requested_bytes.all.peak": 680092736,
+        "requested_bytes.large_pool.allocated": 210328229888,
+        "requested_bytes.large_pool.current": 463910912,
+        "requested_bytes.large_pool.freed": 209864318976,
+        "requested_bytes.large_pool.peak": 626776064,
+        "requested_bytes.small_pool.allocated": 1033381730084,
+        "requested_bytes.small_pool.current": 22136888,
+        "requested_bytes.small_pool.freed": 1033359593196,
+        "requested_bytes.small_pool.peak": 133501696,
+        "reserved_bytes.all.allocated": 805306368,
+        "reserved_bytes.all.current": 805306368,
+        "reserved_bytes.all.freed": 0,
+        "reserved_bytes.all.peak": 805306368,
+        "reserved_bytes.large_pool.allocated": 660602880,
+        "reserved_bytes.large_pool.current": 660602880,
+        "reserved_bytes.large_pool.freed": 0,
+        "reserved_bytes.large_pool.peak": 660602880,
+        "reserved_bytes.small_pool.allocated": 144703488,
+        "reserved_bytes.small_pool.current": 144703488,
+        "reserved_bytes.small_pool.freed": 0,
+        "reserved_bytes.small_pool.peak": 144703488,
+        "segment.all.allocated": 97,
+        "segment.all.current": 97,
+        "segment.all.freed": 0,
+        "segment.all.peak": 97,
+        "segment.large_pool.allocated": 28,
+        "segment.large_pool.current": 28,
+        "segment.large_pool.freed": 0,
+        "segment.large_pool.peak": 28,
+        "segment.small_pool.allocated": 69,
+        "segment.small_pool.current": 69,
+        "segment.small_pool.freed": 0,
+        "segment.small_pool.peak": 69
+    }
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/head_config.json

@@ -0,0 +1,21 @@
+{
+  "config": {
+    "activation_function": "tanh",
+    "bias": true,
+    "dropout_prob": null,
+    "head_type": "classification",
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layers": 2,
+    "num_labels": 2,
+    "use_pooler": false
+  },
+  "hidden_size": 768,
+  "model_class": "BertAdapterModel",
+  "model_name": "bert-base-uncased",
+  "model_type": "bert",
+  "name": "cola",
+  "version": "0.2.1"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/pytorch_adapter.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9163ac81be9e722743c826d9388fa399fb25ae7e609d83b0a6725f868e26f98f
+size 7191062

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/pytorch_model_head.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0c4d71839d147c316ab8631c5c3b238e5632e23a39f4731f02a0bf6bf77a919
+size 2370664

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_1999/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "padding_side": "left",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "config": {
+    "adapter_residual_before_ln": false,
+    "cross_adapter": false,
+    "dropout": 0.0,
+    "factorized_phm_W": true,
+    "factorized_phm_rule": false,
+    "hypercomplex_nonlinearity": "glorot-uniform",
+    "init_weights": "bert",
+    "inv_adapter": null,
+    "inv_adapter_reduction_factor": null,
+    "is_parallel": false,
+    "learn_phm": true,
+    "leave_out": [],
+    "ln_after": false,
+    "ln_before": false,
+    "mh_adapter": true,
+    "non_linearity": "swish",
+    "original_ln_after": true,
+    "original_ln_before": false,
+    "output_adapter": true,
+    "phm_bias": true,
+    "phm_c_init": "normal",
+    "phm_dim": 4,
+    "phm_init_range": 0.0001,
+    "phm_layer": false,
+    "phm_rank": 1,
+    "reduction_factor": 16,
+    "residual_before_ln": true,
+    "scaling": 1.0,
+    "shared_W_phm": false,
+    "shared_phm_rule": true,
+    "use_gating": false
+  },
+  "hidden_size": 768,
+  "model_class": "BertAdapterModel",
+  "model_name": "bert-base-uncased",
+  "model_type": "bert",
+  "name": "cola",
+  "version": "0.2.1"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/all_results.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.542244787638552}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/all_results_val.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.5568270392858016}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/gpu_stats.json

@@ -0,0 +1,130 @@
+{
+    "memory_allocated": 487209472,
+    "max_memory_allocated": 699914240,
+    "memory_reserved": 805306368,
+    "max_memory_reserved": 805306368,
+    "memory_stats": {
+        "active.all.allocated": 5923424,
+        "active.all.current": 520,
+        "active.all.freed": 5922904,
+        "active.all.peak": 777,
+        "active.large_pool.allocated": 225064,
+        "active.large_pool.current": 81,
+        "active.large_pool.freed": 224983,
+        "active.large_pool.peak": 183,
+        "active.small_pool.allocated": 5698360,
+        "active.small_pool.current": 439,
+        "active.small_pool.freed": 5697921,
+        "active.small_pool.peak": 696,
+        "active_bytes.all.allocated": 2476015690752,
+        "active_bytes.all.current": 487209472,
+        "active_bytes.all.freed": 2475528481280,
+        "active_bytes.all.peak": 699914240,
+        "active_bytes.large_pool.allocated": 497251237888,
+        "active_bytes.large_pool.current": 465043456,
+        "active_bytes.large_pool.freed": 496786194432,
+        "active_bytes.large_pool.peak": 646561792,
+        "active_bytes.small_pool.allocated": 1978764452864,
+        "active_bytes.small_pool.current": 22166016,
+        "active_bytes.small_pool.freed": 1978742286848,
+        "active_bytes.small_pool.peak": 133540864,
+        "allocated_bytes.all.allocated": 2476015690752,
+        "allocated_bytes.all.current": 487209472,
+        "allocated_bytes.all.freed": 2475528481280,
+        "allocated_bytes.all.peak": 699914240,
+        "allocated_bytes.large_pool.allocated": 497251237888,
+        "allocated_bytes.large_pool.current": 465043456,
+        "allocated_bytes.large_pool.freed": 496786194432,
+        "allocated_bytes.large_pool.peak": 646561792,
+        "allocated_bytes.small_pool.allocated": 1978764452864,
+        "allocated_bytes.small_pool.current": 22166016,
+        "allocated_bytes.small_pool.freed": 1978742286848,
+        "allocated_bytes.small_pool.peak": 133540864,
+        "allocation.all.allocated": 5923424,
+        "allocation.all.current": 520,
+        "allocation.all.freed": 5922904,
+        "allocation.all.peak": 777,
+        "allocation.large_pool.allocated": 225064,
+        "allocation.large_pool.current": 81,
+        "allocation.large_pool.freed": 224983,
+        "allocation.large_pool.peak": 183,
+        "allocation.small_pool.allocated": 5698360,
+        "allocation.small_pool.current": 439,
+        "allocation.small_pool.freed": 5697921,
+        "allocation.small_pool.peak": 696,
+        "inactive_split.all.allocated": 3209672,
+        "inactive_split.all.current": 47,
+        "inactive_split.all.freed": 3209625,
+        "inactive_split.all.peak": 119,
+        "inactive_split.large_pool.allocated": 191602,
+        "inactive_split.large_pool.current": 19,
+        "inactive_split.large_pool.freed": 191583,
+        "inactive_split.large_pool.peak": 23,
+        "inactive_split.small_pool.allocated": 3018070,
+        "inactive_split.small_pool.current": 28,
+        "inactive_split.small_pool.freed": 3018042,
+        "inactive_split.small_pool.peak": 99,
+        "inactive_split_bytes.all.allocated": 2650731834368,
+        "inactive_split_bytes.all.current": 60147200,
+        "inactive_split_bytes.all.freed": 2650671687168,
+        "inactive_split_bytes.all.peak": 149930496,
+        "inactive_split_bytes.large_pool.allocated": 563739623424,
+        "inactive_split_bytes.large_pool.current": 48758784,
+        "inactive_split_bytes.large_pool.freed": 563690864640,
+        "inactive_split_bytes.large_pool.peak": 75759616,
+        "inactive_split_bytes.small_pool.allocated": 2086992210944,
+        "inactive_split_bytes.small_pool.current": 11388416,
+        "inactive_split_bytes.small_pool.freed": 2086980822528,
+        "inactive_split_bytes.small_pool.peak": 82985472,
+        "max_split_size": -1,
+        "num_alloc_retries": 0,
+        "num_device_alloc": 97,
+        "num_device_free": 0,
+        "num_ooms": 0,
+        "num_sync_all_streams": 0,
+        "oversize_allocations.allocated": 0,
+        "oversize_allocations.current": 0,
+        "oversize_allocations.freed": 0,
+        "oversize_allocations.peak": 0,
+        "oversize_segments.allocated": 0,
+        "oversize_segments.current": 0,
+        "oversize_segments.freed": 0,
+        "oversize_segments.peak": 0,
+        "requested_bytes.all.allocated": 2385732458304,
+        "requested_bytes.all.current": 486047800,
+        "requested_bytes.all.freed": 2385246410504,
+        "requested_bytes.all.peak": 680092736,
+        "requested_bytes.large_pool.allocated": 407212120064,
+        "requested_bytes.large_pool.current": 463910912,
+        "requested_bytes.large_pool.freed": 406748209152,
+        "requested_bytes.large_pool.peak": 626776064,
+        "requested_bytes.small_pool.allocated": 1978520338240,
+        "requested_bytes.small_pool.current": 22136888,
+        "requested_bytes.small_pool.freed": 1978498201352,
+        "requested_bytes.small_pool.peak": 133501696,
+        "reserved_bytes.all.allocated": 805306368,
+        "reserved_bytes.all.current": 805306368,
+        "reserved_bytes.all.freed": 0,
+        "reserved_bytes.all.peak": 805306368,
+        "reserved_bytes.large_pool.allocated": 660602880,
+        "reserved_bytes.large_pool.current": 660602880,
+        "reserved_bytes.large_pool.freed": 0,
+        "reserved_bytes.large_pool.peak": 660602880,
+        "reserved_bytes.small_pool.allocated": 144703488,
+        "reserved_bytes.small_pool.current": 144703488,
+        "reserved_bytes.small_pool.freed": 0,
+        "reserved_bytes.small_pool.peak": 144703488,
+        "segment.all.allocated": 97,
+        "segment.all.current": 97,
+        "segment.all.freed": 0,
+        "segment.all.peak": 97,
+        "segment.large_pool.allocated": 28,
+        "segment.large_pool.current": 28,
+        "segment.large_pool.freed": 0,
+        "segment.large_pool.peak": 28,
+        "segment.small_pool.allocated": 69,
+        "segment.small_pool.current": 69,
+        "segment.small_pool.freed": 0,
+        "segment.small_pool.peak": 69
+    }
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/head_config.json

@@ -0,0 +1,21 @@
+{
+  "config": {
+    "activation_function": "tanh",
+    "bias": true,
+    "dropout_prob": null,
+    "head_type": "classification",
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layers": 2,
+    "num_labels": 2,
+    "use_pooler": false
+  },
+  "hidden_size": 768,
+  "model_class": "BertAdapterModel",
+  "model_name": "bert-base-uncased",
+  "model_type": "bert",
+  "name": "cola",
+  "version": "0.2.1"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/pytorch_adapter.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcd11f8a4b1606457a831d8f275103a705f9326c343919a8a5747ef886e615df
+size 7191062

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/pytorch_model_head.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ba004d6f3992457f038f420e4b26e6f1aa219c44882b5ecc4a0006f0f61f760
+size 2370664

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_3999/tokenizer_config.json

@@ -0,0 +1,56 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "padding_side": "left",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "config": {
+    "adapter_residual_before_ln": false,
+    "cross_adapter": false,
+    "dropout": 0.0,
+    "factorized_phm_W": true,
+    "factorized_phm_rule": false,
+    "hypercomplex_nonlinearity": "glorot-uniform",
+    "init_weights": "bert",
+    "inv_adapter": null,
+    "inv_adapter_reduction_factor": null,
+    "is_parallel": false,
+    "learn_phm": true,
+    "leave_out": [],
+    "ln_after": false,
+    "ln_before": false,
+    "mh_adapter": true,
+    "non_linearity": "swish",
+    "original_ln_after": true,
+    "original_ln_before": false,
+    "output_adapter": true,
+    "phm_bias": true,
+    "phm_c_init": "normal",
+    "phm_dim": 4,
+    "phm_init_range": 0.0001,
+    "phm_layer": false,
+    "phm_rank": 1,
+    "reduction_factor": 16,
+    "residual_before_ln": true,
+    "scaling": 1.0,
+    "shared_W_phm": false,
+    "shared_phm_rule": true,
+    "use_gating": false
+  },
+  "hidden_size": 768,
+  "model_class": "BertAdapterModel",
+  "model_name": "bert-base-uncased",
+  "model_type": "bert",
+  "name": "cola",
+  "version": "0.2.1"
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/all_results.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.5073664747016221}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/all_results_val.json

@@ -0,0 +1 @@
+{"eval_matthews_correlation": 0.552840881500124}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/gpu_stats.json

@@ -0,0 +1,130 @@
+{
+    "memory_allocated": 487209472,
+    "max_memory_allocated": 699914240,
+    "memory_reserved": 805306368,
+    "max_memory_reserved": 805306368,
+    "memory_stats": {
+        "active.all.allocated": 8770831,
+        "active.all.current": 520,
+        "active.all.freed": 8770311,
+        "active.all.peak": 777,
+        "active.large_pool.allocated": 333282,
+        "active.large_pool.current": 81,
+        "active.large_pool.freed": 333201,
+        "active.large_pool.peak": 183,
+        "active.small_pool.allocated": 8437549,
+        "active.small_pool.current": 439,
+        "active.small_pool.freed": 8437110,
+        "active.small_pool.peak": 696,
+        "active_bytes.all.allocated": 3664038001152,
+        "active_bytes.all.current": 487209472,
+        "active_bytes.all.freed": 3663550791680,
+        "active_bytes.all.peak": 699914240,
+        "active_bytes.large_pool.allocated": 736794148864,
+        "active_bytes.large_pool.current": 465043456,
+        "active_bytes.large_pool.freed": 736329105408,
+        "active_bytes.large_pool.peak": 646561792,
+        "active_bytes.small_pool.allocated": 2927243852288,
+        "active_bytes.small_pool.current": 22166016,
+        "active_bytes.small_pool.freed": 2927221686272,
+        "active_bytes.small_pool.peak": 133540864,
+        "allocated_bytes.all.allocated": 3664038001152,
+        "allocated_bytes.all.current": 487209472,
+        "allocated_bytes.all.freed": 3663550791680,
+        "allocated_bytes.all.peak": 699914240,
+        "allocated_bytes.large_pool.allocated": 736794148864,
+        "allocated_bytes.large_pool.current": 465043456,
+        "allocated_bytes.large_pool.freed": 736329105408,
+        "allocated_bytes.large_pool.peak": 646561792,
+        "allocated_bytes.small_pool.allocated": 2927243852288,
+        "allocated_bytes.small_pool.current": 22166016,
+        "allocated_bytes.small_pool.freed": 2927221686272,
+        "allocated_bytes.small_pool.peak": 133540864,
+        "allocation.all.allocated": 8770831,
+        "allocation.all.current": 520,
+        "allocation.all.freed": 8770311,
+        "allocation.all.peak": 777,
+        "allocation.large_pool.allocated": 333282,
+        "allocation.large_pool.current": 81,
+        "allocation.large_pool.freed": 333201,
+        "allocation.large_pool.peak": 183,
+        "allocation.small_pool.allocated": 8437549,
+        "allocation.small_pool.current": 439,
+        "allocation.small_pool.freed": 8437110,
+        "allocation.small_pool.peak": 696,
+        "inactive_split.all.allocated": 4756567,
+        "inactive_split.all.current": 47,
+        "inactive_split.all.freed": 4756520,
+        "inactive_split.all.peak": 119,
+        "inactive_split.large_pool.allocated": 283096,
+        "inactive_split.large_pool.current": 19,
+        "inactive_split.large_pool.freed": 283077,
+        "inactive_split.large_pool.peak": 23,
+        "inactive_split.small_pool.allocated": 4473471,
+        "inactive_split.small_pool.current": 28,
+        "inactive_split.small_pool.freed": 4473443,
+        "inactive_split.small_pool.peak": 99,
+        "inactive_split_bytes.all.allocated": 3923585340928,
+        "inactive_split_bytes.all.current": 60147200,
+        "inactive_split_bytes.all.freed": 3923525193728,
+        "inactive_split_bytes.all.peak": 149930496,
+        "inactive_split_bytes.large_pool.allocated": 836194975744,
+        "inactive_split_bytes.large_pool.current": 48758784,
+        "inactive_split_bytes.large_pool.freed": 836146216960,
+        "inactive_split_bytes.large_pool.peak": 75759616,
+        "inactive_split_bytes.small_pool.allocated": 3087390365184,
+        "inactive_split_bytes.small_pool.current": 11388416,
+        "inactive_split_bytes.small_pool.freed": 3087378976768,
+        "inactive_split_bytes.small_pool.peak": 82985472,
+        "max_split_size": -1,
+        "num_alloc_retries": 0,
+        "num_device_alloc": 97,
+        "num_device_free": 0,
+        "num_ooms": 0,
+        "num_sync_all_streams": 0,
+        "oversize_allocations.allocated": 0,
+        "oversize_allocations.current": 0,
+        "oversize_allocations.freed": 0,
+        "oversize_allocations.peak": 0,
+        "oversize_segments.allocated": 0,
+        "oversize_segments.current": 0,
+        "oversize_segments.freed": 0,
+        "oversize_segments.peak": 0,
+        "requested_bytes.all.allocated": 3531550399964,
+        "requested_bytes.all.current": 486047032,
+        "requested_bytes.all.freed": 3531064352932,
+        "requested_bytes.all.peak": 680092736,
+        "requested_bytes.large_pool.allocated": 604666959872,
+        "requested_bytes.large_pool.current": 463910912,
+        "requested_bytes.large_pool.freed": 604203048960,
+        "requested_bytes.large_pool.peak": 626776064,
+        "requested_bytes.small_pool.allocated": 2926883440092,
+        "requested_bytes.small_pool.current": 22136120,
+        "requested_bytes.small_pool.freed": 2926861303972,
+        "requested_bytes.small_pool.peak": 133501696,
+        "reserved_bytes.all.allocated": 805306368,
+        "reserved_bytes.all.current": 805306368,
+        "reserved_bytes.all.freed": 0,
+        "reserved_bytes.all.peak": 805306368,
+        "reserved_bytes.large_pool.allocated": 660602880,
+        "reserved_bytes.large_pool.current": 660602880,
+        "reserved_bytes.large_pool.freed": 0,
+        "reserved_bytes.large_pool.peak": 660602880,
+        "reserved_bytes.small_pool.allocated": 144703488,
+        "reserved_bytes.small_pool.current": 144703488,
+        "reserved_bytes.small_pool.freed": 0,
+        "reserved_bytes.small_pool.peak": 144703488,
+        "segment.all.allocated": 97,
+        "segment.all.current": 97,
+        "segment.all.freed": 0,
+        "segment.all.peak": 97,
+        "segment.large_pool.allocated": 28,
+        "segment.large_pool.current": 28,
+        "segment.large_pool.freed": 0,
+        "segment.large_pool.peak": 28,
+        "segment.small_pool.allocated": 69,
+        "segment.small_pool.current": 69,
+        "segment.small_pool.freed": 0,
+        "segment.small_pool.peak": 69
+    }
+}

outputs/cola/bert-base-uncased_adapterstrain_val_0.0001_12345_8_10000/step_5999/head_config.json

@@ -0,0 +1,21 @@
+{
+  "config": {
+    "activation_function": "tanh",
+    "bias": true,
+    "dropout_prob": null,
+    "head_type": "classification",
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layers": 2,
+    "num_labels": 2,
+    "use_pooler": false
+  },
+  "hidden_size": 768,
+  "model_class": "BertAdapterModel",
+  "model_name": "bert-base-uncased",
+  "model_type": "bert",
+  "name": "cola",
+  "version": "0.2.1"
+}