Add checkpoint checkpoint-5000

Browse files

Files changed (14) hide show

.gitattributes +1 -0
colbert_linear.pt +3 -0
config.json +28 -0
model.safetensors +3 -0
optimizer.pt +3 -0
rng_state.pth +3 -0
scheduler.pt +3 -0
sentencepiece.bpe.model +3 -0
sparse_linear.pt +3 -0
special_tokens_map.json +51 -0
tokenizer.json +3 -0
tokenizer_config.json +55 -0
trainer_state.json +383 -0
training_args.bin +3 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

colbert_linear.pt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:bf920cbcd5fff1e26737b298e6736fd9a39979c6f3d1f07e662150f124d07cf1
+size 4200349

config.json ADDED Viewed

	@@ -0,0 +1,28 @@

+{
+  "_name_or_path": "/home/mila/u/uemurak/scratch/hf_cache/hub/models--intfloat--multilingual-e5-large-instruct/snapshots/274baa43b0e13e37fafa6428dbc7938e62e5c439",
+  "architectures": [
+    "XLMRobertaModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.2",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250002
+}

model.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:7d0c2157bbf6da25784a393927864740e1286b428f4e8acf4356351943297df6
+size 2239607176

optimizer.pt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:d7010798705c7af2da89fc46f163e7a27c3340cbb58de7aeee3c52202ca125c6
+size 4479470178

rng_state.pth ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:66b828711f6275c1fb5c8a7e665344607c002df80651eedbdb814435815a0e8d
+size 14645

scheduler.pt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:cf10fcc00ed3305c79f3603198ec0cc4095f3e760a66b10105be48049fca5610
+size 1465

sentencepiece.bpe.model ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

sparse_linear.pt ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:4c33a60503ce10d7b0d7ddd4db2f68994b1d1c4a85dff1a8ab0f2f9f37af11af
+size 5973

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,51 @@

+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:8373f9cd3d27591e1924426bcc1c8799bc5a9affc4fc857982c5d66668dd1f41
+size 17082832

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,55 @@

+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}

trainer_state.json ADDED Viewed

	@@ -0,0 +1,383 @@

+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.6862475981334065,
+  "eval_steps": 500,
+  "global_step": 5000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01372495196266813,
+      "grad_norm": 23.43617820739746,
+      "learning_rate": 6.378600823045268e-07,
+      "loss": 3.6307,
+      "step": 100
+    },
+    {
+      "epoch": 0.02744990392533626,
+      "grad_norm": 27.224123001098633,
+      "learning_rate": 1.3237311385459534e-06,
+      "loss": 0.8845,
+      "step": 200
+    },
+    {
+      "epoch": 0.04117485588800439,
+      "grad_norm": 22.287761688232422,
+      "learning_rate": 2.00960219478738e-06,
+      "loss": 0.5517,
+      "step": 300
+    },
+    {
+      "epoch": 0.05489980785067252,
+      "grad_norm": 14.750782012939453,
+      "learning_rate": 2.6954732510288067e-06,
+      "loss": 0.3488,
+      "step": 400
+    },
+    {
+      "epoch": 0.06862475981334065,
+      "grad_norm": 51.949310302734375,
+      "learning_rate": 3.3813443072702336e-06,
+      "loss": 0.3129,
+      "step": 500
+    },
+    {
+      "epoch": 0.08234971177600878,
+      "grad_norm": 10.03300666809082,
+      "learning_rate": 4.06721536351166e-06,
+      "loss": 0.2679,
+      "step": 600
+    },
+    {
+      "epoch": 0.09607466373867692,
+      "grad_norm": 26.423612594604492,
+      "learning_rate": 4.753086419753087e-06,
+      "loss": 0.2911,
+      "step": 700
+    },
+    {
+      "epoch": 0.10979961570134504,
+      "grad_norm": 3.058533191680908,
+      "learning_rate": 5.438957475994513e-06,
+      "loss": 0.2801,
+      "step": 800
+    },
+    {
+      "epoch": 0.12352456766401318,
+      "grad_norm": 18.2351016998291,
+      "learning_rate": 6.12482853223594e-06,
+      "loss": 0.2771,
+      "step": 900
+    },
+    {
+      "epoch": 0.1372495196266813,
+      "grad_norm": 7.775646686553955,
+      "learning_rate": 6.810699588477366e-06,
+      "loss": 0.2446,
+      "step": 1000
+    },
+    {
+      "epoch": 0.15097447158934943,
+      "grad_norm": 10.91329574584961,
+      "learning_rate": 7.496570644718793e-06,
+      "loss": 0.2736,
+      "step": 1100
+    },
+    {
+      "epoch": 0.16469942355201755,
+      "grad_norm": 32.206260681152344,
+      "learning_rate": 8.18244170096022e-06,
+      "loss": 0.2983,
+      "step": 1200
+    },
+    {
+      "epoch": 0.1784243755146857,
+      "grad_norm": 30.968856811523438,
+      "learning_rate": 8.868312757201646e-06,
+      "loss": 0.2933,
+      "step": 1300
+    },
+    {
+      "epoch": 0.19214932747735383,
+      "grad_norm": 31.030960083007812,
+      "learning_rate": 9.554183813443072e-06,
+      "loss": 0.2826,
+      "step": 1400
+    },
+    {
+      "epoch": 0.20587427944002196,
+      "grad_norm": 17.41258430480957,
+      "learning_rate": 9.97331096538051e-06,
+      "loss": 0.2527,
+      "step": 1500
+    },
+    {
+      "epoch": 0.21959923140269008,
+      "grad_norm": 4.282568454742432,
+      "learning_rate": 9.897056580753394e-06,
+      "loss": 0.2468,
+      "step": 1600
+    },
+    {
+      "epoch": 0.23332418336535823,
+      "grad_norm": 14.256071090698242,
+      "learning_rate": 9.820802196126277e-06,
+      "loss": 0.2618,
+      "step": 1700
+    },
+    {
+      "epoch": 0.24704913532802636,
+      "grad_norm": 32.6270866394043,
+      "learning_rate": 9.744547811499163e-06,
+      "loss": 0.2557,
+      "step": 1800
+    },
+    {
+      "epoch": 0.26077408729069446,
+      "grad_norm": 5.828367710113525,
+      "learning_rate": 9.668293426872047e-06,
+      "loss": 0.2713,
+      "step": 1900
+    },
+    {
+      "epoch": 0.2744990392533626,
+      "grad_norm": 19.892574310302734,
+      "learning_rate": 9.592039042244929e-06,
+      "loss": 0.2482,
+      "step": 2000
+    },
+    {
+      "epoch": 0.28822399121603076,
+      "grad_norm": 20.713647842407227,
+      "learning_rate": 9.515784657617814e-06,
+      "loss": 0.2102,
+      "step": 2100
+    },
+    {
+      "epoch": 0.30194894317869886,
+      "grad_norm": 20.811189651489258,
+      "learning_rate": 9.439530272990698e-06,
+      "loss": 0.2071,
+      "step": 2200
+    },
+    {
+      "epoch": 0.315673895141367,
+      "grad_norm": 0.3585197925567627,
+      "learning_rate": 9.363275888363582e-06,
+      "loss": 0.2159,
+      "step": 2300
+    },
+    {
+      "epoch": 0.3293988471040351,
+      "grad_norm": 28.223173141479492,
+      "learning_rate": 9.287021503736465e-06,
+      "loss": 0.2876,
+      "step": 2400
+    },
+    {
+      "epoch": 0.34312379906670326,
+      "grad_norm": 3.1307175159454346,
+      "learning_rate": 9.21076711910935e-06,
+      "loss": 0.2182,
+      "step": 2500
+    },
+    {
+      "epoch": 0.3568487510293714,
+      "grad_norm": 16.897327423095703,
+      "learning_rate": 9.134512734482233e-06,
+      "loss": 0.2483,
+      "step": 2600
+    },
+    {
+      "epoch": 0.3705737029920395,
+      "grad_norm": 3.389308214187622,
+      "learning_rate": 9.058258349855117e-06,
+      "loss": 0.1956,
+      "step": 2700
+    },
+    {
+      "epoch": 0.38429865495470766,
+      "grad_norm": 14.658860206604004,
+      "learning_rate": 8.982003965228e-06,
+      "loss": 0.2176,
+      "step": 2800
+    },
+    {
+      "epoch": 0.3980236069173758,
+      "grad_norm": 15.268556594848633,
+      "learning_rate": 8.905749580600886e-06,
+      "loss": 0.2009,
+      "step": 2900
+    },
+    {
+      "epoch": 0.4117485588800439,
+      "grad_norm": 23.204755783081055,
+      "learning_rate": 8.82949519597377e-06,
+      "loss": 0.2218,
+      "step": 3000
+    },
+    {
+      "epoch": 0.42547351084271207,
+      "grad_norm": 7.059377670288086,
+      "learning_rate": 8.753240811346654e-06,
+      "loss": 0.2085,
+      "step": 3100
+    },
+    {
+      "epoch": 0.43919846280538016,
+      "grad_norm": 3.764357328414917,
+      "learning_rate": 8.676986426719537e-06,
+      "loss": 0.2096,
+      "step": 3200
+    },
+    {
+      "epoch": 0.4529234147680483,
+      "grad_norm": 13.846981048583984,
+      "learning_rate": 8.600732042092421e-06,
+      "loss": 0.2364,
+      "step": 3300
+    },
+    {
+      "epoch": 0.46664836673071647,
+      "grad_norm": 9.728800773620605,
+      "learning_rate": 8.524477657465305e-06,
+      "loss": 0.2173,
+      "step": 3400
+    },
+    {
+      "epoch": 0.48037331869338457,
+      "grad_norm": 7.5195841789245605,
+      "learning_rate": 8.448223272838189e-06,
+      "loss": 0.202,
+      "step": 3500
+    },
+    {
+      "epoch": 0.4940982706560527,
+      "grad_norm": 4.6686224937438965,
+      "learning_rate": 8.371968888211072e-06,
+      "loss": 0.1921,
+      "step": 3600
+    },
+    {
+      "epoch": 0.5078232226187208,
+      "grad_norm": 18.52180290222168,
+      "learning_rate": 8.295714503583956e-06,
+      "loss": 0.2086,
+      "step": 3700
+    },
+    {
+      "epoch": 0.5215481745813889,
+      "grad_norm": 2.1525790691375732,
+      "learning_rate": 8.21946011895684e-06,
+      "loss": 0.1939,
+      "step": 3800
+    },
+    {
+      "epoch": 0.5352731265440571,
+      "grad_norm": 18.92357635498047,
+      "learning_rate": 8.143205734329724e-06,
+      "loss": 0.1995,
+      "step": 3900
+    },
+    {
+      "epoch": 0.5489980785067252,
+      "grad_norm": 2.638871192932129,
+      "learning_rate": 8.06695134970261e-06,
+      "loss": 0.1913,
+      "step": 4000
+    },
+    {
+      "epoch": 0.5627230304693933,
+      "grad_norm": 15.569317817687988,
+      "learning_rate": 7.990696965075493e-06,
+      "loss": 0.186,
+      "step": 4100
+    },
+    {
+      "epoch": 0.5764479824320615,
+      "grad_norm": 8.418330192565918,
+      "learning_rate": 7.914442580448377e-06,
+      "loss": 0.1761,
+      "step": 4200
+    },
+    {
+      "epoch": 0.5901729343947296,
+      "grad_norm": 37.495304107666016,
+      "learning_rate": 7.83818819582126e-06,
+      "loss": 0.205,
+      "step": 4300
+    },
+    {
+      "epoch": 0.6038978863573977,
+      "grad_norm": 16.499755859375,
+      "learning_rate": 7.761933811194144e-06,
+      "loss": 0.1875,
+      "step": 4400
+    },
+    {
+      "epoch": 0.6176228383200659,
+      "grad_norm": 2.1567940711975098,
+      "learning_rate": 7.685679426567028e-06,
+      "loss": 0.1995,
+      "step": 4500
+    },
+    {
+      "epoch": 0.631347790282734,
+      "grad_norm": 3.6063127517700195,
+      "learning_rate": 7.609425041939912e-06,
+      "loss": 0.1703,
+      "step": 4600
+    },
+    {
+      "epoch": 0.6450727422454021,
+      "grad_norm": 10.337509155273438,
+      "learning_rate": 7.5331706573127965e-06,
+      "loss": 0.1889,
+      "step": 4700
+    },
+    {
+      "epoch": 0.6587976942080702,
+      "grad_norm": 5.367345333099365,
+      "learning_rate": 7.45691627268568e-06,
+      "loss": 0.1731,
+      "step": 4800
+    },
+    {
+      "epoch": 0.6725226461707384,
+      "grad_norm": 10.134766578674316,
+      "learning_rate": 7.380661888058563e-06,
+      "loss": 0.1984,
+      "step": 4900
+    },
+    {
+      "epoch": 0.6862475981334065,
+      "grad_norm": 9.244234085083008,
+      "learning_rate": 7.304407503431448e-06,
+      "loss": 0.1799,
+      "step": 5000
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 14572,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:da2e11e29bb8ed02d32b7e47ef55d93abc00bce7e31cc52df79b7b099c62dec8
+size 6225