Upload 19 files

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+dev_dataset.json filter=lfs diff=lfs merge=lfs -text
+dev.txt filter=lfs diff=lfs merge=lfs -text
+train[[:space:]](1).txt filter=lfs diff=lfs merge=lfs -text
+train_dataset.json filter=lfs diff=lfs merge=lfs -text
+training_loss_curve.png filter=lfs diff=lfs merge=lfs -text
+validation_loss_epoch.png filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "bert-base-uncased",
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 32000
+}

dev.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66089906f03b56e7ed0dc30f43e184ca5d1e91eb8dab8171dcea70ee1ac7a31b
+size 115207574

dev_dataset.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:128c5b5299602e442370d26e80f85cd3a5a5ced47554720f7cdce3da8218a45c
+size 233971657

generation_config.json

@@ -0,0 +1,5 @@
+{
+  "_from_model_config": true,
+  "pad_token_id": 0,
+  "transformers_version": "4.47.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a1f3432ebb307b252a2d3c8b21b1521b0673fd8af3bd4168da369ea1131fb10
+size 442627224

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49f644a1d1a073c65e2c786583e4c78c1ddf158d911d4cb3971b61d6e3cb8c0c
+size 885375994

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf4711db6ba08a35de5b6aa86ac4ee280762106cc41c95a818472cd7f9e72577
+size 14512

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1fbe28552655feb434bdaa33f38b145599943af01aa17fa6a019fcbb5081141
+size 14512

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8be275dd0065b027389a4b9844ea49fc7e44e26eb632a0b4587009fde017e62
+size 1064

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config .json

@@ -0,0 +1,58 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

train (1).txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83106240256fc9b0e316ab936a043f4a07b1212e30cc1d23a025634bbb8ba142
+size 468227148

train_dataset.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95fa22eaeec06ca1dbbbcdaf91d0f172c62fcba6e5ec8c14784d76cba8153dfc
+size 949435972

trainer_state.json

@@ -0,0 +1,333 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 20.0,
+  "eval_steps": 500,
+  "global_step": 96980,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "grad_norm": 4.406856536865234,
+      "learning_rate": 4.750206228088266e-05,
+      "loss": 3.599,
+      "step": 4849
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 2.830796480178833,
+      "eval_runtime": 63.7615,
+      "eval_samples_per_second": 608.329,
+      "eval_steps_per_second": 19.024,
+      "step": 4849
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 3.6979801654815674,
+      "learning_rate": 4.5002577851103326e-05,
+      "loss": 2.6769,
+      "step": 9698
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 2.430718183517456,
+      "eval_runtime": 62.9007,
+      "eval_samples_per_second": 616.654,
+      "eval_steps_per_second": 19.284,
+      "step": 9698
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 3.8321845531463623,
+      "learning_rate": 4.2503608991544655e-05,
+      "loss": 2.3727,
+      "step": 14547
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 2.206211566925049,
+      "eval_runtime": 62.7764,
+      "eval_samples_per_second": 617.876,
+      "eval_steps_per_second": 19.323,
+      "step": 14547
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 3.3764877319335938,
+      "learning_rate": 4.000464013198598e-05,
+      "loss": 2.1798,
+      "step": 19396
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 2.0566577911376953,
+      "eval_runtime": 62.7,
+      "eval_samples_per_second": 618.629,
+      "eval_steps_per_second": 19.346,
+      "step": 19396
+    },
+    {
+      "epoch": 5.0,
+      "grad_norm": 3.017418146133423,
+      "learning_rate": 3.750567127242731e-05,
+      "loss": 2.041,
+      "step": 24245
+    },
+    {
+      "epoch": 5.0,
+      "eval_loss": 1.943306803703308,
+      "eval_runtime": 62.7554,
+      "eval_samples_per_second": 618.082,
+      "eval_steps_per_second": 19.329,
+      "step": 24245
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 3.028724431991577,
+      "learning_rate": 3.500567127242731e-05,
+      "loss": 1.9417,
+      "step": 29094
+    },
+    {
+      "epoch": 6.0,
+      "eval_loss": 1.8640257120132446,
+      "eval_runtime": 63.2843,
+      "eval_samples_per_second": 612.916,
+      "eval_steps_per_second": 19.167,
+      "step": 29094
+    },
+    {
+      "epoch": 7.0,
+      "grad_norm": 2.9556593894958496,
+      "learning_rate": 3.2506702412868636e-05,
+      "loss": 1.8625,
+      "step": 33943
+    },
+    {
+      "epoch": 7.0,
+      "eval_loss": 1.7964398860931396,
+      "eval_runtime": 63.0672,
+      "eval_samples_per_second": 615.027,
+      "eval_steps_per_second": 19.233,
+      "step": 33943
+    },
+    {
+      "epoch": 8.0,
+      "grad_norm": 2.954166889190674,
+      "learning_rate": 3.000670241286863e-05,
+      "loss": 1.797,
+      "step": 38792
+    },
+    {
+      "epoch": 8.0,
+      "eval_loss": 1.7490918636322021,
+      "eval_runtime": 63.1115,
+      "eval_samples_per_second": 614.595,
+      "eval_steps_per_second": 19.22,
+      "step": 38792
+    },
+    {
+      "epoch": 9.0,
+      "grad_norm": 2.954655885696411,
+      "learning_rate": 2.7508249123530627e-05,
+      "loss": 1.7491,
+      "step": 43641
+    },
+    {
+      "epoch": 9.0,
+      "eval_loss": 1.7073553800582886,
+      "eval_runtime": 63.2567,
+      "eval_samples_per_second": 613.184,
+      "eval_steps_per_second": 19.176,
+      "step": 43641
+    },
+    {
+      "epoch": 10.0,
+      "grad_norm": 3.0339479446411133,
+      "learning_rate": 2.5009280263971953e-05,
+      "loss": 1.7049,
+      "step": 48490
+    },
+    {
+      "epoch": 10.0,
+      "eval_loss": 1.6738481521606445,
+      "eval_runtime": 63.2477,
+      "eval_samples_per_second": 613.271,
+      "eval_steps_per_second": 19.179,
+      "step": 48490
+    },
+    {
+      "epoch": 11.0,
+      "grad_norm": 3.4147965908050537,
+      "learning_rate": 2.2509795834192618e-05,
+      "loss": 1.6655,
+      "step": 53339
+    },
+    {
+      "epoch": 11.0,
+      "eval_loss": 1.6354576349258423,
+      "eval_runtime": 63.6046,
+      "eval_samples_per_second": 609.83,
+      "eval_steps_per_second": 19.071,
+      "step": 53339
+    },
+    {
+      "epoch": 12.0,
+      "grad_norm": 3.4822511672973633,
+      "learning_rate": 2.0010826974633944e-05,
+      "loss": 1.6295,
+      "step": 58188
+    },
+    {
+      "epoch": 12.0,
+      "eval_loss": 1.6142407655715942,
+      "eval_runtime": 63.1801,
+      "eval_samples_per_second": 613.927,
+      "eval_steps_per_second": 19.199,
+      "step": 58188
+    },
+    {
+      "epoch": 13.0,
+      "grad_norm": 3.052323341369629,
+      "learning_rate": 1.751134254485461e-05,
+      "loss": 1.6022,
+      "step": 63037
+    },
+    {
+      "epoch": 13.0,
+      "eval_loss": 1.595221757888794,
+      "eval_runtime": 63.3841,
+      "eval_samples_per_second": 611.951,
+      "eval_steps_per_second": 19.137,
+      "step": 63037
+    },
+    {
+      "epoch": 14.0,
+      "grad_norm": 3.0843849182128906,
+      "learning_rate": 1.5012373685295938e-05,
+      "loss": 1.5769,
+      "step": 67886
+    },
+    {
+      "epoch": 14.0,
+      "eval_loss": 1.5756992101669312,
+      "eval_runtime": 61.4761,
+      "eval_samples_per_second": 630.944,
+      "eval_steps_per_second": 19.731,
+      "step": 67886
+    },
+    {
+      "epoch": 15.0,
+      "grad_norm": 3.4379537105560303,
+      "learning_rate": 1.2513404825737265e-05,
+      "loss": 1.5571,
+      "step": 72735
+    },
+    {
+      "epoch": 15.0,
+      "eval_loss": 1.554135799407959,
+      "eval_runtime": 62.2027,
+      "eval_samples_per_second": 623.574,
+      "eval_steps_per_second": 19.501,
+      "step": 72735
+    },
+    {
+      "epoch": 16.0,
+      "grad_norm": 3.0627856254577637,
+      "learning_rate": 1.0014435966178593e-05,
+      "loss": 1.5343,
+      "step": 77584
+    },
+    {
+      "epoch": 16.0,
+      "eval_loss": 1.541438102722168,
+      "eval_runtime": 61.9741,
+      "eval_samples_per_second": 625.874,
+      "eval_steps_per_second": 19.573,
+      "step": 77584
+    },
+    {
+      "epoch": 17.0,
+      "grad_norm": 3.1744439601898193,
+      "learning_rate": 7.5154671066199224e-06,
+      "loss": 1.5188,
+      "step": 82433
+    },
+    {
+      "epoch": 17.0,
+      "eval_loss": 1.5268478393554688,
+      "eval_runtime": 61.9172,
+      "eval_samples_per_second": 626.449,
+      "eval_steps_per_second": 19.591,
+      "step": 82433
+    },
+    {
+      "epoch": 18.0,
+      "grad_norm": 3.5251266956329346,
+      "learning_rate": 5.015982676840586e-06,
+      "loss": 1.5037,
+      "step": 87282
+    },
+    {
+      "epoch": 18.0,
+      "eval_loss": 1.5217667818069458,
+      "eval_runtime": 61.3504,
+      "eval_samples_per_second": 632.237,
+      "eval_steps_per_second": 19.772,
+      "step": 87282
+    },
+    {
+      "epoch": 19.0,
+      "grad_norm": 3.248511552810669,
+      "learning_rate": 2.517013817281914e-06,
+      "loss": 1.4927,
+      "step": 92131
+    },
+    {
+      "epoch": 19.0,
+      "eval_loss": 1.511409044265747,
+      "eval_runtime": 62.5116,
+      "eval_samples_per_second": 620.493,
+      "eval_steps_per_second": 19.404,
+      "step": 92131
+    },
+    {
+      "epoch": 20.0,
+      "grad_norm": 2.856531858444214,
+      "learning_rate": 1.7529387502577853e-08,
+      "loss": 1.4862,
+      "step": 96980
+    },
+    {
+      "epoch": 20.0,
+      "eval_loss": 1.5111480951309204,
+      "eval_runtime": 61.8388,
+      "eval_samples_per_second": 627.244,
+      "eval_steps_per_second": 19.616,
+      "step": 96980
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 96980,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 20,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.0420834325692416e+17,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49a71c611b38872245cf1746cb2adc5590d4f3efa157b052d24f39577a822455
+size 5240