DGurgurov commited on
Commit
7c085c1
·
verified ·
1 Parent(s): 8a7483c

Uploading checkpoint-16500 for mbert - yor-latn

Browse files
README.md ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ license: apache-2.0
4
+ base_model: google-bert/bert-base-multilingual-cased
5
+ tags:
6
+ - generated_from_trainer
7
+ metrics:
8
+ - accuracy
9
+ model-index:
10
+ - name: yor-Latn
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # yor-Latn
18
+
19
+ This model is a fine-tuned version of [google-bert/bert-base-multilingual-cased](https://huggingface.co/google-bert/bert-base-multilingual-cased) on an unknown dataset.
20
+ It achieves the following results on the evaluation set:
21
+ - Loss: 0.7111
22
+ - Accuracy: 0.8622
23
+
24
+ ## Model description
25
+
26
+ More information needed
27
+
28
+ ## Intended uses & limitations
29
+
30
+ More information needed
31
+
32
+ ## Training and evaluation data
33
+
34
+ More information needed
35
+
36
+ ## Training procedure
37
+
38
+ ### Training hyperparameters
39
+
40
+ The following hyperparameters were used during training:
41
+ - learning_rate: 0.0001
42
+ - train_batch_size: 16
43
+ - eval_batch_size: 16
44
+ - seed: 42
45
+ - distributed_type: multi-GPU
46
+ - num_devices: 2
47
+ - total_train_batch_size: 32
48
+ - total_eval_batch_size: 32
49
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
50
+ - lr_scheduler_type: linear
51
+ - training_steps: 100000
52
+
53
+
54
+ ### Citation Information
55
+
56
+ If you use this model in your work, please cite the following paper. Additionally, if you require more details on training and performance, refer to the paper:
57
+
58
+ @misc{gurgurov2025smallmodelsbigimpact,
59
+ title={Small Models, Big Impact: Efficient Corpus and Graph-Based Adaptation of Small Multilingual Language Models for Low-Resource Languages},
60
+ author={Daniil Gurgurov and Ivan Vykopal and Josef van Genabith and Simon Ostermann},
61
+ year={2025},
62
+ eprint={2502.10140},
63
+ archivePrefix={arXiv},
64
+ primaryClass={cs.CL},
65
+ url={https://arxiv.org/abs/2502.10140},
66
+ }
67
+
config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "google-bert/bert-base-multilingual-cased",
3
+ "adapters": {
4
+ "adapters": {},
5
+ "config_map": {},
6
+ "fusion_config_map": {},
7
+ "fusions": {}
8
+ },
9
+ "architectures": [
10
+ "BertForMaskedLM"
11
+ ],
12
+ "attention_probs_dropout_prob": 0.1,
13
+ "classifier_dropout": null,
14
+ "directionality": "bidi",
15
+ "hidden_act": "gelu",
16
+ "hidden_dropout_prob": 0.1,
17
+ "hidden_size": 768,
18
+ "initializer_range": 0.02,
19
+ "intermediate_size": 3072,
20
+ "layer_norm_eps": 1e-12,
21
+ "max_position_embeddings": 512,
22
+ "model_type": "bert",
23
+ "num_attention_heads": 12,
24
+ "num_hidden_layers": 12,
25
+ "pad_token_id": 0,
26
+ "pooler_fc_size": 768,
27
+ "pooler_num_attention_heads": 12,
28
+ "pooler_num_fc_layers": 3,
29
+ "pooler_size_per_head": 128,
30
+ "pooler_type": "first_token_transform",
31
+ "position_embedding_type": "absolute",
32
+ "torch_dtype": "float32",
33
+ "transformers_version": "4.45.2",
34
+ "type_vocab_size": 2,
35
+ "use_cache": true,
36
+ "vocab_size": 119547
37
+ }
generation_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "pad_token_id": 0,
4
+ "transformers_version": "4.45.2"
5
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9fe5f75d7a1b6be36560a7e4f4cb5ea2dceb034fa24fb7bdf253b82dbab4f3a1
3
+ size 711921812
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0bb18be482d8a8650d3b234d374fed2f09ae64dc2e0a825759d013849c8fd9b
3
+ size 1423965114
rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:94fbd7761436e9c833f05f957e2fac9af6d9e26f15e827da7110de569ef60d9a
3
+ size 14512
rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fc74469ea6e4fd79dad1c43e5ff3031b2fe13006169852ac80b831e704841341
3
+ size 14512
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1a99ba211dcd8f166f10cdea2d9738b789b8bbac03d39d321722c3b6dfe500ff
3
+ size 1064
special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "[CLS]",
3
+ "mask_token": "[MASK]",
4
+ "pad_token": "[PAD]",
5
+ "sep_token": "[SEP]",
6
+ "unk_token": "[UNK]"
7
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "[PAD]",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "100": {
12
+ "content": "[UNK]",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "101": {
20
+ "content": "[CLS]",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "102": {
28
+ "content": "[SEP]",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "103": {
36
+ "content": "[MASK]",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "clean_up_tokenization_spaces": false,
45
+ "cls_token": "[CLS]",
46
+ "do_lower_case": false,
47
+ "mask_token": "[MASK]",
48
+ "model_max_length": 512,
49
+ "pad_token": "[PAD]",
50
+ "sep_token": "[SEP]",
51
+ "strip_accents": null,
52
+ "tokenize_chinese_chars": true,
53
+ "tokenizer_class": "BertTokenizer",
54
+ "unk_token": "[UNK]"
55
+ }
trainer_state.json ADDED
@@ -0,0 +1,561 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.6883311867713928,
3
+ "best_model_checkpoint": "./model_fine-tune/glot/mbert/yor-Latn/checkpoint-16500",
4
+ "epoch": 67.34693877551021,
5
+ "eval_steps": 500,
6
+ "global_step": 16500,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 2.0408163265306123,
13
+ "grad_norm": 3.014599323272705,
14
+ "learning_rate": 9.95e-05,
15
+ "loss": 1.5111,
16
+ "step": 500
17
+ },
18
+ {
19
+ "epoch": 2.0408163265306123,
20
+ "eval_accuracy": 0.7682061550515823,
21
+ "eval_loss": 1.1437931060791016,
22
+ "eval_runtime": 16.6138,
23
+ "eval_samples_per_second": 117.974,
24
+ "eval_steps_per_second": 3.732,
25
+ "step": 500
26
+ },
27
+ {
28
+ "epoch": 4.081632653061225,
29
+ "grad_norm": 2.2488036155700684,
30
+ "learning_rate": 9.900000000000001e-05,
31
+ "loss": 1.111,
32
+ "step": 1000
33
+ },
34
+ {
35
+ "epoch": 4.081632653061225,
36
+ "eval_accuracy": 0.7948809959721713,
37
+ "eval_loss": 0.9893151521682739,
38
+ "eval_runtime": 16.5211,
39
+ "eval_samples_per_second": 118.636,
40
+ "eval_steps_per_second": 3.753,
41
+ "step": 1000
42
+ },
43
+ {
44
+ "epoch": 6.122448979591836,
45
+ "grad_norm": 2.612577438354492,
46
+ "learning_rate": 9.850000000000001e-05,
47
+ "loss": 0.9843,
48
+ "step": 1500
49
+ },
50
+ {
51
+ "epoch": 6.122448979591836,
52
+ "eval_accuracy": 0.8096590077564759,
53
+ "eval_loss": 0.9186487793922424,
54
+ "eval_runtime": 16.503,
55
+ "eval_samples_per_second": 118.766,
56
+ "eval_steps_per_second": 3.757,
57
+ "step": 1500
58
+ },
59
+ {
60
+ "epoch": 8.16326530612245,
61
+ "grad_norm": 2.055933713912964,
62
+ "learning_rate": 9.8e-05,
63
+ "loss": 0.9099,
64
+ "step": 2000
65
+ },
66
+ {
67
+ "epoch": 8.16326530612245,
68
+ "eval_accuracy": 0.8180220037796107,
69
+ "eval_loss": 0.8695068359375,
70
+ "eval_runtime": 16.5184,
71
+ "eval_samples_per_second": 118.656,
72
+ "eval_steps_per_second": 3.753,
73
+ "step": 2000
74
+ },
75
+ {
76
+ "epoch": 10.204081632653061,
77
+ "grad_norm": 2.095768928527832,
78
+ "learning_rate": 9.75e-05,
79
+ "loss": 0.8553,
80
+ "step": 2500
81
+ },
82
+ {
83
+ "epoch": 10.204081632653061,
84
+ "eval_accuracy": 0.8226890203332577,
85
+ "eval_loss": 0.853586733341217,
86
+ "eval_runtime": 16.0076,
87
+ "eval_samples_per_second": 122.442,
88
+ "eval_steps_per_second": 3.873,
89
+ "step": 2500
90
+ },
91
+ {
92
+ "epoch": 12.244897959183673,
93
+ "grad_norm": 2.289365530014038,
94
+ "learning_rate": 9.7e-05,
95
+ "loss": 0.8136,
96
+ "step": 3000
97
+ },
98
+ {
99
+ "epoch": 12.244897959183673,
100
+ "eval_accuracy": 0.8261254926090332,
101
+ "eval_loss": 0.834091067314148,
102
+ "eval_runtime": 16.0504,
103
+ "eval_samples_per_second": 122.115,
104
+ "eval_steps_per_second": 3.863,
105
+ "step": 3000
106
+ },
107
+ {
108
+ "epoch": 14.285714285714286,
109
+ "grad_norm": 2.0706839561462402,
110
+ "learning_rate": 9.65e-05,
111
+ "loss": 0.7732,
112
+ "step": 3500
113
+ },
114
+ {
115
+ "epoch": 14.285714285714286,
116
+ "eval_accuracy": 0.8323066793474285,
117
+ "eval_loss": 0.7992222309112549,
118
+ "eval_runtime": 15.9988,
119
+ "eval_samples_per_second": 122.509,
120
+ "eval_steps_per_second": 3.875,
121
+ "step": 3500
122
+ },
123
+ {
124
+ "epoch": 16.3265306122449,
125
+ "grad_norm": 2.018846273422241,
126
+ "learning_rate": 9.6e-05,
127
+ "loss": 0.7447,
128
+ "step": 4000
129
+ },
130
+ {
131
+ "epoch": 16.3265306122449,
132
+ "eval_accuracy": 0.8354064746360119,
133
+ "eval_loss": 0.7927262783050537,
134
+ "eval_runtime": 16.0248,
135
+ "eval_samples_per_second": 122.31,
136
+ "eval_steps_per_second": 3.869,
137
+ "step": 4000
138
+ },
139
+ {
140
+ "epoch": 18.367346938775512,
141
+ "grad_norm": 1.8335440158843994,
142
+ "learning_rate": 9.55e-05,
143
+ "loss": 0.7207,
144
+ "step": 4500
145
+ },
146
+ {
147
+ "epoch": 18.367346938775512,
148
+ "eval_accuracy": 0.8355885295341857,
149
+ "eval_loss": 0.7959266901016235,
150
+ "eval_runtime": 16.1763,
151
+ "eval_samples_per_second": 121.165,
152
+ "eval_steps_per_second": 3.833,
153
+ "step": 4500
154
+ },
155
+ {
156
+ "epoch": 20.408163265306122,
157
+ "grad_norm": 2.1816558837890625,
158
+ "learning_rate": 9.5e-05,
159
+ "loss": 0.6936,
160
+ "step": 5000
161
+ },
162
+ {
163
+ "epoch": 20.408163265306122,
164
+ "eval_accuracy": 0.8416515194650666,
165
+ "eval_loss": 0.7648417353630066,
166
+ "eval_runtime": 16.0654,
167
+ "eval_samples_per_second": 122.001,
168
+ "eval_steps_per_second": 3.859,
169
+ "step": 5000
170
+ },
171
+ {
172
+ "epoch": 22.448979591836736,
173
+ "grad_norm": 1.9500852823257446,
174
+ "learning_rate": 9.449999999999999e-05,
175
+ "loss": 0.6715,
176
+ "step": 5500
177
+ },
178
+ {
179
+ "epoch": 22.448979591836736,
180
+ "eval_accuracy": 0.8432883182052985,
181
+ "eval_loss": 0.7536068558692932,
182
+ "eval_runtime": 16.0018,
183
+ "eval_samples_per_second": 122.486,
184
+ "eval_steps_per_second": 3.875,
185
+ "step": 5500
186
+ },
187
+ {
188
+ "epoch": 24.489795918367346,
189
+ "grad_norm": 1.791885495185852,
190
+ "learning_rate": 9.4e-05,
191
+ "loss": 0.6501,
192
+ "step": 6000
193
+ },
194
+ {
195
+ "epoch": 24.489795918367346,
196
+ "eval_accuracy": 0.8429277638834043,
197
+ "eval_loss": 0.7491132616996765,
198
+ "eval_runtime": 16.0055,
199
+ "eval_samples_per_second": 122.458,
200
+ "eval_steps_per_second": 3.874,
201
+ "step": 6000
202
+ },
203
+ {
204
+ "epoch": 26.53061224489796,
205
+ "grad_norm": 1.9352576732635498,
206
+ "learning_rate": 9.350000000000001e-05,
207
+ "loss": 0.635,
208
+ "step": 6500
209
+ },
210
+ {
211
+ "epoch": 26.53061224489796,
212
+ "eval_accuracy": 0.8484582078727005,
213
+ "eval_loss": 0.7326425909996033,
214
+ "eval_runtime": 16.0205,
215
+ "eval_samples_per_second": 122.343,
216
+ "eval_steps_per_second": 3.87,
217
+ "step": 6500
218
+ },
219
+ {
220
+ "epoch": 28.571428571428573,
221
+ "grad_norm": 2.0857200622558594,
222
+ "learning_rate": 9.300000000000001e-05,
223
+ "loss": 0.6189,
224
+ "step": 7000
225
+ },
226
+ {
227
+ "epoch": 28.571428571428573,
228
+ "eval_accuracy": 0.8482937496821441,
229
+ "eval_loss": 0.7334189414978027,
230
+ "eval_runtime": 16.0058,
231
+ "eval_samples_per_second": 122.455,
232
+ "eval_steps_per_second": 3.874,
233
+ "step": 7000
234
+ },
235
+ {
236
+ "epoch": 30.612244897959183,
237
+ "grad_norm": 2.3431918621063232,
238
+ "learning_rate": 9.250000000000001e-05,
239
+ "loss": 0.6035,
240
+ "step": 7500
241
+ },
242
+ {
243
+ "epoch": 30.612244897959183,
244
+ "eval_accuracy": 0.8485175910580918,
245
+ "eval_loss": 0.740364670753479,
246
+ "eval_runtime": 16.0104,
247
+ "eval_samples_per_second": 122.421,
248
+ "eval_steps_per_second": 3.872,
249
+ "step": 7500
250
+ },
251
+ {
252
+ "epoch": 32.6530612244898,
253
+ "grad_norm": 1.8592019081115723,
254
+ "learning_rate": 9.200000000000001e-05,
255
+ "loss": 0.586,
256
+ "step": 8000
257
+ },
258
+ {
259
+ "epoch": 32.6530612244898,
260
+ "eval_accuracy": 0.8498935079850839,
261
+ "eval_loss": 0.7310737371444702,
262
+ "eval_runtime": 16.0169,
263
+ "eval_samples_per_second": 122.371,
264
+ "eval_steps_per_second": 3.871,
265
+ "step": 8000
266
+ },
267
+ {
268
+ "epoch": 34.69387755102041,
269
+ "grad_norm": 2.139171838760376,
270
+ "learning_rate": 9.15e-05,
271
+ "loss": 0.5741,
272
+ "step": 8500
273
+ },
274
+ {
275
+ "epoch": 34.69387755102041,
276
+ "eval_accuracy": 0.8535219936836398,
277
+ "eval_loss": 0.7162159085273743,
278
+ "eval_runtime": 16.0256,
279
+ "eval_samples_per_second": 122.305,
280
+ "eval_steps_per_second": 3.869,
281
+ "step": 8500
282
+ },
283
+ {
284
+ "epoch": 36.734693877551024,
285
+ "grad_norm": 2.130950927734375,
286
+ "learning_rate": 9.1e-05,
287
+ "loss": 0.5613,
288
+ "step": 9000
289
+ },
290
+ {
291
+ "epoch": 36.734693877551024,
292
+ "eval_accuracy": 0.8531061006400094,
293
+ "eval_loss": 0.7184491753578186,
294
+ "eval_runtime": 16.5555,
295
+ "eval_samples_per_second": 118.39,
296
+ "eval_steps_per_second": 3.745,
297
+ "step": 9000
298
+ },
299
+ {
300
+ "epoch": 38.775510204081634,
301
+ "grad_norm": 1.8260786533355713,
302
+ "learning_rate": 9.05e-05,
303
+ "loss": 0.548,
304
+ "step": 9500
305
+ },
306
+ {
307
+ "epoch": 38.775510204081634,
308
+ "eval_accuracy": 0.8545740978201136,
309
+ "eval_loss": 0.7058050036430359,
310
+ "eval_runtime": 16.5238,
311
+ "eval_samples_per_second": 118.617,
312
+ "eval_steps_per_second": 3.752,
313
+ "step": 9500
314
+ },
315
+ {
316
+ "epoch": 40.816326530612244,
317
+ "grad_norm": 1.890523076057434,
318
+ "learning_rate": 9e-05,
319
+ "loss": 0.5395,
320
+ "step": 10000
321
+ },
322
+ {
323
+ "epoch": 40.816326530612244,
324
+ "eval_accuracy": 0.857211986128117,
325
+ "eval_loss": 0.710237443447113,
326
+ "eval_runtime": 16.486,
327
+ "eval_samples_per_second": 118.889,
328
+ "eval_steps_per_second": 3.761,
329
+ "step": 10000
330
+ },
331
+ {
332
+ "epoch": 42.857142857142854,
333
+ "grad_norm": 2.4147567749023438,
334
+ "learning_rate": 8.950000000000001e-05,
335
+ "loss": 0.5266,
336
+ "step": 10500
337
+ },
338
+ {
339
+ "epoch": 42.857142857142854,
340
+ "eval_accuracy": 0.8557428516937341,
341
+ "eval_loss": 0.7127901315689087,
342
+ "eval_runtime": 16.5108,
343
+ "eval_samples_per_second": 118.71,
344
+ "eval_steps_per_second": 3.755,
345
+ "step": 10500
346
+ },
347
+ {
348
+ "epoch": 44.89795918367347,
349
+ "grad_norm": 2.065382242202759,
350
+ "learning_rate": 8.900000000000001e-05,
351
+ "loss": 0.5156,
352
+ "step": 11000
353
+ },
354
+ {
355
+ "epoch": 44.89795918367347,
356
+ "eval_accuracy": 0.8571209622093447,
357
+ "eval_loss": 0.7171084880828857,
358
+ "eval_runtime": 16.5009,
359
+ "eval_samples_per_second": 118.782,
360
+ "eval_steps_per_second": 3.757,
361
+ "step": 11000
362
+ },
363
+ {
364
+ "epoch": 46.93877551020408,
365
+ "grad_norm": 2.3719356060028076,
366
+ "learning_rate": 8.850000000000001e-05,
367
+ "loss": 0.5088,
368
+ "step": 11500
369
+ },
370
+ {
371
+ "epoch": 46.93877551020408,
372
+ "eval_accuracy": 0.8566442539649951,
373
+ "eval_loss": 0.7080183029174805,
374
+ "eval_runtime": 16.52,
375
+ "eval_samples_per_second": 118.644,
376
+ "eval_steps_per_second": 3.753,
377
+ "step": 11500
378
+ },
379
+ {
380
+ "epoch": 48.97959183673469,
381
+ "grad_norm": 1.9313708543777466,
382
+ "learning_rate": 8.800000000000001e-05,
383
+ "loss": 0.497,
384
+ "step": 12000
385
+ },
386
+ {
387
+ "epoch": 48.97959183673469,
388
+ "eval_accuracy": 0.8599520731972987,
389
+ "eval_loss": 0.709360659122467,
390
+ "eval_runtime": 16.5272,
391
+ "eval_samples_per_second": 118.592,
392
+ "eval_steps_per_second": 3.751,
393
+ "step": 12000
394
+ },
395
+ {
396
+ "epoch": 51.02040816326531,
397
+ "grad_norm": 1.9448109865188599,
398
+ "learning_rate": 8.75e-05,
399
+ "loss": 0.4888,
400
+ "step": 12500
401
+ },
402
+ {
403
+ "epoch": 51.02040816326531,
404
+ "eval_accuracy": 0.8609216073157768,
405
+ "eval_loss": 0.706400454044342,
406
+ "eval_runtime": 16.504,
407
+ "eval_samples_per_second": 118.759,
408
+ "eval_steps_per_second": 3.757,
409
+ "step": 12500
410
+ },
411
+ {
412
+ "epoch": 53.06122448979592,
413
+ "grad_norm": 1.7106282711029053,
414
+ "learning_rate": 8.7e-05,
415
+ "loss": 0.4782,
416
+ "step": 13000
417
+ },
418
+ {
419
+ "epoch": 53.06122448979592,
420
+ "eval_accuracy": 0.8604385547461287,
421
+ "eval_loss": 0.7035485506057739,
422
+ "eval_runtime": 16.5812,
423
+ "eval_samples_per_second": 118.206,
424
+ "eval_steps_per_second": 3.739,
425
+ "step": 13000
426
+ },
427
+ {
428
+ "epoch": 55.10204081632653,
429
+ "grad_norm": 2.121896982192993,
430
+ "learning_rate": 8.65e-05,
431
+ "loss": 0.4719,
432
+ "step": 13500
433
+ },
434
+ {
435
+ "epoch": 55.10204081632653,
436
+ "eval_accuracy": 0.8596197153330952,
437
+ "eval_loss": 0.716853678226471,
438
+ "eval_runtime": 16.5563,
439
+ "eval_samples_per_second": 118.384,
440
+ "eval_steps_per_second": 3.745,
441
+ "step": 13500
442
+ },
443
+ {
444
+ "epoch": 57.142857142857146,
445
+ "grad_norm": 1.8903765678405762,
446
+ "learning_rate": 8.6e-05,
447
+ "loss": 0.4639,
448
+ "step": 14000
449
+ },
450
+ {
451
+ "epoch": 57.142857142857146,
452
+ "eval_accuracy": 0.8602637276126747,
453
+ "eval_loss": 0.7129377722740173,
454
+ "eval_runtime": 16.5854,
455
+ "eval_samples_per_second": 118.176,
456
+ "eval_steps_per_second": 3.738,
457
+ "step": 14000
458
+ },
459
+ {
460
+ "epoch": 59.183673469387756,
461
+ "grad_norm": 2.5342397689819336,
462
+ "learning_rate": 8.55e-05,
463
+ "loss": 0.4554,
464
+ "step": 14500
465
+ },
466
+ {
467
+ "epoch": 59.183673469387756,
468
+ "eval_accuracy": 0.8614343212586901,
469
+ "eval_loss": 0.7078380584716797,
470
+ "eval_runtime": 16.0738,
471
+ "eval_samples_per_second": 121.937,
472
+ "eval_steps_per_second": 3.857,
473
+ "step": 14500
474
+ },
475
+ {
476
+ "epoch": 61.224489795918366,
477
+ "grad_norm": 1.810963749885559,
478
+ "learning_rate": 8.5e-05,
479
+ "loss": 0.4437,
480
+ "step": 15000
481
+ },
482
+ {
483
+ "epoch": 61.224489795918366,
484
+ "eval_accuracy": 0.8633158322532457,
485
+ "eval_loss": 0.6998348236083984,
486
+ "eval_runtime": 16.5802,
487
+ "eval_samples_per_second": 118.213,
488
+ "eval_steps_per_second": 3.739,
489
+ "step": 15000
490
+ },
491
+ {
492
+ "epoch": 63.265306122448976,
493
+ "grad_norm": 1.8456369638442993,
494
+ "learning_rate": 8.450000000000001e-05,
495
+ "loss": 0.4424,
496
+ "step": 15500
497
+ },
498
+ {
499
+ "epoch": 63.265306122448976,
500
+ "eval_accuracy": 0.8620835737606731,
501
+ "eval_loss": 0.7072030901908875,
502
+ "eval_runtime": 16.5746,
503
+ "eval_samples_per_second": 118.253,
504
+ "eval_steps_per_second": 3.741,
505
+ "step": 15500
506
+ },
507
+ {
508
+ "epoch": 65.3061224489796,
509
+ "grad_norm": 1.9073472023010254,
510
+ "learning_rate": 8.4e-05,
511
+ "loss": 0.4338,
512
+ "step": 16000
513
+ },
514
+ {
515
+ "epoch": 65.3061224489796,
516
+ "eval_accuracy": 0.8628925222058544,
517
+ "eval_loss": 0.6999377012252808,
518
+ "eval_runtime": 16.5979,
519
+ "eval_samples_per_second": 118.087,
520
+ "eval_steps_per_second": 3.735,
521
+ "step": 16000
522
+ },
523
+ {
524
+ "epoch": 67.34693877551021,
525
+ "grad_norm": 1.9411852359771729,
526
+ "learning_rate": 8.35e-05,
527
+ "loss": 0.4252,
528
+ "step": 16500
529
+ },
530
+ {
531
+ "epoch": 67.34693877551021,
532
+ "eval_accuracy": 0.8659541694954849,
533
+ "eval_loss": 0.6883311867713928,
534
+ "eval_runtime": 16.5927,
535
+ "eval_samples_per_second": 118.124,
536
+ "eval_steps_per_second": 3.737,
537
+ "step": 16500
538
+ }
539
+ ],
540
+ "logging_steps": 500,
541
+ "max_steps": 100000,
542
+ "num_input_tokens_seen": 0,
543
+ "num_train_epochs": 409,
544
+ "save_steps": 500,
545
+ "stateful_callbacks": {
546
+ "TrainerControl": {
547
+ "args": {
548
+ "should_epoch_stop": false,
549
+ "should_evaluate": false,
550
+ "should_log": false,
551
+ "should_save": true,
552
+ "should_training_stop": false
553
+ },
554
+ "attributes": {}
555
+ }
556
+ },
557
+ "total_flos": 1.3911653816991744e+17,
558
+ "train_batch_size": 16,
559
+ "trial_name": null,
560
+ "trial_params": null
561
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff498fd551814afcea75ac95724af9fcf2120852825db2c6741861fa9bb4c0ba
3
+ size 5176
vocab.txt ADDED
The diff for this file is too large to render. See raw diff