Upload folder using huggingface_hub
Browse files- .gitattributes +1 -1
- README.md +272 -0
- config.json +34 -0
- generation_config.json +9 -0
- optimizer.pt +3 -0
- pytorch_model.bin +3 -0
- rng_state.pth +3 -0
- scheduler.pt +3 -0
- sentencepiece.bpe.model +3 -0
- special_tokens_map.json +219 -0
- tokenizer.json +3 -0
- tokenizer_config.json +25 -0
- trainer_state.json +15 -0
- training_args.bin +3 -0
.gitattributes
CHANGED
@@ -25,7 +25,6 @@
|
|
25 |
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
29 |
*.tflite filter=lfs diff=lfs merge=lfs -text
|
30 |
*.tgz filter=lfs diff=lfs merge=lfs -text
|
31 |
*.wasm filter=lfs diff=lfs merge=lfs -text
|
@@ -33,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
25 |
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
|
|
28 |
*.tflite filter=lfs diff=lfs merge=lfs -text
|
29 |
*.tgz filter=lfs diff=lfs merge=lfs -text
|
30 |
*.wasm filter=lfs diff=lfs merge=lfs -text
|
|
|
32 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
33 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
34 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
35 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
@@ -0,0 +1,272 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
language:
|
3 |
+
- ace
|
4 |
+
- acm
|
5 |
+
- acq
|
6 |
+
- aeb
|
7 |
+
- af
|
8 |
+
- ajp
|
9 |
+
- ak
|
10 |
+
- als
|
11 |
+
- am
|
12 |
+
- apc
|
13 |
+
- ar
|
14 |
+
- ars
|
15 |
+
- ary
|
16 |
+
- arz
|
17 |
+
- as
|
18 |
+
- ast
|
19 |
+
- awa
|
20 |
+
- ayr
|
21 |
+
- azb
|
22 |
+
- azj
|
23 |
+
- ba
|
24 |
+
- bm
|
25 |
+
- ban
|
26 |
+
- be
|
27 |
+
- bem
|
28 |
+
- bn
|
29 |
+
- bho
|
30 |
+
- bjn
|
31 |
+
- bo
|
32 |
+
- bs
|
33 |
+
- bug
|
34 |
+
- bg
|
35 |
+
- ca
|
36 |
+
- ceb
|
37 |
+
- cs
|
38 |
+
- cjk
|
39 |
+
- ckb
|
40 |
+
- crh
|
41 |
+
- cy
|
42 |
+
- da
|
43 |
+
- de
|
44 |
+
- dik
|
45 |
+
- dyu
|
46 |
+
- dz
|
47 |
+
- el
|
48 |
+
- en
|
49 |
+
- eo
|
50 |
+
- et
|
51 |
+
- eu
|
52 |
+
- ee
|
53 |
+
- fo
|
54 |
+
- fj
|
55 |
+
- fi
|
56 |
+
- fon
|
57 |
+
- fr
|
58 |
+
- fur
|
59 |
+
- fuv
|
60 |
+
- gaz
|
61 |
+
- gd
|
62 |
+
- ga
|
63 |
+
- gl
|
64 |
+
- gn
|
65 |
+
- gu
|
66 |
+
- ht
|
67 |
+
- ha
|
68 |
+
- he
|
69 |
+
- hi
|
70 |
+
- hne
|
71 |
+
- hr
|
72 |
+
- hu
|
73 |
+
- hy
|
74 |
+
- ig
|
75 |
+
- ilo
|
76 |
+
- id
|
77 |
+
- is
|
78 |
+
- it
|
79 |
+
- jv
|
80 |
+
- ja
|
81 |
+
- kab
|
82 |
+
- kac
|
83 |
+
- kam
|
84 |
+
- kn
|
85 |
+
- ks
|
86 |
+
- ka
|
87 |
+
- kk
|
88 |
+
- kbp
|
89 |
+
- kea
|
90 |
+
- khk
|
91 |
+
- km
|
92 |
+
- ki
|
93 |
+
- rw
|
94 |
+
- ky
|
95 |
+
- kmb
|
96 |
+
- kmr
|
97 |
+
- knc
|
98 |
+
- kg
|
99 |
+
- ko
|
100 |
+
- lo
|
101 |
+
- lij
|
102 |
+
- li
|
103 |
+
- ln
|
104 |
+
- lt
|
105 |
+
- lmo
|
106 |
+
- ltg
|
107 |
+
- lb
|
108 |
+
- lua
|
109 |
+
- lg
|
110 |
+
- luo
|
111 |
+
- lus
|
112 |
+
- lvs
|
113 |
+
- mag
|
114 |
+
- mai
|
115 |
+
- ml
|
116 |
+
- mar
|
117 |
+
- min
|
118 |
+
- mk
|
119 |
+
- mt
|
120 |
+
- mni
|
121 |
+
- mos
|
122 |
+
- mi
|
123 |
+
- my
|
124 |
+
- nl
|
125 |
+
- nn
|
126 |
+
- nb
|
127 |
+
- npi
|
128 |
+
- nso
|
129 |
+
- nus
|
130 |
+
- ny
|
131 |
+
- oc
|
132 |
+
- ory
|
133 |
+
- pag
|
134 |
+
- pa
|
135 |
+
- pap
|
136 |
+
- pbt
|
137 |
+
- pes
|
138 |
+
- plt
|
139 |
+
- pl
|
140 |
+
- pt
|
141 |
+
- prs
|
142 |
+
- quy
|
143 |
+
- ro
|
144 |
+
- rn
|
145 |
+
- ru
|
146 |
+
- sg
|
147 |
+
- sa
|
148 |
+
- sat
|
149 |
+
- scn
|
150 |
+
- shn
|
151 |
+
- si
|
152 |
+
- sk
|
153 |
+
- sl
|
154 |
+
- sm
|
155 |
+
- sn
|
156 |
+
- sd
|
157 |
+
- so
|
158 |
+
- st
|
159 |
+
- es
|
160 |
+
- sc
|
161 |
+
- sr
|
162 |
+
- ss
|
163 |
+
- su
|
164 |
+
- sv
|
165 |
+
- swh
|
166 |
+
- szl
|
167 |
+
- ta
|
168 |
+
- taq
|
169 |
+
- tt
|
170 |
+
- te
|
171 |
+
- tg
|
172 |
+
- tl
|
173 |
+
- th
|
174 |
+
- ti
|
175 |
+
- tpi
|
176 |
+
- tn
|
177 |
+
- ts
|
178 |
+
- tk
|
179 |
+
- tum
|
180 |
+
- tr
|
181 |
+
- tw
|
182 |
+
- tzm
|
183 |
+
- ug
|
184 |
+
- uk
|
185 |
+
- umb
|
186 |
+
- ur
|
187 |
+
- uzn
|
188 |
+
- vec
|
189 |
+
- vi
|
190 |
+
- war
|
191 |
+
- wo
|
192 |
+
- xh
|
193 |
+
- ydd
|
194 |
+
- yo
|
195 |
+
- yue
|
196 |
+
- zh
|
197 |
+
- zsm
|
198 |
+
- zu
|
199 |
+
|
200 |
+
language_details: "ace_Arab, ace_Latn, acm_Arab, acq_Arab, aeb_Arab, afr_Latn, ajp_Arab, aka_Latn, amh_Ethi, apc_Arab, arb_Arab, ars_Arab, ary_Arab, arz_Arab, asm_Beng, ast_Latn, awa_Deva, ayr_Latn, azb_Arab, azj_Latn, bak_Cyrl, bam_Latn, ban_Latn,bel_Cyrl, bem_Latn, ben_Beng, bho_Deva, bjn_Arab, bjn_Latn, bod_Tibt, bos_Latn, bug_Latn, bul_Cyrl, cat_Latn, ceb_Latn, ces_Latn, cjk_Latn, ckb_Arab, crh_Latn, cym_Latn, dan_Latn, deu_Latn, dik_Latn, dyu_Latn, dzo_Tibt, ell_Grek, eng_Latn, epo_Latn, est_Latn, eus_Latn, ewe_Latn, fao_Latn, pes_Arab, fij_Latn, fin_Latn, fon_Latn, fra_Latn, fur_Latn, fuv_Latn, gla_Latn, gle_Latn, glg_Latn, grn_Latn, guj_Gujr, hat_Latn, hau_Latn, heb_Hebr, hin_Deva, hne_Deva, hrv_Latn, hun_Latn, hye_Armn, ibo_Latn, ilo_Latn, ind_Latn, isl_Latn, ita_Latn, jav_Latn, jpn_Jpan, kab_Latn, kac_Latn, kam_Latn, kan_Knda, kas_Arab, kas_Deva, kat_Geor, knc_Arab, knc_Latn, kaz_Cyrl, kbp_Latn, kea_Latn, khm_Khmr, kik_Latn, kin_Latn, kir_Cyrl, kmb_Latn, kon_Latn, kor_Hang, kmr_Latn, lao_Laoo, lvs_Latn, lij_Latn, lim_Latn, lin_Latn, lit_Latn, lmo_Latn, ltg_Latn, ltz_Latn, lua_Latn, lug_Latn, luo_Latn, lus_Latn, mag_Deva, mai_Deva, mal_Mlym, mar_Deva, min_Latn, mkd_Cyrl, plt_Latn, mlt_Latn, mni_Beng, khk_Cyrl, mos_Latn, mri_Latn, zsm_Latn, mya_Mymr, nld_Latn, nno_Latn, nob_Latn, npi_Deva, nso_Latn, nus_Latn, nya_Latn, oci_Latn, gaz_Latn, ory_Orya, pag_Latn, pan_Guru, pap_Latn, pol_Latn, por_Latn, prs_Arab, pbt_Arab, quy_Latn, ron_Latn, run_Latn, rus_Cyrl, sag_Latn, san_Deva, sat_Beng, scn_Latn, shn_Mymr, sin_Sinh, slk_Latn, slv_Latn, smo_Latn, sna_Latn, snd_Arab, som_Latn, sot_Latn, spa_Latn, als_Latn, srd_Latn, srp_Cyrl, ssw_Latn, sun_Latn, swe_Latn, swh_Latn, szl_Latn, tam_Taml, tat_Cyrl, tel_Telu, tgk_Cyrl, tgl_Latn, tha_Thai, tir_Ethi, taq_Latn, taq_Tfng, tpi_Latn, tsn_Latn, tso_Latn, tuk_Latn, tum_Latn, tur_Latn, twi_Latn, tzm_Tfng, uig_Arab, ukr_Cyrl, umb_Latn, urd_Arab, uzn_Latn, vec_Latn, vie_Latn, war_Latn, wol_Latn, xho_Latn, ydd_Hebr, yor_Latn, yue_Hant, zho_Hans, zho_Hant, zul_Latn"
|
201 |
+
|
202 |
+
tags:
|
203 |
+
- nllb
|
204 |
+
- translation
|
205 |
+
license: "cc-by-nc-4.0"
|
206 |
+
datasets:
|
207 |
+
- flores-200
|
208 |
+
metrics:
|
209 |
+
- bleu
|
210 |
+
- spbleu
|
211 |
+
- chrf++
|
212 |
+
inference: false
|
213 |
+
---
|
214 |
+
|
215 |
+
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
216 |
+
should probably proofread and complete it, then remove this comment. -->
|
217 |
+
|
218 |
+
This model is a fine-tuned version of [facebook/nllb-200-3.3B](https://huggingface.co/facebook/nllb-200-3.3B) on the [EasyProject](https://github.com/edchengg/easyproject) dataset.
|
219 |
+
|
220 |
+
### Framework versions
|
221 |
+
|
222 |
+
- Transformers 4.29.2
|
223 |
+
- Pytorch 1.11.0+cu113
|
224 |
+
- Datasets 2.8.0
|
225 |
+
- Tokenizers 0.13.2
|
226 |
+
|
227 |
+
- Paper link: [Frustratingly Easy Label Projection for Cross-lingual Transfer](https://arxiv.org/abs/2211.15613)
|
228 |
+
- Github link: https://github.com/edchengg/easyproject
|
229 |
+
- Please use the transformers==4.29.2 library as Huggingface recently fixed a bug in [NLLB tokenizer](https://github.com/huggingface/transformers/pull/22313)
|
230 |
+
|
231 |
+
# Code
|
232 |
+
```python
|
233 |
+
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
234 |
+
import torch
|
235 |
+
|
236 |
+
tokenizer = AutoTokenizer.from_pretrained(
|
237 |
+
"facebook/nllb-200-distilled-600M", src_lang="eng_Latn")
|
238 |
+
|
239 |
+
print("Loading model")
|
240 |
+
model = AutoModelForSeq2SeqLM.from_pretrained("ychenNLP/nllb-200-distilled-1.3b-easyproject")
|
241 |
+
model.cuda()
|
242 |
+
|
243 |
+
input_chunks = ["A translator always risks inadvertently introducing source-language words, grammar, or syntax into the target-language rendering."]
|
244 |
+
print("Start translation...")
|
245 |
+
output_result = []
|
246 |
+
|
247 |
+
batch_size = 1
|
248 |
+
for idx in tqdm(range(0, len(input_chunks), batch_size)):
|
249 |
+
start_idx = idx
|
250 |
+
end_idx = idx + batch_size
|
251 |
+
inputs = tokenizer(input_chunks[start_idx: end_idx], padding=True, truncation=True, max_length=128, return_tensors="pt").to('cuda')
|
252 |
+
|
253 |
+
with torch.no_grad():
|
254 |
+
translated_tokens = model.generate(**inputs, forced_bos_token_id=tokenizer.lang_code_to_id["zho_Hans"],
|
255 |
+
max_length=128, num_beams=5, num_return_sequences=1, early_stopping=True)
|
256 |
+
|
257 |
+
output = tokenizer.batch_decode(translated_tokens, skip_special_tokens=True)
|
258 |
+
output_result.extend(output)
|
259 |
+
print(output_result)
|
260 |
+
```
|
261 |
+
|
262 |
+
## Citation
|
263 |
+
|
264 |
+
```
|
265 |
+
@inproceedings{chen2023easyproject,
|
266 |
+
title={Frustratingly Easy Label Projection for Cross-lingual Transfer},
|
267 |
+
author={Chen, Yang and Jiang, Chao and Ritter, Alan and Xu, Wei},
|
268 |
+
booktitle={Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Findings)},
|
269 |
+
year={2023}
|
270 |
+
}
|
271 |
+
```
|
272 |
+
|
config.json
ADDED
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_name_or_path": "facebook/nllb-200-distilled-1.3B",
|
3 |
+
"activation_dropout": 0.0,
|
4 |
+
"activation_function": "relu",
|
5 |
+
"architectures": [
|
6 |
+
"M2M100ForConditionalGeneration"
|
7 |
+
],
|
8 |
+
"attention_dropout": 0.1,
|
9 |
+
"bos_token_id": 0,
|
10 |
+
"d_model": 1024,
|
11 |
+
"decoder_attention_heads": 16,
|
12 |
+
"decoder_ffn_dim": 8192,
|
13 |
+
"decoder_layerdrop": 0,
|
14 |
+
"decoder_layers": 24,
|
15 |
+
"decoder_start_token_id": 2,
|
16 |
+
"dropout": 0.1,
|
17 |
+
"encoder_attention_heads": 16,
|
18 |
+
"encoder_ffn_dim": 8192,
|
19 |
+
"encoder_layerdrop": 0,
|
20 |
+
"encoder_layers": 24,
|
21 |
+
"eos_token_id": 2,
|
22 |
+
"init_std": 0.02,
|
23 |
+
"is_encoder_decoder": true,
|
24 |
+
"max_length": 200,
|
25 |
+
"max_position_embeddings": 1024,
|
26 |
+
"model_type": "m2m_100",
|
27 |
+
"num_hidden_layers": 24,
|
28 |
+
"pad_token_id": 1,
|
29 |
+
"scale_embedding": true,
|
30 |
+
"torch_dtype": "float32",
|
31 |
+
"transformers_version": "4.29.2",
|
32 |
+
"use_cache": true,
|
33 |
+
"vocab_size": 256206
|
34 |
+
}
|
generation_config.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_from_model_config": true,
|
3 |
+
"bos_token_id": 0,
|
4 |
+
"decoder_start_token_id": 2,
|
5 |
+
"eos_token_id": 2,
|
6 |
+
"max_length": 200,
|
7 |
+
"pad_token_id": 1,
|
8 |
+
"transformers_version": "4.29.2"
|
9 |
+
}
|
optimizer.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:ddc1e1a5aabcf48363cf287a46d3ff983a5dc64d51acd2e6e4d8b2f2632494f1
|
3 |
+
size 369450217
|
pytorch_model.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:44c94e81263bb6201728a78a19256d9a681f56bd65bc8e2bc62efc83889eb6ea
|
3 |
+
size 5491302146
|
rng_state.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:6facffd74bec5258e8fd79ea06d0ee8b0ef4398f5eb3a89f6e32a315245a08d3
|
3 |
+
size 14503
|
scheduler.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:ed70e833c6c2092be3980743d06828aaf3cf489390300379089a83c5b56e2ebc
|
3 |
+
size 623
|
sentencepiece.bpe.model
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a
|
3 |
+
size 4852054
|
special_tokens_map.json
ADDED
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [
|
3 |
+
"ace_Arab",
|
4 |
+
"ace_Latn",
|
5 |
+
"acm_Arab",
|
6 |
+
"acq_Arab",
|
7 |
+
"aeb_Arab",
|
8 |
+
"afr_Latn",
|
9 |
+
"ajp_Arab",
|
10 |
+
"aka_Latn",
|
11 |
+
"amh_Ethi",
|
12 |
+
"apc_Arab",
|
13 |
+
"arb_Arab",
|
14 |
+
"ars_Arab",
|
15 |
+
"ary_Arab",
|
16 |
+
"arz_Arab",
|
17 |
+
"asm_Beng",
|
18 |
+
"ast_Latn",
|
19 |
+
"awa_Deva",
|
20 |
+
"ayr_Latn",
|
21 |
+
"azb_Arab",
|
22 |
+
"azj_Latn",
|
23 |
+
"bak_Cyrl",
|
24 |
+
"bam_Latn",
|
25 |
+
"ban_Latn",
|
26 |
+
"bel_Cyrl",
|
27 |
+
"bem_Latn",
|
28 |
+
"ben_Beng",
|
29 |
+
"bho_Deva",
|
30 |
+
"bjn_Arab",
|
31 |
+
"bjn_Latn",
|
32 |
+
"bod_Tibt",
|
33 |
+
"bos_Latn",
|
34 |
+
"bug_Latn",
|
35 |
+
"bul_Cyrl",
|
36 |
+
"cat_Latn",
|
37 |
+
"ceb_Latn",
|
38 |
+
"ces_Latn",
|
39 |
+
"cjk_Latn",
|
40 |
+
"ckb_Arab",
|
41 |
+
"crh_Latn",
|
42 |
+
"cym_Latn",
|
43 |
+
"dan_Latn",
|
44 |
+
"deu_Latn",
|
45 |
+
"dik_Latn",
|
46 |
+
"dyu_Latn",
|
47 |
+
"dzo_Tibt",
|
48 |
+
"ell_Grek",
|
49 |
+
"eng_Latn",
|
50 |
+
"epo_Latn",
|
51 |
+
"est_Latn",
|
52 |
+
"eus_Latn",
|
53 |
+
"ewe_Latn",
|
54 |
+
"fao_Latn",
|
55 |
+
"pes_Arab",
|
56 |
+
"fij_Latn",
|
57 |
+
"fin_Latn",
|
58 |
+
"fon_Latn",
|
59 |
+
"fra_Latn",
|
60 |
+
"fur_Latn",
|
61 |
+
"fuv_Latn",
|
62 |
+
"gla_Latn",
|
63 |
+
"gle_Latn",
|
64 |
+
"glg_Latn",
|
65 |
+
"grn_Latn",
|
66 |
+
"guj_Gujr",
|
67 |
+
"hat_Latn",
|
68 |
+
"hau_Latn",
|
69 |
+
"heb_Hebr",
|
70 |
+
"hin_Deva",
|
71 |
+
"hne_Deva",
|
72 |
+
"hrv_Latn",
|
73 |
+
"hun_Latn",
|
74 |
+
"hye_Armn",
|
75 |
+
"ibo_Latn",
|
76 |
+
"ilo_Latn",
|
77 |
+
"ind_Latn",
|
78 |
+
"isl_Latn",
|
79 |
+
"ita_Latn",
|
80 |
+
"jav_Latn",
|
81 |
+
"jpn_Jpan",
|
82 |
+
"kab_Latn",
|
83 |
+
"kac_Latn",
|
84 |
+
"kam_Latn",
|
85 |
+
"kan_Knda",
|
86 |
+
"kas_Arab",
|
87 |
+
"kas_Deva",
|
88 |
+
"kat_Geor",
|
89 |
+
"knc_Arab",
|
90 |
+
"knc_Latn",
|
91 |
+
"kaz_Cyrl",
|
92 |
+
"kbp_Latn",
|
93 |
+
"kea_Latn",
|
94 |
+
"khm_Khmr",
|
95 |
+
"kik_Latn",
|
96 |
+
"kin_Latn",
|
97 |
+
"kir_Cyrl",
|
98 |
+
"kmb_Latn",
|
99 |
+
"kon_Latn",
|
100 |
+
"kor_Hang",
|
101 |
+
"kmr_Latn",
|
102 |
+
"lao_Laoo",
|
103 |
+
"lvs_Latn",
|
104 |
+
"lij_Latn",
|
105 |
+
"lim_Latn",
|
106 |
+
"lin_Latn",
|
107 |
+
"lit_Latn",
|
108 |
+
"lmo_Latn",
|
109 |
+
"ltg_Latn",
|
110 |
+
"ltz_Latn",
|
111 |
+
"lua_Latn",
|
112 |
+
"lug_Latn",
|
113 |
+
"luo_Latn",
|
114 |
+
"lus_Latn",
|
115 |
+
"mag_Deva",
|
116 |
+
"mai_Deva",
|
117 |
+
"mal_Mlym",
|
118 |
+
"mar_Deva",
|
119 |
+
"min_Latn",
|
120 |
+
"mkd_Cyrl",
|
121 |
+
"plt_Latn",
|
122 |
+
"mlt_Latn",
|
123 |
+
"mni_Beng",
|
124 |
+
"khk_Cyrl",
|
125 |
+
"mos_Latn",
|
126 |
+
"mri_Latn",
|
127 |
+
"zsm_Latn",
|
128 |
+
"mya_Mymr",
|
129 |
+
"nld_Latn",
|
130 |
+
"nno_Latn",
|
131 |
+
"nob_Latn",
|
132 |
+
"npi_Deva",
|
133 |
+
"nso_Latn",
|
134 |
+
"nus_Latn",
|
135 |
+
"nya_Latn",
|
136 |
+
"oci_Latn",
|
137 |
+
"gaz_Latn",
|
138 |
+
"ory_Orya",
|
139 |
+
"pag_Latn",
|
140 |
+
"pan_Guru",
|
141 |
+
"pap_Latn",
|
142 |
+
"pol_Latn",
|
143 |
+
"por_Latn",
|
144 |
+
"prs_Arab",
|
145 |
+
"pbt_Arab",
|
146 |
+
"quy_Latn",
|
147 |
+
"ron_Latn",
|
148 |
+
"run_Latn",
|
149 |
+
"rus_Cyrl",
|
150 |
+
"sag_Latn",
|
151 |
+
"san_Deva",
|
152 |
+
"sat_Beng",
|
153 |
+
"scn_Latn",
|
154 |
+
"shn_Mymr",
|
155 |
+
"sin_Sinh",
|
156 |
+
"slk_Latn",
|
157 |
+
"slv_Latn",
|
158 |
+
"smo_Latn",
|
159 |
+
"sna_Latn",
|
160 |
+
"snd_Arab",
|
161 |
+
"som_Latn",
|
162 |
+
"sot_Latn",
|
163 |
+
"spa_Latn",
|
164 |
+
"als_Latn",
|
165 |
+
"srd_Latn",
|
166 |
+
"srp_Cyrl",
|
167 |
+
"ssw_Latn",
|
168 |
+
"sun_Latn",
|
169 |
+
"swe_Latn",
|
170 |
+
"swh_Latn",
|
171 |
+
"szl_Latn",
|
172 |
+
"tam_Taml",
|
173 |
+
"tat_Cyrl",
|
174 |
+
"tel_Telu",
|
175 |
+
"tgk_Cyrl",
|
176 |
+
"tgl_Latn",
|
177 |
+
"tha_Thai",
|
178 |
+
"tir_Ethi",
|
179 |
+
"taq_Latn",
|
180 |
+
"taq_Tfng",
|
181 |
+
"tpi_Latn",
|
182 |
+
"tsn_Latn",
|
183 |
+
"tso_Latn",
|
184 |
+
"tuk_Latn",
|
185 |
+
"tum_Latn",
|
186 |
+
"tur_Latn",
|
187 |
+
"twi_Latn",
|
188 |
+
"tzm_Tfng",
|
189 |
+
"uig_Arab",
|
190 |
+
"ukr_Cyrl",
|
191 |
+
"umb_Latn",
|
192 |
+
"urd_Arab",
|
193 |
+
"uzn_Latn",
|
194 |
+
"vec_Latn",
|
195 |
+
"vie_Latn",
|
196 |
+
"war_Latn",
|
197 |
+
"wol_Latn",
|
198 |
+
"xho_Latn",
|
199 |
+
"ydd_Hebr",
|
200 |
+
"yor_Latn",
|
201 |
+
"yue_Hant",
|
202 |
+
"zho_Hans",
|
203 |
+
"zho_Hant",
|
204 |
+
"zul_Latn"
|
205 |
+
],
|
206 |
+
"bos_token": "<s>",
|
207 |
+
"cls_token": "<s>",
|
208 |
+
"eos_token": "</s>",
|
209 |
+
"mask_token": {
|
210 |
+
"content": "<mask>",
|
211 |
+
"lstrip": true,
|
212 |
+
"normalized": true,
|
213 |
+
"rstrip": false,
|
214 |
+
"single_word": false
|
215 |
+
},
|
216 |
+
"pad_token": "<pad>",
|
217 |
+
"sep_token": "</s>",
|
218 |
+
"unk_token": "<unk>"
|
219 |
+
}
|
tokenizer.json
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e316b82de11d0f951f370943b3c438311629547285129b0b81dadabd01bca665
|
3 |
+
size 17331176
|
tokenizer_config.json
ADDED
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": null,
|
3 |
+
"bos_token": "<s>",
|
4 |
+
"clean_up_tokenization_spaces": true,
|
5 |
+
"cls_token": "<s>",
|
6 |
+
"eos_token": "</s>",
|
7 |
+
"legacy_behaviour": false,
|
8 |
+
"mask_token": {
|
9 |
+
"__type": "AddedToken",
|
10 |
+
"content": "<mask>",
|
11 |
+
"lstrip": true,
|
12 |
+
"normalized": true,
|
13 |
+
"rstrip": false,
|
14 |
+
"single_word": false
|
15 |
+
},
|
16 |
+
"model_max_length": 1024,
|
17 |
+
"pad_token": "<pad>",
|
18 |
+
"sep_token": "</s>",
|
19 |
+
"sp_model_kwargs": {},
|
20 |
+
"src_lang": null,
|
21 |
+
"tgt_lang": null,
|
22 |
+
"tokenizer_class": "NllbTokenizer",
|
23 |
+
"tokenizer_file": null,
|
24 |
+
"unk_token": "<unk>"
|
25 |
+
}
|
trainer_state.json
ADDED
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"best_metric": null,
|
3 |
+
"best_model_checkpoint": null,
|
4 |
+
"epoch": 0.032,
|
5 |
+
"global_step": 200,
|
6 |
+
"is_hyper_param_search": false,
|
7 |
+
"is_local_process_zero": true,
|
8 |
+
"is_world_process_zero": true,
|
9 |
+
"log_history": [],
|
10 |
+
"max_steps": 1000,
|
11 |
+
"num_train_epochs": 1,
|
12 |
+
"total_flos": 584269271728128.0,
|
13 |
+
"trial_name": null,
|
14 |
+
"trial_params": null
|
15 |
+
}
|
training_args.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f8e5dc0388daf440f2e481ba6965671a81ada41e5187563c8330b254ed3b29a6
|
3 |
+
size 4079
|