Upload tokenizer
Browse files- special_tokens_map.json +0 -2
- tokenizer.json +0 -18
- tokenizer_config.json +0 -18
special_tokens_map.json
CHANGED
|
@@ -34,8 +34,6 @@
|
|
| 34 |
"<repo_name>",
|
| 35 |
">>UNUSED_119<<",
|
| 36 |
">>UNUSED_120<<",
|
| 37 |
-
">>UNUSED_215<<",
|
| 38 |
-
">>UNUSED_216<<",
|
| 39 |
">>UNUSED_217<<",
|
| 40 |
">>UNUSED_218<<",
|
| 41 |
">>UNUSED_219<<",
|
|
|
|
| 34 |
"<repo_name>",
|
| 35 |
">>UNUSED_119<<",
|
| 36 |
">>UNUSED_120<<",
|
|
|
|
|
|
|
| 37 |
">>UNUSED_217<<",
|
| 38 |
">>UNUSED_218<<",
|
| 39 |
">>UNUSED_219<<",
|
tokenizer.json
CHANGED
|
@@ -4718,24 +4718,6 @@
|
|
| 4718 |
"rstrip": false,
|
| 4719 |
"normalized": false,
|
| 4720 |
"special": true
|
| 4721 |
-
},
|
| 4722 |
-
{
|
| 4723 |
-
"id": 32768,
|
| 4724 |
-
"content": ">>UNUSED_215<<",
|
| 4725 |
-
"single_word": false,
|
| 4726 |
-
"lstrip": false,
|
| 4727 |
-
"rstrip": false,
|
| 4728 |
-
"normalized": false,
|
| 4729 |
-
"special": true
|
| 4730 |
-
},
|
| 4731 |
-
{
|
| 4732 |
-
"id": 32769,
|
| 4733 |
-
"content": ">>UNUSED_216<<",
|
| 4734 |
-
"single_word": false,
|
| 4735 |
-
"lstrip": false,
|
| 4736 |
-
"rstrip": false,
|
| 4737 |
-
"normalized": false,
|
| 4738 |
-
"special": true
|
| 4739 |
}
|
| 4740 |
],
|
| 4741 |
"normalizer": null,
|
|
|
|
| 4718 |
"rstrip": false,
|
| 4719 |
"normalized": false,
|
| 4720 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4721 |
}
|
| 4722 |
],
|
| 4723 |
"normalizer": null,
|
tokenizer_config.json
CHANGED
|
@@ -4191,22 +4191,6 @@
|
|
| 4191 |
"rstrip": false,
|
| 4192 |
"single_word": false,
|
| 4193 |
"special": true
|
| 4194 |
-
},
|
| 4195 |
-
"32768": {
|
| 4196 |
-
"content": ">>UNUSED_215<<",
|
| 4197 |
-
"lstrip": false,
|
| 4198 |
-
"normalized": false,
|
| 4199 |
-
"rstrip": false,
|
| 4200 |
-
"single_word": false,
|
| 4201 |
-
"special": true
|
| 4202 |
-
},
|
| 4203 |
-
"32769": {
|
| 4204 |
-
"content": ">>UNUSED_216<<",
|
| 4205 |
-
"lstrip": false,
|
| 4206 |
-
"normalized": false,
|
| 4207 |
-
"rstrip": false,
|
| 4208 |
-
"single_word": false,
|
| 4209 |
-
"special": true
|
| 4210 |
}
|
| 4211 |
},
|
| 4212 |
"additional_special_tokens": [
|
|
@@ -4244,8 +4228,6 @@
|
|
| 4244 |
"<repo_name>",
|
| 4245 |
">>UNUSED_119<<",
|
| 4246 |
">>UNUSED_120<<",
|
| 4247 |
-
">>UNUSED_215<<",
|
| 4248 |
-
">>UNUSED_216<<",
|
| 4249 |
">>UNUSED_217<<",
|
| 4250 |
">>UNUSED_218<<",
|
| 4251 |
">>UNUSED_219<<",
|
|
|
|
| 4191 |
"rstrip": false,
|
| 4192 |
"single_word": false,
|
| 4193 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4194 |
}
|
| 4195 |
},
|
| 4196 |
"additional_special_tokens": [
|
|
|
|
| 4228 |
"<repo_name>",
|
| 4229 |
">>UNUSED_119<<",
|
| 4230 |
">>UNUSED_120<<",
|
|
|
|
|
|
|
| 4231 |
">>UNUSED_217<<",
|
| 4232 |
">>UNUSED_218<<",
|
| 4233 |
">>UNUSED_219<<",
|