Upload tokenizer

added_tokens.json

@@ -1,44 +1,43 @@
 {
-  "<af>": 64002,
-  "<am>": 64003,
-  "<ar>": 64004,
-  "<ast>": 64005,
-  "<be>": 64006,
-  "<bg>": 64007,
-  "<bn>": 64008,
-  "<bs>": 64009,
-  "<ca>": 64010,
-  "<cs>": 64011,
-  "<da>": 64012,
-  "<de>": 64013,
-  "<en>": 64014,
-  "<eos>": 64001,
-  "<es>": 64015,
-  "<fr>": 64016,
-  "<gu>": 64017,
-  "<ha>": 64018,
-  "<he>": 64019,
-  "<hi>": 64020,
-  "<is>": 64021,
-  "<it>": 64022,
-  "<kab>": 64023,
-  "<kn>": 64024,
-  "<lb>": 64025,
-  "<mr>": 64026,
-  "<mt>": 64027,
-  "<ne>": 64028,
-  "<nl>": 64029,
-  "<no>": 64030,
-  "<oc>": 64031,
-  "<pl>": 64032,
-  "<pt>": 64033,
-  "<ro>": 64034,
-  "<ru>": 64035,
-  "<sd>": 64036,
-  "<so>": 64037,
-  "<sr>": 64038,
-  "<sv>": 64039,
-  "<ti>": 64040,
-  "<uk>": 64041,
-  "<ur>": 64042
+  "<af>": 64001,
+  "<am>": 64002,
+  "<ar>": 64003,
+  "<ast>": 64004,
+  "<be>": 64005,
+  "<bg>": 64006,
+  "<bn>": 64007,
+  "<bs>": 64008,
+  "<ca>": 64009,
+  "<cs>": 64010,
+  "<da>": 64011,
+  "<de>": 64012,
+  "<en>": 64013,
+  "<es>": 64014,
+  "<fr>": 64015,
+  "<gu>": 64016,
+  "<ha>": 64017,
+  "<he>": 64018,
+  "<hi>": 64019,
+  "<is>": 64020,
+  "<it>": 64021,
+  "<kab>": 64022,
+  "<kn>": 64023,
+  "<lb>": 64024,
+  "<mr>": 64025,
+  "<mt>": 64026,
+  "<ne>": 64027,
+  "<nl>": 64028,
+  "<no>": 64029,
+  "<oc>": 64030,
+  "<pl>": 64031,
+  "<pt>": 64032,
+  "<ro>": 64033,
+  "<ru>": 64034,
+  "<sd>": 64035,
+  "<so>": 64036,
+  "<sr>": 64037,
+  "<sv>": 64038,
+  "<ti>": 64039,
+  "<uk>": 64040,
+  "<ur>": 64041
 }

special_tokens_map.json

@@ -43,14 +43,14 @@
     "<ur>"
   ],
   "bos_token": {
-    "content": "<eos>",
+    "content": "</s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
-    "content": "<eos>",
+    "content": "</s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -41,15 +41,6 @@
     },
     {
       "id": 64001,
-      "content": "<eos>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 64002,
       "content": "<af>",
       "single_word": false,
       "lstrip": false,
@@ -58,7 +49,7 @@
       "special": true
     },
     {
-      "id": 64003,
+      "id": 64002,
       "content": "<am>",
       "single_word": false,
       "lstrip": false,
@@ -67,7 +58,7 @@
       "special": true
     },
     {
-      "id": 64004,
+      "id": 64003,
       "content": "<ar>",
       "single_word": false,
       "lstrip": false,
@@ -76,7 +67,7 @@
       "special": true
     },
     {
-      "id": 64005,
+      "id": 64004,
       "content": "<ast>",
       "single_word": false,
       "lstrip": false,
@@ -85,7 +76,7 @@
       "special": true
     },
     {
-      "id": 64006,
+      "id": 64005,
       "content": "<be>",
       "single_word": false,
       "lstrip": false,
@@ -94,7 +85,7 @@
       "special": true
     },
     {
-      "id": 64007,
+      "id": 64006,
       "content": "<bg>",
       "single_word": false,
       "lstrip": false,
@@ -103,7 +94,7 @@
       "special": true
     },
     {
-      "id": 64008,
+      "id": 64007,
       "content": "<bn>",
       "single_word": false,
       "lstrip": false,
@@ -112,7 +103,7 @@
       "special": true
     },
     {
-      "id": 64009,
+      "id": 64008,
       "content": "<bs>",
       "single_word": false,
       "lstrip": false,
@@ -121,7 +112,7 @@
       "special": true
     },
     {
-      "id": 64010,
+      "id": 64009,
       "content": "<ca>",
       "single_word": false,
       "lstrip": false,
@@ -130,7 +121,7 @@
       "special": true
     },
     {
-      "id": 64011,
+      "id": 64010,
       "content": "<cs>",
       "single_word": false,
       "lstrip": false,
@@ -139,7 +130,7 @@
       "special": true
     },
     {
-      "id": 64012,
+      "id": 64011,
       "content": "<da>",
       "single_word": false,
       "lstrip": false,
@@ -148,7 +139,7 @@
       "special": true
     },
     {
-      "id": 64013,
+      "id": 64012,
       "content": "<de>",
       "single_word": false,
       "lstrip": false,
@@ -157,7 +148,7 @@
       "special": true
     },
     {
-      "id": 64014,
+      "id": 64013,
       "content": "<en>",
       "single_word": false,
       "lstrip": false,
@@ -166,7 +157,7 @@
       "special": true
     },
     {
-      "id": 64015,
+      "id": 64014,
       "content": "<es>",
       "single_word": false,
       "lstrip": false,
@@ -175,7 +166,7 @@
       "special": true
     },
     {
-      "id": 64016,
+      "id": 64015,
       "content": "<fr>",
       "single_word": false,
       "lstrip": false,
@@ -184,7 +175,7 @@
       "special": true
     },
     {
-      "id": 64017,
+      "id": 64016,
       "content": "<gu>",
       "single_word": false,
       "lstrip": false,
@@ -193,7 +184,7 @@
       "special": true
     },
     {
-      "id": 64018,
+      "id": 64017,
       "content": "<ha>",
       "single_word": false,
       "lstrip": false,
@@ -202,7 +193,7 @@
       "special": true
     },
     {
-      "id": 64019,
+      "id": 64018,
       "content": "<he>",
       "single_word": false,
       "lstrip": false,
@@ -211,7 +202,7 @@
       "special": true
     },
     {
-      "id": 64020,
+      "id": 64019,
       "content": "<hi>",
       "single_word": false,
       "lstrip": false,
@@ -220,7 +211,7 @@
       "special": true
     },
     {
-      "id": 64021,
+      "id": 64020,
       "content": "<is>",
       "single_word": false,
       "lstrip": false,
@@ -229,7 +220,7 @@
       "special": true
     },
     {
-      "id": 64022,
+      "id": 64021,
       "content": "<it>",
       "single_word": false,
       "lstrip": false,
@@ -238,7 +229,7 @@
       "special": true
     },
     {
-      "id": 64023,
+      "id": 64022,
       "content": "<kab>",
       "single_word": false,
       "lstrip": false,
@@ -247,7 +238,7 @@
       "special": true
     },
     {
-      "id": 64024,
+      "id": 64023,
       "content": "<kn>",
       "single_word": false,
       "lstrip": false,
@@ -256,7 +247,7 @@
       "special": true
     },
     {
-      "id": 64025,
+      "id": 64024,
       "content": "<lb>",
       "single_word": false,
       "lstrip": false,
@@ -265,7 +256,7 @@
       "special": true
     },
     {
-      "id": 64026,
+      "id": 64025,
       "content": "<mr>",
       "single_word": false,
       "lstrip": false,
@@ -274,7 +265,7 @@
       "special": true
     },
     {
-      "id": 64027,
+      "id": 64026,
       "content": "<mt>",
       "single_word": false,
       "lstrip": false,
@@ -283,7 +274,7 @@
       "special": true
     },
     {
-      "id": 64028,
+      "id": 64027,
       "content": "<ne>",
       "single_word": false,
       "lstrip": false,
@@ -292,7 +283,7 @@
       "special": true
     },
     {
-      "id": 64029,
+      "id": 64028,
       "content": "<nl>",
       "single_word": false,
       "lstrip": false,
@@ -301,7 +292,7 @@
       "special": true
     },
     {
-      "id": 64030,
+      "id": 64029,
       "content": "<no>",
       "single_word": false,
       "lstrip": false,
@@ -310,7 +301,7 @@
       "special": true
     },
     {
-      "id": 64031,
+      "id": 64030,
       "content": "<oc>",
       "single_word": false,
       "lstrip": false,
@@ -319,7 +310,7 @@
       "special": true
     },
     {
-      "id": 64032,
+      "id": 64031,
       "content": "<pl>",
       "single_word": false,
       "lstrip": false,
@@ -328,7 +319,7 @@
       "special": true
     },
     {
-      "id": 64033,
+      "id": 64032,
       "content": "<pt>",
       "single_word": false,
       "lstrip": false,
@@ -337,7 +328,7 @@
       "special": true
     },
     {
-      "id": 64034,
+      "id": 64033,
       "content": "<ro>",
       "single_word": false,
       "lstrip": false,
@@ -346,7 +337,7 @@
       "special": true
     },
     {
-      "id": 64035,
+      "id": 64034,
       "content": "<ru>",
       "single_word": false,
       "lstrip": false,
@@ -355,7 +346,7 @@
       "special": true
     },
     {
-      "id": 64036,
+      "id": 64035,
       "content": "<sd>",
       "single_word": false,
       "lstrip": false,
@@ -364,7 +355,7 @@
       "special": true
     },
     {
-      "id": 64037,
+      "id": 64036,
       "content": "<so>",
       "single_word": false,
       "lstrip": false,
@@ -373,7 +364,7 @@
       "special": true
     },
     {
-      "id": 64038,
+      "id": 64037,
       "content": "<sr>",
       "single_word": false,
       "lstrip": false,
@@ -382,7 +373,7 @@
       "special": true
     },
     {
-      "id": 64039,
+      "id": 64038,
       "content": "<sv>",
       "single_word": false,
       "lstrip": false,
@@ -391,7 +382,7 @@
       "special": true
     },
     {
-      "id": 64040,
+      "id": 64039,
       "content": "<ti>",
       "single_word": false,
       "lstrip": false,
@@ -400,7 +391,7 @@
       "special": true
     },
     {
-      "id": 64041,
+      "id": 64040,
       "content": "<uk>",
       "single_word": false,
       "lstrip": false,
@@ -409,7 +400,7 @@
       "special": true
     },
     {
-      "id": 64042,
+      "id": 64041,
       "content": "<ur>",
       "single_word": false,
       "lstrip": false,
@@ -462,7 +453,7 @@
       },
       {
         "SpecialToken": {
-          "id": "<eos>",
+          "id": "</s>",
           "type_id": 0
         }
       }
@@ -488,28 +479,28 @@
       },
       {
         "SpecialToken": {
-          "id": "<eos>",
+          "id": "</s>",
           "type_id": 0
         }
       }
     ],
     "special_tokens": {
-      "<en>": {
-        "id": "<en>",
+      "</s>": {
+        "id": "</s>",
         "ids": [
-          64014
+          2
         ],
         "tokens": [
-          "<en>"
+          "</s>"
         ]
       },
-      "<eos>": {
-        "id": "<eos>",
+      "<en>": {
+        "id": "<en>",
         "ids": [
-          64001
+          64013
         ],
         "tokens": [
-          "<eos>"
+          "<en>"
         ]
       }
     }

tokenizer_config.json

@@ -33,14 +33,6 @@
       "special": true
     },
     "64001": {
-      "content": "<eos>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "64002": {
       "content": "<af>",
       "lstrip": false,
       "normalized": false,
@@ -48,7 +40,7 @@
       "single_word": false,
       "special": true
     },
-    "64003": {
+    "64002": {
       "content": "<am>",
       "lstrip": false,
       "normalized": false,
@@ -56,7 +48,7 @@
       "single_word": false,
       "special": true
     },
-    "64004": {
+    "64003": {
       "content": "<ar>",
       "lstrip": false,
       "normalized": false,
@@ -64,7 +56,7 @@
       "single_word": false,
       "special": true
     },
-    "64005": {
+    "64004": {
       "content": "<ast>",
       "lstrip": false,
       "normalized": false,
@@ -72,7 +64,7 @@
       "single_word": false,
       "special": true
     },
-    "64006": {
+    "64005": {
       "content": "<be>",
       "lstrip": false,
       "normalized": false,
@@ -80,7 +72,7 @@
       "single_word": false,
       "special": true
     },
-    "64007": {
+    "64006": {
       "content": "<bg>",
       "lstrip": false,
       "normalized": false,
@@ -88,7 +80,7 @@
       "single_word": false,
       "special": true
     },
-    "64008": {
+    "64007": {
       "content": "<bn>",
       "lstrip": false,
       "normalized": false,
@@ -96,7 +88,7 @@
       "single_word": false,
       "special": true
     },
-    "64009": {
+    "64008": {
       "content": "<bs>",
       "lstrip": false,
       "normalized": false,
@@ -104,7 +96,7 @@
       "single_word": false,
       "special": true
     },
-    "64010": {
+    "64009": {
       "content": "<ca>",
       "lstrip": false,
       "normalized": false,
@@ -112,7 +104,7 @@
       "single_word": false,
       "special": true
     },
-    "64011": {
+    "64010": {
       "content": "<cs>",
       "lstrip": false,
       "normalized": false,
@@ -120,7 +112,7 @@
       "single_word": false,
       "special": true
     },
-    "64012": {
+    "64011": {
       "content": "<da>",
       "lstrip": false,
       "normalized": false,
@@ -128,7 +120,7 @@
       "single_word": false,
       "special": true
     },
-    "64013": {
+    "64012": {
       "content": "<de>",
       "lstrip": false,
       "normalized": false,
@@ -136,7 +128,7 @@
       "single_word": false,
       "special": true
     },
-    "64014": {
+    "64013": {
       "content": "<en>",
       "lstrip": false,
       "normalized": false,
@@ -144,7 +136,7 @@
       "single_word": false,
       "special": true
     },
-    "64015": {
+    "64014": {
       "content": "<es>",
       "lstrip": false,
       "normalized": false,
@@ -152,7 +144,7 @@
       "single_word": false,
       "special": true
     },
-    "64016": {
+    "64015": {
       "content": "<fr>",
       "lstrip": false,
       "normalized": false,
@@ -160,7 +152,7 @@
       "single_word": false,
       "special": true
     },
-    "64017": {
+    "64016": {
       "content": "<gu>",
       "lstrip": false,
       "normalized": false,
@@ -168,7 +160,7 @@
       "single_word": false,
       "special": true
     },
-    "64018": {
+    "64017": {
       "content": "<ha>",
       "lstrip": false,
       "normalized": false,
@@ -176,7 +168,7 @@
       "single_word": false,
       "special": true
     },
-    "64019": {
+    "64018": {
       "content": "<he>",
       "lstrip": false,
       "normalized": false,
@@ -184,7 +176,7 @@
       "single_word": false,
       "special": true
     },
-    "64020": {
+    "64019": {
       "content": "<hi>",
       "lstrip": false,
       "normalized": false,
@@ -192,7 +184,7 @@
       "single_word": false,
       "special": true
     },
-    "64021": {
+    "64020": {
       "content": "<is>",
       "lstrip": false,
       "normalized": false,
@@ -200,7 +192,7 @@
       "single_word": false,
       "special": true
     },
-    "64022": {
+    "64021": {
       "content": "<it>",
       "lstrip": false,
       "normalized": false,
@@ -208,7 +200,7 @@
       "single_word": false,
       "special": true
     },
-    "64023": {
+    "64022": {
       "content": "<kab>",
       "lstrip": false,
       "normalized": false,
@@ -216,7 +208,7 @@
       "single_word": false,
       "special": true
     },
-    "64024": {
+    "64023": {
       "content": "<kn>",
       "lstrip": false,
       "normalized": false,
@@ -224,7 +216,7 @@
       "single_word": false,
       "special": true
     },
-    "64025": {
+    "64024": {
       "content": "<lb>",
       "lstrip": false,
       "normalized": false,
@@ -232,7 +224,7 @@
       "single_word": false,
       "special": true
     },
-    "64026": {
+    "64025": {
       "content": "<mr>",
       "lstrip": false,
       "normalized": false,
@@ -240,7 +232,7 @@
       "single_word": false,
       "special": true
     },
-    "64027": {
+    "64026": {
       "content": "<mt>",
       "lstrip": false,
       "normalized": false,
@@ -248,7 +240,7 @@
       "single_word": false,
       "special": true
     },
-    "64028": {
+    "64027": {
       "content": "<ne>",
       "lstrip": false,
       "normalized": false,
@@ -256,7 +248,7 @@
       "single_word": false,
       "special": true
     },
-    "64029": {
+    "64028": {
       "content": "<nl>",
       "lstrip": false,
       "normalized": false,
@@ -264,7 +256,7 @@
       "single_word": false,
       "special": true
     },
-    "64030": {
+    "64029": {
       "content": "<no>",
       "lstrip": false,
       "normalized": false,
@@ -272,7 +264,7 @@
       "single_word": false,
       "special": true
     },
-    "64031": {
+    "64030": {
       "content": "<oc>",
       "lstrip": false,
       "normalized": false,
@@ -280,7 +272,7 @@
       "single_word": false,
       "special": true
     },
-    "64032": {
+    "64031": {
       "content": "<pl>",
       "lstrip": false,
       "normalized": false,
@@ -288,7 +280,7 @@
       "single_word": false,
       "special": true
     },
-    "64033": {
+    "64032": {
       "content": "<pt>",
       "lstrip": false,
       "normalized": false,
@@ -296,7 +288,7 @@
       "single_word": false,
       "special": true
     },
-    "64034": {
+    "64033": {
       "content": "<ro>",
       "lstrip": false,
       "normalized": false,
@@ -304,7 +296,7 @@
       "single_word": false,
       "special": true
     },
-    "64035": {
+    "64034": {
       "content": "<ru>",
       "lstrip": false,
       "normalized": false,
@@ -312,7 +304,7 @@
       "single_word": false,
       "special": true
     },
-    "64036": {
+    "64035": {
       "content": "<sd>",
       "lstrip": false,
       "normalized": false,
@@ -320,7 +312,7 @@
       "single_word": false,
       "special": true
     },
-    "64037": {
+    "64036": {
       "content": "<so>",
       "lstrip": false,
       "normalized": false,
@@ -328,7 +320,7 @@
       "single_word": false,
       "special": true
     },
-    "64038": {
+    "64037": {
       "content": "<sr>",
       "lstrip": false,
       "normalized": false,
@@ -336,7 +328,7 @@
       "single_word": false,
       "special": true
     },
-    "64039": {
+    "64038": {
       "content": "<sv>",
       "lstrip": false,
       "normalized": false,
@@ -344,7 +336,7 @@
       "single_word": false,
       "special": true
     },
-    "64040": {
+    "64039": {
       "content": "<ti>",
       "lstrip": false,
       "normalized": false,
@@ -352,7 +344,7 @@
       "single_word": false,
       "special": true
     },
-    "64041": {
+    "64040": {
       "content": "<uk>",
       "lstrip": false,
       "normalized": false,
@@ -360,7 +352,7 @@
       "single_word": false,
       "special": true
     },
-    "64042": {
+    "64041": {
       "content": "<ur>",
       "lstrip": false,
       "normalized": false,
@@ -412,10 +404,10 @@
     "<uk>",
     "<ur>"
   ],
-  "bos_token": "<eos>",
+  "bos_token": "</s>",
   "clean_up_tokenization_spaces": true,
   "cls_token": null,
-  "eos_token": "<eos>",
+  "eos_token": "</s>",
   "legacy_behaviour": false,
   "mask_token": null,
   "model_max_length": 1000000000000000019884624838656,