Upload tokenizer

tokenizer.json

@@ -33,20 +33,18 @@
   ],
   "normalizer": {
     "type": "Sequence",
-    "normalizers": [
-      {
-        "type": "Replace",
-        "pattern": {
-          "String": " "
-        },
-        "content": "▁"
-      }
-    ]
+    "normalizers": []
   },
   "pre_tokenizer": null,
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
       {
         "Sequence": {
           "id": "A",
@@ -55,12 +53,24 @@
       }
     ],
     "pair": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
       {
         "Sequence": {
           "id": "A",
           "type_id": 0
         }
       },
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 1
+        }
+      },
       {
         "Sequence": {
           "id": "B",
@@ -68,7 +78,17 @@
         }
       }
     ],
-    "special_tokens": {}
+    "special_tokens": {
+      "<s>": {
+        "id": "<s>",
+        "ids": [
+          1
+        ],
+        "tokens": [
+          "<s>"
+        ]
+      }
+    }
   },
   "decoder": {
     "type": "Sequence",

tokenizer_config.json

@@ -1,5 +1,5 @@
 {
-  "add_bos_token": false,
+  "add_bos_token": true,
   "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {