Update tokenizer_config.json

tokenizer_config.json

@@ -1,5 +1,6 @@
 {
-  "max_lenght": 1024,
+  "add_bos_token": false,
+  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<|BOS|>",
@@ -40,6 +41,18 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-  }
+    }
+  },
+  "bos_token": "<|BOS|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "max_length": 1024,
+  "pad_token": "<|pad|>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "use_default_system_prompt": false,
+  "use_fast": true
 }