Update tokenizer_config.json
Browse files- tokenizer_config.json +2 -18
tokenizer_config.json
CHANGED
@@ -8002,22 +8002,6 @@
|
|
8002 |
"rstrip": false,
|
8003 |
"single_word": false,
|
8004 |
"special": true
|
8005 |
-
},
|
8006 |
-
"131072": {
|
8007 |
-
"content": "<SPECIAL_20>",
|
8008 |
-
"lstrip": false,
|
8009 |
-
"normalized": false,
|
8010 |
-
"rstrip": false,
|
8011 |
-
"single_word": false,
|
8012 |
-
"special": true
|
8013 |
-
},
|
8014 |
-
"131073": {
|
8015 |
-
"content": "<SPECIAL_21>",
|
8016 |
-
"lstrip": false,
|
8017 |
-
"normalized": false,
|
8018 |
-
"rstrip": false,
|
8019 |
-
"single_word": false,
|
8020 |
-
"special": true
|
8021 |
}
|
8022 |
},
|
8023 |
"additional_special_tokens": [
|
@@ -8041,8 +8025,8 @@
|
|
8041 |
"[SYSTEM_PROMPT]",
|
8042 |
"[/SYSTEM_PROMPT]",
|
8043 |
"[TOOL_CONTENT]",
|
8044 |
-
"<
|
8045 |
-
"
|
8046 |
"<SPECIAL_22>",
|
8047 |
"<SPECIAL_23>",
|
8048 |
"<SPECIAL_24>",
|
|
|
8002 |
"rstrip": false,
|
8003 |
"single_word": false,
|
8004 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
8005 |
}
|
8006 |
},
|
8007 |
"additional_special_tokens": [
|
|
|
8025 |
"[SYSTEM_PROMPT]",
|
8026 |
"[/SYSTEM_PROMPT]",
|
8027 |
"[TOOL_CONTENT]",
|
8028 |
+
"<think>",
|
8029 |
+
"</think>",
|
8030 |
"<SPECIAL_22>",
|
8031 |
"<SPECIAL_23>",
|
8032 |
"<SPECIAL_24>",
|