{ | |
"added_tokens_decoder": { | |
"163584": { | |
"content": "[BOS]", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163585": { | |
"content": "[EOS]", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163586": { | |
"content": "<|im_end|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163587": { | |
"content": "<|im_user|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163588": { | |
"content": "<|im_assistant|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163594": { | |
"content": "<|im_system|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163601": { | |
"content": "<|im_middle|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163602": { | |
"content": "<|media_start|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163603": { | |
"content": "<|media_content|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163604": { | |
"content": "<|media_end|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163605": { | |
"content": "<|media_pad|>", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163838": { | |
"content": "[PAD]", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
}, | |
"163839": { | |
"content": "[UNK]", | |
"lstrip": false, | |
"normalized": false, | |
"rstrip": false, | |
"single_word": false, | |
"special": true | |
} | |
}, | |
"additional_special_tokens": [ | |
"<|im_end|>", | |
"<|im_user|>", | |
"<|im_assistant|>", | |
"<|im_system|>", | |
"<|im_middle|>", | |
"<|media_start|>", | |
"<|media_content|>", | |
"<|media_end|>", | |
"<|media_pad|>" | |
], | |
"auto_map": { | |
"AutoTokenizer": [ | |
"tokenization_moonshot.TikTokenTokenizer", | |
null | |
] | |
}, | |
"bos_token": "[BOS]", | |
"clean_up_tokenization_spaces": false, | |
"eos_token": "[EOS]", | |
"extra_special_tokens": {}, | |
"model_max_length": 1048576, | |
"pad_token": "[PAD]", | |
"tokenizer_class": "TikTokenTokenizer", | |
"unk_token": "[UNK]" | |
} | |