Delete tokenizer.json
Browse filesdidnt work with https://github.com/huggingface/inference-benchmarker
- tokenizer.json +0 -6
tokenizer.json
DELETED
@@ -1,6 +0,0 @@
|
|
1 |
-
{
|
2 |
-
"model_max_length": 131072,
|
3 |
-
"padding_side": "left",
|
4 |
-
"chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + '</s>'}}{% else %}{{ EOS_TOKEN + message['content'] + EOS_TOKEN }}{% endif %}{% endfor %}",
|
5 |
-
"tokenizer_class": "MistralTokenizer"
|
6 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|