textgeflecht commited on
Commit
2605129
·
verified ·
1 Parent(s): a584589

Delete tokenizer.json

Browse files

didnt work with https://github.com/huggingface/inference-benchmarker

Files changed (1) hide show
  1. tokenizer.json +0 -6
tokenizer.json DELETED
@@ -1,6 +0,0 @@
1
- {
2
- "model_max_length": 131072,
3
- "padding_side": "left",
4
- "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + '</s>'}}{% else %}{{ EOS_TOKEN + message['content'] + EOS_TOKEN }}{% endif %}{% endfor %}",
5
- "tokenizer_class": "MistralTokenizer"
6
- }