anikifoss

/

Kimi-K2-Instruct-DQ4_K

like 4

Updated chate_template to the latest version, you can re-download Kimi-K2-Instruct-DQ4_K-00001-of-00014.gguf or manually apply the patch using instructions below.

cd $PATH_TO_COMPILED_LLAMA_CPP
cd gguf-py
python -m venv venv
source venv/bin/activate
pip install -r ../requirements/requirements-gguf_editor_gui.txt
pip install --editable .

cd $PATH_TO_KIMI_K2

mv Kimi-K2-Instruct-DQ4_K-00001-of-00014.gguf Kimi-K2-Instruct-DQ4_K-00001-of-00014.original.gguf
gguf-new-metadata \
  --chat-template '{%- if tools -%}
  <|im_system|>tool_declare<|im_middle|>{{ tools | tojson }}<|im_end|>
{%- endif -%}
{%- for message in messages -%}
  {%- if loop.first and messages[0]['"'"'role'"'"'] != '"'"'system'"'"' -%}
    <|im_system|>system<|im_middle|>You are Kimi, an AI assistant created by Moonshot AI.<|im_end|>
  {%- endif -%}
  {%- if message['"'"'role'"'"'] == '"'"'system'"'"' -%}
    <|im_system|>system<|im_middle|>
  {%- elif message['"'"'role'"'"'] == '"'"'user'"'"' -%}
    <|im_user|>user<|im_middle|>
  {%- elif message['"'"'role'"'"'] == '"'"'assistant'"'"' -%}
    <|im_assistant|>assistant<|im_middle|>
  {%- elif message['"'"'role'"'"'] == '"'"'tool'"'"' -%}
    <|im_system|>tool<|im_middle|>
  {%- endif -%}
  {%- if message['"'"'role'"'"'] == '"'"'assistant'"'"' and message.get('"'"'tool_calls'"'"') -%}
    {%- if message['"'"'content'"'"'] -%}{{ message['"'"'content'"'"'] }}{%- endif -%}
    <|tool_calls_section_begin|>
    {%- for tool_call in message['"'"'tool_calls'"'"'] -%}
      {%- set formatted_id = tool_call['"'"'id'"'"'] -%}
      <|tool_call_begin|>{{ formatted_id }}<|tool_call_argument_begin|>{% if tool_call['"'"'function'"'"']['"'"'arguments'"'"'] is string %}{{ tool_call['"'"'function'"'"']['"'"'arguments'"'"'] }}{% else %}{{ tool_call['"'"'function'"'"']['"'"'arguments'"'"'] | tojson }}{% endif %}<|tool_call_end|>
    {%- endfor -%}
    <|tool_calls_section_end|>
  {%- elif message['"'"'role'"'"'] == '"'"'tool'"'"' -%}
    ## Return of {{ message.tool_call_id }}
    {{ message['"'"'content'"'"'] }}
  {%- elif message['"'"'content'"'"'] is string -%}
    {{ message['"'"'content'"'"'] }}
  {%- elif message['"'"'content'"'"'] is not none -%}
    {% for content in message['"'"'content'"'"'] -%}
      {% if content['"'"'type'"'"'] == '"'"'image'"'"' or '"'"'image'"'"' in content or '"'"'image_url'"'"' in content -%}
        <|media_start|>image<|media_content|><|media_pad|><|media_end|>
      {% else -%}
        {{ content['"'"'text'"'"'] }}
      {%- endif -%}
    {%- endfor -%}
  {%- endif -%}
  <|im_end|>
{%- endfor -%}
{%- if add_generation_prompt -%}
  <|im_assistant|>assistant<|im_middle|>
{%- endif -%}' \
  Kimi-K2-Instruct-DQ4_K-00001-of-00014.original.gguf \
  Kimi-K2-Instruct-DQ4_K-00001-of-00014.gguf