--- library_name: transformers tags: - axolotl - generated_from_trainer datasets: - AlexHung29629/train_0415_chat_notag - AlexHung29629/glaive-function-calling-v2-mistral - vision_dialogue_dataset-0527.jsonl - AlexHung29629/When2Call_mistral - AlexHung29629/tir_upload - AlexHung29629/tir_upload - AlexHung29629/tir_upload - AlexHung29629/tir_upload model-index: - name: model_step2 results: [] --- [Built with Axolotl](https://github.com/axolotl-ai-cloud/axolotl)
See axolotl config axolotl version: `0.10.0.dev0` ```yaml base_model: /mnt/shared/tp1-an1/alex/FFM_training/placeholder_sft_v2 plugins: - axolotl.integrations.liger.LigerPlugin liger_rope: true liger_rms_norm: true liger_glu_activation: true liger_fused_linear_cross_entropy: true #unfrozen_parameters: # - language_model.* # - multi_modal_projector.* datasets: - path: AlexHung29629/train_0415_chat_notag type: chat_template field_messages: conversations chat_template: jinja chat_template_jinja: "{%- set today = strftime_now(\"%Y-%m-%d\") %}\n{%- set default_system_message = \"You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is \" + today + \".\\n\\nWhen you're not sure about some information, you say that you don't have the information and don't make up anything.\\nIf the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \\\"What are some good restaurants around me?\\\" => \\\"Where are you?\\\" or \\\"When is the next flight to Tokyo\\\" => \\\"Where do you travel from?\\\")\" %}\n{{- bos_token }}\n{%- if messages[0]['role'] == 'system' %}\n {%- if messages[0]['content'] is string %}\n {%- set system_message = messages[0]['content'] %}\n {%- set loop_messages = messages[1:] %}\n {%- else %}\n {%- set system_message = messages[0]['content'][0]['text'] %}\n {%- set loop_messages = messages[1:] %}\n {%- endif %}\n{%- else %}\n {%- set system_message = default_system_message %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- elif tools is not none %}\n {%- set parallel_tool_prompt = \"You are a helpful assistant that can call tools. If you call one or more tools, format them in a single JSON array or objects, where each object is a tool call, not as separate objects outside of an array or multiple arrays. Use the format [{\\\"name\\\": tool call name, \\\"arguments\\\": tool call arguments}, additional tool calls] if you call more than one tool. If you call tools, do not attempt to interpret them or otherwise provide a response until you receive a tool call result that you can interpret for the user.\" %}\n {%- if system_message is defined %}\n {%- set system_message = system_message + \"\\n\\n\" + parallel_tool_prompt %}\n {%- else %}\n {%- set system_message = parallel_tool_prompt %}\n {%- endif %}\n{%- endif %}\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n{%- for message in loop_messages %}\n {%- if message[\"role\"] == \"user\" %}\n {%- if tools is not none and (message == user_messages[-1]) %}\n {{- \"[AVAILABLE_TOOLS] [\" }}\n {%- for tool in tools %}\n {%- set tool = tool.function %}\n {{- '{\"type\": \"function\", \"function\": {' }}\n {%- for key, val in tool.items() if key != \"return\" %}\n {%- if val is string %}\n {{- '\"' + key + '\": \"' + val + '\"' }}\n {%- else %}\n {{- '\"' + key + '\": ' + val|tojson }}\n {%- endif %}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \"}}\" }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" }}\n {%- endif %}\n {%- endfor %}\n {{- \"[/AVAILABLE_TOOLS]\" }}\n {%- endif %}\n {%- if message['content'] is string %}\n {{- '[INST]' + message['content'] + '[/INST]\\n' }}\n {%- else %}\n {{- '[INST]' }}\n {%- for block in message['content'] %}\n {%- if block['type'] == 'text' %}\n {{- block['text'] }}\n {%- elif block['type'] == 'image' or block['type'] == 'image_url' %}\n {{- '[IMG]' }}\n {%- else %}\n {{- raise_exception('Only text and image blocks are supported in message content!') }}\n {%- endif %}\n {%- endfor %}\n {{- '[/INST]\\n' }}\n {%- endif %}\n {%- if enable_thinking is defined %}\n {%- if enable_thinking is false %}\n {{- '\\n' }}\n {%- endif %}\n {%- endif %}\n {%- elif message[\"role\"] == \"tool_calls\" or message.tool_calls is defined %}\n {%- if message.tool_calls is defined %}\n {%- set tool_calls = message.tool_calls %}\n {%- else %}\n {%- set tool_calls = message.content %}\n {%- endif %}\n {{- \"[TOOL_CALLS] [\" }}\n {%- for tool_call in tool_calls %}\n {%- set out = tool_call.function|tojson %}\n {{- out[:-1] }}\n {%- if not tool_call.id is defined or tool_call.id|length < 9 %}\n {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length >= 9! (1)\" + tool_call.id) }}\n {%- endif %}\n {{- ', \"id\": \"' + tool_call.id[-9:] + '\"}' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" + eos_token }}\n {%- endif %}\n {%- endfor %}\n {%- elif message['role'] == 'assistant' %}\n {%- if message['content'] is string %}\n {{- message['content'] + eos_token }}\n {%- else %}\n {{- message['content'][0]['text'] + eos_token }}\n {%- endif %}\n {%- elif message[\"role\"] == \"tool_results\" or message[\"role\"] == \"tool\" %}\n {%- if message.content is defined and message.content.content is defined %}\n {%- set content = message.content.content %}\n {%- else %}\n {%- set content = message.content %}\n {%- endif %}\n {{- '[TOOL_RESULTS] {\"content\": ' + content|string + \", \" }}\n {%- if not message.tool_call_id is defined or message.tool_call_id|length < 9 %}\n {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length >= 9! (2)\" + message.tool_call_id) }}\n {%- endif %}\n {{- '\"call_id\": \"' + message.tool_call_id[-9:] + '\"}[/TOOL_RESULTS]' }}\n {%- else %}\n {{- raise_exception(\"Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n {%- endif %}\n{%- endfor %}" - path: AlexHung29629/glaive-function-calling-v2-mistral type: system_prompt: "" field_system: field_instruction: input field_output: output format: "{instruction}\n\n" no_input_format: "{instruction}\n\n" - path: vision_dialogue_dataset-0527.jsonl type: chat_template field_messages: messages chat_template: jinja chat_template_jinja: "{%- set today = strftime_now(\"%Y-%m-%d\") %}\n{%- set default_system_message = \"You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\\nYour knowledge base was last updated on 2023-10-01. The current date is \" + today + \".\\n\\nWhen you're not sure about some information, you say that you don't have the information and don't make up anything.\\nIf the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \\\"What are some good restaurants around me?\\\" => \\\"Where are you?\\\" or \\\"When is the next flight to Tokyo\\\" => \\\"Where do you travel from?\\\")\" %}\n\n{{- bos_token }}\n\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content'] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set system_message = default_system_message %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\n\n{%- for message in loop_messages %}\n {%- if message['role'] == 'user' %}\n\t {%- if message['content'] is string %}\n {{- '[INST]' + message['content'] + '[/INST]\n\n' }}\n\t {%- else %}\n\t\t {{- '[INST]' }}\n\t\t {%- for block in message['content'] %}\n\t\t\t {%- if block['type'] == 'text' %}\n\t\t\t\t {{- block['text'] }}\n\t\t\t {%- elif block['type'] == 'image' or block['type'] == 'image_url' %}\n\t\t\t\t {{- '[IMG]' }}\n\t\t\t\t{%- else %}\n\t\t\t\t {{- raise_exception('Only text and image blocks are supported in message content!') }}\n\t\t\t\t{%- endif %}\n\t\t\t{%- endfor %}\n\t\t {{- '[/INST]\n\n' }}\n\t\t{%- endif %}\n {%- elif message['role'] == 'system' %}\n {{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }}\n {%- elif message['role'] == 'assistant' %}\n {%- if message['content'] is string %}\n {{- message['content'] }}\n {%- elif message['content'] is iterable %} \n\t\t {%- for block in message['content'] %}\n\t\t\t {%- if block['type'] == 'text' %}\n\t\t\t\t {{- block['text'] }}\n\t\t\t {%- else %}\n\t\t\t\t {{- raise_exception('Only text blocks are supported in assistant message content!') }} {%- endif %}\n\t\t\t \n\t\t\t{%- endfor %} {{- eos_token }} {%- else %}\n {{- raise_exception('Unsupported assistant message content format!') }} \n{%- endif %} \n{%- else %}\n {{- raise_exception('Only user, system and assistant roles are supported!') }}\n {%- endif %}\n{%- endfor %}" - path: AlexHung29629/When2Call_mistral type: system_prompt: "" field_system: system field_instruction: input field_output: output format: "{instruction}\n\n" no_input_format: "{instruction}\n\n" - path: AlexHung29629/tir_upload split: mix type: chat_template chat_template: jinja chat_template_jinja: "{%- set default_system_message = \"You are an advanced language model capable of tool-integrated reasoning. This means that during your internal reasoning process (the \\\"thinking\\\" phase), you are allowed to use external tools\u2014such as Python code execution\u2014to assist with complex calculations, data manipulation, or logical deductions.When using Python code, you must follow this specific format:\\n Wrap the code block within and tags.\\n After execution, the system will return:\\n The result in a markdown block labeled output.\\n A system status block labeled system, indicating remaining executions.\" %}\n{{- bos_token }}\n{%- if messages[0]['role'] == 'system' %}\n {%- if messages[0]['content'] is string %}\n {%- set system_message = messages[0]['content'] %}\n {%- set loop_messages = messages[1:] %}\n {%- else %}\n {%- set system_message = messages[0]['content'][0]['text'] %}\n {%- set loop_messages = messages[1:] %}\n {%- endif %}\n{%- else %}\n {%- set system_message = default_system_message %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- elif tools is not none %}\n {%- set parallel_tool_prompt = \"You are a helpful assistant that can call tools. If you call one or more tools, format them in a single JSON array or objects, where each object is a tool call, not as separate objects outside of an array or multiple arrays. Use the format [{\\\"name\\\": tool call name, \\\"arguments\\\": tool call arguments}, additional tool calls] if you call more than one tool. If you call tools, do not attempt to interpret them or otherwise provide a response until you receive a tool call result that you can interpret for the user.\" %}\n {%- if system_message is defined %}\n {%- set system_message = system_message + \"\\n\\n\" + parallel_tool_prompt %}\n {%- else %}\n {%- set system_message = parallel_tool_prompt %}\n {%- endif %}\n{%- endif %}\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\n\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n{%- for message in loop_messages %}\n {%- if message[\"role\"] == \"AVAILABLE_TOOLS\" %}\n {{- \"[AVAILABLE_TOOLS] \" + message['content'] + \"[/AVAILABLE_TOOLS]\" }}\n {%- elif message[\"role\"] == \"user\" %}\n {%- if message['content'] is string %}\n {{- '[INST]' + message['content'] + '[/INST]' }}\n {%- else %}\n {{- '[INST]' }}\n {%- for block in message['content'] %}\n {%- if block['type'] == 'text' %}\n {{- block['text'] }}\n {%- elif block['type'] == 'image' or block['type'] == 'image_url' %}\n {{- '[IMG]' }}\n {%- else %}\n {{- raise_exception('Only text and image blocks are supported in message content!') }}\n {%- endif %}\n {%- endfor %}\n {{- '[/INST]' }}\n {%- endif %}\n {%- elif message[\"role\"] == \"tool_calls\" or message[\"role\"] == \"tool_call\" or message.tool_calls is defined %}\n {{- \"[TOOL_CALLS] \" + message.content + eos_token }}\n {%- elif message['role'] == 'assistant' %}\n {%- if message['content'] is string %}\n {{- message['content'] + eos_token }}\n {%- else %}\n {{- message['content'][0]['text'] + eos_token }}\n {%- endif %}\n {%- elif message['role'] == 'think' %}\n {{- message['content'] }}\n {%- elif message['role'] == 'ipython' %}\n {{- \"\\n\\n\" }}\n {{- message['content'] }}\n {{- \"\\n\\n\" }}\n {%- elif message['role'] == 'tool_system' %}\n {{- \"```system\\n\" }}\n {{- message['content'] }}\n {{- \"\\n```\\n\" }}\n {%- elif message[\"role\"][:11] == \"tool_result\" %}\n {%- set prev_role = loop_messages[loop.index0 - 1]['role'] %}\n {%- if prev_role == 'ipython' %}\n {{- \"```output\\n\" + message['content'] + \"\\n```\\n\"}}\n {%- else %}\n\t {%- if message.content is defined and message.content.content is defined %}\n\t {%- set content = message.content.content %}\n {%- else %}\n\t {%- set content = message.content %}\n {%- endif %}\n {{- '[TOOL_RESULTS] {\"content\": ' + content|string + \", \" }}\n {{- '\"call_id\": \"' + message[\"role\"][-9:] + '\"}[/TOOL_RESULTS]' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception(message['role'] + \"? Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n {%- endif %}\n{%- endfor %}\n" field_messages: messages roles_to_train: ["assistant", "think", "ipython", "tool_call"] - path: AlexHung29629/tir_upload split: py type: chat_template chat_template: jinja chat_template_jinja: "{%- set default_system_message = \"You are an advanced language model capable of tool-integrated reasoning. This means that during your internal reasoning process (the \\\"thinking\\\" phase), you are allowed to use external tools\u2014such as Python code execution\u2014to assist with complex calculations, data manipulation, or logical deductions.When using Python code, you must follow this specific format:\\n Wrap the code block within and tags.\\n After execution, the system will return:\\n The result in a markdown block labeled output.\\n A system status block labeled system, indicating remaining executions.\" %}\n{{- bos_token }}\n{%- if messages[0]['role'] == 'system' %}\n {%- if messages[0]['content'] is string %}\n {%- set system_message = messages[0]['content'] %}\n {%- set loop_messages = messages[1:] %}\n {%- else %}\n {%- set system_message = messages[0]['content'][0]['text'] %}\n {%- set loop_messages = messages[1:] %}\n {%- endif %}\n{%- else %}\n {%- set system_message = default_system_message %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- elif tools is not none %}\n {%- set parallel_tool_prompt = \"You are a helpful assistant that can call tools. If you call one or more tools, format them in a single JSON array or objects, where each object is a tool call, not as separate objects outside of an array or multiple arrays. Use the format [{\\\"name\\\": tool call name, \\\"arguments\\\": tool call arguments}, additional tool calls] if you call more than one tool. If you call tools, do not attempt to interpret them or otherwise provide a response until you receive a tool call result that you can interpret for the user.\" %}\n {%- if system_message is defined %}\n {%- set system_message = system_message + \"\\n\\n\" + parallel_tool_prompt %}\n {%- else %}\n {%- set system_message = parallel_tool_prompt %}\n {%- endif %}\n{%- endif %}\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\n\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n{%- for message in loop_messages %}\n {%- if message[\"role\"] == \"AVAILABLE_TOOLS\" %}\n {{- \"[AVAILABLE_TOOLS] \" + message['content'] + \"[/AVAILABLE_TOOLS]\" }}\n {%- elif message[\"role\"] == \"user\" %}\n {%- if message['content'] is string %}\n {{- '[INST]' + message['content'] + '[/INST]' }}\n {%- else %}\n {{- '[INST]' }}\n {%- for block in message['content'] %}\n {%- if block['type'] == 'text' %}\n {{- block['text'] }}\n {%- elif block['type'] == 'image' or block['type'] == 'image_url' %}\n {{- '[IMG]' }}\n {%- else %}\n {{- raise_exception('Only text and image blocks are supported in message content!') }}\n {%- endif %}\n {%- endfor %}\n {{- '[/INST]' }}\n {%- endif %}\n {%- elif message[\"role\"] == \"tool_calls\" or message[\"role\"] == \"tool_call\" or message.tool_calls is defined %}\n {{- \"[TOOL_CALLS] \" + message.content + eos_token }}\n {%- elif message['role'] == 'assistant' %}\n {%- if message['content'] is string %}\n {{- message['content'] + eos_token }}\n {%- else %}\n {{- message['content'][0]['text'] + eos_token }}\n {%- endif %}\n {%- elif message['role'] == 'think' %}\n {{- message['content'] }}\n {%- elif message['role'] == 'ipython' %}\n {{- \"\\n\\n\" }}\n {{- message['content'] }}\n {{- \"\\n\\n\" }}\n {%- elif message['role'] == 'tool_system' %}\n {{- \"```system\\n\" }}\n {{- message['content'] }}\n {{- \"\\n```\\n\" }}\n {%- elif message[\"role\"][:11] == \"tool_result\" %}\n {%- set prev_role = loop_messages[loop.index0 - 1]['role'] %}\n {%- if prev_role == 'ipython' %}\n {{- \"```output\\n\" + message['content'] + \"\\n```\\n\"}}\n {%- else %}\n\t {%- if message.content is defined and message.content.content is defined %}\n\t {%- set content = message.content.content %}\n {%- else %}\n\t {%- set content = message.content %}\n {%- endif %}\n {{- '[TOOL_RESULTS] {\"content\": ' + content|string + \", \" }}\n {{- '\"call_id\": \"' + message[\"role\"][-9:] + '\"}[/TOOL_RESULTS]' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception(message['role'] + \"? Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n {%- endif %}\n{%- endfor %}\n" field_messages: messages roles_to_train: ["assistant", "think", "ipython"] - path: AlexHung29629/tir_upload split: mix_xml type: chat_template chat_template: jinja chat_template_jinja: "{%- set default_system_message = \"You are an advanced language model capable of tool-integrated reasoning. This means that during your internal reasoning process (the \\\"thinking\\\" phase), you are allowed to use external tools\u2014such as Python code execution\u2014to assist with complex calculations, data manipulation, or logical deductions.When using Python code, you must follow this specific format:\\n Wrap the code block within and tags.\\n After execution, the system will return:\\n The result in a markdown block labeled output.\\n A system status block labeled system, indicating remaining executions.\\n\\n# Tool Use Formatting\u200b\\n Tool use is formatted using XML-style tags. The tool name is enclosed in opening and closing tags, and each parameter is similarly enclosed within its own set of tags. Here's the structure:\u200b\\n\u200b\\nvalue1\u200b\\nvalue2\u200b\\n...\u200b\\n\u200b\" %}\n{{- bos_token }}\n{%- if messages[0]['role'] == 'system' %}\n {%- if messages[0]['content'] is string %}\n {%- set system_message = messages[0]['content'] %}\n {%- set loop_messages = messages[1:] %}\n {%- else %}\n {%- set system_message = messages[0]['content'][0]['text'] %}\n {%- set loop_messages = messages[1:] %}\n {%- endif %}\n{%- else %}\n {%- set system_message = default_system_message %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- elif tools is not none %}\n {%- set parallel_tool_prompt = \"You are a helpful assistant that can call tools. If you call one or more tools, format them in a single JSON array or objects, where each object is a tool call, not as separate objects outside of an array or multiple arrays. Use the format [{\\\"name\\\": tool call name, \\\"arguments\\\": tool call arguments}, additional tool calls] if you call more than one tool. If you call tools, do not attempt to interpret them or otherwise provide a response until you receive a tool call result that you can interpret for the user.\" %}\n {%- if system_message is defined %}\n {%- set system_message = system_message + \"\\n\\n\" + parallel_tool_prompt %}\n {%- else %}\n {%- set system_message = parallel_tool_prompt %}\n {%- endif %}\n{%- endif %}\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\n\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n{%- for message in loop_messages %}\n {%- if message[\"role\"] == \"AVAILABLE_TOOLS\" %}\n {{- \"[AVAILABLE_TOOLS] \" + message['content'] + \"[/AVAILABLE_TOOLS]\" }}\n {%- elif message[\"role\"] == \"user\" %}\n {%- if message['content'] is string %}\n {{- '[INST]' + message['content'] + '[/INST]' }}\n {%- else %}\n {{- '[INST]' }}\n {%- for block in message['content'] %}\n {%- if block['type'] == 'text' %}\n {{- block['text'] }}\n {%- elif block['type'] == 'image' or block['type'] == 'image_url' %}\n {{- '[IMG]' }}\n {%- else %}\n {{- raise_exception('Only text and image blocks are supported in message content!') }}\n {%- endif %}\n {%- endfor %}\n {{- '[/INST]' }}\n {%- endif %}\n {%- elif message[\"role\"] == \"tool_calls\" or message[\"role\"] == \"tool_call\" or message.tool_calls is defined %}\n {{- message.content + eos_token }}\n {%- elif message['role'] == 'assistant' %}\n {%- if message['content'] is string %}\n {{- message['content'] + eos_token }}\n {%- else %}\n {{- message['content'][0]['text'] + eos_token }}\n {%- endif %}\n {%- elif message['role'] == 'think' %}\n {{- message['content'] }}\n {%- elif message['role'] == 'ipython' %}\n {{- \"\\n\\n\" }}\n {{- message['content'] }}\n {{- \"\\n\\n\" }}\n {%- elif message['role'] == 'tool_system' %}\n {{- \"```system\\n\" }}\n {{- message['content'] }}\n {{- \"\\n```\\n\" }}\n {%- elif message[\"role\"][:11] == \"tool_result\" %}\n {%- set prev_role = loop_messages[loop.index0 - 1]['role'] %}\n {%- if prev_role == 'ipython' %}\n {{- \"```output\\n\" + message['content'] + \"\\n```\\n\"}}\n {%- else %}\n\t {%- if message.content is defined and message.content.content is defined %}\n\t {%- set content = message.content.content %}\n {%- else %}\n\t {%- set content = message.content %}\n {%- endif %}\n {{- '[TOOL_RESULTS] {\"content\": ' + content|string + \"}[/TOOL_RESULTS]\" }}\n {%- endif %}\n {%- else %}\n {{- raise_exception(message['role'] + \"? Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n {%- endif %}\n{%- endfor %}\n" field_messages: messages roles_to_train: ["assistant", "think", "ipython", "tool_call"] - path: AlexHung29629/tir_upload split: mix_yaml type: chat_template chat_template: jinja chat_template_jinja: "{%- set default_system_message = \"You are an advanced language model capable of tool-integrated reasoning. This means that during your internal reasoning process (the \\\"thinking\\\" phase), you are allowed to use external tools\u2014such as Python code execution\u2014to assist with complex calculations, data manipulation, or logical deductions.When using Python code, you must follow this specific format:\\n Wrap the code block within and tags.\\n After execution, the system will return:\\n The result in a markdown block labeled output.\\n A system status block labeled system, indicating remaining executions.\\n\\n# Tool Use Formatting\u200b\\nTool use is formatted using YAML. Here's the structure:\u200b\\n- name: tool_name\\n arguments:\\n - parameter1_name: value1\\n - parameter2_name: value2\u200b\" %}\n{{- bos_token }}\n{%- if messages[0]['role'] == 'system' %}\n {%- if messages[0]['content'] is string %}\n {%- set system_message = messages[0]['content'] %}\n {%- set loop_messages = messages[1:] %}\n {%- else %}\n {%- set system_message = messages[0]['content'][0]['text'] %}\n {%- set loop_messages = messages[1:] %}\n {%- endif %}\n{%- else %}\n {%- set system_message = default_system_message %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- elif tools is not none %}\n {%- set parallel_tool_prompt = \"You are a helpful assistant that can call tools. If you call one or more tools, format them in a single JSON array or objects, where each object is a tool call, not as separate objects outside of an array or multiple arrays. Use the format [{\\\"name\\\": tool call name, \\\"arguments\\\": tool call arguments}, additional tool calls] if you call more than one tool. If you call tools, do not attempt to interpret them or otherwise provide a response until you receive a tool call result that you can interpret for the user.\" %}\n {%- if system_message is defined %}\n {%- set system_message = system_message + \"\\n\\n\" + parallel_tool_prompt %}\n {%- else %}\n {%- set system_message = parallel_tool_prompt %}\n {%- endif %}\n{%- endif %}\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\n\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n{%- for message in loop_messages %}\n {%- if message[\"role\"] == \"AVAILABLE_TOOLS\" %}\n {{- \"[AVAILABLE_TOOLS] \" + message['content'] + \"[/AVAILABLE_TOOLS]\" }}\n {%- elif message[\"role\"] == \"user\" %}\n {%- if message['content'] is string %}\n {{- '[INST]' + message['content'] + '[/INST]' }}\n {%- else %}\n {{- '[INST]' }}\n {%- for block in message['content'] %}\n {%- if block['type'] == 'text' %}\n {{- block['text'] }}\n {%- elif block['type'] == 'image' or block['type'] == 'image_url' %}\n {{- '[IMG]' }}\n {%- else %}\n {{- raise_exception('Only text and image blocks are supported in message content!') }}\n {%- endif %}\n {%- endfor %}\n {{- '[/INST]' }}\n {%- endif %}\n {%- elif message[\"role\"] == \"tool_calls\" or message[\"role\"] == \"tool_call\" or message.tool_calls is defined %}\n {{- message.content + eos_token }}\n {%- elif message['role'] == 'assistant' %}\n {%- if message['content'] is string %}\n {{- message['content'] + eos_token }}\n {%- else %}\n {{- message['content'][0]['text'] + eos_token }}\n {%- endif %}\n {%- elif message['role'] == 'think' %}\n {{- message['content'] }}\n {%- elif message['role'] == 'ipython' %}\n {{- \"\\n\\n\" }}\n {{- message['content'] }}\n {{- \"\\n\\n\" }}\n {%- elif message['role'] == 'tool_system' %}\n {{- \"```system\\n\" }}\n {{- message['content'] }}\n {{- \"\\n```\\n\" }}\n {%- elif message[\"role\"][:11] == \"tool_result\" %}\n {%- set prev_role = loop_messages[loop.index0 - 1]['role'] %}\n {%- if prev_role == 'ipython' %}\n {{- \"```output\\n\" + message['content'] + \"\\n```\\n\"}}\n {%- else %}\n\t {%- if message.content is defined and message.content.content is defined %}\n\t {%- set content = message.content.content %}\n {%- else %}\n\t {%- set content = message.content %}\n {%- endif %}\n {{- '[TOOL_RESULTS] {\"content\": ' + content|string + \"}[/TOOL_RESULTS]\" }}\n {%- endif %}\n {%- else %}\n {{- raise_exception(message['role'] + \"? Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n {%- endif %}\n{%- endfor %}\n" field_messages: messages roles_to_train: ["assistant", "think", "ipython", "tool_call"] dataset_prepared_path: ./sft_dataprep/ val_set_size: 0 output_dir: ./placeholder_sft_fix/ shuffle_merged_datasets: true sequence_len: 32768 sample_packing: true eval_sample_packing: false pad_to_sequence_len: true hub_model_id: AlexHung29629/model_step2 hub_strategy: every_save wandb_project: TP1_2025_05 wandb_entity: wandb_watch: wandb_name: Mistral-24B-SFT-250522_sft use_tensorboard: true gradient_accumulation_steps: 1 micro_batch_size: 1 num_epochs: 2 optimizer: adamw_torch lr_scheduler: cosine learning_rate: 2e-5 max_grad_norm: 1.0 adam_beta1: 0.9 adam_beta2: 0.95 adam_epsilon: 1e-8 bf16: true tf32: false logging_steps: 1 flash_attention: false xformers_attention: false sdp_attention: true warmup_ratio: 0.05 saves_per_epoch: 2 weight_decay: 0 fsdp: - full_shard - auto_wrap fsdp_config: fsdp_limit_all_gathers: true fsdp_sync_module_states: true fsdp_offload_params: true fsdp_use_orig_params: true fsdp_cpu_ram_efficient_loading: true fsdp_activation_checkpointing: true fsdp_transformer_layer_cls_to_wrap: MistralDecoderLayer fsdp_state_dict_type: FULL_STATE_DICT fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP seed: 42 auto_resume_from_checkpoints: true ```

# model_step2 This model was trained from scratch on the AlexHung29629/train_0415_chat_notag, the AlexHung29629/glaive-function-calling-v2-mistral, the vision_dialogue_dataset-0527.jsonl, the AlexHung29629/When2Call_mistral, the AlexHung29629/tir_upload, the AlexHung29629/tir_upload, the AlexHung29629/tir_upload and the AlexHung29629/tir_upload datasets. ## Model description More information needed ## Intended uses & limitations More information needed ## Training and evaluation data More information needed ## Training procedure ### Training hyperparameters The following hyperparameters were used during training: - learning_rate: 2e-05 - train_batch_size: 1 - eval_batch_size: 1 - seed: 42 - distributed_type: multi-GPU - num_devices: 64 - total_train_batch_size: 64 - total_eval_batch_size: 64 - optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments - lr_scheduler_type: cosine - lr_scheduler_warmup_steps: 245 - num_epochs: 2.0 ### Training results ### Framework versions - Transformers 4.52.4 - Pytorch 2.7.0+cu128 - Datasets 3.5.1 - Tokenizers 0.21.1