|
|
--- |
|
|
library_name: transformers |
|
|
license: apache-2.0 |
|
|
base_model: mistralai/Mistral-Nemo-Instruct-2407 |
|
|
tags: |
|
|
- roleplay |
|
|
- conversational |
|
|
- axolotl |
|
|
--- |
|
|
|
|
|
# 残响 MN 12b (第一系列) |
|
|
|
|
|
[English](./README.md) | [简体中文](./README-cn.md) |
|
|
|
|
|
*空气中飘浮着一缕尘埃。它仿佛来自某个逝去的时代，但你无从追溯。它落在你的舌尖，滋味奇妙。* |
|
|
|
|
|
 |
|
|
|
|
|
「残响」是一系列专注于SFW与NSFW角色扮演及对话的微调大语言模型。 |
|
|
|
|
|
## 量化版本 |
|
|
GGUF: |
|
|
- 待补充！ |
|
|
|
|
|
EXL3: |
|
|
- 待补充！ |
|
|
|
|
|
EXL2: |
|
|
- 待补充！ |
|
|
|
|
|
其他格式: |
|
|
- 待补充！ |
|
|
|
|
|
## 推荐配置 |
|
|
对话模板: Mistral v7 Tekken |
|
|
采样器: |
|
|
视效果自行调整！ |
|
|
|
|
|
## 致谢 |
|
|
特别鸣谢 Allura 和 ilya <3 |
|
|
衷心感谢 Axolotl（训练框架）、Mistral（基础模型）、Nebius（GPU资源）的开发者们，以及我的银行（感谢它提供的信用卡） |
|
|
|
|
|
## 其他 |
|
|
|
|
|
[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl) |
|
|
<details><summary>查看 axolotl 配置</summary> |
|
|
|
|
|
axolotl 版本: `0.8.0.dev0` |
|
|
```yaml |
|
|
# === 模型配置 === |
|
|
base_model: mistralai/Mistral-Nemo-Instruct-2407 # 例如 "mistralai/Mistral-Small-24B-Instruct-2501" |
|
|
load_in_8bit: false |
|
|
load_in_4bit: false |
|
|
|
|
|
# === 训练设置 === |
|
|
num_epochs: 2 |
|
|
micro_batch_size: 16 |
|
|
gradient_accumulation_steps: 1 |
|
|
sequence_len: 8192 |
|
|
sample_packing: true |
|
|
pad_to_sequence_len: true |
|
|
|
|
|
# === 超参数配置 === |
|
|
optimizer: apollo_adamw |
|
|
# Apollo-mini 配置: |
|
|
optim_args: "proj=random,rank=1,scale=128.0,scale_type=tensor,update_proj_gap=200" |
|
|
# 标准 Apollo 配置: |
|
|
# optim_args: |
|
|
optim_target_modules: all_linear |
|
|
learning_rate: 1e-5 |
|
|
lr_scheduler: rex |
|
|
weight_decay: 0.01 |
|
|
warmup_ratio: 0.05 |
|
|
|
|
|
# === 数据配置 === |
|
|
datasets: |
|
|
- path: allura-org/inkmix-v3.0 |
|
|
type: chat_template |
|
|
split: train |
|
|
field_messages: conversations |
|
|
message_field_role: from |
|
|
message_field_content: value |
|
|
|
|
|
dataset_prepared_path: last_run_prepared |
|
|
chat_template: jinja |
|
|
chat_template_jinja: | |
|
|
{{- bos_token }}{%- for message in messages %} |
|
|
{%- if message['role'] == 'system' %} |
|
|
{{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }} |
|
|
{%- elif message['role'] == 'user' %} |
|
|
{{- '[INST]' + message['content'] + '[/INST]' }} |
|
|
{%- elif message['role'] == 'assistant' %} |
|
|
{{- message['content'] + eos_token }} |
|
|
{%- endif %} |
|
|
{%- endfor %} |
|
|
|
|
|
# === 插件 === |
|
|
plugins: |
|
|
- axolotl.integrations.liger.LigerPlugin |
|
|
- axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin |
|
|
|
|
|
# === 硬件优化 === |
|
|
gradient_checkpointing: unsloth |
|
|
gradient_checkpointing_kwargs: |
|
|
use_reentrant: false |
|
|
liger_rope: true |
|
|
liger_rms_norm: true |
|
|
liger_glu_activation: true |
|
|
cut_cross_entropy: true |
|
|
torch_compile: true |
|
|
# 多 GPU 时启用: |
|
|
# deepspeed: [DEEPSPEED_CONFIG_PATH] # 例如 "deepspeed_configs/zero3_bf16.json" |
|
|
|
|
|
# === Wandb 追踪 === |
|
|
wandb_project: nemo12b-inkmix-v3 |
|
|
|
|
|
# === 检查点 === |
|
|
saves_per_epoch: 2 |
|
|
save_total_limit: 3 |
|
|
|
|
|
# === 高级设置 === |
|
|
output_dir: offload |
|
|
bf16: auto |
|
|
flash_attention: true |
|
|
train_on_inputs: false |
|
|
group_by_length: false |
|
|
logging_steps: 1 |
|
|
trust_remote_code: true |
|
|
# nemo 原生不支持系统提示 |
|
|
tokens: |
|
|
- "[SYSTEM_PROMPT]" |
|
|
- "[/SYSTEM_PROMPT]" |
|
|
special_tokens: |
|
|
pad_token: "<pad>" |
|
|
|
|
|
``` |
|
|
|
|
|
</details> |
