Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_format.py

@@ -0,0 +1,875 @@
+'''AntGLM Chat-model data format.
+
+格式化 AntGLM 以及各种开源模型的符号系统:
+  - 确定 Chat 模型依赖的文件数据结构协议
+  - 确定单轮/多轮的统一结构
+  - 确定 Chat 符号系统的协议, 包括角色定义、分隔符等
+  - 方便做开源模型依赖的 prompt 转换
+  - 支持工具、代码、推理等支持
+
+参考 FastChat Conversation 对象的设计思路.
+Reference: https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py
+'''
+
+import copy
+import dataclasses
+import logging
+import re
+import uuid
+from copy import deepcopy
+from enum import IntEnum, auto
+from typing import Dict, List, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+class PromptStyle(IntEnum):
+    '''Prompt styles.'''
+
+    # 原始 antglm format 格式, 单轮指令没有结构, 多轮 `第1轮\n用户: xx\n机器人: xx\n`
+    ANTGLM_RAW = auto()
+    # Chat format 格式, 单轮多轮统一为 chat format 格式
+    ANTGLM_CHAT = auto()
+    # 单轮指令没有结构, 只有多轮为 chat format 格式
+    ANTGLM_ONLY_MULTITURN_CHAT = auto()
+    # OpenAI ChatML 格式, 包括千问
+    CHATML = auto()
+    # LLAMA2 格式
+    LLAMA2 = auto()
+    # ChatGLM 1/2 格式
+    CHATGLM = auto()
+    # ChatGLM3 格式
+    CHATGLM3 = auto()
+    # 百川格式
+    BAICHUAN2 = auto()
+
+
+@dataclasses.dataclass
+class Chat:
+    '''Chat 数据符号结构, 格式化 AntGLM 以及各种开源模型的符号系统.
+
+    Examples:
+
+    ```python
+    >>> from antllm.data.chat_format import Chat
+
+    >>> ### 从 json 数据结构创建 chat 对象, 并且 format 结构使用 AntGLM 原始结构
+    >>> input_json = {
+    ...     "messages": [
+    ...         {"role": "HUMAN", "content": "讲一个笑话"},
+    ...         {"role": "ASSISTANT", "content": "为什么猪不能上网？因为它们会被网上的“猪”骗！哈哈哈！"},
+    ...         {"role": "HUMAN", "content": "不好笑，换个程序员的笑话"}
+    ...      ],
+    ... }
+    >>> chat = Chat.from_json(input_json, name='antglm_raw')
+
+    >>> ### 根据 chat 对象创建大模型训练所需 pack 数据
+    >>> pack_data = chat.prompt_pack
+    >>> print(pack_data)
+
+    >>> ### 根据 chat 对象创建大模型训练所需 input, output 数据
+    >>> data = chat.prompt_inout
+    >>> print(data)
+
+    >>> ### 根据 chat 对象创建大模型预测用的 prompt
+    >>> prompt = chat.prompt_str
+    >>> print(prompt)
+
+    >>> ### 从大模型训练数据 {"input": "xx", "output": "xx"} 中创建 chat 对象
+    >>> data = {
+    ...     'input': (
+    ...         '第1轮\n用户: 讲一个笑话\n机器人: 为什么猪不能上网？因为它们会被网上的“猪”骗！哈哈哈！\n'
+    ...         '第2轮\n用户: 不好笑，换个程序员的笑话\n机器人:'
+    ...     ),
+    ...     'output': ''
+    ... }
+    >>> chat = Chat.from_inout(data, name='antglm_raw')
+
+    >>> ### 从大模型 pack 训练数据创建 chat 对象列表
+    >>> pack_data = {
+    ...     'inputs': ['第1轮\n用户: 讲一个笑话\n机器人:', '第2轮\n用户: 不好笑，换个程序员的笑话\n机器人:', '第1轮\n用户: 写首诗\n机器人:'],
+    ...     'outputs': [
+    ...         '为什么猪不能上网？因为它们会被网上的“猪”骗！哈哈哈！\n',
+    ...         '为什么程序员总是喜欢使用黑色主题？因为他们喜欢“黑暗模式”（Dark Mode),这样他们就可以在晚上加班时更好地隐藏自己的错误！',
+    ...         '']
+    ... }
+    >>> chats = Chat.from_pack(pack_data, name='antglm_raw')
+    >>> assert len(chats) == 2
+    >>> print(chats[0])
+    >>> print(chats[1])
+
+    >>> ### 显示总交互轮数 (以用户输出多少次为轮数个数)
+    >>> print(chat.turns_num)
+
+    >>> ### 根据 chat 对象创建 json 格式化输出
+    >>> data_json = chat.to_json()
+    >>> print(data_json)
+
+    >>> ### 增加轮次信息
+    >>> content = (
+    ...     '为什么程序员总是喜欢使用黑色主题？'
+    ...     '因为他们喜欢“黑暗模式”（Dark Mode),这样他们就可以在晚上加班时更好地隐藏自己的错误！'
+    ... )
+    >>> chat.append_message(chat.role_assistant, content)
+
+    >>> ### 将 chat 对象转成 OpenAI ChatCompletion 接口的入参
+    >>> openai_messages = chat.to_openai_api_messages()
+    >>> print(openai_messages)
+
+    >>> ### 复制一个 chat 对象
+    >>> chat_new = chat.copy()
+    ```
+    '''
+
+    # 数据结构名称
+    id: str = None
+
+    # format 支持: antglm_raw, antglm_chat, chatglm1, chatglm2, llama2, qwen, baichuan2
+    name: Optional[str] = None
+
+    # Prompt 风格
+    prompt_style: Optional[PromptStyle] = None
+
+    # System Template 和 message
+    system_template: str = '<role>SYSTEM</role>{}'
+    system_message: str = ''
+
+    # 角色定义
+    role_human: str = 'HUMAN'
+    role_assistant: str = 'ASSISTANT'
+    role_observation: str = 'OBSERVATION'
+    role_template: str = '<role>{}</role>'
+
+    # 每轮符号定义
+    turn_start: str = ''
+    human_end: str = ''
+    assistant_start: str = ''
+    assistant_end: str = ''
+    assistant_end_ids: Optional[List[int]] = None
+    general_role_end: str = ''
+
+    # agent 符号定义
+    tool_template = '<tool>{}</tool>'
+    code_template = '<code>{}</code>'
+    arithemetic_templte = '<arithemetic>{}</arithemetic>'
+    image_template = '<image>{}</image>'
+
+    # All messages. Each item is (role, message).
+    messages: List[Tuple[str, str]] = ()
+
+    # messages 中用于 few-shot messages
+    offset: int = 0
+
+    # 其他 meta data
+    source: Optional[str] = None
+    lang: Optional[str] = None
+    topic: Optional[str] = None
+
+    # 原始 json 数据
+    origin_json: Optional[dict] = None
+
+    @property
+    def support_names(self) -> Dict[str, str]:
+        '''支持的数据对象名称.'''
+        return {
+            'antglm_raw': '原始 antglm format 格式, 单轮指令没有结构, 多轮 `第1轮\\n用户:xx\\n机器人xx\\n`',
+            'antglm_chat': 'Chat format 格式, 单轮多轮统一为 chat format 格式',
+            'chatglm1': 'chatglm1 format',
+            'chatglm2': 'chatglm2 format',
+            'llama2': 'llama2 format',
+            'qwen': '千问 format',
+            'baichuan2': '百川 2 format',
+        }
+
+    @classmethod
+    def from_json(
+        cls,
+        input: dict,
+        name: Optional[str] = None,
+        prompt_style: Optional[PromptStyle] = None,
+    ):
+        '''从文件数据结构到数据对象的转换.
+
+        Params:
+            name: `Optional[str]`, 符号系统名称
+                - format 支持: antglm_raw, antglm_chat, chatglm1, chatglm2, llama2, qwen, baichuan2
+                - 如果指定了 format name, 使用该 name 符号系统, 否则使用 input 中 `name` 字段
+
+            prompt_style: `Optional[PromptStyle]`, 指定 prompt 风格, 默认使用和 name 一致的风格
+
+            input: `dict`, 文件中的 json dict 对象, 协议为:
+                - 既支持 `messages` 字段, 也支持 `turns` 字段
+            {
+                "id": "xxx",
+                "name": "antglm",
+                "source": "xxx",
+                "lang": "xx",
+                "topic": "xx",
+                "system_template": "",
+                "system_message": "xx",
+                "messages": [
+                    {
+                        "role": "HUMAN",
+                        "content": "Hi"
+                    },
+                    {
+                        "role": "ASSISTANT",
+                        "content": "Hello"
+                    },
+                    {
+                        "role": "OBSERVATION",
+                        "content": "xxx"
+                    },
+                    {
+                        "role": "ASSISTANT",
+                        "content": "xxx"
+                    }
+                ],
+                "turns": [
+                    {"HUMAN": "xxx", "OBSERVATION": "xx", "ASSISTANT": "xx"}
+                ]
+            }
+
+        Returns:
+            `Chat` 对象
+        '''
+        _id = input.get('id')
+        if name:
+            _name = name
+        else:
+            _name = input.get('name')
+        source = input.get('source')
+        lang = input.get('lang')
+        topic = input.get('topic')
+        kwargs = {}
+        if 'system_template' in input:
+            kwargs['system_template'] = input['system_template']
+        if 'system_message' in input:
+            kwargs['system_message'] = input['system_message']
+
+        # 转换成 Chat 对象
+        chat = cls(
+            id=_id,
+            name=_name,
+            prompt_style=prompt_style,
+            source=source,
+            lang=lang,
+            topic=topic,
+            origin_json=deepcopy(input),
+            **kwargs,
+        )
+        if 'messages' in input:
+            for msg in input['messages']:
+                if msg['role'] == 'HUMAN':
+                    role = chat.role_human
+                elif msg['role'] == 'OBSERVATION':
+                    role = chat.role_observation
+                elif msg['role'] == 'ASSISTANT':
+                    role = chat.role_assistant
+                else:
+                    raise ValueError(f'不支持数据集中的 role: {msg["role"]}')
+
+                chat.append_message(role, msg['content'])
+
+        elif 'turns' in input:
+            for turn in input['turns']:
+                if 'HUMAN' in turn:
+                    content = turn['HUMAN']
+                    chat.append_message(chat.role_human, content)
+                if 'OBSERVATION' in turn:
+                    content = turn['OBSERVATION']
+                    chat.append_message(chat.role_observation, content)
+                if 'ASSISTANT' in turn:
+                    content = turn['ASSISTANT']
+                    chat.append_message(chat.role_assistant, content)
+
+        return chat
+
+    @classmethod
+    def from_pack(
+        cls,
+        packs: Dict[str, List[str]],
+        name: str,
+        prompt_style: Optional[PromptStyle] = None,
+    ) -> list:
+        '''根据 pack 数据创建 Chat 对象.
+
+        Params:
+            packs: `dict`, pack 样本数据
+                {
+                    'inputs': ['xx', 'xx'],
+                    'outputs': ['xx', 'xx'],
+                }
+
+            name: `str`, 符号系统名称
+            prompt_style: `Optional[PromptStyle]`, 指定 prompt 风格, 默认使用和 name 一致的风格
+        '''
+        chat = cls(name=name, prompt_style=prompt_style)
+        packs = cls._format_packs(packs)
+
+        sys_pattern = re.compile(chat.system_template.format(r'(.*?)'), re.DOTALL)
+        turn_pattern = re.compile(chat.turn_start.format(r'(\d+)'), re.DOTALL)
+        human_pattern = re.compile(chat.role_template.format(chat.role_human).strip(), re.DOTALL)
+        observe_pattern = re.compile(chat.role_template.format(chat.role_observation).strip(), re.DOTALL)
+        assistant_pattern = re.compile(chat.role_template.format(chat.role_assistant).strip(), re.DOTALL)
+
+        chats = []
+        for input, output in zip(packs['input'], packs['output']):
+            # system message
+            sys_match = sys_pattern.search(input)
+            if sys_match and sys_match.group(0):
+                # system 指令只在首轮, 新增 chat 对象
+                if len(chat.messages) > 0:
+                    chats.append(chat)
+                    chat = cls(name=name, prompt_style=prompt_style)
+
+                input = input[sys_match.end() :]
+                chat.system_message = sys_match.group(1)
+
+            # turn start
+            turn_match = turn_pattern.search(input)
+            if turn_match and turn_match.group(0):
+                # 当出现下一个轮次开始信息, 新增 chat 对象
+                if name in ['antglm', 'antglm_raw', 'chatglm2']:
+                    round_start = 1
+                else:
+                    round_start = 0
+
+                if all(
+                    [
+                        len(turn_match.groups()) > 0,
+                        int(turn_match.group(1)) == round_start,
+                        len(chat.messages) > 0,
+                    ]
+                ):
+                    chats.append(chat)
+                    chat = cls(name=name, prompt_style=prompt_style)
+
+                input = input[turn_match.end() :]
+
+            human_iter = human_pattern.finditer(input)
+            observe_iter = observe_pattern.finditer(input)
+            assistant_iter = assistant_pattern.finditer(input)
+            human_match = next(human_iter, None)
+            observe_match = next(observe_iter, None)
+            assistant_match = next(assistant_iter, None)
+
+            if not human_match and not observe_match:
+                # 无 role format
+                chat.append_message(chat.role_human, input)
+
+            while human_match or observe_match:
+                next_human_match = next(human_iter, None)
+                next_observe_match = next(observe_iter, None)
+                input = cls._append_human_observation(
+                    chat,
+                    input,
+                    human_match=human_match,
+                    next_human_match=next_human_match,
+                    observe_match=observe_match,
+                    next_observe_match=next_observe_match,
+                    assistant_match=assistant_match,
+                )
+
+                human_match = next_human_match
+                observe_match = next_observe_match
+                next_human_match = next(human_iter, None)
+                next_observe_match = next(observe_iter, None)
+
+            if output:
+                chat.append_message(chat.role_assistant, output)
+
+        if chat.messages:
+            chats.append(chat)
+
+        return chats
+
+    @classmethod
+    def _append_human_observation(
+        cls,
+        chat,
+        input: str,
+        human_match: Optional[re.Match] = None,
+        next_human_match: Optional[re.Match] = None,
+        observe_match: Optional[re.Match] = None,
+        next_observe_match: Optional[re.Match] = None,
+        assistant_match: Optional[re.Match] = None,
+    ) -> str:
+        '''给 chat 对象增加 human/observation message.'''
+        if observe_match:
+            # observation 在 human 之后
+            if observe_match.span()[0] > observe_match.span()[0]:
+                human_str = input[observe_match.span()[1] : observe_match.span()[0]]
+                observe_str = input[observe_match.span()[1] : assistant_match.span()[0]]
+                chat.append_message(chat.role_human, human_str.strip())
+                input_end = observe_match.span()[1]
+                if observe_match.span()[0] < next_human_match.span()[0]:
+                    chat.append_message(chat.role_observation, observe_str.strip())
+                    input_end = assistant_match.span()[1]
+            else:
+                # observation 在 human 之前
+                human_str = input[observe_match.span()[1] : assistant_match.span()[0]]
+                observe_str = input[observe_match.span()[1] : observe_match.span()[0]]
+                chat.append_message(chat.role_observation, observe_str.strip())
+                input_end = observe_match.span()[1]
+                if observe_match.span()[0] < next_observe_match.span()[0]:
+                    chat.append_message(chat.role_human, human_str.strip())
+                    input_end = assistant_match.span()[1]
+        else:
+            if assistant_match:
+                human_str = input[human_match.span()[1] : assistant_match.span()[0]]
+                input_end = assistant_match.span()[1]
+            else:
+                human_str = input[human_match.span()[1] :]
+                input_end = len(input)
+            chat.append_message(chat.role_human, human_str.strip())
+
+        return input[input_end:]
+
+    @classmethod
+    def from_inout(
+        cls,
+        sample: Dict[str, str],
+        name: str,
+        prompt_style: Optional[PromptStyle] = None,
+    ):
+        '''根据单样本创建一个 Chat 对象.
+
+        Params:
+            sample: `Dict[str, str]`, input/output 数据样本
+                {
+                    "input": "xxx",
+                    "output": "xxx",
+                }
+
+            name: `str`, 符号系统名称
+            prompt_style: `Optional[PromptStyle]`, 指定 prompt 风格, 默认使用和 name 一致的风格
+        '''
+        chat = cls(name=name, prompt_style=prompt_style)
+        input = sample['input']
+        output = sample['output']
+
+        sys_pattern = re.compile(chat.system_template.format(r'(.*?)'), re.DOTALL)
+        turn_pattern = re.compile(chat.turn_start.format(r'(\d+)'), re.DOTALL)
+        human_pattern = re.compile(chat.role_template.format(chat.role_human).strip(), re.DOTALL)
+        observe_pattern = re.compile(chat.role_template.format(chat.role_observation).strip(), re.DOTALL)
+        assistant_pattern = re.compile(chat.role_template.format(chat.role_assistant).strip(), re.DOTALL)
+
+        # 去除轮次信息
+        input = turn_pattern.sub('', input)
+
+        # system message search
+        sys_match = sys_pattern.search(input)
+        if sys_match and sys_match.group(0):
+            input = input[sys_match.end() :]
+            chat.system_message = sys_match.group(1)
+
+        human_iter = human_pattern.finditer(input)
+        observe_iter = observe_pattern.finditer(input)
+        assistant_iter = assistant_pattern.finditer(input)
+        human_match = next(human_iter, None)
+        observe_match = next(observe_iter, None)
+        assistant_match = next(assistant_iter, None)
+        next_human_match = next(human_iter, None)
+        next_observe_match = next(observe_iter, None)
+
+        while any(
+            [
+                human_match,
+                observe_match,
+                assistant_match,
+            ]
+        ):
+
+            # human/observation 先后顺序可能不一样, 并且有可能有多个
+            # 判断 assitant 之前是否还有 human/observation
+            while any(
+                [
+                    human_match and human_match.span()[0] < assistant_match.span()[0],
+                    observe_match and observe_match.span()[0] < assistant_match.span()[0],
+                    next_human_match and next_human_match.span()[0] < assistant_match.span()[0],
+                    next_observe_match and next_observe_match.span()[0] < assistant_match.span()[0],
+                ]
+            ):
+                if not input:
+                    break
+
+                cls._append_human_observation(
+                    chat,
+                    input,
+                    human_match=human_match,
+                    next_human_match=next_human_match,
+                    observe_match=observe_match,
+                    next_observe_match=next_observe_match,
+                    assistant_match=assistant_match,
+                )
+
+                human_match = next_human_match
+                observe_match = next_observe_match
+                next_human_match = next(human_iter, None)
+                next_observe_match = next(observe_iter, None)
+
+            # assistant message
+            if assistant_match and assistant_match.span():
+                if observe_match:
+                    if observe_match.span() and observe_match.span()[0] < human_match.span()[0]:
+                        assistant_str = input[assistant_match.span()[1] : observe_match.span()[0]]
+                elif human_match:
+                    if human_match.span():
+                        assistant_str = input[assistant_match.span()[1] : human_match.span()[0]]
+                else:
+                    assistant_str = input[assistant_match.span()[1] :]
+
+                if assistant_str:
+                    chat.append_message(chat.role_assistant, assistant_str)
+
+            assistant_match = next(assistant_iter, None)
+
+        if output:
+            chat.append_message(chat.role_assistant, output)
+
+        return chat
+
+    def __hash__(self):
+        '''数据对象的 hash 函数.'''
+        return hash(self.id)
+
+    def __post_init__(self):
+        '''对象初始化后的处理, 处理包括:
+        - 根据数据对象名称, 支持转成其他开源数据对象的基本信息
+        '''
+        self.id = str(uuid.uuid4())
+        if not self.messages:
+            self.messages = []
+
+        if not self.name and not self.prompt_style:
+            logger.error('构造 Chat 对象至少包含以下一个入参: `name/prompt_style`.\n\n' '`name` 支持以下 format 名称:')
+            logger.error('\n'.join([f'{k}: {v}' for k, v in self.support_names.items()]))
+            logger.error('\n`prompt_style` 参考 antllm.data.chat_format.PromptStyle')
+            raise ValueError
+
+        if self.name == 'antglm':
+            # 默认 antglm 使用原始 antglm_raw - 第1轮\n用户: xx\n机器人: xx\n
+            self.name = 'antglm_raw'
+
+        if not self.name and self.prompt_style == PromptStyle.ANTGLM_CHAT:
+            logger.info(
+                'Chat 对象入参没有 `name`, 默认使用 `ANTGLM_CHAT`, format:\n'
+                f'role_human: {self.role_human}\n'
+                f'role_assistant: {self.role_assistant}\n'
+                f'role_observation: {self.role_observation}\n'
+                f'role_template: {self.role_template}\n'
+                f'turn_start: {self.turn_start}\n'
+                f'human_end: {self.human_end}\n'
+                f'assistant_start: {self.assistant_start}\n'
+                f'assistant_end: {self.assistant_end}\n'
+                f'assistant_end_ids: {self.assistant_end_ids}\n'
+                f'general_role_end: {self.general_role_end}\n'
+                f'tool_template: {self.tool_template}\n'
+                f'code_template: {self.code_template}\n'
+                f'arithemetic_templte: {self.arithemetic_templte}\n'
+                f'image_template: {self.image_template}\n'
+                f'\n入参 `name` 支持: ``'
+            )
+            return
+
+        if self.name == 'antglm_raw' or self.prompt_style == PromptStyle.ANTGLM_RAW:
+            self.prompt_style = PromptStyle.ANTGLM_RAW
+            self.role_template = '{}'
+            self.role_human = '用户: '
+            self.role_assistant = '机器人: '
+            self.turn_start = '第{}轮\n'
+            self.general_role_end = '\n'
+
+        if self.name in ['chatglm1', 'chatglm2'] or self.prompt_style == PromptStyle.CHATGLM:
+            self.prompt_style = PromptStyle.CHATGLM
+            self.role_template = '{}'
+            self.role_human = '问：'
+            self.role_assistant = '答：'
+            self.turn_start = '[Round {}]\n'
+            if self.name == 'chatglm1':
+                self.general_role_end = '\n'
+            else:
+                self.general_role_end = '\n\n'
+
+        elif self.name == 'chatglm3' or self.prompt_style == PromptStyle.CHATGLM3:
+            self.prompt_style = PromptStyle.CHATGLM3
+            self.system_template = '<|system|>\n {}'
+            self.role_human = '<|user|>\n '
+            self.role_assistant = '<|assistant|>\n '
+            self.role_template = '{}'
+
+        elif self.name == 'llama2' or self.prompt_style == PromptStyle.LLAMA2:
+            self.prompt_style = PromptStyle.LLAMA2
+            self.role_template = '{}'
+            self.system_template = '[INST] <<SYS>>\n{}\n<</SYS>>\n\n'
+            self.role_human = '[INST] '
+            self.role_assistant = '[/INST] '
+            self.human_end = ' '
+            self.assistant_end = ' </s><s>'
+
+        elif self.name == 'qwen':
+            self.prompt_style = PromptStyle.CHATML
+            self.role_template = '{}'
+            self.system_template = '<|im_start|>system\n{}'
+            if not self.system_message:
+                self.system_message = 'You are a helpful assistant.'
+            self.role_human = '<|im_start|>user\n'
+            self.role_assistant = '<|im_start|>assistant\n'
+            self.general_role_end = '<|im_end|>\n'
+
+        elif self.name == 'baichuan':
+            self.prompt_style = PromptStyle.BAICHUAN2
+            self.role_template = '{}'
+            self.system_template = '{}'
+            self.role_human = '<token_id-195>'
+            self.role_assistant = '<token_id-196>'
+
+        if not self.system_template:
+            self.system_template = '{}'
+
+    def readable_messages(self) -> str:
+        '''将 messages 输出为人类可读的字符串, 方便分析数据.'''
+        pass
+
+    @property
+    def prompt_str(self) -> str:
+        '''将 Chat 对象转成 prompt str, 合并 human/assitant 输出为 format 字符串.'''
+        return f'{self.prompt_inout["input"]}{self.prompt_inout["output"]}'
+
+    @classmethod
+    def _format_packs(cls, packs: Dict[str, List[str]]) -> Dict[str, List[str]]:
+        '''格式化 pack 样本, 输出相同 pack inputs, outputs 个数.'''
+        _packs = copy.deepcopy(packs)
+        if len(_packs['input']) - 1 == len(_packs['output']):
+            _packs['output'].append('')
+
+        if len(_packs['input']) != len(_packs['output']):
+            print(packs)
+            raise ValueError(
+                '输入 input 和 output 数量不匹配, '
+                f'input num: {len(packs["input"])}, '
+                f'output num: {len(packs["output"])}'
+            )
+
+        return _packs
+
+    @property
+    def prompt_inout(self) -> Dict[str, str]:
+        '''将 Chat 对象转成 input prompt, output prompt 字符串.
+
+        Returns:
+            `Dict[str, str]`, 示例:
+            {
+                "input": "<role>SYSTEM</role>xxxx<role>HUMAN</role>你好<role>ASSISTANT</role>你好，有什么可以帮您？<role>ASSISTANT</role>",  # noqa
+                "output": "你好，有什么可以帮您？"
+            }
+        '''
+        packs = self._format_packs(self.prompt_pack)
+
+        # 兼容逻辑
+        if self.prompt_style == PromptStyle.ANTGLM_RAW:
+            packs['input'] = [f'{item} ' for item in packs['input']]
+
+        prompt_input = ''.join([f'{x}{y}' for x, y in zip(packs['input'][:-1], packs['output'][:-1])])
+        prompt_input += packs['input'][-1]
+        prompt_output = packs['output'][-1]
+
+        # 兼容逻辑
+        if self.prompt_style == PromptStyle.ANTGLM_RAW:
+            prompt_input = prompt_input.strip()
+
+        return {
+            'input': prompt_input,
+            'output': prompt_output,
+        }
+
+    @property
+    def prompt_pack(self) -> Dict[str, List[str]]:
+        '''将数据对象转成 pack input prompt, output prompt 字符串列表.:
+
+        Returns:
+            `Dict[str, List[str]]`, 示例:
+
+            {
+                "input": [
+                    "<role>SYSTEM</role>xxxx<role>HUMAN</role>你好<role>ASSISTANT</role>",
+                    "<role>HUMAN</role>讲个笑话<role>ASSISTANT</role>",
+                    "<role>OBSERVATION</role>{\"weather\": \"晴\"}<role>ASSISTANT</role>"
+                ],
+                "output": [
+                    "你好，有什么可以帮您？",
+                    "笑话 1",
+                    "今天天气 xxx"
+                ]
+            }
+
+        '''
+        inputs = []
+        outputs = []
+
+        # 最开始 system 构造
+        system_prompt = ''
+        if self.system_message:
+            system_prompt = self.system_template.format(self.system_message)
+
+        if system_prompt:
+            ret = system_prompt + self.general_role_end
+        else:
+            ret = ''
+
+        # 有些 prompt style 单轮指令没有 format
+        if self.prompt_style in [
+            PromptStyle.ANTGLM_RAW,
+            PromptStyle.ANTGLM_ONLY_MULTITURN_CHAT,
+        ]:
+            if len(self.messages) <= 2:
+                output = ''
+                for role, message in self.messages:
+                    if role == self.role_assistant:
+                        output = message
+                    else:
+                        input = ret + message
+                return {
+                    'input': [input],
+                    'output': [output],
+                }
+
+        # 多轮对话
+        if self.name in ['antglm_raw', 'chatglm2']:
+            round_start = 1
+        else:
+            round_start = 0
+
+        for i, (role, message) in enumerate(self.messages):
+            # 轮次信息
+            if self.name in ['antglm_raw', 'chatglm1', 'chatglm2']:
+                if i % 2 == 0:
+                    ret += self.turn_start.format(i // 2 + round_start)
+
+            # 角色 + 内容
+            role_end = self.general_role_end
+            if role == self.role_assistant and self.assistant_end:
+                role_end = self.assistant_end
+            elif self.human_end:
+                role_end = self.human_end
+
+            ret += self.role_template.format(role) + message + role_end
+
+            if role == self.role_assistant:
+                # output 只保留实际 assistant 内容
+                if not message:
+                    outputs.append('')
+                else:
+                    outputs.append(message + role_end)
+                # input 需要连接 assistant role
+                inputs[-1] += ret[: -len(message + role_end)]
+            elif all(
+                [
+                    role == self.role_observation,
+                    len(self.messages) > 1,
+                    self.messages[i - 1][0] != self.role_assistant,
+                ]
+            ):
+                # observation 之前不是 assistant, 需要将 observation 和上一个 input 连接一起
+                continue
+            else:
+                inputs.append(ret)
+            ret = ''
+
+            # 最后一轮不是机器人回复, 需要拼接机器人 role, 用于模型生成
+            if i == len(self.messages) - 1 and role != self.role_assistant:
+                inputs[-1] += self.role_template.format(self.role_assistant).strip()
+
+        # 兼容逻辑, 去除 inputs 最后空格符号
+        if self.prompt_style == PromptStyle.ANTGLM_RAW:
+            inputs = [item.strip() for item in inputs]
+
+        return {
+            'input': inputs,
+            'output': outputs,
+        }
+
+    @property
+    def turns_num(self) -> int:
+        '''和机器人的交互轮数, 以用户输出多少次为轮数个数.'''
+        return sum([1 if msg[0] == self.role_human else 0 for msg in self.messages])
+
+    def to_json(self) -> dict:
+        '''输出 chat json dict 格式, 包含不同角色和机器人交互的每轮信息.
+
+        Returns
+            `List[dict]`, {
+                "id": "xx",
+                "messages": [
+                    {"role": "HUMAN", "content": "xxx"}
+                ]
+                "turns": [
+                    {"HUMAN": "xx", "OBSERVATION": "xx", "ASSISTANT": "xx"}
+                ]
+            }
+        '''
+        turns = []
+        messages = []
+        turn = {}
+        for msg in self.messages:
+            if msg[0] == self.role_assistant:
+                messages.append({'role': 'ASSISTANT', 'content': msg[1]})
+                turn['ASSISTANT'] = msg[1]
+                turns.append(turn)
+                turn = {}
+
+            if msg[0] == self.role_human:
+                messages.append({'role': 'HUMAN', 'content': msg[1]})
+                turn['HUMAN'] = msg[1]
+
+            if msg[0] == self.role_observation:
+                messages.append({'role': 'OBSERVATION', 'content': msg[1]})
+                turn['OBSERVATION'] = msg[1]
+
+        if self.messages[-1][0] == self.role_human:
+            messages.append({'role': 'ASSISTANT', 'content': ''})
+            turn['ASSISTANT'] = ''
+            turns.append(turn)
+
+        result = self.origin_json or {}
+        result.update(
+            {
+                'id': self.id,
+                'name': self.name,
+                'source': self.source,
+                'lang': self.lang,
+                'topic': self.topic,
+                'system_template': self.system_template,
+                'system_message': self.system_message,
+                'turns': turns,
+                'messages': messages,
+            }
+        )
+
+        return result
+
+    def set_system_message(self, system_message: str):
+        '''Set the system message.'''
+        self.system_message = system_message
+
+    def append_message(self, role: str, message: str):
+        '''Append a new message.'''
+        if not message:
+            message = ''
+        self.messages.append([role, message])
+
+    def to_openai_api_messages(self) -> List[dict]:
+        '''Convert the conversation to OpenAI chat completion format.'''
+        ret = [{'role': 'system', 'content': self.system_message}]
+
+        for i, (_, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append({'role': 'user', 'content': msg})
+            else:
+                if msg is not None:
+                    ret.append({'role': 'assistant', 'content': msg})
+        return ret
+
+    def copy(self):
+        return copy.deepcopy(self)

chat_template.jinja

@@ -0,0 +1 @@
+{% for message in messages %}{% set role = message['role'] | lower %}{% if role == 'user' %}{% set role = 'HUMAN' %}{% endif %}{% set role = role | upper %}{{ '<role>' + role + '</role>' + message['content'] }}{% endfor %}{% if add_generation_prompt %}{{ '<role>ASSISTANT</role><think>' }}{% endif %}

config.json

@@ -0,0 +1,125 @@
+{
+  "architectures": [
+    "BailingMoeLinearV2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_bailing_moe_linear_v2.BailingMoeLinearV2Config",
+    "AutoModel": "modeling_bailing_moe_linear_v2.BailingMoeLinearV2Model",
+    "AutoModelForCausalLM": "modeling_bailing_moe_linear_v2.BailingMoeLinearV2ForCausalLM"
+  },
+  "embedding_dropout": 0.0,
+  "eos_token_id": 156892,
+  "first_k_dense_replace": 1,
+  "group_norm_size": 4,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 9216,
+  "layer_group_size": 8,
+  "linear_silu": false,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 20,
+  "model_type": "bailing_moe_linear",
+  "moe_intermediate_size": 1024,
+  "moe_router_enable_expert_bias": true,
+  "moe_shared_expert_intermediate_size": 1024,
+  "mtp_loss_scaling_factor": 0,
+  "n_group": 8,
+  "norm_topk_prob": true,
+  "num_attention_heads": 32,
+  "num_experts": 256,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 4,
+  "num_nextn_predict_layers": 0,
+  "num_shared_experts": 1,
+  "output_dropout": 0.0,
+  "output_router_logits": false,
+  "pad_token_id": 156892,
+  "partial_rotary_factor": 0.5,
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "format": "pack-quantized",
+        "input_activations": null,
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": "weight",
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 128,
+          "num_bits": 4,
+          "observer": "minmax",
+          "observer_kwargs": {},
+          "strategy": "group",
+          "symmetric": true,
+          "type": "int"
+        }
+      }
+    },
+    "format": "pack-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "model.layers.0.attention.dense",
+      "model.layers.1.attention.dense",
+      "model.layers.2.attention.dense",
+      "model.layers.3.attention.dense",
+      "model.layers.4.attention.dense",
+      "model.layers.5.attention.dense",
+      "model.layers.6.attention.dense",
+      "model.layers.7.attention.dense",
+      "model.layers.8.attention.dense",
+      "model.layers.9.attention.dense",
+      "model.layers.10.attention.dense",
+      "model.layers.11.attention.dense",
+      "model.layers.12.attention.dense",
+      "model.layers.13.attention.dense",
+      "model.layers.14.attention.dense",
+      "model.layers.15.attention.dense",
+      "model.layers.16.attention.dense",
+      "model.layers.17.attention.dense",
+      "model.layers.18.attention.dense",
+      "model.layers.19.attention.dense",
+      "model.layers.20.attention.dense",
+      "model.layers.21.attention.dense",
+      "model.layers.22.attention.dense",
+      "model.layers.23.attention.dense",
+      "model.layers.24.attention.dense",
+      "model.layers.25.attention.dense",
+      "model.layers.26.attention.dense",
+      "model.layers.27.attention.dense",
+      "model.layers.28.attention.dense",
+      "model.layers.29.attention.dense",
+      "model.layers.30.attention.dense",
+      "model.layers.31.attention.dense",
+      "lm_head"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.11.0"
+  },
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 600000,
+  "routed_scaling_factor": 2.5,
+  "router_dtype": "fp32",
+  "score_function": "sigmoid",
+  "tie_word_embeddings": false,
+  "topk_group": 4,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.2",
+  "use_bias": false,
+  "use_cache": true,
+  "use_qk_norm": true,
+  "use_qkv_bias": false,
+  "use_rmsnorm": true,
+  "vocab_size": 157184
+}

configuration_bailing_moe_linear_v2.py

@@ -0,0 +1,93 @@
+"""Bailing MoE V2 model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+
+
+class BailingMoeLinearV2Config(PretrainedConfig):
+    model_type = "bailing_moe_linear"
+
+    def __init__(
+        self,
+        vocab_size=157184,
+        hidden_size=2048,
+        intermediate_size=5120,
+        num_hidden_layers=20,
+        num_attention_heads=16,
+        num_key_value_heads=4,
+        hidden_act="silu",
+        use_qkv_bias=False,  # bailing only
+        use_bias=False,  # bailing only
+        rms_norm_eps=1e-06,
+        tie_word_embeddings=False,  # PretrainedConfig key, here change default value.
+        embedding_dropout=0.0,
+        attention_dropout=0.0,
+        output_dropout=0.0,
+        initializer_range=0.02,
+        max_position_embeddings=32768,
+        rope_theta=600000.0,
+        use_cache=True,
+        max_window_layers=20,
+        rope_scaling=None,
+        pad_token_id=156892,
+        eos_token_id=156892,
+        num_experts=256,
+        num_shared_experts=1,
+        num_experts_per_tok=8,
+        n_group=8,
+        topk_group=4,
+        moe_intermediate_size=512,
+        first_k_dense_replace=1,
+        head_dim=128,
+        output_router_logits=False,
+        use_qk_norm=True,
+        num_nextn_predict_layers=0,
+        mtp_loss_scaling_factor=0,
+        moe_router_enable_expert_bias=True,
+        routed_scaling_factor=1.0,
+        layer_group_size=1,
+        group_norm_size=1,
+        linear_silu=False,
+        **kwargs,
+    ):
+        self.num_hidden_layers = num_hidden_layers
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.use_qkv_bias = use_qkv_bias
+        self.use_bias = use_bias
+        self.rms_norm_eps = rms_norm_eps
+        self.embedding_dropout = embedding_dropout
+        self.attention_dropout = attention_dropout
+        self.output_dropout = output_dropout
+        self.num_nextn_predict_layers = num_nextn_predict_layers
+        self.mtp_loss_scaling_factor = mtp_loss_scaling_factor
+        self.initializer_range = initializer_range
+        self.max_position_embeddings = max_position_embeddings
+        self.rope_theta = rope_theta
+        self.use_cache = use_cache
+        self.max_window_layers = max_window_layers
+        self.head_dim = head_dim or self.hidden_size // self.num_attention_heads
+        self.rope_scaling = rope_scaling
+        self.use_qk_norm = use_qk_norm
+        self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
+        self.routed_scaling_factor = routed_scaling_factor
+
+        # MoE configs
+        self.num_experts = num_experts
+        self.num_shared_experts = num_shared_experts
+        self.num_experts_per_tok = num_experts_per_tok
+        self.n_group = n_group
+        self.topk_group = topk_group
+        self.moe_intermediate_size = moe_intermediate_size
+        self.first_k_dense_replace = first_k_dense_replace
+        self.output_router_logits = output_router_logits
+
+        # Linear configs
+        self.layer_group_size = layer_group_size
+        self.group_norm_size = group_norm_size
+        self.linear_silu = linear_silu
+
+        super().__init__(pad_token_id=pad_token_id, eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs)

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 156892,
+  "pad_token_id": 156892,
+  "transformers_version": "4.55.2"
+}

model-00001-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75aebe06c673bf7894bfc82148f6c06bc393ab72e25ad198979fb2ca3edb537d
+size 4999517128

model-00002-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d7e77f7301952f14afd5c409e61dc440a902dbee8f616684d04faacdc369a10
+size 4998826464

model-00003-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03ac7cbfdf56d9863486fd31deaa5dd23de27d04f55a2fc266654a41d8cd0746
+size 4999885432

model-00004-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8160f67efca711502555a0e8a7c1b644f83ced76a9f86f9cc7563de38515b2bd
+size 4999893824

model-00005-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a503208f4ca5df698606db9ce8111591d284977bfa05bac22b7dbd0f876f407
+size 4999896528

model-00006-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d178d4c06764adec8c0a241fcdae07d76ddd05d008193713523355214fea72c6
+size 4999892128

model-00007-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcbd4e19c993442ccdc6e12eb003a6becaac46686152c82935a70cacf732439b
+size 4999896672

model-00008-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d65e4af7f90ad87ffa052a9ca9a32a970bab396a5f8bdabb9adbd33b71d32385
+size 4999892384

model-00009-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4aaacbfbc3de52ed319bb21dcc84852412ff5663595fb3791c0325fb48beb9ab
+size 4999896792

model-00010-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14dc330663c849b185fbfb05dbcc8c7eddbf0c8abaaacd0c3bc9814042a75482
+size 4998833392

model-00011-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19717ed540bc6368259358d4458e709bf164fed71a17390b8a309b5533e32e2a
+size 4999892176

model-00012-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a1413da426840e35ea7a57ee01da102db665b30882e31a963c0770211eb93f1
+size 1488776768

modeling_bailing_moe_linear_v2.py

@@ -0,0 +1,1758 @@
+# coding=utf-8
+# Copyright 2025 Antgroup and The HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch BailingMoE model."""
+
+import math
+import warnings
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.modeling_attn_mask_utils import (
+    AttentionMaskConverter,
+    _prepare_4d_attention_mask,
+    _prepare_4d_causal_attention_mask,
+    _prepare_4d_causal_attention_mask_for_sdpa,
+)
+from transformers.modeling_outputs import MoeModelOutputWithPast
+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
+from transformers.modeling_utils import PreTrainedModel
+from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS, is_torch_greater_or_equal_than_1_13
+from transformers.utils import (
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    is_flash_attn_2_available,
+    is_flash_attn_greater_or_equal_2_10,
+    logging,
+    replace_return_docstrings,
+)
+from transformers.utils.import_utils import is_torch_fx_available
+from .configuration_bailing_moe_linear_v2 import BailingMoeLinearV2Config
+from transformers.generation.utils import GenerationMixin
+from dataclasses import dataclass
+from transformers.utils import ModelOutput
+
+
+if is_flash_attn_2_available():
+    from flash_attn import flash_attn_func, flash_attn_varlen_func
+    from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input  # noqa
+
+from fla.ops.simple_gla.fused_recurrent import fused_recurrent_simple_gla
+from fla.ops.simple_gla.chunk import chunk_simple_gla
+
+
+# This makes `_prepare_4d_causal_attention_mask` a leaf function in the FX graph.
+# It means that the function will not be traced through and simply appear as a node in the graph.
+if is_torch_fx_available():
+    if not is_torch_greater_or_equal_than_1_13:
+        import torch.fx
+
+    _prepare_4d_causal_attention_mask = torch.fx.wrap(_prepare_4d_causal_attention_mask)
+
+
+logger = logging.get_logger(__name__)
+
+_CONFIG_FOR_DOC = "BailingMoeLinearV2Config"
+
+
+def roll_tensor(tensor, shifts=-1, dims=-1, fill_value=0):
+    """Roll the tensor input along the given dimension(s).
+    Inserted elements are set to be 0.0.
+    """
+    rolled_tensor = torch.roll(tensor, shifts=shifts, dims=dims)
+    rolled_tensor.select(dims, shifts).fill_(fill_value)
+    return rolled_tensor, rolled_tensor.sum()
+
+
+@dataclass
+class MoEV2CausalLMOutputWithPast(ModelOutput):
+    """
+    Base class for causal language model (or autoregressive) outputs as well as Mixture of Expert's router hidden
+    states terms, to train a MoE model.
+    Args:
+        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
+            Language modeling loss (for next-token prediction).
+        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+        past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).
+            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
+            `past_key_values` input) to speed up sequential decoding.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        z_loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided):
+            z_loss for the sparse modules.
+        aux_loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided):
+            aux_loss for the sparse modules.
+        router_logits (`tuple(torch.FloatTensor)`, *optional*, returned when `output_router_logits=True` is passed or when `config.add_router_probs=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, sequence_length, num_experts)`.
+            Router logits of the encoder model, useful to compute the auxiliary loss and the z_loss for the sparse
+            modules.
+    """
+
+    loss: Optional[torch.FloatTensor] = None
+    logits: Optional[torch.FloatTensor] = None
+    past_key_values: Optional[Cache] = None
+    hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[tuple[torch.FloatTensor, ...]] = None
+    z_loss: Optional[torch.FloatTensor] = None
+    aux_loss: Optional[torch.FloatTensor] = None
+    router_logits: Optional[tuple[torch.FloatTensor]] = None
+    mtp_loss: Optional[torch.FloatTensor] = None
+    mtp_logits: Optional[tuple[torch.FloatTensor, ...]] = None
+
+
+class MoeV2ModelOutputWithPast(MoeModelOutputWithPast):
+
+    def __init__(self, mtp_hidden_states=None, **kwargs):
+        super().__init__(**kwargs)
+        self.mtp_hidden_states = mtp_hidden_states
+
+
+def _get_unpad_data(attention_mask):
+    seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
+    indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = seqlens_in_batch.max().item()
+    cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+
+
+def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
+    warnings.warn(
+        "Calling `transformers.models.BailingMoeV2.modeling_BailingMoeV2._prepare_4d_attention_mask` is deprecated and will be removed in v4.37. Use `transformers.modeling_attn_mask_utils._prepare_4d_attention_mask"
+    )
+    return _prepare_4d_attention_mask(mask=mask, dtype=dtype, tgt_len=tgt_len)
+
+
+def _make_causal_mask(
+    input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
+):
+    warnings.warn(
+        "Calling `transformers.models.BailingMoeV2.modeling_BailingMoeV2._make_causal_mask` is deprecated and will be removed in v4.37. Use `transformers.models.BailingMoeV2.modeling_BailingMoeV2.AttentionMaskConverter._make_causal_mask"
+    )
+    return AttentionMaskConverter._make_causal_mask(
+        input_ids_shape=input_ids_shape, dtype=dtype, device=device, past_key_values_length=past_key_values_length
+    )
+
+
+class BailingMoeV2RMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        """
+        BailingMoeV2RMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+
+class BailingMoeV2GroupRMSNorm(nn.Module):
+    def __init__(self, hidden_size, group_norm_size, eps=1e-6):
+        """
+        BailingMoeV2RMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.group_norm_size = group_norm_size
+        assert hidden_size % group_norm_size == 0, "hidden_size must be divisible by group_norm_size"
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        input_shape = hidden_states.size()
+        group_input_shape = input_shape[:-1] + (self.group_norm_size, input_shape[-1] // self.group_norm_size)
+        hidden_states = hidden_states.view(group_input_shape)
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype).view(input_shape)
+
+
+ALL_LAYERNORM_LAYERS.append(BailingMoeV2RMSNorm)
+
+
+class BailingMoeV2RotaryEmbedding(nn.Module):
+    def __init__(self, config: BailingMoeLinearV2Config, device=None):
+        super().__init__()
+        # BC: "rope_type" was originally "type"
+        if hasattr(config, "rope_scaling") and config.rope_scaling is not None:
+            self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
+        else:
+            self.rope_type = "default"
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
+
+        self.config = config
+        self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
+
+        inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.original_inv_freq = self.inv_freq
+
+    @torch.no_grad()
+    @dynamic_rope_update  # power user: used with advanced RoPE types (e.g. dynamic rope)
+    def forward(self, x, position_ids):
+        inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)
+        position_ids_expanded = position_ids[:, None, :].float()
+
+        device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
+        with torch.autocast(device_type=device_type, enabled=False):  # Force float32
+            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+            emb = torch.cat((freqs, freqs), dim=-1)
+            cos = emb.cos() * self.attention_scaling
+            sin = emb.sin() * self.attention_scaling
+
+        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+
+# Copied from transformers.models.llama.modeling_llama.rotate_half
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb
+def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
+    """Applies Rotary Position Embedding to the query and key tensors.
+    Args:
+        q (`torch.Tensor`): The query tensor.
+        k (`torch.Tensor`): The key tensor.
+        cos (`torch.Tensor`): The cosine part of the rotary embedding.
+        sin (`torch.Tensor`): The sine part of the rotary embedding.
+        unsqueeze_dim (`int`, *optional*, defaults to 1):
+            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+    Returns:
+        `tuple(torch.Tensor)` comprising the query and key tensors rotated using the Rotary Position Embedding.
+    """
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
+
+    # Keep half or full tensor for later concatenation
+    rotary_dim = cos.shape[-1]
+    q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]
+    k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]
+
+    # Apply rotary embeddings on the first half or full tensor
+    q_embed = (q_rot * cos) + (rotate_half(q_rot) * sin)
+    k_embed = (k_rot * cos) + (rotate_half(k_rot) * sin)
+
+    # Concatenate back to full shape
+    q_embed = torch.cat([q_embed, q_pass], dim=-1)
+    k_embed = torch.cat([k_embed, k_pass], dim=-1)
+    return q_embed, k_embed
+
+
+class BailingMoeV2MLP(nn.Module):
+    def __init__(self, config: BailingMoeLinearV2Config, intermediate_size: int):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = intermediate_size
+
+        self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
+        self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
+        self.act_fn = ACT2FN[config.hidden_act]
+
+    def forward(self, x):
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+
+class BailingMoeV2Gate(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.top_k = config.num_experts_per_tok
+        self.num_experts = config.num_experts
+
+        self.n_group = config.n_group
+        self.topk_group = config.topk_group
+
+        # topk selection algorithm
+        self.gating_dim = config.hidden_size
+        self.weight = nn.Parameter(torch.empty((self.num_experts, self.gating_dim)))
+        self.routed_scaling_factor = config.routed_scaling_factor
+
+        self.register_buffer("expert_bias", torch.zeros((self.num_experts)))
+        self.reset_parameters()
+
+    def reset_parameters(self) -> None:
+        import torch.nn.init as init
+
+        init.kaiming_uniform_(self.weight, a=math.sqrt(5))
+
+    def group_limited_topk(
+        self,
+        scores: torch.Tensor,
+    ):
+        num_tokens, _ = scores.size()
+        # Organize the experts into groups
+        group_scores = scores.view(num_tokens, self.n_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
+        group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1]
+        group_mask = torch.zeros_like(group_scores)
+        group_mask.scatter_(1, group_idx, 1)
+
+        # Mask the experts based on selection groups
+        score_mask = (
+            group_mask.unsqueeze(-1)
+            .expand(num_tokens, self.n_group, self.num_experts // self.n_group)
+            .reshape(num_tokens, -1)
+        )
+
+        masked_scores = scores.masked_fill(~score_mask.bool(), float('-inf'))
+        probs, top_indices = torch.topk(masked_scores, k=self.top_k, dim=-1)
+
+        return probs, top_indices
+
+    def forward(self, hidden_states):
+        # compute gating score
+        hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
+        logits = F.linear(hidden_states.type(torch.float32), self.weight.type(torch.float32))
+
+        scores = torch.sigmoid(logits.float()).type_as(logits)
+
+        scores_for_routing = scores + self.expert_bias
+        _, topk_idx = self.group_limited_topk(scores_for_routing)
+
+        scores = torch.gather(scores, dim=1, index=topk_idx).type_as(logits)
+
+        topk_weight = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20) if self.top_k > 1 else scores
+        topk_weight = topk_weight * self.routed_scaling_factor
+
+        return topk_idx, topk_weight, logits
+
+
+class BailingMoeV2SparseMoeBlock(nn.Module):
+    """
+    A mixed expert module containing shared experts.
+    """
+
+    def __init__(self, config: BailingMoeLinearV2Config):
+        super().__init__()
+        self.config = config
+        self.num_experts_per_tok = config.num_experts_per_tok
+        self._setup_experts()
+        self.gate = BailingMoeV2Gate(config)
+        if config.num_shared_experts is not None:
+            self.shared_experts = BailingMoeV2MLP(
+                config=config, intermediate_size=config.moe_intermediate_size * config.num_shared_experts
+            )
+
+    def _setup_experts(self):
+        self.experts = nn.ModuleList(
+            [
+                BailingMoeV2MLP(config=self.config, intermediate_size=self.config.moe_intermediate_size)
+                for _ in range(self.config.num_experts)
+            ]
+        )
+
+    def forward(self, hidden_states):
+        identity = hidden_states
+        bsz, seq_len, h = hidden_states.shape
+        topk_idx, topk_weight, router_logits = self.gate(hidden_states)
+        hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
+        flat_topk_idx = topk_idx.view(-1)
+        if self.training:
+            hidden_states = hidden_states.repeat_interleave(self.num_experts_per_tok, dim=0)
+            y = torch.empty_like(hidden_states)
+            for i, expert in enumerate(self.experts):
+                y[flat_topk_idx == i] = expert(hidden_states[flat_topk_idx == i])
+            y = (y.view(*topk_weight.shape, -1) * topk_weight.unsqueeze(-1)).sum(dim=1)
+            y = y.to(hidden_states.dtype).view(bsz, seq_len, h)
+        else:
+            y = self.moe_infer(hidden_states, topk_idx, topk_weight).view(bsz, seq_len, h)
+        if self.config.num_shared_experts is not None:
+            y = y + self.shared_experts(identity)
+        return y, (router_logits.view(bsz, seq_len, -1), topk_idx.view(bsz, seq_len, -1))
+
+    @torch.no_grad()
+    def moe_infer(self, x, topk_ids, topk_weight):
+        cnts = topk_ids.new_zeros((topk_ids.shape[0], len(self.experts)))
+        cnts.scatter_(1, topk_ids, 1)
+        tokens_per_expert = cnts.sum(dim=0)
+        idxs = topk_ids.view(-1).argsort()
+        sorted_tokens = x[idxs // topk_ids.shape[1]]
+        tokens_per_expert = tokens_per_expert.cpu().numpy()
+        outputs = []
+        start_idx = 0
+        for i, num_tokens in enumerate(tokens_per_expert):
+            end_idx = start_idx + num_tokens
+            if num_tokens == 0:
+                continue
+            expert = self.experts[i]
+            tokens_for_this_expert = sorted_tokens[start_idx:end_idx]
+            expert_out = expert(tokens_for_this_expert)
+            outputs.append(expert_out.to(x.device))
+            start_idx = end_idx
+
+        outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
+        new_x = torch.empty_like(outs)
+        new_x[idxs] = outs
+        final_out = (
+            new_x.view(*topk_ids.shape, -1)
+            .type(topk_weight.dtype)
+            .mul_(topk_weight.unsqueeze(dim=-1))
+            .sum(dim=1)
+            .type(new_x.dtype)
+        )
+        return final_out
+
+
+# Copied from transformers.models.llama.modeling_llama.repeat_kv
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int, head_first: bool = True) -> torch.Tensor:  
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). If head_first is True, the hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    if n_rep == 1:
+        return hidden_states
+    if head_first:
+        batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+        hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+        return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+    else:
+        batch, slen, num_key_value_heads, head_dim = hidden_states.shape
+        hidden_states = hidden_states[:, :, :, None, :].expand(batch, slen, num_key_value_heads, n_rep, head_dim)
+        return hidden_states.reshape(batch, slen, num_key_value_heads * n_rep, head_dim)
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaAttention with Llama->BailingMoeV2
+class BailingMoeV2Attention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: BailingMoeLinearV2Config, layer_idx: Optional[int] = None):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        if layer_idx is None:
+            logger.warning_once(
+                f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will "
+                "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
+                "when creating this class."
+            )
+
+        self.attention_dropout = config.attention_dropout
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = config.head_dim or self.hidden_size // self.num_heads
+        partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
+        self.rope_dim = int(self.head_dim * partial_rotary_factor)
+        self.num_key_value_heads = config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.max_position_embeddings = config.max_position_embeddings
+        self.rope_theta = config.rope_theta
+        self.is_causal = True
+
+        self.query_key_value = nn.Linear(
+            self.hidden_size,
+            (self.num_heads + 2 * self.num_key_value_heads) * self.head_dim,
+            bias=config.use_qkv_bias,
+        )
+
+        if self.config.use_qk_norm:
+            self.query_layernorm = BailingMoeV2RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+            self.key_layernorm = BailingMoeV2RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+        self.dense = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.use_bias)
+
+    def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
+        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+
+        bsz, q_len, _ = hidden_states.size()
+
+        qkv = self.query_key_value(hidden_states)
+        qkv = qkv.view(bsz, q_len, self.num_heads + 2 * self.num_key_value_heads, self.head_dim)
+
+        query_states, key_states, value_states = qkv.split(
+            [self.num_heads, self.num_key_value_heads, self.num_key_value_heads], dim=-2
+        )
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        if self.config.use_qk_norm:
+            query_states = self.query_layernorm(query_states)
+            key_states = self.key_layernorm(key_states)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        if past_key_value is not None:
+            if self.layer_idx is None:
+                raise ValueError(
+                    f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
+                    "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
+                    "with a layer index."
+                )
+            cache_kwargs = {"sin": sin, "cos": cos}
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+        kv_seq_len = key_states.shape[-2]
+        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+            attn_weights = attn_weights + attention_mask
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+
+        attn_output = attn_output.reshape(bsz, q_len, -1)
+
+        attn_output = self.dense(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2 with Llama->BailingMoeV2
+class BailingMoeV2FlashAttention2(BailingMoeV2Attention):
+    """
+    BailingMoeV2 flash attention module. This module inherits from `BailingMoeV2Attention` as the weights of the module stays
+    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
+    flash attention and deal with padding tokens in case the input contains any of them.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
+        # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
+        # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
+        self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        # BailingMoeV2FlashAttention2 attention does not support output_attentions
+        output_attentions = False
+
+        bsz, q_len, _ = hidden_states.size()
+
+        # Flash attention requires the input to have the shape
+        # batch_size x seq_length x head_dim x hidden_dim
+        # therefore we just need to keep the original shape
+
+        qkv = self.query_key_value(hidden_states)
+        qkv = qkv.view(bsz, q_len, self.num_heads + 2 * self.num_key_value_heads, self.head_dim)
+
+        query_states, key_states, value_states = qkv.split(
+            [self.num_heads, self.num_key_value_heads, self.num_key_value_heads], dim=-2
+        )
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        if self.config.use_qk_norm:
+            query_states = self.query_layernorm(query_states)
+            key_states = self.key_layernorm(key_states)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos}
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
+        # to be able to avoid many of these transpose/reshape/view.
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        dropout_rate = self.attention_dropout if self.training else 0.0
+
+        # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+        # therefore the input hidden states gets silently cast in float32. Hence, we need
+        # cast them back in the correct dtype just to be sure everything works as expected.
+        # This might slow down training & inference so it is recommended to not cast the LayerNorms
+        # in fp32. (BailingMoeV2RMSNorm handles it correctly)
+
+        input_dtype = query_states.dtype
+        if input_dtype == torch.float32:
+            # Handle the case where the model is quantized
+            if hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            elif torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            else:
+                target_dtype = self.query_key_value.weight.dtype
+
+            logger.warning_once(
+                f"The input hidden states seems to be silently casted in float32, this might be related to"
+                f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                f" {target_dtype}."
+            )
+
+            query_states = query_states.to(target_dtype)
+            key_states = key_states.to(target_dtype)
+            value_states = value_states.to(target_dtype)
+
+        attn_output = self._flash_attention_forward(
+            query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate
+        )
+
+        attn_output = attn_output.reshape(bsz, q_len, -1).contiguous()
+        attn_output = self.dense(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+    def _flash_attention_forward(
+        self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            attention_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`int`, *optional*):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+            query_length (`int`):
+                The length of the query sequence in terms of tokens. This represents the number of tokens in the
+                `query_states` tensor along the sequence dimension. It is used to determine the effective sequence
+                length for attention computations.
+        """
+        if not self._flash_attn_uses_top_left_mask:
+            causal = self.is_causal
+        else:
+            # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in BailingMoeV2FlashAttention2 __init__.
+            causal = self.is_causal and query_length != 1
+
+        # Contains at least one padding token in the sequence
+        if attention_mask is not None:
+            batch_size = query_states.shape[0]
+            query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
+                query_states, key_states, value_states, attention_mask, query_length
+            )
+
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+            attn_output_unpad = flash_attn_varlen_func(
+                query_states,
+                key_states,
+                value_states,
+                cu_seqlens_q=cu_seqlens_q,
+                cu_seqlens_k=cu_seqlens_k,
+                max_seqlen_q=max_seqlen_in_batch_q,
+                max_seqlen_k=max_seqlen_in_batch_k,
+                dropout_p=dropout,
+                softmax_scale=softmax_scale,
+                causal=causal,
+            )
+
+            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+        else:
+            attn_output = flash_attn_func(
+                query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal
+            )
+
+        return attn_output
+
+    def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+        batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
+
+        key_layer = index_first_axis(
+            key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+        value_layer = index_first_axis(
+            value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+        if query_length == kv_seq_len:
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k
+            )
+            cu_seqlens_q = cu_seqlens_k
+            max_seqlen_in_batch_q = max_seqlen_in_batch_k
+            indices_q = indices_k
+        elif query_length == 1:
+            max_seqlen_in_batch_q = 1
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )  # There is a memcpy here, that is very bad.
+            indices_q = cu_seqlens_q[:-1]
+            query_layer = query_layer.squeeze(1)
+        else:
+            # The -q_len: slice assumes left padding.
+            attention_mask = attention_mask[:, -query_length:]
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
+
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q,
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->BailingMoeV2
+class BailingMoeV2SdpaAttention(BailingMoeV2Attention):
+    """
+    BailingMoeV2 attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
+    `BailingMoeV2Attention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
+    SDPA API.
+    """
+
+    # Adapted from BailingMoeV2Attention.forward
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        if output_attentions:
+            # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.
+            logger.warning_once(
+                "BailingMoeV2Model is using BailingMoeV2SdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "
+                'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
+            )
+            return super().forward(
+                hidden_states=hidden_states,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_value=past_key_value,
+                output_attentions=output_attentions,
+                use_cache=use_cache,
+            )
+
+        bsz, q_len, _ = hidden_states.size()
+
+        qkv = self.query_key_value(hidden_states)
+        qkv = qkv.view(bsz, q_len, self.num_heads + 2 * self.num_key_value_heads, self.head_dim)
+
+        query_states, key_states, value_states = qkv.split(
+            [self.num_heads, self.num_key_value_heads, self.num_key_value_heads], dim=-2
+        )
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        if self.config.use_qk_norm:
+            query_states = self.query_layernorm(query_states)
+            key_states = self.key_layernorm(key_states)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos}
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        if attention_mask is not None:
+            kv_seq_len = key_states.shape[-2]
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+
+        # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
+        # Reference: https://github.com/pytorch/pytorch/issues/112577.
+        if query_states.device.type == "cuda" and attention_mask is not None:
+            query_states = query_states.contiguous()
+            key_states = key_states.contiguous()
+            value_states = value_states.contiguous()
+
+        attn_output = torch.nn.functional.scaled_dot_product_attention(
+            query_states,
+            key_states,
+            value_states,
+            attn_mask=attention_mask,
+            dropout_p=self.attention_dropout if self.training else 0.0,
+            # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1.
+            is_causal=self.is_causal and attention_mask is None and q_len > 1,
+        )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, -1)
+
+        attn_output = self.dense(attn_output)
+
+        return attn_output, None, past_key_value
+
+
+ATTENTION_CLASSES = {
+    "eager": BailingMoeV2Attention,
+    "flash_attention_2": BailingMoeV2FlashAttention2,
+    "sdpa": BailingMoeV2SdpaAttention,
+}
+
+
+class BailingMoeV2LinearAttention(nn.Module):
+    """
+    BailingMoeAttention implements a linear attention mechanism based on Lightning Attention-2
+    (https://arxiv.org/abs/2401.04658) with efficient computation using flash-linear-attention operators.
+    
+    The implementation leverages optimized kernels from the flash-linear-attention library
+    (https://github.com/fla-org/flash-linear-attention) for maximum performance.
+    """
+    def __init__(self, config: BailingMoeLinearV2Config, layer_idx: Optional[int] = None):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        if layer_idx is None:
+            logger.warning_once(
+                f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will "
+                "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
+                "when creating this class."
+            )
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = config.head_dim or self.hidden_size // self.num_heads
+        self.num_key_value_heads = config.num_attention_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0
+        self.rope_dim = int(self.head_dim * partial_rotary_factor)
+
+        self.use_qk_norm = getattr(config, "use_qk_norm", False)
+        self.rms_norm_eps = getattr(config, "rms_norm_eps", 1e-5)
+        self.mode = 'chunk'
+
+        self.query_key_value = nn.Linear(
+            self.hidden_size,
+            (self.num_heads + 2 * self.num_key_value_heads) * self.head_dim,
+            bias=config.use_qkv_bias,
+        )
+
+        if self.config.use_qk_norm:
+            self.query_layernorm = BailingMoeV2RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+            self.key_layernorm = BailingMoeV2RMSNorm(self.head_dim, eps=config.rms_norm_eps)
+
+        self.rotary_emb = BailingMoeV2RotaryEmbedding(config=config)
+
+        self.dense = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=config.use_bias)
+
+        self.g_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.g_norm = BailingMoeV2GroupRMSNorm(self.num_heads * self.head_dim, group_norm_size=config.group_norm_size, eps=self.rms_norm_eps)
+        slope = - BailingMoeV2LinearAttention.build_slope_tensor(self.num_heads) * (1 - (self.layer_idx - 1) / (self.config.num_hidden_layers - 1) + 1e-5)
+        self.register_buffer('slope', slope, persistent=False)
+
+        self.lightning_attn_ops = {
+            'chunk': chunk_simple_gla,
+            'fused_recurrent': fused_recurrent_simple_gla
+        }
+
+    @staticmethod
+    def build_slope_tensor(n_attention_heads: int):
+        """
+        Build a tensor of slopes for Lightning Attention-2 as described in the paper:
+        "Lightning Attention-2: A Free Lunch for Handling Unlimited Sequence Lengths in Large Language Models"
+        (https://arxiv.org/abs/2401.04658)
+        
+        This function computes the slope values that control the decay rate of attention scores
+        based on the number of attention heads. The slopes are designed to have specific
+        mathematical properties that work optimally when the number of heads is a power of 2.
+        
+        For non-power-of-2 head counts, a workaround is implemented to maintain similar properties.
+        
+        Args:
+            n_attention_heads (int): Number of attention heads in the model
+            
+        Returns:
+            torch.Tensor: A tensor of shape [n_attention_heads] containing the computed slopes
+            
+        Note:
+            Code copied from: https://github.com/OpenNLPLab/lightning-attention/blob/d15c38529bbd5c2c82b44ddda3cac885825aa873/lightning_attn/utils/utils.py#L6
+        """    
+        def get_slopes(n):
+            def get_slopes_power_of_2(n):
+                start = 2 ** (-(2 ** -(math.log2(n) - 3)))
+                ratio = start
+                return [start * ratio ** i for i in range(n)]
+
+            if math.log2(n).is_integer():
+                return get_slopes_power_of_2(
+                    n)  # In the paper, we only train models that have 2^a heads for some a. This function has
+            else:  # some good properties that only occur when the input is a power of 2. To maintain that even
+                closest_power_of_2 = 2 ** math.floor(
+                    math.log2(n))  # when the number of heads is not a power of 2, we use this workaround.
+                return (get_slopes_power_of_2(closest_power_of_2)
+                        + get_slopes(2 * closest_power_of_2)[0::2][:n - closest_power_of_2])
+
+        slopes = torch.tensor(get_slopes(n_attention_heads), dtype=torch.float)
+        return slopes
+
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        if attention_mask is not None:
+            assert len(attention_mask.shape) == 2, (
+                "Expected attention_mask as a 0-1 matrix with shape [batch_size, seq_len] "
+                "for padding purposes (0 indicating padding). "
+                "Arbitrary attention masks of shape [batch_size, seq_len, seq_len] are not allowed."
+            )
+
+        # launching the triton kernel for just one token will actually be slower
+        mode = 'fused_recurrent' if hidden_states.shape[1] <= 64 else self.mode
+
+        # Currently output_attentions can only be False, returning attention weights is not supported
+        assert not output_attentions, "output_attentions can only be False, returning attention weights is not supported"
+
+        bsz, q_len, _ = hidden_states.size()
+        device = hidden_states.device
+
+        qkv = self.query_key_value(hidden_states)
+        qkv = qkv.view(bsz, q_len, self.num_heads + 2 * self.num_key_value_heads, self.head_dim)
+        query_states, key_states, value_states = qkv.split(
+            [self.num_heads, self.num_key_value_heads, self.num_key_value_heads], dim=-2
+        )
+        if self.config.use_qk_norm:
+            query_states = self.query_layernorm(query_states)
+            key_states = self.key_layernorm(key_states)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, unsqueeze_dim=2)
+
+        if self.num_key_value_groups > 1:
+            # [bsz, q_len, n_kv_heads, head_dim] -> [bsz, q_len, n_heads, head_dim]
+            key_states = repeat_kv(key_states, self.num_key_value_groups, head_first=False)
+            value_states = repeat_kv(value_states, self.num_key_value_groups, head_first=False)
+
+        recurrent_state = None
+        if past_key_value is not None and isinstance(past_key_value, Cache):
+            # ensure the cache list is long enough
+            while len(past_key_value.layers) <= self.layer_idx:
+                past_key_value.layers.append(DynamicLayer())
+
+            if past_key_value.layers[self.layer_idx].keys is not None:
+                recurrent_state = past_key_value.layers[self.layer_idx].keys
+                # ensure recurrent_state is on the same device as hidden_states
+                if recurrent_state.device != hidden_states.device:
+                    recurrent_state = recurrent_state.to(device).contiguous()
+
+        if recurrent_state is None:
+            # dealing with left-padding
+            if attention_mask is not None and use_cache:
+                value_states = value_states.mul_(attention_mask[:, -q_len:, None, None])
+
+        o, recurrent_state = self.lightning_attn_ops[mode](
+            q=query_states,
+            k=key_states,
+            v=value_states,
+            g=self.slope[None, None, :].expand(bsz, q_len, self.num_heads),
+            initial_state=recurrent_state,
+            output_final_state=use_cache,
+        )
+
+        o = o.reshape(bsz, q_len, -1)
+        o = self.g_norm(o)
+        g_proj = self.g_proj(hidden_states)
+        o = o * torch.sigmoid_(g_proj)
+        o = self.dense(o)
+
+        if use_cache and past_key_value is not None and isinstance(past_key_value, Cache):
+            target_device = None
+            for cache in past_key_value.layers:
+                if cache.keys is not None:
+                    target_device = cache.keys.device
+                    break
+            if target_device is None:
+                target_device = recurrent_state.device
+
+            # move to target device
+            if recurrent_state.device != target_device:
+                recurrent_state = recurrent_state.to(target_device)
+
+            past_key_value.layers[self.layer_idx].keys = recurrent_state
+
+        return o, None, past_key_value
+
+
+class BailingMoeV2MTPLayer(nn.Module):
+    def __init__(self, config: BailingMoeLinearV2Config, layer_idx: int):
+        super().__init__()
+        self.layer_idx = layer_idx
+        self.input_layernorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.enorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
+        self.post_attention_layernorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.attention = ATTENTION_CLASSES[config._attn_implementation](config=config, layer_idx=layer_idx)
+        self.mlp = BailingMoeV2SparseMoeBlock(config)
+
+        self.hnorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.final_layernorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        input_embeds,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        output_router_logits: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
+        **kwargs,
+    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+        input_embeds = self.enorm(input_embeds)
+        hidden_states = self.hnorm(hidden_states)
+        hidden_states = self.eh_proj(torch.cat([input_embeds, hidden_states], dim=-1))
+        residual = hidden_states
+
+        hidden_states = self.input_layernorm(hidden_states)
+
+        # Self Attention
+        hidden_states, self_attn_weights, present_key_value = self.attention(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_value=past_key_value,
+            output_attentions=output_attentions,
+            position_embeddings=position_embeddings,
+            use_cache=use_cache,
+        )
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        if isinstance(hidden_states, tuple):
+            hidden_states, router_logits = hidden_states
+        else:
+            router_logits = None
+        hidden_states = residual + hidden_states.to(residual.device)
+        hidden_states = self.final_layernorm(hidden_states)
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        if use_cache:
+            outputs += (present_key_value,)
+
+        if output_router_logits:
+            outputs += (router_logits,)
+
+        return outputs
+
+
+class BailingMoeLinearV2DecoderLayer(nn.Module):
+    def __init__(self, config: BailingMoeLinearV2Config, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.attention_layer_type = "attention" if (layer_idx + 1) % config.layer_group_size == 0 or \
+            layer_idx >= config.num_hidden_layers // config.layer_group_size * config.layer_group_size else "linear_attention"
+
+        if self.attention_layer_type == "attention":
+            self.attention = ATTENTION_CLASSES[config._attn_implementation](config=config, layer_idx=layer_idx)
+        else:
+            self.attention = BailingMoeV2LinearAttention(
+                config=config,
+                layer_idx=layer_idx
+            )
+
+        self.mlp = (
+            BailingMoeV2SparseMoeBlock(config)
+            if (config.num_experts is not None and layer_idx >= config.first_k_dense_replace)
+            else BailingMoeV2MLP(config=config, intermediate_size=config.intermediate_size)
+        )
+        self.input_layernorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        output_router_logits: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
+        **kwargs,
+    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            attention_mask (`torch.FloatTensor`, *optional*):
+                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
+                query_sequence_length, key_sequence_length)` if default attention is used.
+            position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+                config.n_positions - 1]`.
+            past_key_value (`Tuple(torch.FloatTensor)`, *optional*):
+                cached past key and value projection states
+            output_attentions (`bool`, *optional*):
+                Whether to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            output_router_logits (`bool`, *optional*):
+                Whether or not to return the logits of all the routers. They are useful for computing the router loss,
+                and should not be returned during inference.
+            use_cache (`bool`, *optional*):
+                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+                (see `past_key_values`).
+        """
+        residual = hidden_states
+
+        hidden_states = self.input_layernorm(hidden_states)
+
+        # Self Attention
+        if self.attention_layer_type == "attention":
+            hidden_states, self_attn_weights, present_key_value = self.attention(
+                hidden_states=hidden_states,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_value=past_key_value,
+                output_attentions=output_attentions,
+                position_embeddings=position_embeddings,
+                use_cache=use_cache,
+            )
+        else:
+            batch_size, seq_len = hidden_states.shape[0], hidden_states.shape[1]
+            device = hidden_states.device
+
+            if attention_mask is None:
+                # if attention_mask is None, create a full mask
+                attention_mask = torch.ones((batch_size, seq_len), dtype=torch.int32, device=device)
+            elif attention_mask.dim() == 4 and attention_mask.shape[1] == 1:
+                attention_mask = attention_mask[:, 0, -1, :].to(torch.int32)
+                attention_mask = (attention_mask > -1e4).to(torch.int32)
+            elif attention_mask.dim() == 2:
+                attention_mask = attention_mask.to(torch.int32)
+            else:
+                raise ValueError(f"Unsupported mask dimension: {attention_mask.shape}")
+
+            hidden_states, self_attn_weights, present_key_value = self.attention(
+                hidden_states=hidden_states,
+                attention_mask=attention_mask,
+                past_key_value=past_key_value,
+                position_ids=position_ids,
+                use_cache=use_cache,
+                output_attentions=output_attentions,
+                position_embeddings=position_embeddings,
+            )
+
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        if isinstance(hidden_states, tuple):
+            hidden_states, router_logits = hidden_states
+        else:
+            router_logits = None
+        hidden_states = residual + hidden_states.to(residual.device)
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        if use_cache:
+            outputs += (present_key_value,)
+
+        if output_router_logits:
+            outputs += (router_logits,)
+
+        return outputs
+
+
+BAILINGMOEV2_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+    Parameters:
+        config ([`BailingMoeLinearV2Config`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+@add_start_docstrings(
+    "The bare BailingMoeV2 Model outputting raw hidden-states without any specific head on top.",
+    BAILINGMOEV2_START_DOCSTRING,
+)
+class BailingMoeV2PreTrainedModel(PreTrainedModel):
+    config_class = BailingMoeLinearV2Config
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["BailingMoeLinearV2DecoderLayer"]
+    _skip_keys_device_placement = "past_key_values"
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+    _supports_cache_class = True
+
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+
+BAILINGMOEV2_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+            it.
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+            [What are input IDs?](../glossary#input-ids)
+        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+            [What are attention masks?](../glossary#attention-mask)
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+            If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
+            `past_key_values`).
+            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+            information on the default strategy.
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.n_positions - 1]`.
+            [What are position IDs?](../glossary#position-ids)
+        past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
+            Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+            blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
+            returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
+            Two formats are allowed:
+            - a [`~cache_utils.Cache`] instance;
+            - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
+            cache format.
+            The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
+            legacy cache format will be returned.
+            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
+            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
+            of shape `(batch_size, sequence_length)`.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+@add_start_docstrings(
+    "The bare BailingMoeV2 Model outputting raw hidden-states without any specific head on top.",
+    BAILINGMOEV2_START_DOCSTRING,
+)
+class BailingMoeLinearV2Model(BailingMoeV2PreTrainedModel):
+    """
+    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`BailingMoeLinearV2DecoderLayer`]
+    Args:
+        config: BailingMoeLinearV2Config
+    """
+
+    def __init__(self, config: BailingMoeLinearV2Config):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+        self.num_nextn_predict_layers = config.num_nextn_predict_layers
+
+        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = []
+        for layer_idx in range(config.num_hidden_layers + config.num_nextn_predict_layers):
+            layer_cls = BailingMoeLinearV2DecoderLayer if layer_idx < config.num_hidden_layers else BailingMoeV2MTPLayer
+            self.layers.append(layer_cls(config, layer_idx))
+
+        self.layers = nn.ModuleList(self.layers)
+
+        self._use_sdpa = config._attn_implementation == "sdpa"
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+        self.norm = BailingMoeV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = BailingMoeV2RotaryEmbedding(config=config)
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.word_embeddings
+
+    def set_input_embeddings(self, value):
+        self.word_embeddings = value
+
+    @add_start_docstrings_to_model_forward(BAILINGMOEV2_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        output_router_logits: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs,
+    ) -> Union[Tuple, MoeV2ModelOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        output_router_logits = (
+            output_router_logits if output_router_logits is not None else self.config.output_router_logits
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # retrieve input_ids and inputs_embeds
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            batch_size, seq_length = input_ids.shape[:2]
+        elif inputs_embeds is not None:
+            batch_size, seq_length = inputs_embeds.shape[:2]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`transformers."
+                )
+                use_cache = False
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache()
+
+        if inputs_embeds is None:
+            inputs_embeds = self.word_embeddings(input_ids)
+
+        softmax_attention_layer_id = self.config.layer_group_size - 1
+        past_seen_tokens = past_key_values.get_seq_length(layer_idx=softmax_attention_layer_id) if past_key_values is not None else 0
+
+        if position_ids is None:
+            position_ids = torch.arange(
+                past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
+            )
+            position_ids = position_ids.unsqueeze(0)
+
+        if self._use_flash_attention_2:
+            # 2d mask is passed through the layers
+            attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
+        elif self._use_sdpa and not output_attentions:
+            # output_attentions=True can not be supported when using SDPA, and we fall back on
+            # the manual implementation that requires a 4D causal mask in all cases.
+            attention_mask = _prepare_4d_causal_attention_mask_for_sdpa(
+                attention_mask,
+                (batch_size, seq_length),
+                inputs_embeds,
+                past_seen_tokens,
+            )
+        else:
+            # 4d mask is passed through the layers
+            attention_mask = _prepare_4d_causal_attention_mask(
+                attention_mask, (batch_size, seq_length), inputs_embeds, past_seen_tokens
+            )
+
+        # embed positions
+        hidden_states = inputs_embeds
+
+        # create position embeddings to be shared across the decoder layers
+        position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+        # decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        all_router_logits = () if output_router_logits else None
+        next_decoder_cache = None
+        layers = self.layers[: -self.num_nextn_predict_layers] if self.num_nextn_predict_layers > 0 else self.layers
+        mtp_layers = self.layers[-self.num_nextn_predict_layers :] if self.num_nextn_predict_layers > 0 else None
+
+        for decoder_layer in layers:
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    decoder_layer.__call__,
+                    hidden_states,
+                    attention_mask,
+                    position_ids,
+                    past_key_values,
+                    output_attentions,
+                    output_router_logits,
+                    use_cache,
+                    position_embeddings,
+                )
+            else:
+                layer_outputs = decoder_layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    past_key_value=past_key_values,
+                    output_attentions=output_attentions,
+                    output_router_logits=output_router_logits,
+                    use_cache=use_cache,
+                    position_embeddings=position_embeddings,
+                )
+            hidden_states = layer_outputs[0]
+
+            if use_cache:
+                next_decoder_cache = layer_outputs[2 if output_attentions else 1]
+
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
+
+            if output_router_logits and layer_outputs[-1] is not None:
+                all_router_logits += (layer_outputs[-1],)
+
+        hidden_states = self.norm(hidden_states)
+        main_hidden_states = hidden_states
+
+        # add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (main_hidden_states,)
+
+        mtp_hidden_states = None
+
+        if mtp_layers:
+            for decoder_layer in mtp_layers:
+                input_ids, _ = roll_tensor(input_ids, shifts=-1, dims=-1)
+                inputs_embeds = self.word_embeddings(input_ids)
+
+                if self.gradient_checkpointing and self.training:
+                    layer_outputs = self._gradient_checkpointing_func(
+                        decoder_layer.__call__,
+                        inputs_embeds,
+                        hidden_states,
+                        attention_mask,
+                        position_ids,
+                        past_key_values,
+                        output_attentions,
+                        output_router_logits,
+                        use_cache,
+                        position_embeddings,
+                    )
+                else:
+                    layer_outputs = decoder_layer(
+                        inputs_embeds,
+                        hidden_states,
+                        attention_mask=attention_mask,
+                        position_ids=position_ids,
+                        past_key_value=past_key_values,
+                        output_attentions=output_attentions,
+                        output_router_logits=output_router_logits,
+                        use_cache=use_cache,
+                        position_embeddings=position_embeddings,
+                    )
+                if mtp_hidden_states is None:
+                    mtp_hidden_states = []
+                hidden_states = layer_outputs[0]
+                mtp_hidden_states.append(hidden_states)
+
+                if output_hidden_states:
+                    all_hidden_states += (hidden_states,)
+
+                if use_cache:
+                    next_decoder_cache = layer_outputs[2 if output_attentions else 1]
+
+                if output_attentions:
+                    all_self_attns += (layer_outputs[1],)
+
+                if output_router_logits and layer_outputs[-1] is not None:
+                    all_router_logits += (layer_outputs[-1],)
+
+        next_cache = None
+        if use_cache:
+            next_cache = next_decoder_cache
+        if not return_dict:
+            return tuple(
+                v
+                for v in [main_hidden_states, next_cache, all_hidden_states, all_self_attns, all_router_logits]
+                if v is not None
+            )
+        return MoeV2ModelOutputWithPast(
+            last_hidden_state=main_hidden_states,
+            past_key_values=next_cache,
+            hidden_states=all_hidden_states,
+            mtp_hidden_states=mtp_hidden_states,
+            attentions=all_self_attns,
+            router_logits=all_router_logits,
+        )
+
+
+class BailingMoeLinearV2ForCausalLM(BailingMoeV2PreTrainedModel, GenerationMixin):
+    _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config: BailingMoeLinearV2Config):
+        super().__init__(config)
+        self.model = BailingMoeLinearV2Model(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.num_nextn_predict_layers = config.num_nextn_predict_layers
+        self.mtp_loss_scaling_factor = config.mtp_loss_scaling_factor
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.word_embeddings
+
+    def set_input_embeddings(self, value):
+        self.model.word_embeddings = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def set_decoder(self, decoder):
+        self.model = decoder
+
+    def get_decoder(self):
+        return self.model
+
+    @add_start_docstrings_to_model_forward(BAILINGMOEV2_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=MoEV2CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        output_router_logits: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs,
+    ) -> Union[Tuple, MoEV2CausalLMOutputWithPast]:
+        r"""
+        Args:
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+        Returns:
+        Example:
+        ```python
+        >>> from transformers import AutoTokenizer
+        >>> model = BailingMoeLinearV2ForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
+        >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
+        >>> prompt = "Hey, are you conscious? Can you talk to me?"
+        >>> inputs = tokenizer(prompt, return_tensors="pt")
+        >>> # Generate
+        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
+        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
+        ```"""
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        output_router_logits = (
+            output_router_logits if output_router_logits is not None else self.config.output_router_logits
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            output_router_logits=output_router_logits,
+            return_dict=return_dict,
+            **kwargs,
+        )
+
+        loss = None
+        all_mtp_loss = None
+        aux_loss = None
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()
+
+        if labels is not None:
+            loss = self.loss_function(logits, labels, self.config.vocab_size, **kwargs)
+
+        all_mtp_logits = None
+        if self.num_nextn_predict_layers > 0:
+            mtp_hidden_states = outputs.mtp_hidden_states
+            shift_labels_mtp = None
+            for i in range(self.num_nextn_predict_layers):
+                mtp_hidden_states = mtp_hidden_states[i]
+                mtp_logits = self.lm_head(mtp_hidden_states).float()
+                if all_mtp_logits is None:
+                    all_mtp_logits = []
+                all_mtp_logits.append(mtp_logits)
+                if labels is not None:
+                    if shift_labels_mtp is None:
+                        shift_labels_mtp = labels.clone()
+                    shift_labels_mtp, _ = roll_tensor(shift_labels_mtp, shifts=-1, dims=-1, fill_value=-100)
+                    mtp_logits_ = mtp_logits.view(-1, self.config.vocab_size)
+                    mtp_loss = self.loss_function(mtp_logits_, shift_labels_mtp.to(mtp_logits_.device).view(-1), self.config.vocab_size, **kwargs)
+                    if loss is not None:
+                        loss += self.mtp_loss_scaling_factor * mtp_loss
+                    else:
+                        loss = self.mtp_loss_scaling_factor * mtp_loss
+
+                    if all_mtp_loss is None:
+                        all_mtp_loss = []
+                    all_mtp_loss.append(mtp_loss)
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            if output_router_logits:
+                output = (aux_loss,) + output
+            return (loss,) + output if loss is not None else output
+
+        return MoEV2CausalLMOutputWithPast(
+            loss=loss,
+            mtp_loss=all_mtp_loss,
+            aux_loss=aux_loss,
+            logits=logits,
+            mtp_logits=all_mtp_logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            router_logits=outputs.router_logits,
+        )

recipe.yaml

@@ -0,0 +1,26 @@
+default_stage:
+  default_modifiers:
+    GPTQModifier:
+      config_groups:
+        config_group:
+          targets: [Linear]
+          weights:
+            num_bits: 4
+            type: int
+            symmetric: true
+            group_size: 128
+            strategy: group
+            block_structure: null
+            dynamic: false
+            actorder: weight
+            observer: minmax
+            observer_kwargs: {}
+          input_activations: null
+          output_activations: null
+          format: null
+      targets: [Linear]
+      ignore: [lm_head, 're:.*mlp.gate$', 're:.*shared_experts*$', 're:.*.dense$']
+      sequential_update: true
+      block_size: 128
+      dampening_frac: 0.01
+      offload_hessians: false

special_tokens_map.json

@@ -0,0 +1,41 @@
+{
+  "additional_special_tokens": [
+    "<role>",
+    "</role>"
+  ],
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "gmask_token": {
+    "content": "[gMASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenization_bailing.py

@@ -0,0 +1,1068 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# Copyright (c) Ant Group. All rights reserved.
+
+import itertools
+from typing import Any, Dict, List, Optional, Union
+
+import torch
+from transformers import PreTrainedTokenizerFast
+from transformers.tokenization_utils_base import AddedToken, BatchEncoding
+from transformers.utils import TensorType, logging
+
+logger = logging.get_logger(__name__)
+
+
+def is_system(msg):
+    return msg['role'].lower() == 'system'
+
+
+def is_user(msg):
+    return msg['role'].lower() in ['human', 'user']
+
+
+def is_assistant(msg):
+    return msg['role'].lower() == 'assistant'
+
+
+def _convert_to_conversation(query, system=None):
+    conversation = []
+    if system:
+        conversation.append({"role": "SYSTEM", "content": system})
+    if isinstance(query, str):
+        conversation.append({"role": "HUMAN", "content": query})
+    elif isinstance(query, List):
+        conversation.extend(query)
+    elif isinstance(query, Dict):
+        if "messages" in query:
+            conversation.extend(query["messages"])
+            if "system_message" in query and len(conversation) > 0 and not is_system(conversation[0]):
+                conversation.insert(0, {"role": "SYSTEM", "content": query["system_message"]})
+        else:
+            conversation.append(query)
+    return conversation
+
+
+class BailingTokenizer(PreTrainedTokenizerFast):
+    is_bailing_tokenizer = True
+    model_input_names = ["input_ids", "attention_mask"]
+    slow_tokenizer_class = None
+
+    # add gmask_token
+    SPECIAL_TOKENS_ATTRIBUTES = [
+        "bos_token",
+        "eos_token",
+        "unk_token",
+        "sep_token",
+        "pad_token",
+        "cls_token",
+        "mask_token",
+        "gmask_token",
+        "additional_special_tokens",
+    ]
+
+    def __init__(
+        self,
+        vocab_file=None,
+        merges_file=None,
+        tokenizer_file=None,
+        clean_up_tokenization_spaces=False,
+        bos_token="<|startoftext|>",
+        eos_token="<|endoftext|>",
+        cls_token="[CLS]",
+        pad_token="<|endoftext|>",
+        gmask_token="[gMASK]",
+        add_bos_token=False,
+        add_eos_token=False,
+        **kwargs,
+    ):
+        self.add_bos_token = add_bos_token
+
+        self._gmask_token = (
+            AddedToken(gmask_token, lstrip=False, rstrip=False, normalized=False)
+            if isinstance(gmask_token, str)
+            else gmask_token
+        )
+
+        self._sop_token = (
+            AddedToken(bos_token, lstrip=False, rstrip=False, normalized=False)
+            if isinstance(bos_token, str)
+            else bos_token
+        )
+
+        self._eop_token = (
+            AddedToken(eos_token, lstrip=False, rstrip=False, normalized=False)
+            if isinstance(eos_token, str)
+            else eos_token
+        )
+
+        super().__init__(
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            tokenizer_file=tokenizer_file,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            cls_token=cls_token,
+            pad_token=pad_token,
+            gmask_token=gmask_token,
+            add_bos_token=add_bos_token,
+            add_eos_token=add_eos_token,
+            **kwargs,
+        )
+
+        self.check_special_tokens()
+
+    def check_special_tokens(self):
+        '''
+        eos_token, cls_token, mask_token
+        special tokens should init, check special token is not None
+        '''
+        for name, special_token in zip(
+            ['eos', 'bos', 'cls', 'gmask'],
+            [self.eos_token, self.bos_token, self.cls_token, self.gmask_token],
+        ):
+            assert special_token is not None, f'should init special token [{name}] in tokenizer_config.json'
+
+    @property
+    def gmask_token(self) -> Optional[str]:
+        if self._gmask_token is None:
+            if self.verbose:
+                logger.error("Using gmask_token, but it is not set yet.")
+            return None
+        return str(self._gmask_token)
+
+    @gmask_token.setter
+    def gmask_token(self, value):
+        if not isinstance(value, (str, AddedToken)) and value is not None:
+            raise ValueError("Cannot set a non-string value as the gmask token")
+        self._gmask_token = value
+
+    @property
+    def gmask_token_id(self) -> Optional[int]:
+        if self._gmask_token is None:
+            return None
+        return self.convert_tokens_to_ids(self.gmask_token)
+
+    @property
+    def sop_token(self) -> Optional[str]:
+        if self._sop_token is None:
+            if self.verbose:
+                logger.error("Using sop_token, but it is not set yet.")
+            return None
+        return str(self._sop_token)
+
+    @sop_token.setter
+    def sop_token(self, value):
+        if not isinstance(value, (str, AddedToken)) and value is not None:
+            raise ValueError("Cannot set a non-string value as the sop token")
+        self._sop_token = value
+
+    @property
+    def sop_token_id(self) -> Optional[int]:
+        if self._sop_token is None:
+            return None
+        return self.convert_tokens_to_ids(self.sop_token)
+
+    @property
+    def eop_token(self) -> Optional[str]:
+        if self._eop_token is None:
+            if self.verbose:
+                logger.error("Using eop_token, but it is not set yet.")
+            return None
+        return str(self._eop_token)
+
+    @eop_token.setter
+    def eop_token(self, value):
+        if not isinstance(value, (str, AddedToken)) and value is not None:
+            raise ValueError("Cannot set a non-string value as the eop token")
+        self._eop_token = value
+
+    @property
+    def eop_token_id(self) -> Optional[int]:
+        if self._eop_token is None:
+            return None
+        return self.convert_tokens_to_ids(self.eop_token)
+
+    @property
+    def vocab_size(self):
+        return len(self.get_vocab())
+
+    def _chat_from_json(self, chat, chat_format="antglm_chat", system=None):
+        msgs = chat if "messages" not in chat else chat["messages"]
+        _msgs = []
+        sys_msg = None
+        for msg in msgs:
+            if is_system(msg):
+                sys_msg = msg['content']
+            else:
+                _msgs.append(msg)
+        chat = {"messages": _msgs}
+        system = system or sys_msg
+        if system:
+            chat['system_message'] = system
+        from .chat_format import Chat
+
+        return Chat.from_json(chat, name=chat_format)
+
+    def apply_chat_template(
+        self,
+        conversation: Union[List[Dict[str, str]], List[List[Dict[str, str]]]],
+        tools: Optional[List[Dict]] = None,
+        documents: Optional[List[Dict[str, str]]] = None,
+        chat_template: Optional[str] = None,
+        add_generation_prompt: bool = False,
+        system: str = None,  # only used for legacy chatml
+        tokenize=False,
+        padding: bool = False,
+        truncation: bool = False,
+        max_length: Optional[int] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        return_dict: bool = False,
+        return_assistant_tokens_mask: bool = False,
+        tokenizer_kwargs: Optional[Dict[str, Any]] = None,
+        **kwargs,
+    ):
+        if hasattr(self, "chat_template") and self.chat_template:
+            if isinstance(conversation, Dict) and "messages" in conversation:
+                conversation = conversation["messages"]
+            # use transformers built-in method
+            return super().apply_chat_template(
+                conversation=conversation,
+                tools=tools,
+                documents=documents,
+                chat_template=chat_template,
+                add_generation_prompt=add_generation_prompt,
+                tokenize=tokenize,
+                padding=padding,
+                truncation=truncation,
+                return_tensors=return_tensors,
+                return_dict=return_dict,
+                return_assistant_tokens_mask=return_assistant_tokens_mask,
+                tokenizer_kwargs=tokenizer_kwargs,
+            )
+
+        # 非chat_template方式后续将不再支持。
+        logger.warning("Please set chat_template in tokenizer_config.json!")
+
+        chat_format = kwargs.get('chat_format', 'antglm_chat')
+
+        is_batched = False
+
+        if isinstance(conversation, List) and (
+            isinstance(conversation[0], (list, tuple)) or "messages" in conversation[0]
+        ):
+            conversations = conversation
+            is_batched = True
+
+        if not is_batched:
+            conversations = [conversation]
+
+        rendered = []
+        for chat in conversations:
+            rendered_chat = self._chat_from_json(chat, chat_format=chat_format, system=system).prompt_str
+            rendered.append(rendered_chat)
+
+        if not is_batched:
+            rendered = rendered[0]
+
+        if tokenize:
+            out = self(
+                rendered,
+                padding=padding,
+                truncation=truncation,
+                max_length=max_length,
+                add_special_tokens=False,
+                return_tensors=return_tensors,
+            )
+            if return_dict:
+                return out
+            else:
+                return out["input_ids"]
+        else:
+            return rendered
+
+    def _build_position_ids(
+        self,
+        mask_pos: int,
+        bos_pos: int,
+        max_output_length: int,
+        rotary_type: Optional[str] = "none",
+        **kwargs,
+    ) -> List[List[int]]:
+        window_size = kwargs.get("window_size", 1024) - 1
+        block_position_ids = [0] * bos_pos
+
+        # 获得mask所在的位置，用于后面output positionid的构造
+        if "1d" in rotary_type:
+            position_ids = list(range(bos_pos)) + list(range(mask_pos + 1, mask_pos + max_output_length + 2))
+            block_position_ids = block_position_ids + list(range(1, max_output_length + 2))
+        elif "2d" in rotary_type:
+            # 后面input_ids要加一个bos_id
+            position_ids = list(range(bos_pos))
+            position_ids = position_ids + [mask_pos] * (1 + max_output_length)
+            block_position_ids = block_position_ids + list(range(1, max_output_length + 2))
+        else:
+            # build position ids
+            position_ids = []
+            repeat_times = bos_pos // window_size
+            for _ in range(repeat_times):
+                position_ids += list(range(window_size))
+            position_ids += list(range(bos_pos - window_size * repeat_times))
+            # need consider additional bos_id after input_ids
+            mask_pos = position_ids[-1]
+            position_ids += [mask_pos] * (max_output_length + 1)
+
+            block_repeat_times = max_output_length // (window_size - 1)
+            additional_block_position_ids = []
+            for _ in range(block_repeat_times):
+                additional_block_position_ids += list(range(1, window_size))
+            additional_block_position_ids += list(
+                range(1, max_output_length + 2 - (window_size - 1) * block_repeat_times)
+            )
+            block_position_ids = block_position_ids + additional_block_position_ids
+
+        position_ids = [position_ids, block_position_ids]
+        return position_ids
+
+    def _build_inputs_for_generation(
+        self,
+        input_ids: List[int],
+        max_input_length=None,
+        left_truncate=True,
+        max_output_length=1024,
+        rotary_type="none",
+        unidirectional_attention: bool = True,
+        attention_dtype=None,
+        **kwargs,
+    ):
+        if max_input_length and len(input_ids) > max_input_length:
+            if left_truncate:
+                input_ids = input_ids[-max_input_length:]
+            else:
+                input_ids = input_ids[:max_input_length]
+
+        is_left_padding = input_ids[0] == self.eos_token_id
+        if not unidirectional_attention:
+            if input_ids[0] != self.cls_token_id:
+                input_ids = [self.cls_token_id] + input_ids
+
+            if self.gmask_token_id not in set(input_ids):
+                input_ids = input_ids + [self.gmask_token_id]
+
+            mask_pos = input_ids.index(self.gmask_token_id)
+            sep = len(input_ids)
+        else:
+            if self.add_bos_token:
+                input_ids = input_ids + [self.bos_token_id]
+                if self.eos_token_id in input_ids:
+                    mask_pos = input_ids.index(self.eos_token_id) - 1
+                else:
+                    mask_pos = len(input_ids) - 1
+                sep = len(input_ids) - 1
+            else:
+                sep = len(input_ids)
+                if self.eos_token_id in input_ids:
+                    if is_left_padding:
+                        ori_input_ids = input_ids
+                        input_ids = input_ids[::-1]
+                    mask_pos = input_ids.index(self.eos_token_id) - 1
+                    mask_pos = max(0, mask_pos)  # for empty sequence
+                    if is_left_padding:
+                        input_ids = ori_input_ids
+                        mask_pos = sep - 1 - mask_pos  # the first non-eos token
+
+                else:
+                    mask_pos = len(input_ids) - 1
+
+        position_ids = self._build_position_ids(mask_pos, sep, max_output_length, rotary_type, **kwargs)
+
+        if is_left_padding:
+            position_ids[0] = [max(0, i - mask_pos) for i in range(len(position_ids[0]))]
+
+        # 后面input_ids要加一个bos_id
+        total_length = sep + max_output_length
+        if self.add_bos_token:
+            total_length += 1
+
+        def build_mask_matrix(seq_length, sep, mask_pos, unidirectional_attention):
+            # 长序列使用bool类型节省显存
+            if unidirectional_attention:
+                attention_mask = torch.ones([seq_length, seq_length], dtype=attention_dtype)
+                attention_mask = torch.tril(attention_mask)
+                if is_left_padding:
+                    attention_mask[:, :mask_pos] = 0
+                else:
+                    attention_mask[:, mask_pos + 1 : sep] = 0
+            else:
+                attention_mask = torch.zeros([seq_length, seq_length], dtype=attention_dtype)
+                attention_mask[:, : mask_pos + 1] = 1
+                for i in range(sep, total_length):
+                    attention_mask[i, sep : i + 1] = 1
+            return attention_mask
+
+        if self.add_bos_token:
+            attention_mask = build_mask_matrix(total_length, sep + 1, mask_pos, unidirectional_attention)
+        else:
+            attention_mask = build_mask_matrix(total_length, sep, mask_pos, unidirectional_attention)
+        attention_mask = torch.unsqueeze(attention_mask, dim=0)
+        attention_mask = torch.unsqueeze(attention_mask, dim=1)
+        if attention_dtype is None:
+            attention_mask = attention_mask.long()
+        inputs = {
+            "input_ids": torch.Tensor([input_ids]).long(),
+            "position_ids": torch.Tensor([position_ids]).long(),
+            "attention_mask": attention_mask,
+        }
+        return BatchEncoding(inputs)
+
+    def build_inputs_for_generation(
+        self,
+        input_ids: Union[List[int], List[List[int]], torch.Tensor],
+        max_input_length=None,
+        left_truncate=True,
+        max_output_length=1024,
+        rotary_type="1d",
+        unidirectional_attention=True,
+        attention_dtype=None,
+        **kwargs,
+    ):
+        if isinstance(input_ids, torch.Tensor):
+            input_ids = input_ids.tolist()
+
+        if isinstance(input_ids[0], list):
+            input_ids_list = []
+            position_ids_list = []
+            attention_mask_list = []
+            for _input_ids in input_ids:
+                inputs = self._build_inputs_for_generation(
+                    _input_ids,
+                    max_input_length=max_input_length,
+                    left_truncate=left_truncate,
+                    max_output_length=max_output_length,
+                    rotary_type=rotary_type,
+                    unidirectional_attention=unidirectional_attention,
+                    attention_dtype=attention_dtype,
+                    **kwargs,
+                )
+                input_ids_list.append(inputs['input_ids'])
+                position_ids_list.append(inputs['position_ids'])
+                attention_mask_list.append(inputs["attention_mask"])
+
+            max_ids_length = max([input.size(1) for input in input_ids_list])
+
+            for i in range(len(input_ids)):
+                cur_ids_length = input_ids_list[i].size(1)
+                if cur_ids_length < max_ids_length:
+                    # pad input ids
+                    pad_input_ids = input_ids_list[i].new_zeros((1, max_ids_length - cur_ids_length))
+                    input_ids_list[i] = torch.cat([pad_input_ids, input_ids_list[i]], dim=-1)
+
+                    # pad postition ids with left pad
+                    # 0, 1, 2, 3, 4 ... -> 0, ..., 0, 1, 2, 3, 4, ...
+                    pad_position_ids = input_ids_list[i].new_zeros((1, 2, max_ids_length - cur_ids_length))
+                    position_ids_list[i] = torch.cat([pad_position_ids, position_ids_list[i]], dim=-1)
+
+                    # pad generation attention mask with left and bottom pad
+                    new_attention_mask = input_ids_list[i].new_zeros(
+                        1,
+                        1,
+                        max_ids_length + max_output_length,
+                        max_ids_length + max_output_length,
+                    )
+                    new_attention_mask[
+                        :,
+                        :,
+                        max_ids_length - cur_ids_length :,
+                        max_ids_length - cur_ids_length :,
+                    ] = attention_mask_list[i]
+                    attention_mask_list[i] = new_attention_mask.contiguous()
+
+            input_ids_list = torch.cat(input_ids_list, dim=0)
+            position_ids_list = torch.cat(position_ids_list, dim=0)
+            attention_mask_list = torch.cat(attention_mask_list, dim=0)
+
+            inputs = {
+                "input_ids": input_ids_list,
+                "position_ids": position_ids_list,
+                "attention_mask": attention_mask_list,
+            }
+
+            return BatchEncoding(inputs)
+        else:
+            return self._build_inputs_for_generation(
+                input_ids,
+                max_input_length=max_input_length,
+                left_truncate=left_truncate,
+                max_output_length=max_output_length,
+                rotary_type=rotary_type,
+                unidirectional_attention=unidirectional_attention,
+                **kwargs,
+            )
+
+    def _build_inputs_for_train(
+        self,
+        inputs: Union[str, List[str]],
+        outputs: Union[str, List[str]],
+        new_conversation_offset: List[int] = None,
+        max_length: int = 2048,
+        rotary_type: str = "1d",
+        left_truncate: bool = True,
+        unidirectional_attention: bool = True,
+        isolation_position_ids: bool = False,
+        padding: bool = True,
+        use_fa2: bool = True,
+        use_packed: bool = True,
+        use_baichuan_packed: bool = False,
+        skip_truncated_turn: bool = False,
+        return_attention_mask: bool = True,
+    ):
+        r"""
+        Build tensor input for model training. If inputs and outputs are list, will pack them.
+
+        Args:
+            inputs (str, List[str], List[Dict], List[List[Dict]]): the input prompts.
+            outputs (str, List[str]): the output responses.
+            max_length (int, Optional): the maximum length of the final input ids for training. Default: 2048
+            rotary_type (str, Optional): the rotary type of position embedding. Default: 1d
+            left_truncate (bool, Optional): whether truncate the inputs from left. Default: True
+            use_fa2 (bool, Optional): whether to build attention mask under flash attention 2.
+            new_conversation_offset (List[int], Optional): 第idx条样本是全新的对话，[0, 1]代表：inputs[0]和outputs[0]是一个对话，inputs[1]和outputs[1]是一个对话.
+        """
+        if use_packed and use_baichuan_packed and unidirectional_attention:
+            return self._build_baichuan_inputs_for_train(
+                inputs,
+                outputs,
+                new_conversation_offset,
+                max_length,
+                rotary_type,
+                left_truncate,
+                skip_truncated_turn,
+                use_fa2,
+                padding,
+            )
+        if isinstance(inputs, str):
+            inputs = [inputs]
+        if isinstance(outputs, str):
+            outputs = [outputs]
+
+        assert len(inputs) == len(outputs)
+
+        input_ids = [self(item)['input_ids'] for item in inputs]
+        output_ids = [self(item)['input_ids'] for item in outputs]
+
+        packed_input_ids = []
+        packed_output_ids = []
+        if new_conversation_offset is None:
+            new_conversation_offset = list(range(0, len(inputs)))
+        assert 0 in new_conversation_offset, f"没有0，请检查new_conversation_offset: {new_conversation_offset}"
+        current_len = 0
+
+        for idx, (input, output) in enumerate(zip(input_ids, output_ids)):
+            num_special_tokens = 0
+            if not unidirectional_attention:
+                if idx in new_conversation_offset:
+                    # cls and gmask
+                    num_special_tokens += 2
+                else:
+                    # only gmask
+                    num_special_tokens += 1
+            else:
+                # sop and eos
+                if self.add_bos_token:
+                    num_special_tokens += 2
+                else:
+                    num_special_tokens += 1
+
+            # truncate
+            if len(input) + len(output) + current_len > max_length - num_special_tokens:
+                if not use_packed or use_fa2 and unidirectional_attention:
+                    attention_mask = torch.tensor(0)
+                elif use_fa2:
+                    attention_mask = -1 * torch.ones([2, max_length])
+                else:
+                    attention_mask = torch.tril(torch.ones([max_length, max_length]))
+                # 返回一个空的样本，该样本不参与训练
+                default_return = {
+                    'input_ids': (torch.ones(max_length) * self.eos_token_id).long(),
+                    'position_ids': torch.zeros(2, max_length).long(),
+                    'attention_mask': (attention_mask.long()),
+                    'labels': (torch.ones(max_length) * -100).long(),
+                }
+                # 如果不截断，直接返回
+                if skip_truncated_turn:
+                    if current_len == 0:
+                        return default_return
+                    else:
+                        break
+                left_len = max_length - num_special_tokens - current_len
+                # 如果截断，只截断prompt
+                if left_len - len(output) > 0:
+                    if left_truncate:
+                        input = input[-(left_len - len(output)) :]
+                    else:
+                        input = input[: left_len - len(output)]
+                else:
+                    # response超过left_len，直接返回
+                    if current_len == 0:
+                        return default_return
+                    else:
+                        break
+            if unidirectional_attention:
+                packed_input_ids.append(list(input))
+            else:
+                if num_special_tokens == 4:
+                    packed_input_ids.append([self.cls_token_id] + list(input) + [self.gmask_token_id])
+                else:
+                    packed_input_ids.append(list(input) + [self.gmask_token_id])
+
+            packed_output_ids.append(list(output) + [self.eos_token_id])
+            current_len += len(input) + len(output) + num_special_tokens
+
+        assert current_len <= max_length
+
+        if use_packed:
+            # pack模式
+            def build_mask_matrix(seq_length, sep):
+                # https://github.com/pytorch/pytorch/issues/101932, fix triu/tril bf16 support
+                m = torch.ones((1, seq_length, seq_length))
+                mask = torch.arange(1, m.shape[-1] + 1).reshape(1, -1, 1).to(m.device)
+                ids = torch.arange(1, m.shape[-1] + 1).reshape(1, 1, -1).expand(1, m.shape[-1], -1).to(m.device)
+                m = (ids <= mask).type_as(m)
+
+                m[0, :, : int(sep)] = 1
+                m = m.squeeze(0)
+                return m
+
+            tokens = []
+            attention_mask_list = []
+            input_length_list = []
+            position_id_list = []
+            block_position_id_list = []
+            for input, output in zip(packed_input_ids, packed_output_ids):
+                if self.add_bos_token:
+                    data = input + [self.sop_token_id] + output
+                    mask_pos = len(input) - 1
+                else:
+                    data = input + output
+                    mask_pos = len(input) - 2
+                if return_attention_mask:
+                    if unidirectional_attention:
+                        attention_mask = build_mask_matrix(len(data), 0)
+                    else:
+                        attention_mask = build_mask_matrix(len(data), len(input))
+                    attention_mask = attention_mask.squeeze((0, 1))
+
+                    attention_mask_list.append(attention_mask)
+                input_length_list.append(len(input))
+                tokens += data
+
+                sop_pos = mask_pos + 1
+                position_ids, block_position_ids = self._build_position_ids(
+                    mask_pos=mask_pos, bos_pos=sop_pos, max_output_length=len(output), rotary_type=rotary_type
+                )
+
+                position_id_list.append(position_ids)
+                block_position_id_list.append(block_position_ids)
+
+            labels = []
+            for i in range(len(packed_input_ids)):
+                if self.add_bos_token:
+                    labels += [-100] * len(packed_input_ids[i]) + packed_output_ids[i] + [-100]
+                else:
+                    labels += [-100] * (len(packed_input_ids[i]) - 1) + packed_output_ids[i] + [-100]
+
+            total_len = 0
+            if use_fa2:
+                pack_attention_mask = -1 * torch.ones([2, current_len])
+            else:
+                pack_attention_mask = torch.tril(torch.ones([current_len, current_len]))
+
+            pack_position_ids = []
+            pack_block_position_ids = []
+            total_len = 0
+            max_index = 0
+            for i in range(len(position_id_list)):
+
+                if use_fa2:
+                    pack_attention_mask[0][i] = total_len
+                    pack_attention_mask[1][i] = total_len + input_length_list[i]
+                else:
+                    pack_attention_mask[
+                        total_len : total_len + attention_mask.shape[0],
+                        total_len : total_len + attention_mask.shape[0],
+                    ] = attention_mask
+                position_ids = [pid + max_index for pid in position_id_list[i]]
+                block_position_ids = block_position_id_list[i]
+                pack_position_ids.extend(position_ids)
+                pack_block_position_ids.extend(block_position_ids)
+                if not isolation_position_ids:
+                    max_index = pack_position_ids[-1] + 1
+                total_len += len(position_id_list[i])
+            position_ids = [pack_position_ids, pack_block_position_ids]
+        else:
+            # 单输入模式
+            # 真多轮下，一条样本可能会有好几轮对话，此时需要获取第一条样本的结束位置
+            if len(new_conversation_offset) > 1:
+                end_idx = new_conversation_offset[1]
+            else:
+                end_idx = 1
+            input, output = list(itertools.chain(*packed_input_ids[:end_idx])), list(
+                itertools.chain(*packed_output_ids[:end_idx])
+            )
+            if self.add_bos_token:
+                tokens = input + [self.sop_token_id] + output
+            else:
+                tokens = input + output
+
+            if self.add_bos_token:
+                labels = [-100] * len(input) + output + [-100]
+                position_ids = self._build_position_ids(
+                    mask_pos=len(input) - 1, bos_pos=len(input), max_output_length=len(output), rotary_type=rotary_type
+                )
+            else:
+                labels = [-100] * (len(input) - 1) + output + [-100]
+                position_ids = self._build_position_ids(
+                    mask_pos=len(input) - 2,
+                    bos_pos=len(input) - 1,
+                    max_output_length=len(output),
+                    rotary_type=rotary_type,
+                )
+            attention_mask = len(input)
+        assert current_len == len(tokens)
+
+        # 最大长度补全
+        if max_length > 0 and len(tokens) < max_length and padding:
+            pad_length = max_length - len(tokens)
+            tokens += [self.pad_token_id] * pad_length
+            labels.extend([-100] * pad_length)
+            position_ids[0] += [0] * pad_length
+            position_ids[1] += [0] * pad_length
+
+            if use_packed:
+                if use_fa2:
+                    new_attention_mask = -1 * torch.ones([2, max_length])
+                    new_attention_mask[:, :current_len] = pack_attention_mask
+                else:
+                    new_attention_mask = torch.tril(torch.ones([max_length, max_length]))
+                    new_attention_mask[:current_len, :current_len] = pack_attention_mask
+                pack_attention_mask = new_attention_mask.contiguous()
+
+        assert len(tokens) == len(labels)
+
+        if max_length > 0 and padding:
+            assert len(tokens) == max_length
+
+        if use_fa2 and unidirectional_attention:
+            # pack_attention_mask = torch.zeros([1], dtype=torch.long)
+            pack_attention_mask = torch.tensor(0)
+
+        if use_packed:
+            if not use_fa2:
+                attention_mask = pack_attention_mask.unsqueeze(0).long()
+            else:
+                attention_mask = pack_attention_mask
+        else:
+            attention_mask = torch.tensor(attention_mask).long()
+        return {
+            'input_ids': torch.tensor(tokens).long(),
+            'position_ids': torch.tensor(position_ids).long(),
+            'attention_mask': attention_mask,
+            'labels': torch.tensor(labels).long(),
+        }
+
+    def _build_baichuan_inputs_for_train(
+        self,
+        inputs: Union[str, List[str]],
+        outputs: Union[str, List[str]],
+        new_conversation_offset: List[int] = None,
+        max_length: int = 2048,
+        rotary_type: str = "1d",
+        left_truncate: bool = True,
+        skip_truncated_turn: bool = True,
+        use_fa2: bool = True,
+        padding: bool = True,
+    ):
+        '''
+        input:  <role> HUMAN </role> u1 <role>  ASSISTANT </role> a11 a12            <role> HUMAN </role> u2 <role> ASSISTANT </role> a21 a22           <|endoftext|> <role> HUMAN </role> u1 <role>  ASSISTANT </role> a11 a12            <role> HUMAN </role> u2 <role> ASSISTANT </role> a21 a22           <|endoftext|>
+        output: x      x     x       x  x       x         a11     a12 <|endoftext|>  x      x     x       x  x      x         a21     a22 <|endoftext|> x             x      x     x       x  x       x         a11     a12 <|endoftext|>  x      x     x       x  x      x         a21     a22 <|endoftext|> x
+        只适用真多轮+pack数据训练单向模型，需要打开use_true_multiturn
+        '''
+        if isinstance(inputs, str):
+            inputs = [inputs]
+        if isinstance(outputs, str):
+            outputs = [outputs]
+        assert len(inputs) == len(outputs)
+
+        input_ids = [self(item)['input_ids'] for item in inputs]
+        output_ids = [self(item)['input_ids'] for item in outputs]
+
+        packed_input_ids = []
+        packed_output_ids = []
+
+        if new_conversation_offset is None:
+            new_conversation_offset = list(range(0, len(inputs)))
+        assert 0 in new_conversation_offset, f"没有0，请检查new_conversation_offset: {new_conversation_offset}"
+        current_len = 0
+
+        for idx, (input, output) in enumerate(zip(input_ids, output_ids)):
+            num_special_tokens = 0
+            if idx != 0 and idx in new_conversation_offset:
+                # 在input_ids加入eos，只有第0条样本不加
+                num_special_tokens += 1
+
+            # truncate
+            if len(input) + len(output) + current_len > max_length - num_special_tokens:
+                if use_fa2:
+                    attention_mask = torch.tensor(0)
+                else:
+                    attention_mask = torch.tril(torch.ones([max_length, max_length]))
+                # 返回一个空的样本，该样本不参与训练
+                default_return = {
+                    'input_ids': (torch.ones(max_length) * self.eos_token_id).long(),
+                    'position_ids': torch.zeros(2, max_length).long(),
+                    'attention_mask': (attention_mask.long()),
+                    'labels': (torch.ones(max_length) * -100).long(),
+                }
+
+                # 如果不截断，直接返回
+                if skip_truncated_turn:
+                    if current_len == 0:
+                        return default_return
+                    else:
+                        break
+                left_len = max_length - num_special_tokens - current_len
+                # 如果截断，只截断prompt
+                if left_len - len(output) > 0:
+                    if left_truncate:
+                        input = input[-(left_len - len(output)) :]
+                    else:
+                        input = input[: left_len - len(output)]
+                else:
+                    # response超过left_len，直接返回
+                    if current_len == 0:
+                        return default_return
+                    else:
+                        break
+            # 这里拼的是input_ids
+            if num_special_tokens == 1:
+                packed_input_ids.append([self.eos_token_id] + list(input))
+            else:
+                packed_input_ids.append(list(input))
+            packed_output_ids.append(list(output))
+            current_len += len(input) + len(output) + num_special_tokens
+        assert current_len <= max_length
+
+        def build_mask_matrix(seq_length, sep):
+            # https://github.com/pytorch/pytorch/issues/101932, fix triu/tril bf16 support
+            m = torch.ones((1, seq_length, seq_length))
+            mask = torch.arange(1, m.shape[-1] + 1).reshape(1, -1, 1).to(m.device)
+            ids = torch.arange(1, m.shape[-1] + 1).reshape(1, 1, -1).expand(1, m.shape[-1], -1).to(m.device)
+            m = (ids <= mask).type_as(m)
+
+            m[0, :, : int(sep)] = 1
+            m = m.squeeze(0)
+            return m
+
+        tokens = []
+        attention_mask_list = []
+        position_id_list = []
+        block_position_id_list = []
+        token_lens = []
+        for input, output in zip(packed_input_ids, packed_output_ids):
+            data = input + output
+            if not use_fa2:
+                attention_mask = build_mask_matrix(len(data), 0)
+                attention_mask_list.append(attention_mask)
+            tokens += data
+            token_lens.append(len(data))
+
+            position_ids, block_position_ids = self._build_position_ids(
+                mask_pos=len(input) - 2, bos_pos=len(input) - 1, max_output_length=len(output), rotary_type=rotary_type
+            )
+
+            position_id_list.append(position_ids)
+            block_position_id_list.append(block_position_ids)
+
+        labels = []
+        for i in range(len(packed_input_ids)):
+            labels += [-100] * (len(packed_input_ids[i]) - 1) + packed_output_ids[i] + [self.eos_token_id]
+
+        total_len = 0
+        if use_fa2:
+            pack_attention_mask = torch.Tensor([[0], [1]])
+        else:
+            pack_attention_mask = torch.tril(torch.ones([max_length, max_length]))
+
+        pack_position_ids = []
+        pack_block_position_ids = []
+        total_len = 0
+        max_index = 0
+        for i in range(len(token_lens)):
+            if not use_fa2:
+                attention_mask = attention_mask_list[i]
+                pack_attention_mask[
+                    total_len : total_len + attention_mask.shape[0], total_len : total_len + attention_mask.shape[0]
+                ] = attention_mask
+            position_ids = [pid + max_index for pid in position_id_list[i]]
+            block_position_ids = block_position_id_list[i]
+            pack_position_ids.extend(position_ids)
+            pack_block_position_ids.extend(block_position_ids)
+            max_index = pack_position_ids[-1] + 1
+            total_len += token_lens[i]
+        position_ids = [pack_position_ids, pack_block_position_ids]
+
+        if max_length > 0 and len(tokens) < max_length and padding:
+            pad_length = max_length - len(tokens)
+            tokens += [self.pad_token_id] * pad_length
+            labels.extend([-100] * pad_length)
+            position_ids[0] += [0] * pad_length
+            position_ids[1] += [0] * pad_length
+
+        assert len(tokens) == len(labels)
+
+        if not use_fa2:
+            attention_mask = pack_attention_mask.unsqueeze(0).long()
+        else:
+            attention_mask = torch.tensor(0)
+        return {
+            'input_ids': torch.tensor(tokens).long(),
+            'position_ids': torch.tensor(position_ids).long(),
+            'attention_mask': attention_mask,
+            'labels': torch.tensor(labels).long(),
+        }
+
+    def build_inputs_for_train(
+        self,
+        data: Union[Dict, List[Dict]],
+        new_conversation_offset: List[int] = None,
+        chat_format="antglm_chat",
+        is_chat_format=True,  # 如果传入的是字符串，用于说明是否已经是
+        use_true_multiturn=False,
+        max_length: int = 2048,
+        rotary_type: str = "1d",
+        left_truncate: bool = True,
+        unidirectional_attention: bool = True,
+        isolation_position_ids: bool = False,
+        padding: bool = True,
+        use_fa2: bool = True,
+        use_packed: bool = True,
+        use_baichuan_packed: bool = False,
+        skip_truncated_turn: bool = False,
+        return_attention_mask: bool = True,
+    ):
+        r"""
+        Build tensor input for model training. If inputs and outputs are list, will pack them.
+
+        Args:
+            inputs (str, List[str], List[Dict], List[List[Dict]]): the input prompts.
+            outputs (str, List[str]): the output responses.
+            new_conversation_offset (List[int]): the offset index of the new conversation turn.
+            is_chat_format (bool): whether the input is already chatml format
+            max_length (int, Optional): the maximum length of the final input ids for training. Default: 2048
+            rotary_type (str, Optional): the rotary type of position embedding. Default: 1d
+            left_truncate (bool, Optional): whether truncate the inputs from left. Default: True
+            use_fa2 (bool, Optional): whether to build attention mask under flash attention 2.
+        """
+        if isinstance(data, List):
+            # chatml list
+            _inputs = []
+            _outputs = []
+            new_conversation_offset = []
+            for _input in data:
+                if use_true_multiturn:
+                    chat = self._chat_from_json(_input, chat_format=chat_format)
+                    chat_data = chat.prompt_pack
+                    new_conversation_offset.append(len(_inputs))
+                    _inputs.extend(chat_data['input'])
+                    _outputs.extend(chat_data['output'])
+                else:
+                    _conversation = _convert_to_conversation(_input)
+                    assert is_assistant(_conversation[-1])
+
+                    _inputs.append(
+                        self.apply_chat_template(_conversation[:-1], tokenize=False, add_generation_prompt=True)
+                    )
+                    _outputs.append(_conversation[-1]['content'])
+
+            return self._build_inputs_for_train(
+                inputs=_inputs,
+                outputs=_outputs,
+                new_conversation_offset=new_conversation_offset,
+                max_length=max_length,
+                rotary_type=rotary_type,
+                left_truncate=left_truncate,
+                unidirectional_attention=unidirectional_attention,
+                isolation_position_ids=isolation_position_ids,
+                padding=padding,
+                use_fa2=use_fa2,
+                use_packed=use_packed,
+                use_baichuan_packed=use_baichuan_packed,
+                skip_truncated_turn=skip_truncated_turn,
+                return_attention_mask=return_attention_mask,
+            )
+        elif isinstance(data, Dict):
+            if 'messages' in data:
+                # chatml format
+                if use_true_multiturn:
+                    chat = self._chat_from_json(data, chat_format=chat_format)
+                    chat_data = chat.prompt_pack
+                else:
+                    _conversation = _convert_to_conversation(data)
+                    assert is_assistant(_conversation[-1])
+
+                    chat_data = {
+                        "input": self.apply_chat_template(
+                            _conversation[:-1], tokenize=False, add_generation_prompt=True
+                        ),
+                        "output": _conversation[-1]['content'],
+                    }
+
+                return self._build_inputs_for_train(
+                    inputs=chat_data['input'],
+                    outputs=chat_data['output'],
+                    max_length=max_length,
+                    rotary_type=rotary_type,
+                    left_truncate=left_truncate,
+                    unidirectional_attention=unidirectional_attention,
+                    isolation_position_ids=isolation_position_ids,
+                    padding=padding,
+                    use_fa2=use_fa2,
+                    use_packed=use_packed,
+                    use_baichuan_packed=use_baichuan_packed,
+                    skip_truncated_turn=skip_truncated_turn,
+                    return_attention_mask=return_attention_mask,
+                )
+            else:
+                inputs = data['input']
+                outputs = data['output']
+
+                if isinstance(inputs, str):
+                    inputs = [inputs]
+                if isinstance(outputs, str):
+                    outputs = [outputs]
+
+                if not is_chat_format and chat_format:
+                    inputs = [
+                        self.apply_chat_template(
+                            [{"role": "HUMAN", "content": item}], tokenize=False, chat_format=chat_format
+                        )
+                        for item in inputs
+                    ]
+
+                return self._build_inputs_for_train(
+                    inputs=inputs,
+                    outputs=outputs,
+                    new_conversation_offset=new_conversation_offset,
+                    max_length=max_length,
+                    rotary_type=rotary_type,
+                    left_truncate=left_truncate,
+                    unidirectional_attention=unidirectional_attention,
+                    isolation_position_ids=isolation_position_ids,
+                    padding=padding,
+                    use_fa2=use_fa2,
+                    use_packed=use_packed,
+                    use_baichuan_packed=use_baichuan_packed,
+                    skip_truncated_turn=skip_truncated_turn,
+                    return_attention_mask=return_attention_mask,
+                )

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdcadf59ad1db38dde175f2a82d3ec2dde15986ac1f81aef69c5cdd03afc6e1b
+size 12205847

tokenizer_config.json

@@ -0,0 +1,2124 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "156891": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156892": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156893": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156894": {
+      "content": "[gMASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156895": {
+      "content": "<|reserved_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156896": {
+      "content": "<|reserved_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156897": {
+      "content": "<|reserved_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156898": {
+      "content": "<|reserved_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156899": {
+      "content": "<|reserved_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156900": {
+      "content": "<|reserved_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156901": {
+      "content": "<|reserved_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156902": {
+      "content": "<|reserved_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156903": {
+      "content": "<|reserved_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156904": {
+      "content": "<|reserved_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156905": {
+      "content": "<|reserved_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156906": {
+      "content": "<|reserved_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156907": {
+      "content": "<|reserved_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156908": {
+      "content": "<|reserved_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156909": {
+      "content": "<|reserved_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156910": {
+      "content": "<|reserved_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156911": {
+      "content": "<|reserved_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156912": {
+      "content": "<|reserved_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156913": {
+      "content": "<|reserved_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156914": {
+      "content": "<|reserved_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156915": {
+      "content": "<|reserved_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156916": {
+      "content": "<|reserved_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156917": {
+      "content": "<|reserved_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156918": {
+      "content": "<|reserved_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156919": {
+      "content": "<|reserved_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156920": {
+      "content": "<|reserved_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156921": {
+      "content": "<|reserved_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156922": {
+      "content": "<|reserved_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156923": {
+      "content": "<|reserved_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156924": {
+      "content": "<|reserved_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156925": {
+      "content": "<|reserved_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156926": {
+      "content": "<|reserved_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156927": {
+      "content": "<|reserved_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156928": {
+      "content": "<|reserved_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156929": {
+      "content": "<|reserved_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156930": {
+      "content": "<|reserved_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156931": {
+      "content": "<|reserved_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156932": {
+      "content": "<|reserved_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156933": {
+      "content": "<|reserved_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156934": {
+      "content": "<|reserved_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156935": {
+      "content": "<|reserved_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156936": {
+      "content": "<|reserved_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156937": {
+      "content": "<|reserved_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156938": {
+      "content": "<|reserved_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156939": {
+      "content": "<|reserved_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156940": {
+      "content": "<|reserved_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156941": {
+      "content": "<|reserved_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156942": {
+      "content": "<|reserved_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156943": {
+      "content": "<|reserved_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156944": {
+      "content": "<|reserved_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156945": {
+      "content": "<|reserved_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156946": {
+      "content": "<|reserved_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156947": {
+      "content": "<|reserved_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156948": {
+      "content": "<|reserved_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156949": {
+      "content": "<|reserved_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156950": {
+      "content": "<|reserved_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156951": {
+      "content": "<|reserved_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156952": {
+      "content": "<|reserved_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156953": {
+      "content": "<|reserved_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156954": {
+      "content": "<|reserved_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156955": {
+      "content": "<|reserved_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156956": {
+      "content": "<|reserved_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156957": {
+      "content": "<|reserved_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156958": {
+      "content": "<|reserved_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156959": {
+      "content": "<|reserved_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156960": {
+      "content": "<|reserved_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156961": {
+      "content": "<|reserved_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156962": {
+      "content": "<|reserved_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156963": {
+      "content": "<|reserved_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156964": {
+      "content": "<|reserved_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156965": {
+      "content": "<|reserved_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156966": {
+      "content": "<|reserved_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156967": {
+      "content": "<|reserved_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156968": {
+      "content": "<|reserved_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156969": {
+      "content": "<|reserved_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156970": {
+      "content": "<|reserved_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156971": {
+      "content": "<|reserved_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156972": {
+      "content": "<|reserved_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156973": {
+      "content": "<|reserved_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156974": {
+      "content": "<|reserved_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156975": {
+      "content": "<|reserved_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156976": {
+      "content": "<|reserved_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156977": {
+      "content": "<|reserved_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156978": {
+      "content": "<|reserved_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156979": {
+      "content": "<|reserved_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156980": {
+      "content": "<|reserved_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156981": {
+      "content": "<|reserved_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156982": {
+      "content": "<|reserved_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156983": {
+      "content": "<|reserved_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156984": {
+      "content": "<|reserved_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156985": {
+      "content": "<|reserved_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156986": {
+      "content": "<|reserved_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156987": {
+      "content": "<|reserved_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156988": {
+      "content": "<|reserved_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156989": {
+      "content": "<|reserved_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156990": {
+      "content": "<|reserved_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156991": {
+      "content": "<|reserved_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156992": {
+      "content": "<|reserved_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156993": {
+      "content": "<|reserved_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156994": {
+      "content": "<|reserved_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156995": {
+      "content": "<|reserved_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156996": {
+      "content": "<|reserved_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156997": {
+      "content": "<|reserved_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156998": {
+      "content": "<|reserved_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "156999": {
+      "content": "<|reserved_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157000": {
+      "content": "<|reserved_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157001": {
+      "content": "<|reserved_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157002": {
+      "content": "<|reserved_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157003": {
+      "content": "<|reserved_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157004": {
+      "content": "<|reserved_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157005": {
+      "content": "<|reserved_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157006": {
+      "content": "<|reserved_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157007": {
+      "content": "<|reserved_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157008": {
+      "content": "<|reserved_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157009": {
+      "content": "<|reserved_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157010": {
+      "content": "<|reserved_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157011": {
+      "content": "<|reserved_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157012": {
+      "content": "<|reserved_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157013": {
+      "content": "<|reserved_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157014": {
+      "content": "<|reserved_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157015": {
+      "content": "<|reserved_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157016": {
+      "content": "<|reserved_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157017": {
+      "content": "<|reserved_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157018": {
+      "content": "<|reserved_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157019": {
+      "content": "<|reserved_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157020": {
+      "content": "<|reserved_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157021": {
+      "content": "<|reserved_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157022": {
+      "content": "<|reserved_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157023": {
+      "content": "<|reserved_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157024": {
+      "content": "<|reserved_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157025": {
+      "content": "<|reserved_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157026": {
+      "content": "<|reserved_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157027": {
+      "content": "<|reserved_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157028": {
+      "content": "<|reserved_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157029": {
+      "content": "<|reserved_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157030": {
+      "content": "<|reserved_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157031": {
+      "content": "<|reserved_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157032": {
+      "content": "<|reserved_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157033": {
+      "content": "<|reserved_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157034": {
+      "content": "<|reserved_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157035": {
+      "content": "<|reserved_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157036": {
+      "content": "<|reserved_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157037": {
+      "content": "<|reserved_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157038": {
+      "content": "<|reserved_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157039": {
+      "content": "<|reserved_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157040": {
+      "content": "<|reserved_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157041": {
+      "content": "<|reserved_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157042": {
+      "content": "<|reserved_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157043": {
+      "content": "<|reserved_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157044": {
+      "content": "<|reserved_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157045": {
+      "content": "<|reserved_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157046": {
+      "content": "<|reserved_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157047": {
+      "content": "<|reserved_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157048": {
+      "content": "<|reserved_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157049": {
+      "content": "<|reserved_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157050": {
+      "content": "<|reserved_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157051": {
+      "content": "<|reserved_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157052": {
+      "content": "<|reserved_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157053": {
+      "content": "<|reserved_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157054": {
+      "content": "<|reserved_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157055": {
+      "content": "<|reserved_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157056": {
+      "content": "<|reserved_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157057": {
+      "content": "<|reserved_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157058": {
+      "content": "<|reserved_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157059": {
+      "content": "<|reserved_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157060": {
+      "content": "<|reserved_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157061": {
+      "content": "<|reserved_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157062": {
+      "content": "<|reserved_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157063": {
+      "content": "<|reserved_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157064": {
+      "content": "<|reserved_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157065": {
+      "content": "<|reserved_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157066": {
+      "content": "<|reserved_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157067": {
+      "content": "<|reserved_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157068": {
+      "content": "<|reserved_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157069": {
+      "content": "<|reserved_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157070": {
+      "content": "<|reserved_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157071": {
+      "content": "<|reserved_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157072": {
+      "content": "<|reserved_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157073": {
+      "content": "<|reserved_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157074": {
+      "content": "<|reserved_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157075": {
+      "content": "<|reserved_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157076": {
+      "content": "<|reserved_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157077": {
+      "content": "<|reserved_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157078": {
+      "content": "<|reserved_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157079": {
+      "content": "<|reserved_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157080": {
+      "content": "<|reserved_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157081": {
+      "content": "<|reserved_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157082": {
+      "content": "<|reserved_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157083": {
+      "content": "<|reserved_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157084": {
+      "content": "<|reserved_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157085": {
+      "content": "<|reserved_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157086": {
+      "content": "<|reserved_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157087": {
+      "content": "<|reserved_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157088": {
+      "content": "<|reserved_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157089": {
+      "content": "<|reserved_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157090": {
+      "content": "<|reserved_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157091": {
+      "content": "<|reserved_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157092": {
+      "content": "<|reserved_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157093": {
+      "content": "<|reserved_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157094": {
+      "content": "<|reserved_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157095": {
+      "content": "<|reserved_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157096": {
+      "content": "<|reserved_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157097": {
+      "content": "<|reserved_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157098": {
+      "content": "<|reserved_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157099": {
+      "content": "<|reserved_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157100": {
+      "content": "<|reserved_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157101": {
+      "content": "<|reserved_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157102": {
+      "content": "<|reserved_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157103": {
+      "content": "<|reserved_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157104": {
+      "content": "<|reserved_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157105": {
+      "content": "<|reserved_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157106": {
+      "content": "<|reserved_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157107": {
+      "content": "<|reserved_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157108": {
+      "content": "<|reserved_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157109": {
+      "content": "<|reserved_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157110": {
+      "content": "<|reserved_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157111": {
+      "content": "<|reserved_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157112": {
+      "content": "<|reserved_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157113": {
+      "content": "<|reserved_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157114": {
+      "content": "<|reserved_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157115": {
+      "content": "<|reserved_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157116": {
+      "content": "<|reserved_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157117": {
+      "content": "<|reserved_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157118": {
+      "content": "<|reserved_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157119": {
+      "content": "<|reserved_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157120": {
+      "content": "<|reserved_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157121": {
+      "content": "<|reserved_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157122": {
+      "content": "<|reserved_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157123": {
+      "content": "<|reserved_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157124": {
+      "content": "<|reserved_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157125": {
+      "content": "<|reserved_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157126": {
+      "content": "<|reserved_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157127": {
+      "content": "<|reserved_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157128": {
+      "content": "<|reserved_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157129": {
+      "content": "<|reserved_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157130": {
+      "content": "<|reserved_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157131": {
+      "content": "<|reserved_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157132": {
+      "content": "<|reserved_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157133": {
+      "content": "<|reserved_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157134": {
+      "content": "<|reserved_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157135": {
+      "content": "<|reserved_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157136": {
+      "content": "<|reserved_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157137": {
+      "content": "<|reserved_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157138": {
+      "content": "<|reserved_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157139": {
+      "content": "<|reserved_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157140": {
+      "content": "<|reserved_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157141": {
+      "content": "<|reserved_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157142": {
+      "content": "<|reserved_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157143": {
+      "content": "<|reserved_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157144": {
+      "content": "<|reserved_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157145": {
+      "content": "<|reserved_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157146": {
+      "content": "<|reserved_token_251|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157147": {
+      "content": "<|reserved_token_252|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157148": {
+      "content": "<|reserved_token_253|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157149": {
+      "content": "<|reserved_token_254|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157150": {
+      "content": "<|reserved_token_255|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157151": {
+      "content": "<role>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "157152": {
+      "content": "</role>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<role>",
+    "</role>"
+  ],
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_bailing.BailingTokenizer",
+      null
+    ]
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "fast_tokenizer": true,
+  "gmask_token": "[gMASK]",
+  "merges_file": null,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "BailingTokenizer",
+  "trust_remote_code": true
+}