init

Chat/added_tokens.json

@@ -0,0 +1,59 @@
+{
+  "</asr>": 151682,
+  "</box>": 151670,
+  "</image>": 151666,
+  "</image_id>": 151678,
+  "</point>": 151674,
+  "</quad>": 151672,
+  "</query>": 151684,
+  "</ref>": 151668,
+  "</slice>": 151676,
+  "</tool_call>": 151658,
+  "</unit>": 151680,
+  "<asr>": 151681,
+  "<box>": 151669,
+  "<image>": 151665,
+  "<image_id>": 151677,
+  "<point>": 151673,
+  "<quad>": 151671,
+  "<query>": 151683,
+  "<ref>": 151667,
+  "<reserved_43>": 151698,
+  "<reserved_53>": 151699,
+  "<slice>": 151675,
+  "<tool_call>": 151657,
+  "<unit>": 151679,
+  "<|audio_end|>": 151687,
+  "<|audio_start|>": 151685,
+  "<|audio|>": 151686,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|interrupt|>": 151695,
+  "<|listen|>": 151693,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|speak|>": 151694,
+  "<|spk_bos|>": 151688,
+  "<|spk_eos|>": 151690,
+  "<|spk|>": 151689,
+  "<|tts_bos|>": 151691,
+  "<|tts_eos|>": 151692,
+  "<|vad_end|>": 151697,
+  "<|vad_start|>": 151696,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

Chat/config.json

@@ -0,0 +1,194 @@
+{
+  "_name_or_path": "pretrained_models/CosyVoice2-0.5B/Chat",
+  "architectures": [
+    "MiniCPMO"
+  ],
+
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "use_sliding_window": false,
+  "vocab_size": 151700,
+  "batch_vision_input": true,
+  "drop_vision_last_layer": false,
+  "image_size": 448,
+
+  "audio_chunk_length": 1.0,
+  "audio_config": {
+    "_name_or_path": "openai/whisper-medium",
+    "architectures": [
+      "MiniCPMWhisperEncoder"
+    ],
+    "begin_suppress_tokens": [
+      220,
+      50257
+    ],
+    "bos_token_id": 50257,
+    "d_model": 1024,
+    "decoder_attention_heads": 16,
+    "decoder_ffn_dim": 4096,
+    "decoder_layers": 24,
+    "decoder_start_token_id": 50258,
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layers": 24,
+    "eos_token_id": 50257,
+    "forced_decoder_ids": [
+      [
+        1,
+        50259
+      ],
+      [
+        2,
+        50359
+      ],
+      [
+        3,
+        50363
+      ]
+    ],
+    "max_length": 448,
+    "model_type": "whisper",
+    "num_hidden_layers": 24,
+    "pad_token_id": 50257,
+    "suppress_tokens": [
+      1,
+      2,
+      7,
+      8,
+      9,
+      10,
+      14,
+      25,
+      26,
+      27,
+      28,
+      29,
+      31,
+      58,
+      59,
+      60,
+      61,
+      62,
+      63,
+      90,
+      91,
+      92,
+      93,
+      359,
+      503,
+      522,
+      542,
+      873,
+      893,
+      902,
+      918,
+      922,
+      931,
+      1350,
+      1853,
+      1982,
+      2460,
+      2627,
+      3246,
+      3253,
+      3268,
+      3536,
+      3846,
+      3961,
+      4183,
+      4667,
+      6585,
+      6647,
+      7273,
+      9061,
+      9383,
+      10428,
+      10929,
+      11938,
+      12033,
+      12331,
+      12562,
+      13793,
+      14157,
+      14635,
+      15265,
+      15618,
+      16553,
+      16604,
+      18362,
+      18956,
+      20075,
+      21675,
+      22520,
+      26130,
+      26161,
+      26435,
+      28279,
+      29464,
+      31650,
+      32302,
+      32470,
+      36865,
+      42863,
+      47425,
+      49870,
+      50254,
+      50258,
+      50358,
+      50359,
+      50360,
+      50361,
+      50362
+    ],
+    "torch_dtype": "bfloat16"
+  },
+  "audio_pool_step": 2,
+  "auto_map": {
+    "AutoConfig": "configuration_minicpm.MiniCPMOConfig",
+    "AutoModel": "modeling_minicpmo.MiniCPMO",
+    "AutoModelForCausalLM": "modeling_minicpmo.MiniCPMO"
+  },
+  "chunk_input": true,
+  "listen_speak_type": "asr",
+  "model_type": "minicpmo",
+  "patch_size": 14,
+  "query_num": 64,
+  "slice_config": {
+    "max_slice_nums": 9,
+    "model_type": "minicpmv"
+  },
+  "slice_mode": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "tts_config": {
+    "model_type": "conditional_chattts",
+    "llm_dim": 3584
+  },
+  "use_cache": true,
+  "use_image_id": true,
+  "version": 2.6,
+  "vision_batch_size": 16,
+  "vision_config": {
+    "hidden_size": 1152,
+    "image_size": 980,
+    "intermediate_size": 4304,
+    "model_type": "siglip_vision_model",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 27,
+    "patch_size": 14
+  }
+}

Chat/configuration_minicpm.py

@@ -0,0 +1,210 @@
+# coding=utf-8
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import Union
+
+from transformers import PretrainedConfig
+from transformers import Qwen2Config
+from transformers import WhisperConfig
+from transformers.utils import logging
+
+from .modeling_navit_siglip import SiglipVisionConfig
+
+logger = logging.get_logger(__name__)
+
+
+class MiniCPMVSliceConfig(PretrainedConfig):
+    model_type = "minicpmv"
+
+    def __init__(
+        self,
+        patch_size=14,
+        max_slice_nums=9,
+        scale_resolution=448,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.patch_size = patch_size
+        self.max_slice_nums = max_slice_nums
+        self.scale_resolution = scale_resolution
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+        cls._set_token_in_kwargs(kwargs)
+
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+        if config_dict.get("model_type") == "minicpmv":
+            config_dict = config_dict["slice_config"]
+
+        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+            logger.warning(
+                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+            )
+
+        return cls.from_dict(config_dict, **kwargs)
+
+
+class ConditionalChatTTSConfig(PretrainedConfig):
+    model_type = "conditional_chattts"
+
+    def __init__(
+        self,
+        llm_dim: int = 2560,
+        hidden_size: int = 768,
+        intermediate_size: int = 3072,
+        num_attention_heads: int = 12,
+        num_hidden_layers: int = 20,
+        max_position_embeddings: int = 4096,
+        num_audio_tokens: int = 626,
+        num_text_tokens: int = 21178,
+        num_mel_bins: int = 100,
+        num_vq: int = 4,
+        use_speaker_embedding: bool = True,
+        use_llm_hidden_state: bool = False,
+        spk_emb_token_id: int = 21143,
+        num_spk_embs: int = 1,
+        audio_bos_token_id: int = 21132,
+        text_eos_token_id: int = 21133,
+        use_text: bool = True,
+        streaming: bool = True,
+        streaming_text_chunk_size: int = 10,
+        streaming_text_reserved_len: int = 300,
+        streaming_audio_chunk_size: int = 50,
+        attn_implementation: str = "sdpa",
+        use_mlp: bool = True,
+        aug_loss_weight: bool = True,
+        do_sample: bool = True,
+        top_p: float = 0.7,
+        top_k: int = 20,
+        repetition_penalty: float = 1.0,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.llm_dim = llm_dim
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_attention_heads = num_attention_heads
+        self.num_hidden_layers = num_hidden_layers
+        self.max_position_embeddings = max_position_embeddings
+        self.num_audio_tokens = num_audio_tokens
+        self.num_text_tokens = num_text_tokens
+        self.num_mel_bins = num_mel_bins
+        self.num_vq = num_vq
+        self.use_speaker_embedding = use_speaker_embedding
+        self.use_llm_hidden_state = use_llm_hidden_state
+        self.spk_emb_token_id = spk_emb_token_id
+        self.num_spk_embs = num_spk_embs
+        self.audio_bos_token_id = audio_bos_token_id
+        self.text_eos_token_id = text_eos_token_id
+        self.use_text = use_text
+        self.streaming = streaming
+        self.streaming_text_chunk_size = streaming_text_chunk_size
+        self.streaming_text_reserved_len = streaming_text_reserved_len
+        self.streaming_audio_chunk_size = streaming_audio_chunk_size
+        self.attn_implementation = attn_implementation
+        self.use_mlp = use_mlp
+        self.aug_loss_weight = aug_loss_weight
+        self.do_sample = do_sample
+        self.top_p = top_p
+        self.top_k = top_k
+        self.repetition_penalty = repetition_penalty
+
+
+class MiniCPMOConfig(Qwen2Config):
+    model_type = "minicpmo"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    default_vision_config = {
+        "hidden_size": 1152,
+        "image_size": 980,
+        "intermediate_size": 4304,
+        "model_type": "siglip",
+        "num_attention_heads": 16,
+        "num_hidden_layers": 27,
+        "patch_size": 14,
+    }
+
+    def __init__(
+        self,
+        use_cache=True,
+        query_num=64,
+        image_size=448,
+        drop_vision_last_layer=True,
+        batch_vision_input=True,
+        slice_config=None,
+        vision_config=None,
+        audio_config=None,
+        tts_config=None,
+        use_image_id=True,
+        vision_batch_size=16,
+        audio_pool_step=2,
+        audio_chunk_length=1.0,
+        stream_input=False,
+        init_vision=True,
+        init_audio=True,
+        init_tts=True,
+        **kwargs,
+    ):
+        self.use_cache = use_cache
+        self.query_num = query_num
+        self.image_size = image_size
+        self.drop_vision_last_layer = drop_vision_last_layer
+        self.batch_vision_input = batch_vision_input
+        self.use_image_id = use_image_id
+        self.vision_batch_size = vision_batch_size
+        self.audio_pool_step = audio_pool_step
+        self.audio_chunk_length = audio_chunk_length
+        self.stream_input = stream_input
+        self.init_vision = init_vision
+        self.init_audio = init_audio
+        self.init_tts = init_tts
+
+        if slice_config is None:
+            self.slice_config = MiniCPMVSliceConfig(max_slice_nums=1)
+        else:
+            self.slice_config = MiniCPMVSliceConfig(**slice_config)
+        self.slice_mode = True
+
+        # same as HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit add tgt_sizes
+        if vision_config is None:
+            self.vision_config = SiglipVisionConfig(**self.default_vision_config)
+            # logger.info("vision_config is None, using default vision config")
+        elif isinstance(vision_config, dict):
+            self.vision_config = SiglipVisionConfig(**vision_config)
+        elif isinstance(vision_config, SiglipVisionConfig):
+            self.vision_config = vision_config
+
+        # same as openai/whisper-medium add use_cache
+        if audio_config is None:
+            self.audio_config = WhisperConfig()
+        elif isinstance(audio_config, dict):
+            self.audio_config = WhisperConfig(**audio_config)
+        elif isinstance(audio_config, WhisperConfig):
+            self.audio_config = audio_config
+
+        if tts_config is None:
+            self.tts_config = ConditionalChatTTSConfig()
+        elif isinstance(tts_config, dict):
+            self.tts_config = ConditionalChatTTSConfig(**tts_config)
+        elif isinstance(tts_config, ConditionalChatTTSConfig):
+            self.tts_config = tts_config
+
+        self.patch_size = self.vision_config.patch_size
+
+        super().__init__(**kwargs)

Chat/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "transformers_version": "4.44.2"
+}

Chat/image_processing_minicpmv.py

@@ -0,0 +1,407 @@
+# coding=utf-8
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Union
+
+import numpy as np
+import PIL
+import PIL.Image
+import PIL.ImageSequence
+import torch
+from PIL import Image
+from transformers import AutoImageProcessor
+from transformers.image_processing_utils import BaseImageProcessor
+from transformers.image_processing_utils import BatchFeature
+from transformers.image_transforms import to_channel_dimension_format
+from transformers.image_utils import ChannelDimension
+from transformers.image_utils import infer_channel_dimension_format
+from transformers.image_utils import is_torch_tensor
+from transformers.image_utils import to_numpy_array
+from transformers.image_utils import valid_images
+from transformers.utils import is_torch_device
+from transformers.utils import is_torch_dtype
+from transformers.utils import requires_backends
+from transformers.utils import TensorType
+
+
+def recursive_converter(converter, value):
+    if isinstance(value, list):
+        new_value = []
+        for v in value:
+            new_value += [recursive_converter(converter, v)]
+        return new_value
+    else:
+        return converter(value)
+
+
+class MiniCPMOBatchFeature(BatchFeature):
+    r"""
+    Extend from BatchFeature for supporting various image size
+    """
+
+    def __init__(self, data: Optional[Dict[str, Any]] = None, tensor_type: Union[None, str, TensorType] = None):
+        super().__init__(data)
+        self.convert_to_tensors(tensor_type=tensor_type)
+
+    def convert_to_tensors(self, tensor_type: Optional[Union[str, TensorType]] = None):
+        if tensor_type is None:
+            return self
+
+        is_tensor, as_tensor = self._get_is_as_tensor_fns(tensor_type)
+
+        def converter(value):
+            try:
+                if not is_tensor(value):
+                    tensor = as_tensor(value)
+                    return tensor
+            except:  # noqa E722
+                if key == "overflowing_values":
+                    raise ValueError("Unable to create tensor returning overflowing values of different lengths. ")
+                raise ValueError(
+                    "Unable to create tensor, you should probably activate padding "
+                    "with 'padding=True' to have batched tensors with the same length."
+                )
+
+        for key, value in self.items():
+            self[key] = recursive_converter(converter, value)
+        return self
+
+    def to(self, *args, **kwargs) -> "MiniCPMOBatchFeature":
+        requires_backends(self, ["torch"])
+        import torch
+
+        def cast_tensor(v):
+            # check if v is a floating point
+            if torch.is_floating_point(v):
+                # cast and send to device
+                return v.to(*args, **kwargs)
+            elif device is not None:
+                return v.to(device=device)
+            else:
+                return v
+
+        new_data = {}
+        device = kwargs.get("device")
+        # Check if the args are a device or a dtype
+        if device is None and len(args) > 0:
+            # device should be always the first argument
+            arg = args[0]
+            if is_torch_dtype(arg):
+                # The first argument is a dtype
+                pass
+            elif isinstance(arg, str) or is_torch_device(arg) or isinstance(arg, int):
+                device = arg
+            else:
+                # it's something else
+                raise ValueError(f"Attempting to cast a BatchFeature to type {str(arg)}. This is not supported.")
+        # We cast only floating point tensors to avoid issues with tokenizers casting `LongTensor` to `FloatTensor`
+        for k, v in self.items():
+            new_data[k] = recursive_converter(cast_tensor, v)
+        self.data = new_data
+        return self
+
+
+class MiniCPMVImageProcessor(BaseImageProcessor):
+    model_input_names = ["pixel_values"]
+
+    def __init__(self, max_slice_nums=9, scale_resolution=448, patch_size=14, **kwargs):
+        super().__init__(**kwargs)
+        self.max_slice_nums = max_slice_nums
+        self.scale_resolution = scale_resolution
+        self.patch_size = patch_size
+        self.use_image_id = kwargs.pop("use_image_id", False)
+        self.image_feature_size = kwargs.pop("image_feature_size", 64)
+        self.im_start_token = kwargs.pop("im_start", "<image>")
+        self.im_end_token = kwargs.pop("im_end", "</image>")
+        self.slice_start_token = kwargs.pop("slice_start", "<slice>")
+        self.slice_end_token = kwargs.pop("slice_end", "</slice>")
+        self.unk_token = kwargs.pop("unk", "<unk>")
+        self.im_id_start = kwargs.pop("im_id_start", "<image_id>")
+        self.im_id_end = kwargs.pop("im_id_end", "</image_id>")
+        self.slice_mode = kwargs.pop("slice_mode", True)
+
+        self.mean = np.array(kwargs.pop("norm_mean", [0.5, 0.5, 0.5]))
+        self.std = np.array(kwargs.pop("norm_std", [0.5, 0.5, 0.5]))
+        self.version = kwargs.pop("version", 2.0)
+
+    def ensure_divide(self, length, patch_size):
+        return max(round(length / patch_size) * patch_size, patch_size)
+
+    def find_best_resize(self, original_size, scale_resolution, patch_size, allow_upscale=False):
+        width, height = original_size
+        if (width * height > scale_resolution * scale_resolution) or allow_upscale:
+            r = width / height
+            height = int(scale_resolution / math.sqrt(r))
+            width = int(height * r)
+        best_width = self.ensure_divide(width, patch_size)
+        best_height = self.ensure_divide(height, patch_size)
+        return (best_width, best_height)
+
+    def get_refine_size(self, original_size, grid, scale_resolution, patch_size, allow_upscale=False):
+        width, height = original_size
+        grid_x, grid_y = grid
+
+        refine_width = self.ensure_divide(width, grid_x)
+        refine_height = self.ensure_divide(height, grid_y)
+
+        grid_width = refine_width / grid_x
+        grid_height = refine_height / grid_y
+
+        best_grid_size = self.find_best_resize(
+            (grid_width, grid_height), scale_resolution, patch_size, allow_upscale=allow_upscale
+        )
+        refine_size = (best_grid_size[0] * grid_x, best_grid_size[1] * grid_y)
+        return refine_size
+
+    def split_to_patches(self, image, grid):
+        patches = []
+        width, height = image.size
+        grid_x = int(width / grid[0])
+        grid_y = int(height / grid[1])
+        for i in range(0, height, grid_y):
+            images = []
+            for j in range(0, width, grid_x):
+                box = (j, i, j + grid_x, i + grid_y)
+                patch = image.crop(box)
+                images.append(patch)
+            patches.append(images)
+        return patches
+
+    def slice_image(self, image, max_slice_nums=9, scale_resolution=448, patch_size=14, never_split=False):
+        original_size = image.size
+        source_image = None
+        best_grid = self.get_sliced_grid(original_size, max_slice_nums, never_split)
+        patches = []
+
+        if best_grid is None:
+            # dont need to slice, upsample
+            best_size = self.find_best_resize(original_size, scale_resolution, patch_size, allow_upscale=True)
+            source_image = image.resize(best_size, resample=Image.Resampling.BICUBIC)
+        else:
+            # source image, down-sampling and ensure divided by patch_size
+            best_resize = self.find_best_resize(original_size, scale_resolution, patch_size)
+            source_image = image.copy().resize(best_resize, resample=Image.Resampling.BICUBIC)
+            refine_size = self.get_refine_size(
+                original_size, best_grid, scale_resolution, patch_size, allow_upscale=True
+            )
+            refine_image = image.resize(refine_size, resample=Image.Resampling.BICUBIC)
+            patches = self.split_to_patches(refine_image, best_grid)
+
+        return source_image, patches, best_grid
+
+    def get_grid_placeholder(self, grid):
+        if grid is None:
+            return ""
+        slice_image_placeholder = (
+            self.slice_start_token + self.unk_token * self.image_feature_size + self.slice_end_token
+        )
+
+        cols = grid[0]
+        rows = grid[1]
+        slices = []
+        for i in range(rows):
+            lines = []
+            for j in range(cols):
+                lines.append(slice_image_placeholder)
+            slices.append("".join(lines))
+
+        slice_placeholder = "\n".join(slices)
+        return slice_placeholder
+
+    def get_image_id_placeholder(self, idx=0):
+        return f"{self.im_id_start}{idx}{self.im_id_end}"
+
+    def get_sliced_images(self, image, max_slice_nums=None):
+        slice_images = []
+
+        if not self.slice_mode:
+            return [image]
+
+        max_slice_nums = self.max_slice_nums if max_slice_nums is None else int(max_slice_nums)
+        assert max_slice_nums > 0
+        source_image, patches, sliced_grid = self.slice_image(
+            image, max_slice_nums, self.scale_resolution, self.patch_size  # default: 9  # default: 448  # default: 14
+        )
+
+        slice_images.append(source_image)
+        if len(patches) > 0:
+            for i in range(len(patches)):
+                for j in range(len(patches[0])):
+                    slice_images.append(patches[i][j])
+        return slice_images
+
+    def get_sliced_grid(self, image_size, max_slice_nums, nerver_split=False):
+        original_width, original_height = image_size
+        log_ratio = math.log(original_width / original_height)
+        ratio = original_width * original_height / (self.scale_resolution * self.scale_resolution)
+        multiple = min(math.ceil(ratio), max_slice_nums)
+        if multiple <= 1 or nerver_split:
+            return None
+        candidate_split_grids_nums = []
+        for i in [multiple - 1, multiple, multiple + 1]:
+            if i == 1 or i > max_slice_nums:
+                continue
+            candidate_split_grids_nums.append(i)
+
+        candidate_grids = []
+        for split_grids_nums in candidate_split_grids_nums:
+            m = 1
+            while m <= split_grids_nums:
+                if split_grids_nums % m == 0:
+                    candidate_grids.append([m, split_grids_nums // m])
+                m += 1
+
+        best_grid = [1, 1]
+        min_error = float("inf")
+        for grid in candidate_grids:
+            error = abs(log_ratio - math.log(grid[0] / grid[1]))
+            if error < min_error:
+                best_grid = grid
+                min_error = error
+
+        return best_grid
+
+    def get_slice_image_placeholder(self, image_size, image_idx=0, max_slice_nums=None, use_image_id=None):
+        max_slice_nums = self.max_slice_nums if max_slice_nums is None else int(max_slice_nums)
+        assert max_slice_nums > 0
+        grid = self.get_sliced_grid(image_size=image_size, max_slice_nums=max_slice_nums)
+
+        image_placeholder = self.im_start_token + self.unk_token * self.image_feature_size + self.im_end_token
+        use_image_id = self.use_image_id if use_image_id is None else bool(use_image_id)
+        if use_image_id:
+            final_placeholder = self.get_image_id_placeholder(image_idx) + image_placeholder
+        else:
+            final_placeholder = image_placeholder
+
+        if self.slice_mode:
+            final_placeholder = final_placeholder + self.get_grid_placeholder(grid=grid)
+        return final_placeholder
+
+    def to_pil_image(self, image, rescale=None) -> PIL.Image.Image:
+        """
+        Converts `image` to a PIL Image. Optionally rescales it and puts the channel dimension back as the last axis if
+        needed.
+
+        Args:
+            image (`PIL.Image.Image` or `numpy.ndarray` or `torch.Tensor`):
+                The image to convert to the PIL Image format.
+            rescale (`bool`, *optional*):
+                Whether or not to apply the scaling factor (to make pixel values integers between 0 and 255). Will
+                default to `True` if the image type is a floating type, `False` otherwise.
+        """
+        if isinstance(image, PIL.Image.Image):
+            return image
+        if is_torch_tensor(image):
+            image = image.numpy()
+
+        if isinstance(image, np.ndarray):
+            if rescale is None:
+                # rescale default to the array being of floating type.
+                rescale = isinstance(image.flat[0], np.floating)
+            # If the channel as been moved to first dim, we put it back at the end.
+            if image.ndim == 3 and image.shape[0] in [1, 3]:
+                image = image.transpose(1, 2, 0)
+            if rescale:
+                image = image * 255
+            image = image.astype(np.uint8)
+            return PIL.Image.fromarray(image)
+        return image
+
+    def reshape_by_patch(self, image):
+        """
+        :param image: shape [3, H, W]
+        :param patch_size:
+        :return: [3, patch_size, HW/patch_size]
+        """
+        image = torch.from_numpy(image)
+        patch_size = self.patch_size
+        patches = torch.nn.functional.unfold(image, (patch_size, patch_size), stride=(patch_size, patch_size))
+
+        patches = patches.reshape(image.size(0), patch_size, patch_size, -1)
+        patches = patches.permute(0, 1, 3, 2).reshape(image.size(0), patch_size, -1)
+        return patches.numpy()
+
+    def preprocess(
+        self,
+        images: Union[Image.Image, List[Image.Image], List[List[Image.Image]]],
+        do_pad: Optional[bool] = True,
+        max_slice_nums: int = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        **kwargs,
+    ) -> MiniCPMOBatchFeature:
+        if isinstance(images, Image.Image):
+            images_list = [[images]]
+        elif isinstance(images[0], Image.Image):
+            images_list = [images]
+        else:
+            images_list = images
+
+        new_images_list = []
+        image_sizes_list = []
+        tgt_sizes_list = []
+
+        for _images in images_list:
+            if _images is None or len(_images) == 0:
+                new_images_list.append([])
+                image_sizes_list.append([])
+                tgt_sizes_list.append([])
+                continue
+            if not valid_images(_images):
+                raise ValueError(
+                    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
+                    "torch.Tensor, tf.Tensor or jax.ndarray."
+                )
+
+            _images = [self.to_pil_image(image).convert("RGB") for image in _images]
+            input_data_format = infer_channel_dimension_format(np.array(_images[0]))
+
+            new_images = []
+            image_sizes = [image.size for image in _images]
+            tgt_sizes = []
+            for image in _images:
+                image_patches = self.get_sliced_images(image, max_slice_nums)
+                image_patches = [to_numpy_array(image).astype(np.float32) / 255 for image in image_patches]
+                image_patches = [
+                    self.normalize(image=image, mean=self.mean, std=self.std, input_data_format=input_data_format)
+                    for image in image_patches
+                ]
+                image_patches = [
+                    to_channel_dimension_format(image, ChannelDimension.FIRST, input_channel_dim=input_data_format)
+                    for image in image_patches
+                ]
+                for slice_image in image_patches:
+                    new_images.append(self.reshape_by_patch(slice_image))
+                    tgt_sizes.append(
+                        np.array((slice_image.shape[1] // self.patch_size, slice_image.shape[2] // self.patch_size))
+                    )
+
+            if tgt_sizes:
+                tgt_sizes = np.vstack(tgt_sizes)
+
+            new_images_list.append(new_images)
+            image_sizes_list.append(image_sizes)
+            tgt_sizes_list.append(tgt_sizes)
+        return MiniCPMOBatchFeature(
+            data={"pixel_values": new_images_list, "image_sizes": image_sizes_list, "tgt_sizes": tgt_sizes_list},
+            tensor_type=return_tensors,
+        )
+
+
+AutoImageProcessor.register("MiniCPMVImageProcessor", MiniCPMVImageProcessor)

Chat/model.safetensors.index.json

@@ -0,0 +1,1167 @@
+{
+  "metadata": {
+    "total_size": 17349994056
+  },
+  "weight_map": {
+    "apm.conv1.bias": "model-00004-of-00004.safetensors",
+    "apm.conv1.weight": "model-00004-of-00004.safetensors",
+    "apm.conv2.bias": "model-00004-of-00004.safetensors",
+    "apm.conv2.weight": "model-00004-of-00004.safetensors",
+    "apm.embed_positions.weight": "model-00004-of-00004.safetensors",
+    "apm.layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.0.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.0.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.0.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.0.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.0.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.0.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.0.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.0.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.0.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.0.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.0.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.0.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.1.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.1.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.1.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.1.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.1.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.1.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.1.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.1.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.1.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.1.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.1.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.1.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.10.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.10.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.10.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.10.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.10.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.10.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.10.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.10.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.10.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.10.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.10.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.10.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.10.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.10.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.10.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.11.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.11.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.11.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.11.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.11.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.11.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.11.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.11.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.11.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.11.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.11.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.11.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.11.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.11.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.11.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.12.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.12.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.12.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.12.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.12.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.12.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.12.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.12.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.12.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.12.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.12.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.12.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.12.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.12.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.12.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.13.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.13.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.13.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.13.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.13.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.13.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.13.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.13.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.13.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.13.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.13.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.13.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.13.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.13.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.13.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.14.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.14.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.14.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.14.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.14.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.14.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.14.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.14.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.14.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.14.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.14.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.14.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.14.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.14.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.14.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.15.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.15.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.15.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.15.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.15.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.15.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.15.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.15.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.15.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.15.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.15.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.15.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.15.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.15.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.15.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.16.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.16.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.16.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.16.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.16.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.16.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.16.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.16.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.16.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.16.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.16.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.16.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.16.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.16.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.16.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.17.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.17.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.17.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.17.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.17.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.17.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.17.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.17.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.17.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.17.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.17.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.17.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.17.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.17.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.17.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.18.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.18.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.18.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.18.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.18.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.18.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.18.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.18.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.18.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.18.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.18.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.18.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.18.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.18.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.18.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.19.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.19.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.19.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.19.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.19.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.19.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.19.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.19.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.19.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.19.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.19.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.19.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.19.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.19.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.19.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.2.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.2.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.2.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.2.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.2.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.2.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.2.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.2.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.2.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.2.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.2.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.2.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.20.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.20.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.20.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.20.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.20.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.20.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.20.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.20.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.20.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.20.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.20.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.20.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.20.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.20.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.20.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.21.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.21.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.21.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.21.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.21.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.21.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.21.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.21.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.21.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.21.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.21.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.21.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.21.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.21.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.21.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.22.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.22.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.22.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.22.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.22.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.22.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.22.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.22.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.22.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.22.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.22.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.22.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.22.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.22.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.22.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.23.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.23.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.23.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.23.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.23.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.23.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.23.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.23.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.23.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.23.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.23.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.23.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.23.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.23.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.3.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.3.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.3.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.3.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.3.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.3.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.3.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.3.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.3.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.3.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.3.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.3.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.4.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.4.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.4.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.4.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.4.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.4.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.4.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.4.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.4.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.4.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.4.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.4.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.4.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.4.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.4.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.5.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.5.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.5.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.5.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.5.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.5.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.5.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.5.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.5.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.5.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.5.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.5.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.5.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.5.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.5.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.6.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.6.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.6.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.6.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.6.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.6.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.6.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.6.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.6.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.6.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.6.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.6.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.6.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.6.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.6.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.7.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.7.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.7.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.7.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.7.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.7.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.7.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.7.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.7.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.7.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.7.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.7.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.7.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.7.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.7.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.8.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.8.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.8.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.8.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.8.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.8.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.8.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.8.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.8.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.8.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.8.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.8.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.8.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.8.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.8.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.9.fc1.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.9.fc1.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.9.fc2.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.9.fc2.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.9.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.9.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.9.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.9.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.9.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.9.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.9.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.9.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.9.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "apm.layers.9.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "apm.layers.9.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "audio_projection_layer.linear1.bias": "model-00004-of-00004.safetensors",
+    "audio_projection_layer.linear1.weight": "model-00004-of-00004.safetensors",
+    "audio_projection_layer.linear2.bias": "model-00004-of-00004.safetensors",
+    "audio_projection_layer.linear2.weight": "model-00004-of-00004.safetensors",
+    "llm.lm_head.weight": "model-00004-of-00004.safetensors",
+    "llm.model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "llm.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "llm.model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "llm.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "llm.model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "llm.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "llm.model.norm.weight": "model-00003-of-00004.safetensors",
+    "resampler.attn.in_proj_bias": "model-00004-of-00004.safetensors",
+    "resampler.attn.in_proj_weight": "model-00004-of-00004.safetensors",
+    "resampler.attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "resampler.attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "resampler.kv_proj.weight": "model-00004-of-00004.safetensors",
+    "resampler.ln_kv.bias": "model-00004-of-00004.safetensors",
+    "resampler.ln_kv.weight": "model-00004-of-00004.safetensors",
+    "resampler.ln_post.bias": "model-00004-of-00004.safetensors",
+    "resampler.ln_post.weight": "model-00004-of-00004.safetensors",
+    "resampler.ln_q.bias": "model-00004-of-00004.safetensors",
+    "resampler.ln_q.weight": "model-00004-of-00004.safetensors",
+    "resampler.proj": "model-00004-of-00004.safetensors",
+    "resampler.query": "model-00004-of-00004.safetensors",
+    "vpm.embeddings.patch_embedding.bias": "model-00004-of-00004.safetensors",
+    "vpm.embeddings.patch_embedding.weight": "model-00004-of-00004.safetensors",
+    "vpm.embeddings.position_embedding.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.10.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.11.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.12.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.13.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.14.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.15.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.16.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.17.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.18.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.19.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.20.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.21.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.22.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.23.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.24.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.4.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.5.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.6.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.7.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.8.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "vpm.encoder.layers.9.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "vpm.post_layernorm.bias": "model-00004-of-00004.safetensors",
+    "vpm.post_layernorm.weight": "model-00004-of-00004.safetensors"
+  }
+}

Chat/modeling_minicpmo.py

@@ -0,0 +1,1996 @@
+# coding=utf-8
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import math
+import os
+import types
+from collections.abc import Iterator
+from copy import deepcopy
+from dataclasses import dataclass
+from threading import Thread
+from typing import List
+from typing import Literal
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+import numpy as np
+import soundfile as sf
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.nn.utils.parametrize as P
+from huggingface_hub import hf_hub_download
+from PIL import Image
+from torch.nn.utils.parametrizations import weight_norm
+from tqdm import tqdm
+from transformers import AutoProcessor
+from transformers import BertTokenizerFast
+from transformers import LlamaConfig
+from transformers import LlamaModel
+# from transformers import LogitsWarper
+from transformers import LogitsProcessor
+from transformers import PreTrainedModel
+from transformers import Qwen2ForCausalLM
+from transformers import Qwen2PreTrainedModel
+from transformers import TextIteratorStreamer
+from transformers import TopKLogitsWarper
+from transformers import TopPLogitsWarper
+from transformers.cache_utils import Cache
+from transformers.cache_utils import DynamicCache
+from transformers.cache_utils import EncoderDecoderCache
+from transformers.cache_utils import StaticCache
+from transformers.modeling_outputs import BaseModelOutputWithPast
+from transformers.modeling_outputs import ModelOutput
+from transformers.models.whisper.modeling_whisper import ACT2FN
+from transformers.models.whisper.modeling_whisper import WHISPER_ATTENTION_CLASSES
+from transformers.models.whisper.modeling_whisper import WhisperConfig
+from transformers.models.whisper.modeling_whisper import WhisperEncoder
+
+try:
+    from vector_quantize_pytorch import GroupedResidualFSQ
+    from vocos import Vocos
+    from vocos.pretrained import instantiate_class
+
+    _tts_deps = True
+except:
+    _tts_deps = False
+
+from .configuration_minicpm import ConditionalChatTTSConfig
+from .configuration_minicpm import MiniCPMOConfig
+from .modeling_navit_siglip import SiglipVisionTransformer
+from .image_processing_minicpmv import MiniCPMOBatchFeature
+from .resampler import Resampler
+from .utils import NumberToTextConverter
+from .utils import sentence_end
+from .utils import VoiceChecker
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class OmniOutput(ModelOutput):
+    text: Optional[Union[str, List[str], Iterator]] = None
+    spk_embeds: Optional[torch.FloatTensor] = None
+    audio_wav: Optional[np.ndarray] = None
+    sampling_rate: Optional[int] = None
+
+
+class MiniCPMOPreTrainedModel(Qwen2PreTrainedModel):
+    config_class = MiniCPMOConfig
+
+
+class MiniCPMO(MiniCPMOPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.llm = Qwen2ForCausalLM(config)
+        self.llm.prepare_inputs_for_generation = types.MethodType(prepare_inputs_for_generation, self.llm)  # patch llm
+
+        self.embed_dim = self.llm.config.hidden_size
+        # init vision module
+        if self.config.init_vision:
+            self.vpm = self.init_vision_module()
+            self.vision_dim = self.vpm.embed_dim
+            self.resampler = self.init_resampler(self.embed_dim, self.vision_dim)
+
+        # init audio module
+        if self.config.init_audio:
+            self.apm = self.init_audio_module()
+            audio_output_dim = int(self.apm.config.encoder_ffn_dim // 4)
+            self.audio_avg_pooler = nn.AvgPool1d(self.config.audio_pool_step, stride=self.config.audio_pool_step)
+            self.audio_projection_layer = MultiModalProjector(in_dim=audio_output_dim, out_dim=self.embed_dim)
+            self.audio_encoder_layer = -1
+
+        # init tts module
+        # if self.config.init_tts:
+        #     assert _tts_deps, "please make sure vector_quantize_pytorch and vocos are installed."
+        #     self.tts = self.init_tts_module()
+
+        self.processor = AutoProcessor.from_pretrained(self.config._name_or_path, trust_remote_code=True)
+
+        self.terminators = ["<|im_end|>", "<|endoftext|>"]
+
+        self.default_tts_chat_template = "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n<|spk_bos|><|spk|><|spk_eos|><|tts_bos|>' }}{% endif %}"
+        self.force_no_stop = False
+
+        # for stream api
+        self.reset_session()
+
+    def reset_session(self):
+        self.session_id = None
+        self.new_user_msg = True
+        self.llm_generated = False
+        self.llm_generate_completed = False
+        self.llm_past_key_values = None
+        self.audio_past_key_values = None  # apm kv cache
+
+    def init_tts(
+        self,
+        tts_text_tokenizer_path=None,
+        vocos_ckpt_path=None,
+    ):
+        """
+        load tts tokenizer and vocos
+        1. try load form local 2. try load from huggingface
+        """
+        from .processing_minicpmo import ChatTTSProcessor
+
+        if tts_text_tokenizer_path is None:
+            tts_text_tokenizer_path = os.path.join(self.config._name_or_path, "assets/chattts_tokenizer")
+        if not os.path.exists(tts_text_tokenizer_path):
+            # try from hf model_id
+            tts_text_tokenizer_path = "openbmb/chattts_tokenizer"
+
+        tts_text_tokenizer = BertTokenizerFast.from_pretrained(tts_text_tokenizer_path)
+        self.tts_processor = ChatTTSProcessor(text_tokenizer=tts_text_tokenizer)
+
+        if vocos_ckpt_path is None:
+            vocos_ckpt_path = os.path.join(self.config._name_or_path, "assets/Vocos.pt")
+        if not os.path.exists(vocos_ckpt_path):
+            vocos_ckpt_path = hf_hub_download(repo_id="openbmb/MiniCPM-o-2_6", subfolder="assets", filename="Vocos.pt")
+
+        assert os.path.exists(vocos_ckpt_path)
+        self.vocos = self.initialize_vocos(vocos_ckpt_path)
+
+    def initialize_vocos(self, ckpt_path):
+        feature_extractor = instantiate_class(
+            args=(),
+            init={
+                "class_path": "vocos.feature_extractors.MelSpectrogramFeatures",
+                "init_args": {"sample_rate": 24000, "n_fft": 1024, "hop_length": 256, "n_mels": 100},
+            },
+        )
+        backbone = instantiate_class(
+            args=(),
+            init={
+                "class_path": "vocos.models.VocosBackbone",
+                "init_args": {"input_channels": 100, "dim": 512, "intermediate_dim": 1536, "num_layers": 8},
+            },
+        )
+        head = instantiate_class(
+            args=(),
+            init={"class_path": "vocos.heads.ISTFTHead", "init_args": {"dim": 512, "n_fft": 1024, "hop_length": 256}},
+        )
+        vocos = Vocos(feature_extractor, backbone, head).to("cuda").eval().to(torch.float32)
+        vocos.load_state_dict(torch.load(ckpt_path, weights_only=True, mmap=True))
+        return vocos
+
+    def init_vision_module(self):
+        if self.config._attn_implementation == "flash_attention_2":
+            self.config.vision_config._attn_implementation = "flash_attention_2"
+        else:
+            self.config.vision_config._attn_implementation = "eager"
+        model = SiglipVisionTransformer(self.config.vision_config)
+        if self.config.drop_vision_last_layer:
+            model.encoder.layers = model.encoder.layers[:-1]
+
+        setattr(model, "embed_dim", model.embeddings.embed_dim)
+        setattr(model, "patch_size", model.embeddings.patch_size)
+
+        return model
+
+    def init_resampler(self, embed_dim, vision_dim):
+        return Resampler(
+            num_queries=self.config.query_num,
+            embed_dim=embed_dim,
+            num_heads=embed_dim // 128,
+            kv_dim=vision_dim,
+            adaptive=True,
+        )
+
+    def init_audio_module(self):
+        model = MiniCPMWhisperEncoder(self.config.audio_config)
+        return model
+
+    def init_tts_module(self):
+        model = ConditionalChatTTS(self.config.tts_config)
+        return model
+
+    def get_input_embeddings(self):
+        return self.llm.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.llm.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.llm.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.llm.lm_head = new_embeddings
+
+    def set_decoder(self, decoder):
+        self.llm = decoder
+
+    def get_decoder(self):
+        return self.llm
+
+    def subsequent_chunk_mask(
+        self,
+        size: int,
+        chunk_size: int,
+        num_left_chunks: int = -1,
+        device: torch.device = torch.device("cpu"),
+        num_lookhead: int = 0,
+    ) -> torch.Tensor:
+        """Create mask for subsequent steps (size, size) with chunk size,
+        this is for streaming encoder
+
+        Args:
+            size (int): size of mask
+            chunk_size (int): size of chunk
+            num_left_chunks (int): number of left chunks
+                <0: use full chunk
+                >=0: use num_left_chunks
+            device (torch.device): "cpu" or "cuda" or torch.Tensor.device
+
+        Returns:
+            torch.Tensor: mask
+
+        Examples:
+            >>> subsequent_chunk_mask(4, 2)
+            [[1, 1, 0, 0],
+            [1, 1, 0, 0],
+            [1, 1, 1, 1],
+            [1, 1, 1, 1]]
+        """
+        ret = torch.zeros(size, size, device=device, dtype=torch.bool)
+        for i in range(size):
+            if num_left_chunks < 0:
+                start = 0
+            else:
+                start = max((i // chunk_size - num_left_chunks) * chunk_size, 0)
+            ending = min((i // chunk_size + 1) * chunk_size + num_lookhead, size)
+            ret[i, start:ending] = True
+        return ret
+
+    def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor):
+        """
+        Computes the output length of the convolutional layers and the output length of the audio encoder
+        """
+        input_lengths_after_cnn = (input_lengths - 1) // 2 + 1
+        input_lengths_after_pooling = (
+            input_lengths_after_cnn - self.config.audio_pool_step
+        ) // self.config.audio_pool_step + 1
+        input_lengths_after_pooling = input_lengths_after_pooling.to(dtype=torch.int32)
+
+        return input_lengths_after_cnn, input_lengths_after_pooling
+
+    def get_vllm_embedding(self, data):
+        """
+        Compute all visual embeddings, and set into llm embeddings.
+        Args:
+            data: Dict
+                tgt_sizes: image size after patch embedding
+                pixel_values: image features
+                image_bound: position of each picture corresponding to input_ids
+                input_ids: full input_ids, include placeholder
+        Returns:
+                embedding with vision, vision_hidden_states
+        """
+        if "vision_hidden_states" not in data:
+            dtype = self.llm.model.embed_tokens.weight.dtype
+            device = self.llm.model.embed_tokens.weight.device
+            tgt_sizes = data["tgt_sizes"]
+            pixel_values_list = data["pixel_values"]
+            vision_hidden_states = []
+            all_pixel_values = []
+            img_cnt = []
+            for pixel_values in pixel_values_list:
+                img_cnt.append(len(pixel_values))
+                all_pixel_values.extend([i.flatten(end_dim=1).permute(1, 0) for i in pixel_values])
+
+            # exist image
+            if all_pixel_values:
+                tgt_sizes = [tgt_size for tgt_size in tgt_sizes if isinstance(tgt_size, torch.Tensor)]
+                tgt_sizes = torch.vstack(tgt_sizes).type(torch.int32)
+
+                max_patches = torch.max(tgt_sizes[:, 0] * tgt_sizes[:, 1])
+
+                all_pixel_values = torch.nn.utils.rnn.pad_sequence(
+                    all_pixel_values, batch_first=True, padding_value=0.0
+                )
+                B, L, _ = all_pixel_values.shape
+                all_pixel_values = all_pixel_values.permute(0, 2, 1).reshape(B, 3, -1, L)
+
+                patch_attn_mask = torch.zeros((B, 1, max_patches), dtype=torch.bool, device=device)
+                for i in range(B):
+                    patch_attn_mask[i, 0, : tgt_sizes[i][0] * tgt_sizes[i][1]] = True
+
+                vision_batch_size = self.config.vision_batch_size
+                all_pixel_values = all_pixel_values.type(dtype)
+                if B > vision_batch_size:
+                    hs = []
+                    for i in range(0, B, vision_batch_size):
+                        start_idx = i
+                        end_idx = i + vision_batch_size
+                        tmp_hs = self.vpm(
+                            all_pixel_values[start_idx:end_idx],
+                            patch_attention_mask=patch_attn_mask[start_idx:end_idx],
+                            tgt_sizes=tgt_sizes[start_idx:end_idx],
+                        ).last_hidden_state
+                        hs.append(tmp_hs)
+                    vision_embedding = torch.cat(hs, dim=0)
+                else:
+                    vision_embedding = self.vpm(
+                        all_pixel_values, patch_attention_mask=patch_attn_mask, tgt_sizes=tgt_sizes
+                    ).last_hidden_state
+                vision_embedding = self.resampler(vision_embedding, tgt_sizes)
+
+                start = 0
+                for pixel_values in pixel_values_list:
+                    img_cnt = len(pixel_values)
+                    if img_cnt > 0:
+                        vision_hidden_states.append(vision_embedding[start : start + img_cnt])
+                        start += img_cnt
+                    else:
+                        vision_hidden_states.append([])
+            else:  # no image
+                if self.training:
+                    dummy_image = torch.zeros((1, 3, 224, 224), device=device, dtype=dtype)
+                    tgt_sizes = torch.Tensor(
+                        [[(224 // self.config.patch_size), math.ceil(224 / self.config.patch_size)]]
+                    ).type(torch.int32)
+                    dummy_feature = self.resampler(self.vpm(dummy_image).last_hidden_state, tgt_sizes)
+                else:
+                    dummy_feature = []
+                for _ in range(len(pixel_values_list)):
+                    vision_hidden_states.append(dummy_feature)
+
+        else:
+            vision_hidden_states = data["vision_hidden_states"]
+
+        if hasattr(self.llm.config, "scale_emb"):
+            vllm_embedding = self.llm.model.embed_tokens(data["input_ids"]) * self.llm.config.scale_emb
+        else:
+            vllm_embedding = self.llm.model.embed_tokens(data["input_ids"])
+
+        new_vllm_embedding = vllm_embedding.clone()
+        
+        vision_hidden_states = [
+            i.type(vllm_embedding.dtype) if isinstance(i, torch.Tensor) else i for i in vision_hidden_states
+        ]
+        
+        bs = len(data["input_ids"])
+        for i in range(bs):
+            cur_vs_hs = vision_hidden_states[i]
+            if len(cur_vs_hs) > 0:
+                cur_vllm_emb = vllm_embedding[i]
+                cur_image_bound = data["image_bound"][i]
+                if len(cur_image_bound) > 0:
+                    image_indices = torch.stack(
+                        [torch.arange(r[0], r[1], dtype=torch.long) for r in cur_image_bound]
+                    ).to(vllm_embedding.device)
+
+                    new_vllm_embedding[i] = cur_vllm_emb.scatter(
+                        0,
+                        image_indices.view(-1, 1).repeat(1, cur_vllm_emb.shape[-1]),
+                        cur_vs_hs.view(-1, cur_vs_hs.shape[-1]),
+                    )
+
+                elif self.training:
+                    new_vllm_embedding[i] += cur_vs_hs[0].mean() * 0
+
+        return new_vllm_embedding, vision_hidden_states
+
+    def get_audio_embedding_streaming(self, data):
+        r"""
+        Extract audio embeddings in a streaming manner using cached key-value pairs.
+
+        This method processes incoming audio features incrementally and stores/updates `past_key_values`
+        for faster inference on subsequent audio frames. It only supports batch_size=1 and is intended
+        for streaming scenarios.
+
+        Args:
+            data (dict):
+                - **"audio_features"** (`torch.FloatTensor`): Input mel-spectrograms of shape `(batch_size, 80, frames)`.
+                - **"audio_feature_lens"** (List[List[int]]): Lengths of each audio segment for each item in the batch.
+
+        Returns:
+            List[List[torch.Tensor]]: audio embeddings
+        """
+        wavforms = data.get("audio_features", [])  # (bs, 80, frames) or [], multi audios need filled in advance
+        audio_feature_lens_raw = data.get("audio_feature_lens", [])  # list, [[x1, x2], [y1], [z1]]
+
+        # exist audio
+        if len(wavforms) > 0:
+            audio_feature_lens = torch.hstack(audio_feature_lens_raw)
+            batch_size, _, max_mel_seq_len = wavforms.shape
+            assert batch_size == 1
+            max_seq_len = (max_mel_seq_len - 1) // 2 + 1
+
+            if self.audio_past_key_values is not None:
+                cache_length = self.audio_past_key_values[0][0].shape[2]
+                apm_max_len = self.apm.embed_positions.weight.shape[0]
+                if cache_length + max_seq_len >= apm_max_len:
+                    logger.warning(
+                        f"audio_past_key_values length {cache_length + max_seq_len} exceed {apm_max_len}, reset."
+                    )
+                    self.audio_past_key_values = None
+
+            audio_outputs = self.apm(wavforms, past_key_values=self.audio_past_key_values, use_cache=True)
+            audio_states = audio_outputs.last_hidden_state  # [:, :audio_feat_lengths, :]
+            self.audio_past_key_values = audio_outputs.past_key_values
+
+            audio_embeds = self.audio_projection_layer(audio_states)
+
+            audio_embeds = audio_embeds.transpose(1, 2)
+            audio_embeds = self.audio_avg_pooler(audio_embeds)
+            audio_embeds = audio_embeds.transpose(1, 2)
+
+            _, feature_lens_after_pooling = self._get_feat_extract_output_lengths(audio_feature_lens)
+
+            num_audio_tokens = feature_lens_after_pooling
+
+            final_audio_embeds = []
+            idx = 0
+            for i in range(len(audio_feature_lens_raw)):
+                target_audio_embeds = []
+                for _ in range(len(audio_feature_lens_raw[i])):
+                    target_audio_embeds.append(audio_embeds[idx, : num_audio_tokens[idx], :])
+                    idx += 1
+                final_audio_embeds.append(target_audio_embeds)
+            return final_audio_embeds
+        else:
+            return []
+
+    def get_audio_embedding(self, data, chunk_length=-1, dummy=True):
+        r"""
+        Extract full audio embeddings with optional chunk-based attention.
+
+        This method computes embeddings for all audio frames at once, either using full attention (when
+        `chunk_length` is -1) or chunk-based attention (when `chunk_length` is a positive number). It does
+        not use key-value caching and is suitable for non-streaming inference.
+
+        Args:
+            data (dict):
+                - **"audio_features"** (`torch.FloatTensor`): Input mel-spectrograms of shape `(batch_size, 80, frames)`.
+                - **"audio_feature_lens"** (List[List[int]]): Lengths of each audio segment for each item in the batch.
+            chunk_length (int, optional): Determines whether to use full attention (-1) or chunk-based
+                attention (>0) during embedding computation.
+
+        Returns:
+            List[List[torch.Tensor]]: audio embeddings
+        """
+        
+        wavforms = data.get("audio_features", [])  # (bs, 80, frames) or [], multi audios need filled in advance
+        audio_feature_lens_raw = data.get("audio_feature_lens", [])  # list, [[x1, x2], [y1], [z1]]
+
+        # exist audio
+        if len(wavforms) > 0:
+            audio_feature_lens = torch.hstack(audio_feature_lens_raw)
+            batch_size, _, max_mel_seq_len = wavforms.shape
+            max_seq_len = (max_mel_seq_len - 1) // 2 + 1
+
+            # Create a sequence tensor of shape (batch_size, max_seq_len)
+            seq_range = (
+                torch.arange(0, max_seq_len, dtype=audio_feature_lens.dtype, device=audio_feature_lens.device)
+                .unsqueeze(0)
+                .expand(batch_size, max_seq_len)
+            )
+            lengths_expand = audio_feature_lens.unsqueeze(1).expand(batch_size, max_seq_len)
+            # Create mask
+            padding_mask = seq_range >= lengths_expand  # 1 for padded values
+
+            audio_attention_mask_ = padding_mask.view(batch_size, 1, 1, max_seq_len).expand(
+                batch_size, 1, max_seq_len, max_seq_len
+            )
+            audio_attention_mask = audio_attention_mask_.to(
+                dtype=self.apm.conv1.weight.dtype, device=self.apm.conv1.weight.device
+            )
+
+            if chunk_length > 0:
+                chunk_num_frame = int(chunk_length * 50)
+                chunk_mask = self.subsequent_chunk_mask(
+                    size=max_seq_len,
+                    chunk_size=chunk_num_frame,
+                    num_left_chunks=-1,
+                    device=audio_attention_mask_.device,
+                )
+                audio_attention_mask_ = torch.logical_or(audio_attention_mask_, torch.logical_not(chunk_mask))
+
+            audio_attention_mask[audio_attention_mask_] = float("-inf")
+            audio_states = self.apm(
+                wavforms, output_hidden_states=True, attention_mask=audio_attention_mask
+            ).hidden_states[self.audio_encoder_layer]
+            audio_embeds = self.audio_projection_layer(audio_states)
+
+            audio_embeds = audio_embeds.transpose(1, 2)
+            audio_embeds = self.audio_avg_pooler(audio_embeds)
+            audio_embeds = audio_embeds.transpose(1, 2)
+
+            _, feature_lens_after_pooling = self._get_feat_extract_output_lengths(audio_feature_lens)
+
+            num_audio_tokens = feature_lens_after_pooling
+
+            final_audio_embeds = []
+            idx = 0
+            for i in range(len(audio_feature_lens_raw)):
+                target_audio_embeds = []
+                for _ in range(len(audio_feature_lens_raw[i])):
+                    target_audio_embeds.append(audio_embeds[idx, : num_audio_tokens[idx], :])
+                    idx += 1
+                final_audio_embeds.append(target_audio_embeds)
+            return final_audio_embeds
+        elif self.training and dummy:
+            dtype = self.apm.embed_positions.weight.dtype
+            device = self.apm.embed_positions.weight.device
+
+            dummy_wavs = torch.zeros((1, 80, 100), device=device, dtype=dtype)
+            audio_states = self.apm(dummy_wavs, output_hidden_states=True).hidden_states[self.audio_encoder_layer]
+
+            audio_embeds = self.audio_projection_layer(audio_states)
+
+            audio_embeds = audio_embeds.transpose(1, 2)
+            audio_embeds = self.audio_avg_pooler(audio_embeds)
+            audio_embeds = audio_embeds.transpose(1, 2)
+            return [audio_embeds]
+
+        else:
+            return []
+
+    def get_omni_embedding(self, data, input_embeddings, chunk_length=-1, stream_input=False):
+        """
+        Args:
+            data:
+            input_embeddings:
+            chunk_length: whisper use full attention or chunk attention
+            stream_input: use streaming audio embedding
+        Returns:
+            final embeddings with audio feature
+        """
+        if stream_input:
+            audio_embeddings = self.get_audio_embedding_streaming(data)
+        else:
+            audio_embeddings = self.get_audio_embedding(data, chunk_length)
+
+        bs = len(input_embeddings)
+        if len(data.get("audio_features", [])) > 0:
+            assert len(audio_embeddings) == len(input_embeddings)
+            if len(audio_embeddings) > 0:
+                audio_bounds = data["audio_bounds"]
+
+                if self.config.chunk_input:
+                    for i in range(bs):
+                        if not audio_embeddings[i]:
+                            continue
+                        audio_embs = torch.cat(audio_embeddings[i], dim=0).to(
+                            device=input_embeddings.device, dtype=input_embeddings.dtype
+                        )
+                        audio_start_pos = 0
+                        for bound in audio_bounds[i]:
+                            audio_len = bound[1] - bound[0]
+                            input_embeddings[i, bound[0] : bound[1]] = audio_embs[
+                                audio_start_pos : audio_start_pos + audio_len, :
+                            ]
+                            audio_start_pos += audio_len
+                else:
+                    for i in range(bs):
+                        audio_embs = audio_embeddings[i]
+                        bounds = audio_bounds[i]
+                        for embs, bound in zip(audio_embs, bounds):
+                            audio_indices = torch.arange(bound[0], bound[1], dtype=torch.long).to(
+                                input_embeddings.device
+                            )
+
+                            if embs.shape[0] != len(audio_indices):
+                                raise ValueError(
+                                    f"Shape mismatch: Trying to assign embeddings of shape {embs.shape} "
+                                    f"to input indices of length {len(audio_indices)}"
+                                )
+                            input_embeddings[i, audio_indices] = embs.to(input_embeddings.dtype)
+        elif self.training:
+            for i in range(bs):
+                # dummy audio_embeddings
+                input_embeddings = input_embeddings + audio_embeddings[0].mean() * 0
+
+        return input_embeddings
+
+    def forward(self, data, **kwargs):
+        vllm_embedding, vision_hidden_states = self.get_vllm_embedding(data)
+
+        if self.config.init_audio:
+            vllm_embedding = self.get_omni_embedding(
+                data, input_embeddings=vllm_embedding, chunk_length=self.config.audio_chunk_length
+            )
+
+        position_ids = data["position_ids"]
+        if position_ids.dtype != torch.int64:
+            position_ids = position_ids.long()
+
+        # compatible with llama factory
+        for key in ["input_ids", "inputs_embeds", "position_ids"]:
+            if key in kwargs:
+                del kwargs[key]
+
+        return self.llm(input_ids=None, position_ids=position_ids, inputs_embeds=vllm_embedding, **kwargs)
+
+    def _decode(self, inputs_embeds, tokenizer, attention_mask, **kwargs):
+        kwargs.pop("output_hidden_states", None)
+        kwargs.pop("return_dict_in_generate", None)
+        terminators = [tokenizer.convert_tokens_to_ids(i) for i in self.terminators]
+        outputs = self.llm.generate(
+            inputs_embeds=inputs_embeds,
+            pad_token_id=0,
+            eos_token_id=terminators,
+            attention_mask=attention_mask,
+            output_hidden_states=True,
+            return_dict_in_generate=True,
+            **kwargs,
+        )
+
+        return outputs
+
+    def _decode_stream(self, inputs_embeds, tokenizer, **kwargs):
+        terminators = [tokenizer.convert_tokens_to_ids(i) for i in self.terminators]
+        streamer = TextIteratorStreamer(tokenizer=tokenizer)
+        generation_kwargs = {
+            "inputs_embeds": inputs_embeds,
+            "pad_token_id": 0,
+            "eos_token_id": terminators,
+            "streamer": streamer,
+        }
+        generation_kwargs.update(kwargs)
+
+        thread = Thread(target=self.llm.generate, kwargs=generation_kwargs)
+        thread.start()
+
+        return streamer
+
+    def _decode_text(self, result_ids, tokenizer):
+        terminators = [tokenizer.convert_tokens_to_ids(i) for i in self.terminators]
+        result_text = []
+        for result in result_ids:
+            result = result[result != 0]
+            if result[0] == tokenizer.bos_id:
+                result = result[1:]
+            if result[-1] in terminators:
+                result = result[:-1]
+            result_text.append(tokenizer.decode(result))
+        return result_text
+
+    def get_sys_prompt(self, ref_audio=None, mode="default", language="zh"):
+        """
+        Choose different system prompts according to different tasks
+        Args:
+            ref_audio: if ref_audio is not None, will use the voice cloning prompts, and the voice
+                       generated by the model will refer to the timbre of ref audio
+            mode:
+                "default": default system prompt and not refer to any task
+                "omni": input video and audio simultaneously
+                "audio_assistant": Default voice-only mode, the model will use the ref_audio's voice to reply user's question as a helpful assistant.
+                "audio_roleplay": Roleplay voice-only mode, the model will use the ref_audio's voice to reply, and also role-play the character based on the audio prompt.
+                "voice_cloning": TTS mode, the model will clone the voice of ref_audio.
+            language: prompts language, the model has the ability to automatically select the response language
+                    based on the question language
+        Returns:
+
+        """
+        if ref_audio is not None:
+            assert isinstance(ref_audio, np.ndarray), "ref_audio error"
+        if mode == "omni":
+            if language == "zh":
+                sys_prompt = "你是一个AI助手。你能接受视频，音频和文本输入并输出语音和文本。"
+                vc_prompt_prefix = sys_prompt + "模仿输入音频中的声音特征。"
+                vc_prompt_suffix = "作为助手，你将使用这种声音风格说话。"
+            else:
+                sys_prompt = "You are a helpful assistant. You can accept video, audio and text input and output voice and text. "
+                vc_prompt_prefix = sys_prompt + "Clone the voice in the provided audio prompt."
+                vc_prompt_suffix = "As an assistant, you will speak using this voice style."
+
+            if ref_audio is not None:
+                sys_msgs = {"role": "user", "content": [vc_prompt_prefix, ref_audio, vc_prompt_suffix]}
+
+            else:
+                sys_msgs = {"role": "user", "content": [sys_prompt]}
+
+            return sys_msgs
+        elif mode == "audio_assistant":
+            if language == "zh":
+                vc_prompt_prefix = "模仿输入音频中的声音特征。"
+                vc_prompt_suffix = "作为助手，你将使用这种声音风格说话。"
+            else:
+                vc_prompt_prefix = "Use the voice in the audio prompt to synthesize new content."
+                vc_prompt_suffix = "You are a helpful assistant with the above voice style."
+
+            if ref_audio is not None:
+                sys_msgs = {"role": "user", "content": [vc_prompt_prefix, ref_audio, vc_prompt_suffix]}
+
+            else:
+                logger.warning(
+                    "Warning: ref_audio is None, speech generation will be performed based on the default voice."
+                )
+                sys_msgs = {"role": "user", "content": ["Use the <reserved_53> voice.", vc_prompt_suffix]}
+
+            return sys_msgs
+        elif mode == "audio_roleplay":
+            if language == "zh":
+                vc_prompt_prefix = "模仿输入音频中的声音特征。"
+                vc_prompt_suffix = "假装你是上述音频中的人物，与我进行对话。"
+            else:
+                vc_prompt_prefix = "Clone the voice in the provided audio prompt."
+                vc_prompt_suffix = "Try to role-play the character based on the audio prompt above."
+
+            if ref_audio is not None:
+                sys_msgs = {"role": "user", "content": [vc_prompt_prefix, ref_audio, vc_prompt_suffix]}
+            else:
+                print("Warning: ref_audio is None, speech generation will be performed based on the default voice.")
+                sys_msgs = {"role": "user", "content": ["Use the <reserved_53> voice.", vc_prompt_suffix]}
+
+            return sys_msgs
+        elif mode == "voice_cloning":
+            if language == "zh":
+                vc_prompt_prefix = "模仿输入音频中的声音特征。"
+            else:
+                vc_prompt_prefix = "Clone the voice in the provided audio prompt."
+
+            if ref_audio is not None:
+                sys_msgs = {"role": "user", "content": [vc_prompt_prefix, ref_audio]}
+            else:
+                raise ValueError("ref_audio con't be None in voice_cloning mode.")
+
+            return sys_msgs
+        else:
+            sys_prompt = "You are a helpful assistant. You can accept audio and text input and output voice and text."
+            sys_msgs = {"role": "user", "content": [sys_prompt]}
+
+            return sys_msgs
+
+    def generate(
+        self,
+        input_ids=None,
+        pixel_values=None,
+        tgt_sizes=None,
+        audio_features=[],
+        audio_feature_lens=None,
+        image_bound=None,
+        audio_bounds=None,
+        spk_bounds=None,
+        attention_mask=None,
+        tokenizer=None,
+        vision_hidden_states=None,
+        stream=False,
+        decode_text=True,
+        **kwargs,
+    ):
+        assert input_ids is not None
+        assert len(input_ids) == len(pixel_values)
+
+        model_inputs = {
+            "input_ids": input_ids,
+            "audio_features": audio_features,
+            "audio_feature_lens": audio_feature_lens,
+            "image_bound": image_bound,
+            "audio_bounds": audio_bounds,
+            "spk_bounds": spk_bounds,
+        }
+
+        if vision_hidden_states is None:
+            model_inputs["pixel_values"] = pixel_values
+            model_inputs["tgt_sizes"] = tgt_sizes
+        else:
+            model_inputs["vision_hidden_states"] = vision_hidden_states
+
+        model_output = {}
+        with torch.inference_mode():
+            model_inputs["inputs_embeds"], vision_hidden_states = self.get_vllm_embedding(model_inputs)
+            model_inputs["inputs_embeds"] = self.get_omni_embedding(
+                model_inputs,
+                input_embeddings=model_inputs["inputs_embeds"],
+                chunk_length=self.config.audio_chunk_length,
+            )
+
+            if stream:
+                result = self._decode_stream(model_inputs["inputs_embeds"], tokenizer, **kwargs)
+                # if stream return TextIteratorStreamer and output is empty
+                outputs = {}
+            else:
+                outputs = self._decode(model_inputs["inputs_embeds"], tokenizer, attention_mask, **kwargs) #怎么每次要调用config
+
+                result = self._decode_text(outputs.sequences, tokenizer)
+                
+        if decode_text is False:
+            return outputs
+        
+        
+        return result, outputs
+
+    def chat(
+        self,
+        image=None,
+        msgs=None,
+        tokenizer=None,
+        processor=None,
+        vision_hidden_states=None,
+        max_new_tokens=2048,
+        min_new_tokens=0,
+        sampling=True,
+        max_inp_length=32768,
+        stream=False,
+        chunk_input=True,
+        omni_input=False,
+        max_slice_nums=None,
+        use_image_id=None,
+        use_tts_template=False,
+        generate_audio=False,
+        return_spk_embed=False,
+        return_dict=False,
+        output_audio_path=None,
+        **kwargs,
+    ):
+        """
+        Unified chat function
+
+        Args:
+            image: use for batch_size=1 vqa, It is not recommended to continue to use this parameter
+            msgs: the input chat msgs, support text: (string)  / image: (PIL.Image) / audio (numpy.ndarray)
+            tokenizer: tokenizer for llm
+            processor: if None, use the default processor
+            max_new_tokens: the maximum length of the generation
+            min_new_tokens: the minimum length of the generation
+            sampling: whether to use sampling decoding or beam search decoding
+            max_inp_length: the maximum length of input
+            stream: whether to return generator, only used when tts is not required
+            chunk_input: whether to split audio into 1s chunks
+            omni_input: determine whether it is omni mode
+            max_slice_nums: control the maximum number of image slices
+            use_image_id: for video understanding or omni understanding, use_image_id should be False
+            use_tts_template: if the msgs contain audio, use_tts_template should be True
+            generate_audio: whether to generate audio output, only used when return_dict=True
+            return_spk_embed: whether to return spk embedding, only used when return_dict=True
+            return_dict: whether to return dict
+            output_audio_path: audio save path when generate_audio
+            **kwargs:
+        """
+        if isinstance(msgs[0], list):
+            batched = True
+        else:
+            batched = False
+
+        if generate_audio or return_spk_embed:
+            return_dict = True
+
+        msgs_list = msgs
+        images_list = image
+
+        if batched is False:
+            images_list, msgs_list = [images_list], [msgs_list]
+        else:
+            assert images_list is None, "Please integrate image to msgs when using batch inference."
+            images_list = [None] * len(msgs_list)
+        assert len(images_list) == len(msgs_list), "The batch dim of images_list and msgs_list should be the same."
+
+        if processor is None:
+            if self.processor is None:
+                self.processor = AutoProcessor.from_pretrained(self.config._name_or_path, trust_remote_code=True)
+            processor = self.processor
+        assert (
+            self.config.query_num == processor.image_processor.image_feature_size
+        ), "These two values should be the same. Check `config.json` and `preprocessor_config.json`."
+        assert (
+            self.config.patch_size == processor.image_processor.patch_size
+        ), "These two values should be the same. Check `config.json` and `preprocessor_config.json`."
+        assert (
+            self.config.use_image_id == processor.image_processor.use_image_id
+        ), "These two values should be the same. Check `config.json` and `preprocessor_config.json`."
+        assert (
+            self.config.slice_config.max_slice_nums == processor.image_processor.max_slice_nums
+        ), "These two values should be the same. Check `config.json` and `preprocessor_config.json`."
+        assert (
+            self.config.slice_mode == processor.image_processor.slice_mode
+        ), "These two values should be the same. Check `config.json` and `preprocessor_config.json`."
+
+        prompts_lists = []
+        input_images_list = []
+        input_audios_list = []
+        audio_parts_list = []
+
+        for image, msgs in zip(images_list, msgs_list):
+            if isinstance(msgs, str):
+                msgs = json.loads(msgs)
+            copy_msgs = deepcopy(msgs)
+
+            assert len(msgs) > 0, "msgs is empty"
+            assert sampling or not stream, "if use stream mode, make sure sampling=True"
+
+            if image is not None and isinstance(copy_msgs[0]["content"], str):
+                copy_msgs[0]["content"] = [image, copy_msgs[0]["content"]]
+
+            images = []
+            audios = []
+            audio_parts = []
+            for i, msg in enumerate(copy_msgs):
+                role = msg["role"]
+                content = msg["content"]
+                assert role in ["system", "user", "assistant"]
+                if i == 0:
+                    assert role in ["user", "system"], "The role of first msg should be user"
+                if isinstance(content, str):
+                    content = [content]
+                cur_msgs = []
+                for c in content:
+                    if isinstance(c, Image.Image):
+                        images.append(c)
+                        cur_msgs.append("(<image>./</image>)")
+                    elif isinstance(c, np.ndarray):  # audio
+                        audios.append(c)
+                        audio_parts.append(i)
+                        cur_msgs.append("(<audio>./</audio>)")
+                        use_tts_template = True
+                    elif isinstance(c, str):
+                        cur_msgs.append(c)
+                if omni_input:
+                    msg["content"] = "".join(cur_msgs)
+                else:
+                    msg["content"] = "\n".join(cur_msgs)
+
+            prompts_lists.append(
+                processor.tokenizer.apply_chat_template(
+                    copy_msgs,
+                    tokenize=False,
+                    add_generation_prompt=True,
+                    chat_template=self.default_tts_chat_template if use_tts_template else None,
+                )
+            )
+            input_images_list.append(images)
+            input_audios_list.append(audios)
+            audio_parts_list.append(audio_parts)
+
+        inputs = processor(
+            prompts_lists,
+            input_images_list,
+            input_audios_list,
+            audio_parts_list,
+            max_slice_nums=max_slice_nums,
+            use_image_id=use_image_id,
+            chunk_input=chunk_input,
+            return_tensors="pt",
+            max_length=max_inp_length,
+        ).to(self.device)
+
+        if sampling:
+            generation_config = {
+                "top_p": 0.8,
+                "top_k": 100,
+                "temperature": 0.7,
+                "do_sample": True,
+                "repetition_penalty": 1.05,
+            }
+        else:
+            generation_config = {
+                "num_beams": 3,
+                "repetition_penalty": 1.2,
+            }
+
+        if min_new_tokens > 0:
+            generation_config["min_new_tokens"] = min_new_tokens
+
+        generation_config.update((k, kwargs[k]) for k in generation_config.keys() & kwargs.keys())
+
+        inputs.pop("image_sizes")
+        with torch.inference_mode():
+            res, outputs = self.generate(
+                **inputs,
+                tokenizer=tokenizer,
+                max_new_tokens=max_new_tokens,
+                vision_hidden_states=vision_hidden_states,
+                stream=stream,
+                **generation_config,
+            )
+
+        if stream:
+
+            def stream_gen():
+                for text in res:
+                    for term in self.terminators:
+                        text = text.replace(term, "")
+                    yield text
+
+            if return_dict:
+                return OmniOutput(text=stream_gen())
+            else:
+                return stream_gen()
+
+        else:
+            spk_embeds = wav_numpy = sr = None
+
+            if batched:
+                answer = res
+            else:
+                answer = res[0]
+
+                if use_tts_template and generate_audio:
+                    mel_spec = self._generate_mel_spec(inputs, outputs, answer)
+                    wav_numpy, sr = self.decode_mel_to_audio(mel_spec, output_audio_path)
+
+            if return_spk_embed:
+                spk_embeds = self._get_last_spk_embeds(inputs, outputs)
+
+            if isinstance(answer, list):
+                answer = [i.replace(tokenizer.tts_end, "") for i in answer]
+            else:
+                answer = answer.replace(tokenizer.tts_end, "")
+
+            if return_dict:
+                return OmniOutput(text=answer, spk_embeds=spk_embeds, audio_wav=wav_numpy, sampling_rate=sr)
+            else:
+                return answer
+
+    def _decode_hidden(self, result_ids, last_hidden_states, tokenizer):
+        terminators = [tokenizer.convert_tokens_to_ids(i) for i in self.terminators] #self.terminators=['<|im_end|>', '<|endoftext|>']
+        hidden_states = torch.concat([h[:,-1:] for h in last_hidden_states],dim=1)
+        hidden_states_unpad = []
+        result_text_unpad = []
+        text_token_len = []
+        for id, result in enumerate(result_ids):
+            hidden_states_i = hidden_states[id, result != 0, :]
+            result = result[result!=0]
+            if result[0] == tokenizer.bos_id:
+                result = result[1:]
+                hidden_states_i = hidden_states_i[1:]
+            if result[-1] in terminators:
+                result = result[:-1]
+                hidden_states_i = hidden_states_i[:-1]
+            if result[-1] == 151692:
+                #'<|tts_eos|>'
+                result = result[:-1]
+                hidden_states_i = hidden_states_i[:-1]
+            result_text_unpad.append(tokenizer.decode(result))
+            hidden_states_unpad.append(hidden_states_i)
+            text_token_len.append(len(result))
+        return text_token_len, hidden_states, hidden_states_unpad, result_text_unpad
+    
+    def get_hidden(
+        self,
+        image=None,
+        msgs=None,
+        tokenizer=None,
+        processor=None,
+        vision_hidden_states=None,
+        max_new_tokens=2048,
+        min_new_tokens=0,
+        sampling=True,
+        max_inp_length=32768,
+        stream=False,
+        chunk_input=True,
+        omni_input=False,
+        max_slice_nums=None,
+        use_image_id=None,
+        use_tts_template=False,
+        generate_audio=False,
+        return_spk_embed=False,
+        **kwargs,
+    ):
+        if isinstance(msgs[0], list):
+            batched = True
+        else:
+            batched = False
+
+        if generate_audio or return_spk_embed:
+            return_dict = True
+
+        msgs_list = msgs
+        images_list = image
+
+        if batched is False:
+            images_list, msgs_list = [images_list], [msgs_list]
+        else:
+            assert images_list is None, "Please integrate image to msgs when using batch inference."
+            images_list = [None] * len(msgs_list)
+        assert len(images_list) == len(msgs_list), "The batch dim of images_list and msgs_list should be the same."
+
+        if processor is None:
+            if self.processor is None:
+                self.processor = AutoProcessor.from_pretrained(self.config._name_or_path, trust_remote_code=True)
+            processor = self.processor
+
+        assert (
+            self.config.query_num == processor.image_processor.image_feature_size
+        ), "These two values should be the same. Check `config.json` and `preprocessor_config.json`."
+        assert (
+            self.config.patch_size == processor.image_processor.patch_size
+        ), "These two values should be the same. Check `config.json` and `preprocessor_config.json`."
+        assert (
+            self.config.use_image_id == processor.image_processor.use_image_id
+        ), "These two values should be the same. Check `config.json` and `preprocessor_config.json`."
+        assert (
+            self.config.slice_config.max_slice_nums == processor.image_processor.max_slice_nums
+        ), "These two values should be the same. Check `config.json` and `preprocessor_config.json`."
+        assert (
+            self.config.slice_mode == processor.image_processor.slice_mode
+        ), "These two values should be the same. Check `config.json` and `preprocessor_config.json`."
+
+        prompts_lists = []
+        input_images_list = []
+        input_audios_list = []
+        audio_parts_list = []
+        for image, msgs in zip(images_list, msgs_list):
+            if isinstance(msgs, str):
+                msgs = json.loads(msgs)
+            copy_msgs = deepcopy(msgs)
+
+            assert len(msgs) > 0, "msgs is empty"
+            assert sampling or not stream, "if use stream mode, make sure sampling=True"
+
+            # if image is not None and isinstance(copy_msgs[0]["content"], str):
+            #     copy_msgs[0]["content"] = [image, copy_msgs[0]["content"]]
+
+            images = []
+            audios = []
+            audio_parts = []
+            for i, msg in enumerate(copy_msgs):
+                role = msg["role"]
+                content = msg["content"]
+                assert role in ["system", "user", "assistant"]
+                if i == 0:
+                    assert role in ["user", "system"], "The role of first msg should be user"
+                if isinstance(content, str):
+                    content = [content]
+                cur_msgs = []
+                for c in content:
+                    if isinstance(c, Image.Image):
+                        images.append(c)
+                        cur_msgs.append("(<image>./</image>)")
+                    elif isinstance(c, np.ndarray):  # audio
+                        audios.append(c)
+                        audio_parts.append(i)
+                        cur_msgs.append("(<audio>./</audio>)")
+                        use_tts_template = True
+                    elif isinstance(c, str):
+                        cur_msgs.append(c)
+                if omni_input:
+                    msg["content"] = "".join(cur_msgs)
+                else:
+                    msg["content"] = "\n".join(cur_msgs)
+
+            prompts_lists.append(
+                processor.tokenizer.apply_chat_template(
+                    copy_msgs,
+                    tokenize=False,
+                    add_generation_prompt=True,
+                    chat_template=self.default_tts_chat_template if use_tts_template else None,
+                )
+            )
+            input_images_list.append(images)
+            input_audios_list.append(audios)
+            audio_parts_list.append(audio_parts)
+
+        inputs = processor(
+            prompts_lists,
+            input_images_list,
+            input_audios_list,
+            audio_parts_list,
+            max_slice_nums=max_slice_nums,
+            use_image_id=use_image_id,
+            chunk_input=chunk_input,
+            return_tensors="pt",
+            max_length=max_inp_length,
+        ).to(self.device)
+
+        if sampling:
+            generation_config = {
+                "top_p": 0.8,
+                "top_k": 100,
+                "temperature": 0.7,
+                "do_sample": True,
+                "repetition_penalty": 1.05,
+            }
+        else:
+            generation_config = {
+                "num_beams": 3,
+                "repetition_penalty": 1.2,
+            }
+
+        if min_new_tokens > 0:
+            generation_config["min_new_tokens"] = min_new_tokens
+
+        generation_config.update((k, kwargs[k]) for k in generation_config.keys() & kwargs.keys())
+
+        inputs.pop("image_sizes")
+        # with torch.inference_mode():
+        with torch.no_grad():
+            res, outputs = self.generate(
+                **inputs,
+                tokenizer=tokenizer,
+                max_new_tokens=max_new_tokens,
+                vision_hidden_states=vision_hidden_states,
+                stream=stream,
+                **generation_config,
+            )
+
+        last_hidden_states = [hs[-1] for hs in outputs.hidden_states]
+        text_token = deepcopy(outputs.sequences)
+        text_token_len, hidden_states, hidden_states_unpad, text_unpad = self._decode_hidden(text_token, last_hidden_states, tokenizer)
+        for id in range(len(text_token)):
+            len_ = text_token_len[id]
+            text_token[id, len_:] = 0
+            hidden_states[id, len_:] = 0
+        max_len = max(text_token_len)
+        text_token = text_token[:, :max_len]
+        hidden_states = hidden_states[:, :max_len]
+
+        return text_unpad, text_token, torch.Tensor(text_token_len).to(torch.int32), hidden_states
+
+    def get_hidden_forward(self,data,**kwargs,):
+        
+        vllm_embedding, vision_hidden_states = self.get_vllm_embedding(data)
+
+        if self.config.init_audio:
+            vllm_embedding = self.get_omni_embedding(
+                data, input_embeddings=vllm_embedding, chunk_length=self.config.audio_chunk_length
+            )
+
+        position_ids = data["position_ids"]
+        if position_ids.dtype != torch.int64:
+            position_ids = position_ids.long()
+
+        # compatible with llama factory
+        for key in ["input_ids", "inputs_embeds", "position_ids"]:
+            if key in kwargs:
+                del kwargs[key]
+
+        outputs = self.llm(input_ids=None, position_ids=position_ids, inputs_embeds=vllm_embedding, output_hidden_states=True, **kwargs)
+        
+        ##计算损失
+        loss_fct = nn.CrossEntropyLoss()
+        logits = outputs.logits.view(-1,self.config.vocab_size).contiguous()
+        labels = data['target'].view(-1).long().contiguous()
+        # Enable model parallelism
+        labels = labels.to(logits.device)
+        loss = loss_fct(logits, labels)
+
+        ##得到隐藏层特征（根据对话拆分多轮）
+        last_hidden_states = outputs.hidden_states[-1] #(batch_size, s, 3584)
+        batch_size = last_hidden_states.shape[0]
+        new_hidden_states = []
+        text_token = []
+        text_token_len = []
+        for batch_id in range(batch_size):
+            st_id = -1
+            end_id = -1
+            for id in range(len(data['target'][batch_id])):
+                if data['target'][batch_id][id] != -100 and data['target'][batch_id][id] != 151645 and st_id==-1:
+                    st_id = id+1 #+1是因为target[0]='\n',要去掉
+                if data['target'][batch_id][id] == 151645 and st_id!=-1: #tokenizer.eos_id
+                    end_id = id
+                    new_hidden_states.append(last_hidden_states[batch_id:batch_id+1,st_id:end_id])
+                    text_token.append(data['target'][batch_id:batch_id+1,st_id:end_id])
+                    text_token_len.append(end_id-st_id)
+                    
+                    st_id = -1
+                    
+        ##根据filter过滤不满足要求的answer
+        # assert sum([len(filter_i) for filter_i in data['filter']]) == len(text_token), f"filter data error! filter:{data['filter']}, {data['target']},{len(text_token)}"
+        filter_bool = [filter_i for batch_filter_i in data['filter'] for filter_i in batch_filter_i]
+        
+        if sum(filter_bool) == 0:
+            #没有满足条件的文本可用于训练tts
+            return None, None, None,loss
+        new_hidden_states = [new_hidden_states[i] for i in range(len(filter_bool)) if filter_bool[i]]
+        text_token = [text_token[i] for i in range(len(filter_bool)) if filter_bool[i]]
+        text_token_len = [text_token_len[i] for i in range(len(filter_bool)) if filter_bool[i]]
+        
+        max_len = np.max(text_token_len)
+        ##padding
+        for id, new_hidden_state in enumerate(new_hidden_states):
+            # new_hidden_state (1,s,3584)
+            pad_num = max_len-new_hidden_state.shape[1]
+            if pad_num==0:
+                continue
+            
+            new_hidden_states[id] = torch.cat( 
+                    [
+                        new_hidden_state,
+                        torch.zeros((1, pad_num, new_hidden_state.shape[-1]), device=new_hidden_state.device),
+                    ],
+                    dim=1,
+                ) #(1,max_len,3584)
+            text_token[id] = torch.cat([text_token[id],torch.zeros((1, pad_num), dtype=text_token[id].dtype, device=text_token[id].device)],dim=1,) #(1,max_len)
+        new_hidden_states =  torch.cat(new_hidden_states, dim=0) #(batch_size,max_len,3584)
+        text_token =  torch.cat(text_token, dim=0) #(batch_size,max_len)
+        text_token_len = torch.tensor(text_token_len, dtype=torch.int32, device=text_token.device)
+        ##################debug############################
+        # from transformers import AutoTokenizer
+        # tokenizer_path = '/mnt/afs/zhoufangru/agent/end2end/pretrained_models/MiniCPM-o-2_6'
+        # tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True)
+        # output_ids = torch.argmax(outputs.logits, dim=-1)
+        # batch_id = 0
+        # tokenizer.decode(output_ids[batch_id][data['target'][batch_id]!=-100])[1:]
+        ##################debug############################
+        return text_token, text_token_len, new_hidden_states,loss
+
+    @torch.inference_mode()
+    def streaming_prefill(
+        self,
+        session_id,
+        msgs,
+        tokenizer,
+        omni_input=True,
+        max_slice_nums=None,
+        ls_temperature=1.0,
+        **kwargs,
+    ):
+        """
+        Streaming video/audio input and output audio stream, Only support batch_size=1
+        Args:
+            session_id: Note: new connection should use a new session_id
+        """
+        assert session_id is not None
+        if self.session_id is None or session_id != self.session_id:  # new session
+            self.is_first = True
+        else:
+            self.is_first = False
+
+        images = []
+        audios = []
+
+        assert len(msgs) == 1
+        copy_msgs = deepcopy(msgs)
+        msg = copy_msgs[0]
+
+        assert msg["role"] in ["system", "user", "assistant"]
+
+        content = msg["content"]
+        cur_msgs = []
+        for j, c in enumerate(content):
+            if isinstance(c, Image.Image):
+                images.append(c)
+                cur_msgs.append("(<image>./</image>)")
+            elif isinstance(c, np.ndarray):  # audio
+                audios.append(c)
+                cur_msgs.append("(<audio>./</audio>)")
+            elif isinstance(c, str):
+                cur_msgs.append(c)
+            else:
+                logger.error("Invalid content type:", c)
+
+        cur_contents = "".join(cur_msgs) if omni_input else "\n".join(cur_msgs)
+        if not self.is_first and self.new_user_msg and msg["role"] == "user":  # new user add im_start
+            if self.llm_generated:
+                if self.llm_generate_completed:
+                    msg["content"] = "<|im_end|>\n<|im_start|>user\n" + cur_contents
+                else:  # break llm gen, add tts_eos
+                    msg["content"] = "<|tts_eos|><|im_end|>\n<|im_start|>user\n" + cur_contents
+            else:
+                msg["content"] = "<|im_start|>user\n" + cur_contents
+            self.new_user_msg = False
+        else:
+            msg["content"] = cur_contents
+
+        if msg["role"] in ["system", "assistant"]:
+            self.new_user_msg = True
+            self.audio_past_key_values = None  # apm kv cache
+
+        if self.is_first:
+            # init pask_key_values
+            logger.info(f"new session_id: {session_id}, reset kv cache")
+            self.reset_session()
+            self.session_id = session_id
+
+            prompt = tokenizer.apply_chat_template(
+                copy_msgs, tokenize=False, add_generation_prompt=False, chat_template=self.default_tts_chat_template
+            )
+            add_special_tokens = True  # add bos
+        else:
+            prompt = copy_msgs[0]["content"]
+            add_special_tokens = False
+
+        model_inputs = self.processor(
+            [prompt],
+            [images],
+            [audios],
+            max_slice_nums=1 if max_slice_nums is None else max_slice_nums,
+            use_image_id=False,
+            chunk_input=True,
+            return_tensors="pt",
+            max_length=None,
+            sampling_rate=16000,
+            add_special_tokens=add_special_tokens,
+        ).to(self.device)
+
+        # 1. prepare input embeddings
+        model_inputs["inputs_embeds"], _ = self.get_vllm_embedding(model_inputs)
+        # get audio embedding with audio_past_key_values
+        inputs_embeds = self.get_omni_embedding(
+            model_inputs, input_embeddings=model_inputs["inputs_embeds"], stream_input=True
+        )
+
+        if self.is_first:
+            # clean audio_past_key_values after first prefill
+            self.audio_past_key_values = None
+
+        if self.llm_past_key_values is not None:
+            cache_length = self.llm_past_key_values[0][0].shape[2]
+        else:
+            cache_length = 0
+
+        attention_mask = torch.ones((1, cache_length + inputs_embeds.shape[1]), dtype=torch.bool, device=self.device)
+
+        # 2. do prefill and predict listen/speak label
+        outputs = self.llm(
+            past_key_values=self.llm_past_key_values,
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            position_ids=None,  # position_ids,
+            use_cache=True,
+            return_dict=True,
+        )
+        self.llm_past_key_values = outputs["past_key_values"]
+        return
+
+    @torch.inference_mode()
+    def streaming_generate(
+        self,
+        session_id,
+        tokenizer,
+        max_new_tokens=512,
+        min_new_tokens=0,
+        sampling=True,
+        enable_regenerate=False,
+        **kwargs,
+    ):
+        """
+        Streaming video/audio input and output audio stream
+        Args:
+        """
+        if sampling:
+            generation_config = {
+                "top_p": 0.8,
+                "top_k": 100,
+                "temperature": 0.7,
+                "do_sample": True,
+                "repetition_penalty": 1.05,
+            }
+        else:
+            generation_config = {
+                "num_beams": 3,
+                "repetition_penalty": 1.2,
+            }
+        generation_config["min_new_tokens"] = min_new_tokens
+        generation_config.update((k, kwargs[k]) for k in generation_config.keys() & kwargs.keys())
+
+        # do generate
+        # reset buffer
+        self.new_user_msg = True
+        self.llm_generated = True
+        self.llm_generate_completed = False
+        self.audio_past_key_values = None  # apm kv cache
+
+        terminators = [tokenizer.convert_tokens_to_ids(i) for i in self.terminators]
+        generate_prompt = "<|im_end|>\n<|im_start|>assistant\n<|spk_bos|><|spk|><|spk_eos|><|tts_bos|>"
+        input_ids = tokenizer(generate_prompt, return_tensors="pt", add_special_tokens=False)["input_ids"].cuda()
+
+        spk_start_idx = torch.where(input_ids[0] == tokenizer.spk_start_id)[0]
+        spk_end_idx = torch.where(input_ids[0] == tokenizer.spk_end_id)[0]
+        spk_bounds = [
+            torch.hstack([(spk_start_idx + 1).unsqueeze(-1), spk_end_idx.unsqueeze(-1)])
+        ]  # List[Tensor], (1,2)
+
+        cache_length = past_length = self.llm_past_key_values[0][0].shape[2]
+        attention_mask = torch.ones((1, cache_length + input_ids.shape[1]), dtype=torch.bool, device=self.device)
+
+        generation_config["max_new_tokens"] = max_new_tokens
+        streamer = self.llm_generate_chunk(input_ids, attention_mask, tokenizer, terminators, generation_config)
+        return streamer
+
+    def llm_generate_chunk(self, input_ids, attention_mask, tokenizer, terminators, generation_config):
+        def check_uncompleted_token(ids):
+            cur_text = tokenizer.decode(ids)
+            end = len(ids)
+            while cur_text[-1] == "�":
+                end -= 1
+                if end == 0:
+                    break
+                cur_text = tokenizer.decode(ids[:end])
+            return end
+
+        max_new_tokens = int(generation_config.pop("max_new_tokens", 2048))
+        new_len = 0
+        eos = False
+        left_ids = None
+
+        while True:
+            outputs = self.llm.generate(
+                input_ids=input_ids,
+                past_key_values=self.llm_past_key_values,
+                attention_mask=attention_mask,
+                use_cache=True,
+                max_new_tokens=3,  # reduce first token delay
+                pad_token_id=0,
+                output_hidden_states=True,
+                return_dict_in_generate=True,
+                eos_token_id=terminators,
+                **generation_config,
+            )
+            if outputs.sequences[0, -1] in terminators:
+                eos = True
+            input_len = input_ids.shape[1]
+            cur_ids = outputs.sequences[:, input_len:] #(batch_size,max_new_tokens)
+            cur_hidden_states = torch.concat([hidden_states[-1][:, -1:] for hidden_states in outputs.hidden_states],dim=1) #(batch_size, max_new_tokens, 3584)
+            new_len += cur_ids.shape[1]
+
+            if left_ids is not None and left_ids.shape[1] > 0:
+                cur_ids = torch.cat([left_ids, cur_ids], dim=1)
+            end = check_uncompleted_token(cur_ids[0])
+            left_ids = cur_ids[:, end:]
+            cur_ids = cur_ids[:, :end]
+            if 151692 in cur_ids[0].cpu().tolist():
+                #<|tts_eos|>
+                end = cur_ids[0].cpu().tolist().index(151692)
+                eos = True
+                cur_ids = cur_ids[:, :end]
+            cur_hidden_states = cur_hidden_states[:, :end]
+            text = self._decode_text(cur_ids, tokenizer)[0] if end > 0 else ""
+            self.llm_past_key_values = outputs.past_key_values
+            input_ids = outputs.sequences[:, -1:]
+            cache_length = past_length = self.llm_past_key_values[0][0].shape[2]
+            attention_mask = torch.ones((1, cache_length + input_ids.shape[1]), dtype=torch.bool, device=self.device)
+
+            res = {"text": text, "text_token":cur_ids, "hidden_states": cur_hidden_states}
+
+            yield res
+
+            if eos:
+                self.llm_generate_completed = True
+                break
+            
+            if new_len >= max_new_tokens:
+                logger.debug(f"LLM generation {new_len} exceeds max_new_tokens({max_new_tokens}), break.")
+                break
+
+
+
+class MultiModalProjector(nn.Module):
+    def __init__(self, in_dim, out_dim):
+        super().__init__()
+        self.linear1 = nn.Linear(in_features=in_dim, out_features=out_dim, bias=True)
+        self.relu = nn.ReLU()
+        self.linear2 = nn.Linear(in_features=out_dim, out_features=out_dim, bias=True)
+
+    def forward(self, audio_features):
+        hidden_states = self.relu(self.linear1(audio_features))
+        hidden_states = self.linear2(hidden_states)
+        return hidden_states
+
+def prepare_inputs_for_generation(
+    self,
+    input_ids,
+    past_key_values=None,
+    attention_mask=None,
+    inputs_embeds=None,
+    cache_position=None,
+    position_ids=None,
+    use_cache=True,
+    **kwargs,
+):
+    if past_key_values is not None:
+        if isinstance(past_key_values, Cache):
+            cache_length = past_key_values.get_seq_length()
+            past_length = past_key_values.seen_tokens
+        else:
+            cache_length = past_length = past_key_values[0][0].shape[2]
+
+        # Keep only the unprocessed tokens:
+        # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
+        # some of the inputs are exclusivelly passed as part of the cache (e.g. when passing input_embeds as
+        # input)
+        if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+            input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
+        # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
+        # input_ids based on the past_length.
+        elif past_length < input_ids.shape[1]:
+            input_ids = input_ids[:, past_length:]
+        # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+
+    if attention_mask is not None and position_ids is None:
+        # create position_ids on the fly for batch generation
+        position_ids = attention_mask.long().cumsum(-1) - 1
+        position_ids.masked_fill_(attention_mask == 0, 1)
+        if past_key_values:
+            position_ids = position_ids[:, -input_ids.shape[1] :]
+
+            # This clo≠clo≠clone call is needed to avoid recapturing cuda graphs with →rch.comπ≤→rch.comπ≤torch.compile's  mode=reduce−overheadmode=reduce-overheadmode="reduce-overhead, as otherwise the input positionidspositionidsposition_ids would have various stride during the decoding. Here, simply using .contiguous().contiguous().contiguous() is not sufficient as in the batch size = 1 case, positionidspositionidsposition_ids is already contiguous but with varying stride which retriggers a capture.
+            position_ids = position_ids.clone(memory_format=torch.contiguous_format)
+
+    # if ∈putsembeds∈putsembedsinputs_embeds are passed, we only want to use them in the 1st generation step
+    if inputs_embeds is not None and cache_position[0] == 0:
+        model_inputs = {"inputs_embeds": inputs_embeds, "input_ids": None}
+    else:
+        # The clone here is for the same reason as for positionidspositionidsposition_ids.
+        model_inputs = {"input_ids": input_ids.clone(memory_format=torch.contiguous_format), "inputs_embeds": None}
+
+    if isinstance(past_key_values, StaticCache) and attention_mask.ndim == 2:
+        if model_inputs["inputs_embeds"] is not None:
+            batch_size, sequence_length, _ = model_inputs["inputs_embeds"].shape
+            device = model_inputs["inputs_embeds"].device
+        else:
+            batch_size, sequence_length = model_inputs["input_ids"].shape
+            device = model_inputs["input_ids"].device
+
+        dtype = self.lm_head.weight.dtype
+        min_dtype = torch.finfo(dtype).min
+
+        attention_mask = _prepare_4d_causal_attention_mask_with_cache_position(
+            attention_mask,
+            sequence_length=sequence_length,
+            target_length=past_key_values.get_max_length(),
+            dtype=dtype,
+            device=device,
+            min_dtype=min_dtype,
+            cache_position=cache_position,
+            batch_size=batch_size,
+        )
+
+    model_inputs.update(
+        {
+            "position_ids": position_ids,
+            # "cache_position": cache_position,
+            "past_key_values": past_key_values,
+            "use_cache": use_cache,
+            "attention_mask": attention_mask,
+        }
+    )
+    return model_inputs
+
+
+# Copied from transformers.models.whisper.modeling_whisper.WhisperEncoderLayer and add use_cache for streaming inference
+class MiniCPMWhisperEncoderLayer(nn.Module):
+    def __init__(self, config: WhisperConfig, layer_idx: int = None):
+        super().__init__()
+        self.embed_dim = config.d_model #1024
+        self.self_attn = WHISPER_ATTENTION_CLASSES[config._attn_implementation](
+            embed_dim=self.embed_dim,
+            num_heads=config.encoder_attention_heads,
+            dropout=config.attention_dropout,
+            config=config,
+            layer_idx=layer_idx,
+        )
+        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
+        self.dropout = config.dropout
+        self.activation_fn = ACT2FN[config.activation_function]
+        self.activation_dropout = config.activation_dropout
+        self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim)
+        self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim)
+        self.final_layer_norm = nn.LayerNorm(self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor,
+        layer_head_mask: torch.Tensor,
+        output_attentions: bool = False,
+        past_key_values: Optional[EncoderDecoderCache] = None,
+        use_cache: Optional[bool] = False,
+    ) -> torch.Tensor:
+        r"""
+        Args:
+            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, embed_dim)`):
+                Hidden states to be fed into the encoder layer.
+            attention_mask (`torch.FloatTensor` of shape `(batch_size, 1, tgt_len, src_len)`):
+                Attention mask where padding elements are indicated by large negative values.
+            layer_head_mask (`torch.FloatTensor` of shape `(encoder_attention_heads,)`):
+                Mask to nullify selected heads of the attention modules.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attention weights.
+            past_key_values (`EncoderDecoderCache`, *optional*):
+                Past key-value pairs used for incremental decoding.
+            use_cache (`bool`, *optional*):
+                Whether or not to return updated `past_key_values` for caching.
+
+        Returns:
+            A tuple of shape `(hidden_states, optional(attn_weights), optional(past_key_values))`.
+        """
+        residual = hidden_states
+        hidden_states = self.self_attn_layer_norm(hidden_states)
+        hidden_states, attn_weights, past_key_values = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            layer_head_mask=layer_head_mask,
+            output_attentions=output_attentions,
+            past_key_value=past_key_values,
+        )
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.final_layer_norm(hidden_states)
+        hidden_states = self.activation_fn(self.fc1(hidden_states))
+        hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training)
+        hidden_states = self.fc2(hidden_states)
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+        hidden_states = residual + hidden_states
+
+        if hidden_states.dtype == torch.float16 and (
+            torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
+        ):
+            clamp_value = torch.finfo(hidden_states.dtype).max - 1000
+            hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        if use_cache:
+            outputs += (past_key_values,)
+
+        return outputs
+
+
+
+# Copied from from transformers.models.whisper.modeling_whisper.WhisperEncoder and add use_cache for streaming inference
+class MiniCPMWhisperEncoder(WhisperEncoder):
+
+    def __init__(self, config: WhisperConfig):
+        # print(config)
+        super().__init__(config)
+        self.layers = nn.ModuleList(
+            [MiniCPMWhisperEncoderLayer(config, layer_idx=i) for i in range(config.encoder_layers)]
+        )
+
+    def forward(
+        self,
+        input_features,
+        attention_mask=None,
+        head_mask=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+        past_key_values: Optional[EncoderDecoderCache] = None,
+        use_cache: Optional[bool] = None,
+    ):
+        r"""
+        Forward pass of the Whisper encoder.
+
+        Args:
+            input_features (`torch.FloatTensor` of shape `(batch_size, feature_size, sequence_length)`):
+                Float values of log-mel features extracted from the raw audio waveform. Typically generated
+                by a feature extractor (e.g., `WhisperFeatureExtractor`) that processes `.flac` or `.wav`
+                files into padded 2D mel spectrogram frames. These features are projected via convolution layers
+                (`conv1` and `conv2`) and then transformed into embeddings for the encoder.
+
+            attention_mask (`torch.Tensor`, *optional*):
+                Not used by Whisper for masking `input_features`, but included for API compatibility with
+                other models. If provided, it is simply ignored within the model. By default, Whisper
+                effectively ignores silence in the input log-mel spectrogram.
+
+            head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
+                Mask to nullify selected attention heads. The elements should be either 1 or 0, where:
+                - 1 indicates the head is **not masked**,
+                - 0 indicates the head is **masked** (i.e., the attention head is dropped).
+
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attention tensors of all encoder layers. If set to `True`, the
+                returned tuple (or `BaseModelOutputWithPast`) will contain an additional element with
+                attention weights for each encoder layer.
+
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. If set to `True`, the returned
+                tuple (or `BaseModelOutputWithPast`) will contain a tuple of hidden states, including the
+                initial embedding output as well as the outputs of each layer.
+
+            return_dict (`bool`, *optional*):
+                Whether or not to return a `BaseModelOutputWithPast` (a subclass of `ModelOutput`) instead
+                of a plain tuple. If set to `True`, the output will be a `BaseModelOutputWithPast` object,
+                otherwise it will be a tuple.
+
+            past_key_values (`EncoderDecoderCache`, *optional*):
+                When using caching for faster inference, this is an object that stores the key-value pairs
+                for attention states. If provided, the model will append new states to the existing cache
+                and return the updated cache. This speeds up sequential decoding or chunked inference.
+
+                - If `past_key_values` is `None`, no past states are used or returned.
+                - If `past_key_values` is not `None` and `use_cache=True`, the model will use the provided
+                cache and return the updated cache (as `next_encoder_cache`).
+
+            use_cache (`bool`, *optional*):
+                Whether or not the model should use caching (`past_key_values`) to speed up processing
+                during inference. When set to `True`, the model will:
+                - Inspect and use `past_key_values` if provided.
+                - Return updated `past_key_values` (under the name `next_encoder_cache` in
+                    `BaseModelOutputWithPast`).
+
+        Returns:
+            `BaseModelOutputWithPast` or `tuple` (depending on `return_dict`):
+                If `return_dict=True`, a `BaseModelOutputWithPast` is returned, which contains:
+                - **last_hidden_state** (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+                The output of the final encoder layer.
+                - **hidden_states** (`tuple(torch.FloatTensor)`, *optional*, returned if `output_hidden_states=True`):
+                Hidden states of the model at each layer (including the initial projection).
+                - **attentions** (`tuple(torch.FloatTensor)`, *optional*, returned if `output_attentions=True`):
+                Attention weights from each encoder layer.
+                - **past_key_values** (an object of type `EncoderDecoderCache` or `None`, *optional*):
+                Updated cache of key-value pairs if `use_cache=True`.
+
+                If `return_dict=False`, a tuple is returned, where the format is:
+                `(last_hidden_state, hidden_states, attentions)`, with `hidden_states` and `attentions`
+                only present if their respective `output_*` arguments are set to `True`.
+
+        Example:
+            >>> from transformers import AutoFeatureExtractor, WhisperConfig, WhisperForConditionalGeneration
+            >>> import torch
+
+            >>> # Load a feature extractor and a Whisper model
+            >>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-tiny.en")
+            >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en")
+
+            >>> # Assume you have audio (list of floats or numpy array) loaded from a file
+            >>> # Then extract the mel features:
+            >>> input_features = feature_extractor(audio, sampling_rate=16000, return_tensors="pt").input_features
+
+            >>> # Forward pass
+            >>> outputs = model.encoder(
+            ...     input_features=input_features,
+            ...     output_hidden_states=True,
+            ...     output_attentions=True,
+            ...     use_cache=True
+            ... )
+
+            >>> # Retrieve the last hidden state
+            >>> last_hidden_state = outputs.last_hidden_state
+            >>> print(last_hidden_state.shape)
+            torch.Size([batch_size, seq_length, hidden_size])
+
+            >>> # Retrieve the intermediate hidden states if output_hidden_states=True
+            >>> all_encoder_hidden_states = outputs.hidden_states
+
+            >>> # Retrieve attention weights if output_attentions=True
+            >>> all_encoder_attentions = outputs.attentions
+
+            >>> # Retrieve updated past key values if use_cache=True
+            >>> encoder_cache = outputs.past_key_values
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # Ignore copy
+        input_features = input_features.to(dtype=self.conv1.weight.dtype, device=self.conv1.weight.device)
+
+        inputs_embeds = nn.functional.gelu(self.conv1(input_features))
+        inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
+
+        inputs_embeds = inputs_embeds.permute(0, 2, 1)
+        # import ipdb; ipdb.set_trace()
+        embed_pos = self.embed_positions.weight
+
+        if embed_pos.shape[0] == 0:
+            #分布式训练
+            params_to_gather = [param for param in self.embed_positions.parameters()]
+            with deepspeed.zero.GatheredParameters(params_to_gather, modifier_rank=0):
+                embed_pos = deepcopy(self.embed_positions.weight)
+                # import ipdb; ipdb.set_trace()
+            
+        past_key_values_length = 0
+        if use_cache:
+            if past_key_values is None:
+                past_key_values = EncoderDecoderCache(DynamicCache(), DynamicCache())
+            elif isinstance(past_key_values, list):
+                past_key_values = EncoderDecoderCache(DynamicCache.from_legacy_cache(past_key_values), DynamicCache())
+            elif isinstance(past_key_values, DynamicCache):
+                past_key_values = EncoderDecoderCache(past_key_values, DynamicCache())
+            else:
+                pass
+            past_key_values_length = past_key_values.self_attention_cache.get_usable_length(inputs_embeds.shape[1])
+            if inputs_embeds.shape[1] + past_key_values_length > embed_pos.shape[0]:
+                logger.warning("seems the audio is longer than 30s. repeating the last part of the audio")
+                embed_pos_front = embed_pos[past_key_values_length:, :]
+                embed_pos = torch.cat(
+                    (
+                        embed_pos_front,
+                        torch.repeat_interleave(
+                            embed_pos[-1, :].unsqueeze(0),
+                            inputs_embeds.shape[1] - embed_pos.shape[0] + past_key_values_length,
+                            dim=0,
+                        ),
+                    )
+                )
+            else:
+                embed_pos = embed_pos[past_key_values_length : inputs_embeds.shape[1] + past_key_values_length, :]
+        else:
+            embed_pos = embed_pos[: inputs_embeds.shape[1], :]
+
+        hidden_states = inputs_embeds + embed_pos
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        # check if head_mask has a correct number of layers specified if desired
+        if head_mask is not None:
+            assert head_mask.size()[0] == (
+                len(self.layers)
+            ), f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}."
+
+        for idx, encoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
+            to_drop = False
+            if self.training:
+                dropout_probability = torch.rand([])
+                if dropout_probability < self.layerdrop:  # skip the layer
+                    to_drop = True
+
+            # Ignore copy
+            if to_drop:
+                layer_outputs = (None, None)
+            else:
+                if self.gradient_checkpointing and self.training:
+                    layer_outputs = self._gradient_checkpointing_func(
+                        encoder_layer.__call__,
+                        hidden_states,
+                        attention_mask,
+                        (head_mask[idx] if head_mask is not None else None),
+                        output_attentions,
+                        past_key_values,
+                        use_cache,
+                    )
+                else:
+                    layer_outputs = encoder_layer(
+                        hidden_states,
+                        attention_mask,
+                        layer_head_mask=(head_mask[idx] if head_mask is not None else None),
+                        output_attentions=output_attentions,
+                        past_key_values=past_key_values,
+                        use_cache=use_cache,
+                    )
+
+                hidden_states = layer_outputs[0]
+
+            if use_cache:
+                next_encoder_cache = layer_outputs[2 if output_attentions else 1]
+            else:
+                next_encoder_cache = None
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        hidden_states = self.layer_norm(hidden_states)
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            hidden_states=encoder_states,
+            attentions=all_attentions,
+            past_key_values=next_encoder_cache,
+        )
+

Chat/modeling_navit_siglip.py

@@ -0,0 +1,940 @@
+# coding=utf-8
+# Copyright 2024 Google AI and The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" PyTorch Siglip model. """
+# Copied from  HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit and add tgt_sizes
+
+
+import math
+import os
+import warnings
+from dataclasses import dataclass
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn.init import _calculate_fan_in_and_fan_out
+from transformers.activations import ACT2FN
+from transformers.configuration_utils import PretrainedConfig
+from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
+from transformers.modeling_outputs import BaseModelOutput
+from transformers.modeling_outputs import BaseModelOutputWithPooling
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import add_start_docstrings
+from transformers.utils import add_start_docstrings_to_model_forward
+from transformers.utils import is_flash_attn_2_available
+from transformers.utils import logging
+from transformers.utils import ModelOutput
+from transformers.utils import replace_return_docstrings
+
+logger = logging.get_logger(__name__)
+
+
+class SiglipVisionConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`SiglipVisionModel`]. It is used to instantiate a
+    Siglip vision encoder according to the specified arguments, defining the model architecture. Instantiating a
+    configuration with the defaults will yield a similar configuration to that of the vision encoder of the Siglip
+    [google/siglip-base-patch16-224](https://huggingface.co/google/siglip-base-patch16-224) architecture.
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+    Args:
+        hidden_size (`int`, *optional*, defaults to 768):
+            Dimensionality of the encoder layers and the pooler layer.
+        intermediate_size (`int`, *optional*, defaults to 3072):
+            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+        num_hidden_layers (`int`, *optional*, defaults to 12):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 12):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_channels (`int`, *optional*, defaults to 3):
+            Number of channels in the input images.
+        image_size (`int`, *optional*, defaults to 224):
+            The size (resolution) of each image.
+        patch_size (`int`, *optional*, defaults to 16):
+            The size (resolution) of each patch.
+        hidden_act (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the layer normalization layers.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+    Example:
+    ```python
+    >>> from transformers import SiglipVisionConfig, SiglipVisionModel
+    >>> # Initializing a SiglipVisionConfig with google/siglip-base-patch16-224 style configuration
+    >>> configuration = SiglipVisionConfig()
+    >>> # Initializing a SiglipVisionModel (with random weights) from the google/siglip-base-patch16-224 style configuration
+    >>> model = SiglipVisionModel(configuration)
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "siglip_vision_model"
+
+    def __init__(
+        self,
+        hidden_size=768,
+        intermediate_size=3072,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        num_channels=3,
+        image_size=224,
+        patch_size=16,
+        hidden_act="gelu_pytorch_tanh",
+        layer_norm_eps=1e-6,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.image_size = image_size
+        self.attention_dropout = attention_dropout
+        self.layer_norm_eps = layer_norm_eps
+        self.hidden_act = hidden_act
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
+        cls._set_token_in_kwargs(kwargs)
+
+        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+
+        # get the vision config dict if we are loading from SiglipConfig
+        if config_dict.get("model_type") == "siglip":
+            config_dict = config_dict["vision_config"]
+
+        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
+            logger.warning(
+                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
+                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
+            )
+
+        return cls.from_dict(config_dict, **kwargs)
+
+
+_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224"
+
+SIGLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [
+    "google/siglip-base-patch16-224",
+    # See all SigLIP models at https://huggingface.co/models?filter=siglip
+]
+
+if is_flash_attn_2_available():
+    from flash_attn import flash_attn_func
+    from flash_attn import flash_attn_varlen_func
+    from flash_attn.bert_padding import index_first_axis  # noqa
+    from flash_attn.bert_padding import pad_input
+    from flash_attn.bert_padding import unpad_input
+
+
+# Copied from transformers.models.llama.modeling_llama._get_unpad_data
+def _get_unpad_data(attention_mask):
+    seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
+    indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = seqlens_in_batch.max().item()
+    cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+
+
+def _trunc_normal_(tensor, mean, std, a, b):
+    # Cut & paste from PyTorch official master until it's in a few official releases - RW
+    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+    def norm_cdf(x):
+        # Computes standard normal cumulative distribution function
+        return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
+
+    if (mean < a - 2 * std) or (mean > b + 2 * std):
+        warnings.warn(
+            "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+            "The distribution of values may be incorrect.",
+            stacklevel=2,
+        )
+
+    # Values are generated by using a truncated uniform distribution and
+    # then using the inverse CDF for the normal distribution.
+    # Get upper and lower cdf values
+    l = norm_cdf((a - mean) / std)
+    u = norm_cdf((b - mean) / std)
+
+    # Uniformly fill tensor with values from [l, u], then translate to
+    # [2l-1, 2u-1].
+    tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+    # Use inverse cdf transform for normal distribution to get truncated
+    # standard normal
+    if tensor.dtype in [torch.float16, torch.bfloat16]:
+        # The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
+        og_dtype = tensor.dtype
+        tensor = tensor.to(torch.float32)
+        tensor.erfinv_()
+        tensor = tensor.to(og_dtype)
+    else:
+        tensor.erfinv_()
+
+    # Transform to proper mean, std
+    tensor.mul_(std * math.sqrt(2.0))
+    tensor.add_(mean)
+
+    # Clamp to ensure it's in the proper range
+    if tensor.dtype == torch.float16:
+        # The `clamp_` op is not (yet?) defined in float16+cpu
+        tensor = tensor.to(torch.float32)
+        tensor.clamp_(min=a, max=b)
+        tensor = tensor.to(torch.float16)
+    else:
+        tensor.clamp_(min=a, max=b)
+
+
+def trunc_normal_tf_(
+    tensor: torch.Tensor, mean: float = 0.0, std: float = 1.0, a: float = -2.0, b: float = 2.0
+) -> torch.Tensor:
+    """Fills the input Tensor with values drawn from a truncated
+    normal distribution. The values are effectively drawn from the
+    normal distribution :math:`\\mathcal{N}(\text{mean}, \text{std}^2)`
+    with values outside :math:`[a, b]` redrawn until they are within
+    the bounds. The method used for generating the random values works
+    best when :math:`a \\leq \text{mean} \\leq b`.
+    NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where the
+    bounds [a, b] are applied when sampling the normal distribution with mean=0, std=1.0
+    and the result is subsquently scaled and shifted by the mean and std args.
+    Args:
+        tensor: an n-dimensional `torch.Tensor`
+        mean: the mean of the normal distribution
+        std: the standard deviation of the normal distribution
+        a: the minimum cutoff value
+        b: the maximum cutoff value
+    """
+    with torch.no_grad():
+        _trunc_normal_(tensor, 0, 1.0, a, b)
+        tensor.mul_(std).add_(mean)
+
+
+def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
+    fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
+    if mode == "fan_in":
+        denom = fan_in
+    elif mode == "fan_out":
+        denom = fan_out
+    elif mode == "fan_avg":
+        denom = (fan_in + fan_out) / 2
+
+    variance = scale / denom
+
+    if distribution == "truncated_normal":
+        # constant is stddev of standard normal truncated to (-2, 2)
+        trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
+    elif distribution == "normal":
+        with torch.no_grad():
+            tensor.normal_(std=math.sqrt(variance))
+    elif distribution == "uniform":
+        bound = math.sqrt(3 * variance)
+        with torch.no_grad():
+            tensor.uniform_(-bound, bound)
+    else:
+        raise ValueError(f"invalid distribution {distribution}")
+
+
+def lecun_normal_(tensor):
+    variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
+
+
+def default_flax_embed_init(tensor):
+    variance_scaling_(tensor, mode="fan_in", distribution="normal")
+
+
+@dataclass
+# Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with CLIP->Siglip
+class SiglipVisionModelOutput(ModelOutput):
+    """
+    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.
+    Args:
+        image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
+            The image embeddings obtained by applying the projection layer to the pooler_output.
+        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+    """
+
+    image_embeds: Optional[torch.FloatTensor] = None
+    last_hidden_state: torch.FloatTensor = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+
+
+class SiglipVisionEmbeddings(nn.Module):
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.image_size = config.image_size
+        self.patch_size = config.patch_size
+
+        self.patch_embedding = nn.Conv2d(
+            in_channels=config.num_channels,
+            out_channels=self.embed_dim,
+            kernel_size=self.patch_size,
+            stride=self.patch_size,
+            padding="valid",
+        )
+
+        self.num_patches_per_side = self.image_size // self.patch_size
+        self.num_patches = self.num_patches_per_side**2
+        self.num_positions = self.num_patches
+        self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
+
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor,
+        patch_attention_mask: torch.BoolTensor,
+        tgt_sizes: Optional[torch.IntTensor] = None,
+    ) -> torch.Tensor:
+        batch_size = pixel_values.size(0)
+
+        patch_embeds = self.patch_embedding(pixel_values)
+        embeddings = patch_embeds.flatten(2).transpose(1, 2)
+
+        max_im_h, max_im_w = pixel_values.size(2), pixel_values.size(3)
+        max_nb_patches_h, max_nb_patches_w = max_im_h // self.patch_size, max_im_w // self.patch_size
+        boundaries = torch.arange(1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side)
+        position_ids = torch.full(
+            size=(
+                batch_size,
+                max_nb_patches_h * max_nb_patches_w,
+            ),
+            fill_value=0,
+        )
+
+        for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
+            if tgt_sizes is not None:
+                nb_patches_h = tgt_sizes[batch_idx][0]
+                nb_patches_w = tgt_sizes[batch_idx][1]
+            else:
+                nb_patches_h = p_attn_mask[:, 0].sum()
+                nb_patches_w = p_attn_mask[0].sum()
+
+            fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
+            fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
+
+            bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True)
+            bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True)
+
+            pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w).flatten()
+            position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
+
+        position_ids = position_ids.to(self.position_embedding.weight.device)
+
+        embeddings = embeddings + self.position_embedding(position_ids)
+        return embeddings
+
+
+class SiglipAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+                f" {self.num_heads})."
+            )
+        self.scale = self.head_dim**-0.5
+        self.dropout = config.attention_dropout
+
+        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        """Input shape: Batch x Time x Channel"""
+
+        batch_size, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+
+        k_v_seq_len = key_states.shape[-2]
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.scale
+
+        if attn_weights.size() != (batch_size, self.num_heads, q_len, k_v_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (batch_size, 1, q_len, k_v_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(batch_size, 1, q_len, k_v_seq_len)}, but is {attention_mask.size()}"
+                )
+            attn_weights = attn_weights + attention_mask
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_weights = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (batch_size, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(batch_size, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
+
+        attn_output = self.out_proj(attn_output)
+
+        return attn_output, attn_weights
+
+
+class SiglipFlashAttention2(SiglipAttention):
+    """
+    Llama flash attention module. This module inherits from `LlamaAttention` as the weights of the module stays
+    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
+    flash attention and deal with padding tokens in case the input contains any of them.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.is_causal = False  # Hack to make sure we don't use a causal mask
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        output_attentions = False
+
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        # Flash attention requires the input to have the shape
+        # batch_size x seq_length x head_dim x hidden_dim
+        # therefore we just need to keep the original shape
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+        # cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+        # query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+        # if past_key_value is not None:
+        #     cache_kwargs = {"sin": sin, "cos": cos}  # Specific to RoPE models
+        #     key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
+        # to be able to avoid many of these transpose/reshape/view.
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        dropout_rate = self.dropout if self.training else 0.0
+
+        # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+        # therefore the input hidden states gets silently casted in float32. Hence, we need
+        # cast them back in the correct dtype just to be sure everything works as expected.
+        # This might slowdown training & inference so it is recommended to not cast the LayerNorms
+        # in fp32. (LlamaRMSNorm handles it correctly)
+
+        input_dtype = query_states.dtype
+        if input_dtype == torch.float32:
+            if torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            # Handle the case where the model is quantized
+            elif hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            else:
+                target_dtype = self.q_proj.weight.dtype
+
+            logger.warning_once(
+                "The input hidden states seems to be silently casted in float32, this might be related to the fact"
+                " you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                f" {target_dtype}."
+            )
+
+            query_states = query_states.to(target_dtype)
+            key_states = key_states.to(target_dtype)
+            value_states = value_states.to(target_dtype)
+
+        attn_output = self._flash_attention_forward(
+            query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate
+        )
+
+        attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
+        attn_output = self.out_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights
+
+    def _flash_attention_forward(
+        self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            attention_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`int`, *optional*):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+        """
+
+        # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
+        causal = self.is_causal and query_length != 1
+
+        # Contains at least one padding token in the sequence
+        if attention_mask is not None:
+            batch_size = query_states.shape[0]
+            query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
+                query_states, key_states, value_states, attention_mask, query_length
+            )
+
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+            attn_output_unpad = flash_attn_varlen_func(
+                query_states,
+                key_states,
+                value_states,
+                cu_seqlens_q=cu_seqlens_q,
+                cu_seqlens_k=cu_seqlens_k,
+                max_seqlen_q=max_seqlen_in_batch_q,
+                max_seqlen_k=max_seqlen_in_batch_k,
+                dropout_p=dropout,
+                softmax_scale=softmax_scale,
+                causal=causal,
+            )
+
+            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+        else:
+            attn_output = flash_attn_func(
+                query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal
+            )
+
+        return attn_output
+
+    def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+        batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
+
+        key_layer = index_first_axis(
+            key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+        value_layer = index_first_axis(
+            value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+        if query_length == kv_seq_len:
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k
+            )
+            cu_seqlens_q = cu_seqlens_k
+            max_seqlen_in_batch_q = max_seqlen_in_batch_k
+            indices_q = indices_k
+        elif query_length == 1:
+            max_seqlen_in_batch_q = 1
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )  # There is a memcpy here, that is very bad.
+            indices_q = cu_seqlens_q[:-1]
+            query_layer = query_layer.squeeze(1)
+        else:
+            # The -q_len: slice assumes left padding.
+            attention_mask = attention_mask[:, -query_length:]
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
+
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q,
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
+class SiglipMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.activation_fn = ACT2FN[config.hidden_act]
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer with CLIP->Siglip
+class SiglipEncoderLayer(nn.Module):
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__()
+        self.embed_dim = config.hidden_size
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+        self.self_attn = SiglipAttention(config) if not self._use_flash_attention_2 else SiglipFlashAttention2(config)
+        self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+        self.mlp = SiglipMLP(config)
+        self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor,
+        output_attentions: Optional[bool] = False,
+    ) -> Tuple[torch.FloatTensor]:
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`):
+                Input to the layer of shape `(batch, seq_len, embed_dim)`.
+            attention_mask (`torch.FloatTensor`):
+                Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
+            output_attentions (`bool`, *optional*, defaults to `False`):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+        """
+        residual = hidden_states
+
+        hidden_states = self.layer_norm1(hidden_states)
+        hidden_states, attn_weights = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+        )
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.layer_norm2(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        return outputs
+
+
+class SiglipPreTrainedModel(PreTrainedModel):
+    """
+    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
+    models.
+    """
+
+    config_class = SiglipVisionConfig
+    base_model_prefix = "siglip"
+    supports_gradient_checkpointing = True
+
+    def _init_weights(self, module):
+        """Initialize the weights"""
+
+        if isinstance(module, SiglipVisionEmbeddings):
+            width = self.config.hidden_size
+            nn.init.normal_(module.position_embedding.weight, std=1 / np.sqrt(width))
+        elif isinstance(module, nn.Embedding):
+            default_flax_embed_init(module.weight)
+        elif isinstance(module, SiglipAttention):
+            nn.init.normal_(module.q_proj.weight)
+            nn.init.normal_(module.k_proj.weight)
+            nn.init.normal_(module.v_proj.weight)
+            nn.init.normal_(module.out_proj.weight)
+            nn.init.zeros_(module.q_proj.bias)
+            nn.init.zeros_(module.k_proj.bias)
+            nn.init.zeros_(module.v_proj.bias)
+            nn.init.zeros_(module.out_proj.bias)
+        elif isinstance(module, SiglipMLP):
+            nn.init.normal_(module.fc1.weight)
+            nn.init.normal_(module.fc2.weight)
+            nn.init.normal_(module.fc1.bias, std=1e-6)
+            nn.init.normal_(module.fc2.bias, std=1e-6)
+        elif isinstance(module, (nn.Linear, nn.Conv2d)):
+            lecun_normal_(module.weight)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+
+
+SIGLIP_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+    Parameters:
+        config ([`SiglipVisionConfig`]): Model configuration class with all the parameters of the model.
+            Initializing with a config file does not load the weights associated with the model, only the
+            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+SIGLIP_VISION_INPUTS_DOCSTRING = r"""
+    Args:
+        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+            Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
+            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPEncoder with CLIP->Siglip
+class SiglipEncoder(nn.Module):
+    """
+    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+    [`SiglipEncoderLayer`].
+    Args:
+        config: SiglipConfig
+    """
+
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__()
+        self.config = config
+        self.layers = nn.ModuleList([SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.gradient_checkpointing = False
+
+    # Ignore copy
+    def forward(
+        self,
+        inputs_embeds,
+        attention_mask: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutput]:
+        r"""
+        Args:
+            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+                than the model's internal embedding lookup matrix.
+            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+                - 1 for tokens that are **not masked**,
+                - 0 for tokens that are **masked**.
+                [What are attention masks?](../glossary#attention-mask)
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+                for more detail.
+            return_dict (`bool`, *optional*):
+                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        hidden_states = inputs_embeds
+        for encoder_layer in self.layers:
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    encoder_layer.__call__,
+                    hidden_states,
+                    attention_mask,
+                    output_attentions,
+                )
+            else:
+                layer_outputs = encoder_layer(
+                    hidden_states,
+                    attention_mask,
+                    output_attentions=output_attentions,
+                )
+
+            hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+        return BaseModelOutput(last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions)
+
+
+@add_start_docstrings("""The vision model from SigLIP without any head or projection on top.""", SIGLIP_START_DOCSTRING)
+class SiglipVisionTransformer(SiglipPreTrainedModel):
+    config_class = SiglipVisionConfig
+    main_input_name = "pixel_values"
+    _supports_flash_attn_2 = True
+    _no_split_modules = []
+
+    def __init__(self, config: SiglipVisionConfig):
+        super().__init__(config)
+        self.config = config
+        embed_dim = config.hidden_size
+
+        self.embeddings = SiglipVisionEmbeddings(config)
+        self.encoder = SiglipEncoder(config)
+        self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self) -> nn.Module:
+        return self.embeddings.patch_embedding
+
+    @add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=SiglipVisionConfig)
+    def forward(
+        self,
+        pixel_values,
+        patch_attention_mask: Optional[torch.BoolTensor] = None,
+        tgt_sizes: Optional[torch.IntTensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPooling]:
+        r"""
+        Returns:
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        batch_size = pixel_values.size(0)
+        if patch_attention_mask is None:
+            patch_attention_mask = torch.ones(
+                size=(
+                    batch_size,
+                    pixel_values.size(2) // self.config.patch_size,
+                    pixel_values.size(3) // self.config.patch_size,
+                ),
+                dtype=torch.bool,
+                device=pixel_values.device,
+            )
+
+        hidden_states = self.embeddings(
+            pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, tgt_sizes=tgt_sizes
+        )
+
+        patch_attention_mask = patch_attention_mask.view(batch_size, -1)
+        # The call to `_upad_input` in `_flash_attention_forward` is expensive
+        # So when the `patch_attention_mask` is full of 1s (i.e. attending to the whole sequence),
+        # avoiding passing the attention_mask, which is equivalent to attending to the full sequence
+        if not torch.any(~patch_attention_mask):
+            attention_mask = None
+        else:
+            attention_mask = (
+                _prepare_4d_attention_mask(patch_attention_mask, hidden_states.dtype)
+                if not self._use_flash_attention_2
+                else patch_attention_mask
+            )
+
+        encoder_outputs = self.encoder(
+            inputs_embeds=hidden_states,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        last_hidden_state = encoder_outputs[0]
+        last_hidden_state = self.post_layernorm(last_hidden_state)
+
+        if not return_dict:
+            return (last_hidden_state, None) + encoder_outputs[1:]
+
+        return BaseModelOutputWithPooling(
+            last_hidden_state=last_hidden_state,
+            pooler_output=None,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )

Chat/preprocessor_config.json

@@ -0,0 +1,24 @@
+{
+    "image_processor_type": "MiniCPMVImageProcessor",
+    "auto_map": {
+        "AutoProcessor": "processing_minicpmo.MiniCPMOProcessor",
+        "AutoImageProcessor": "image_processing_minicpmv.MiniCPMVImageProcessor"
+      },
+    "processor_class": "MiniCPMOProcessor",
+    "max_slice_nums": 9,
+    "scale_resolution": 448,
+    "patch_size": 14,
+    "use_image_id": true,
+    "image_feature_size": 64,
+    "im_start": "<image>",
+    "im_end": "</image>",
+    "slice_start": "<slice>",
+    "slice_end": "</slice>",
+    "unk": "<unk>",
+    "im_id_start": "<image_id>",
+    "im_id_end": "</image_id>",
+    "slice_mode": true,
+    "norm_mean": [0.5, 0.5, 0.5],
+    "norm_std": [0.5, 0.5, 0.5],
+    "version": 2.6
+}

Chat/processing_minicpmo.py

@@ -0,0 +1,505 @@
+# coding=utf-8
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Processor class for MiniCPMO.
+"""
+
+import math
+import re
+from typing import List
+from typing import Literal
+from typing import Optional
+from typing import Union
+
+import numpy as np
+import torch
+import torchaudio
+from transformers.image_utils import ImageInput
+from transformers.processing_utils import ProcessorMixin
+from transformers.tokenization_utils_base import PreTokenizedInput
+from transformers.tokenization_utils_base import TextInput
+from transformers.utils import TensorType
+
+from .image_processing_minicpmv import MiniCPMOBatchFeature
+
+
+class MiniCPMOProcessor(ProcessorMixin):
+    r"""
+    Constructs a MiniCPMV processor which wraps a MiniCPMV image processor and a MiniCPMV tokenizer into a single processor.
+
+    [`MiniCPMVProcessor`] offers all the functionalities of [`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the
+    [`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`] for more information.
+
+    Args:
+        image_processor ([`MiniCPMVImageProcessor`], *optional*):
+            The image processor is a required input.
+        tokenizer ([`LlamaTokenizerWrapper`], *optional*):
+            The tokenizer is a required input.
+    """
+
+    attributes = ["image_processor", "feature_extractor", "tokenizer"]
+    feature_extractor_class = "WhisperFeatureExtractor"
+    image_processor_class = "AutoImageProcessor"
+    tokenizer_class = "AutoTokenizer"
+
+    def __init__(self, image_processor=None, feature_extractor=None, tokenizer=None):
+        super().__init__(image_processor, feature_extractor, tokenizer)
+        self.version = image_processor.version
+
+    def __call__(
+        self,
+        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]],
+        images: ImageInput = None,
+        audios: Union[np.ndarray, List[np.ndarray], List[List[np.ndarray]]] = None,
+        audio_parts: Optional[list] = None,
+        max_length: Optional[int] = None,
+        do_pad: Optional[bool] = True,
+        max_slice_nums: int = None,
+        use_image_id: bool = True,
+        chunk_input: bool = False,
+        return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
+        sampling_rate: Optional[int] = 16000,
+        **kwargs,
+    ) -> MiniCPMOBatchFeature:
+        if images is not None:
+            image_inputs = self.image_processor(
+                images, do_pad=do_pad, max_slice_nums=max_slice_nums, return_tensors=return_tensors
+            )
+        else:
+            image_inputs = None
+
+        if audios is not None:
+            audio_features, audio_feature_lens, audio_phs = self.audio_feature_extract(
+                audios, audio_parts, chunk_input, sampling_rate
+            )
+        else:
+            audio_features, audio_feature_lens, audio_phs = [], [], []
+
+        model_inputs = self._convert_omni_to_inputs(
+            image_inputs,
+            audio_phs,
+            text,
+            max_slice_nums=max_slice_nums,
+            use_image_id=use_image_id,
+            max_length=max_length,
+            **kwargs,
+        )
+
+        model_inputs["audio_features"] = audio_features
+        model_inputs["audio_feature_lens"] = audio_feature_lens
+
+        return MiniCPMOBatchFeature(data={**model_inputs})
+
+    def get_audio_placeholder(self, audio_lens, chunk_input, chunk_length):
+        pool_step = 2
+        feature_lens = math.ceil(audio_lens / self.feature_extractor.hop_length)
+
+        feature_lens = (feature_lens - 1) // 2 + 1
+        output_lens = (feature_lens - pool_step) // pool_step + 1
+
+        if chunk_input:
+            fbank_feat_in_chunk = int(chunk_length * 100)
+            cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1
+            audio_embeds_in_chunk = (cnn_feat_in_chunk - pool_step) // pool_step + 1
+            num_audio_chunks = (output_lens + audio_embeds_in_chunk - 1) // audio_embeds_in_chunk
+
+            place_holders = ""
+            total_unk_len = 0
+            for _ in range(num_audio_chunks):
+                unk_len = min(audio_embeds_in_chunk, output_lens - total_unk_len)
+                place_holders += self.tokenizer.audio_start + "<unk>" * unk_len + self.tokenizer.audio_end
+                total_unk_len += unk_len
+            audio_placeholder = place_holders
+        else:
+            audio_placeholder = self.tokenizer.audio_start + "<unk>" * output_lens + self.tokenizer.audio_end
+
+        return audio_placeholder
+
+    def audio_feature_extract(
+        self,
+        audios: Union[np.ndarray, List[np.ndarray], List[List[np.ndarray]]],
+        audio_parts: Optional[list] = None,
+        chunk_input: Optional[bool] = False,
+        sampling_rate: Optional[int] = None,
+        chunk_length: Optional[int] = 1,
+        **kwargs,
+    ):
+        if isinstance(audios, np.ndarray):
+            audios_list = [[audios]]
+        elif isinstance(audios[0], np.ndarray):
+            audios_list = [audios]
+        else:
+            audios_list = audios
+
+        if audio_parts is not None:
+            assert len(audio_parts) == len(audios_list)
+            for parts, audios in zip(audio_parts, audios_list):
+                assert len(parts) == len(audios)
+
+        audio_feature_lens_list = []
+        audio_ph_list = []
+
+        audio_features_all = []
+
+        # audio placeholder not dependent on audio_parts
+        for audios in audios_list:
+            if audios:
+                audio_ph_list.append([self.get_audio_placeholder(len(a), chunk_input, chunk_length) for a in audios])
+            else:
+                audio_ph_list.append([])
+
+        for idx, audios in enumerate(audios_list):
+            if audio_parts is not None:
+                # same audio part merge
+                audio_part = audio_parts[idx]
+                merge_audio = []
+                cur_audio = []
+                for aid, (part, audio) in enumerate(zip(audio_part, audios)):
+                    if aid == 0 or audio_part[aid] == audio_part[aid - 1]:
+                        cur_audio.append(audio)
+                    else:
+                        merge_audio.append(np.hstack(cur_audio))
+                        cur_audio = [audio]
+                if cur_audio:
+                    merge_audio.append(np.hstack(cur_audio))
+
+            else:
+                merge_audio = audios
+
+            audio_feature_lens = []
+
+            # If the audio exceeds 30 seconds, split it into chunks every 30 seconds.
+            final_merge_audio = []
+            max_audio_inp_len = 30 * sampling_rate
+            for audio in merge_audio:
+                if len(audio) <= max_audio_inp_len:
+                    final_merge_audio.append(audio)
+                else:
+                    for i in range(math.ceil(len(audio) / max_audio_inp_len)):
+                        final_merge_audio.append(audio[i * max_audio_inp_len : (i + 1) * max_audio_inp_len])
+
+            if audios:
+                audio_inputs = self.feature_extractor(
+                    final_merge_audio,
+                    sampling_rate=sampling_rate,
+                    return_attention_mask=True,
+                    padding="max_length",
+                    return_tensors="pt",
+                    **kwargs,
+                )
+                audio_feature = audio_inputs["input_features"]
+                actual_lens = audio_inputs["attention_mask"].sum(dim=1)
+
+                for feat, lens in zip(audio_feature, actual_lens):
+                    audio_features_all.append(feat[:, :lens])
+                    audio_feature_lens.append(lens)
+
+                audio_feature_lens = torch.hstack(audio_feature_lens)
+                audio_feature_lens_list.append(audio_feature_lens)
+            else:
+                audio_feature_lens_list.append([])
+
+        if audio_features_all:
+            audio_features = [i.permute(1, 0) for i in audio_features_all]
+            audio_features = torch.nn.utils.rnn.pad_sequence(
+                audio_features, batch_first=True, padding_value=0.0
+            ).permute(0, 2, 1)
+        else:
+            audio_features = []
+
+        return audio_features, audio_feature_lens_list, audio_ph_list
+
+    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        output_ids = args[0]
+        result_text = []
+        for result in output_ids:
+            result = result[result != 0]
+            if result[0] == self.tokenizer.bos_id:
+                result = result[1:]
+            if result[-1] == self.tokenizer.eos_id:
+                result = result[:-1]
+            result_text.append(self.tokenizer.decode(result, *args[1:], **kwargs).strip())
+        return result_text
+        # return self.tokenizer.batch_decode(*args, **kwargs)
+
+    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        result = args[0]
+        result = result[result != 0]
+        if result[0] == self.tokenizer.bos_id:
+            result = result[1:]
+        if result[-1] == self.tokenizer.eos_id or (
+            hasattr(self.tokenizer, "eot_id") and result[-1] == self.tokenizer.eot_id
+        ):
+            result = result[:-1]
+        return self.tokenizer.decode(result, *args[1:], **kwargs).strip()
+
+    def _convert(self, input_str, max_inp_length: Optional[int] = None, **kwargs):
+        input_ids = self.tokenizer.encode(input_str, **kwargs)
+        if max_inp_length is not None:
+            input_ids = input_ids[:max_inp_length]
+        input_ids = torch.tensor(input_ids, dtype=torch.int32)
+
+        ## image bound
+        start_cond = (input_ids == self.tokenizer.im_start_id) | (input_ids == self.tokenizer.slice_start_id)
+        end_cond = (input_ids == self.tokenizer.im_end_id) | (input_ids == self.tokenizer.slice_end_id)
+
+        image_start_idx = torch.where(start_cond)[0]
+        image_start_idx += 1
+        image_end_idx = torch.where(end_cond)[0]
+
+        valid_image_nums = max(len(image_start_idx), len(image_end_idx))
+
+        image_bounds = torch.hstack(
+            [
+                image_start_idx[:valid_image_nums].unsqueeze(-1),
+                image_end_idx[:valid_image_nums].unsqueeze(-1),
+            ]
+        )
+
+        ##  audio bound
+        audio_start_idx = torch.where(input_ids == self.tokenizer.audio_start_id)[0]
+        audio_end_idx = torch.where(input_ids == self.tokenizer.audio_end_id)[0]
+        assert len(audio_start_idx) == len(audio_end_idx)
+        audio_bounds = torch.hstack([(audio_start_idx + 1).unsqueeze(-1), audio_end_idx.unsqueeze(-1)])
+
+        spk_start_idx = torch.where(input_ids == self.tokenizer.spk_start_id)[0]
+        spk_end_idx = torch.where(input_ids == self.tokenizer.spk_end_id)[0]
+        assert len(spk_start_idx) == len(spk_end_idx)
+        spk_bounds = torch.hstack([(spk_start_idx + 1).unsqueeze(-1), spk_end_idx.unsqueeze(-1)])
+
+        return input_ids, image_bounds, audio_bounds, spk_bounds
+
+    def _convert_omni_to_inputs(
+        self,
+        images,
+        audio_phs,
+        texts: Union[str, List[str]],
+        truncation=None,
+        max_length=None,
+        max_slice_nums=None,
+        use_image_id=None,
+        return_tensors=None,
+        **kwargs,
+    ):
+        if images is None and audio_phs is None:
+            model_inputs = self.tokenizer(
+                texts, return_tensors=return_tensors, truncation=truncation, max_length=max_length, **kwargs
+            )
+            return MiniCPMOBatchFeature(data={**model_inputs})
+
+        image_tag = "(<image>./</image>)"
+        image_pattern = "\(<image>./</image>\)"
+        audio_tag = "(<audio>./</audio>)"
+        audio_pattern = "\(<audio>./</audio>\)"
+        split_pattern = f"({image_pattern}|{audio_pattern})"
+
+        if isinstance(texts, str):
+            texts = [texts]
+
+        bs = len(texts)
+        if images is not None:
+            images, image_sizes, tgt_sizes = images["pixel_values"], images["image_sizes"], images["tgt_sizes"]
+        else:
+            images, image_sizes, tgt_sizes = [[]] * bs, [[]] * bs, [[]] * bs
+
+        input_ids_list = []
+        image_bounds_list = []
+        audio_bounds_list = []
+        spk_bounds_list = []
+
+        for index, text in enumerate(texts):
+            text_chunks = re.split(split_pattern, text)
+
+            image_tags = re.findall(image_pattern, text)
+            audio_tags = re.findall(audio_pattern, text)
+
+            if image_tags:
+                assert images is not None
+                assert len(image_tags) == len(image_sizes[index])
+            if audio_tags:
+                assert audio_phs is not None
+                assert len(audio_tags) == len(audio_phs[index])
+
+            image_id = 0
+            audio_id = 0
+            for i, chunk in enumerate(text_chunks):
+                if chunk == image_tag:
+                    image_placeholder = self.image_processor.get_slice_image_placeholder(
+                        image_sizes[index][image_id], image_id, max_slice_nums, use_image_id
+                    )
+                    image_id += 1
+                    text_chunks[i] = image_placeholder
+                elif chunk == audio_tag:
+                    audio_placeholder = audio_phs[index][audio_id]
+                    audio_id += 1
+                    text_chunks[i] = audio_placeholder
+
+            final_text = "".join(text_chunks)
+            input_ids, image_bounds, audio_bounds, spk_bounds = self._convert(final_text, max_length, **kwargs)
+
+            input_ids_list.append(input_ids)
+            image_bounds_list.append(image_bounds)
+            audio_bounds_list.append(audio_bounds)
+            spk_bounds_list.append(spk_bounds)
+
+        padded_input_ids, padding_lengths = self.pad(input_ids_list, padding_side="left")
+        attention_mask = torch.ones_like(padded_input_ids, dtype=torch.bool)
+        for i, length in enumerate(padding_lengths):
+            image_bounds_list[i] = image_bounds_list[i] + length
+            audio_bounds_list[i] = audio_bounds_list[i] + length
+            spk_bounds_list[i] = spk_bounds_list[i] + length
+            attention_mask[i, :length] = False
+
+        data = {
+            "input_ids": padded_input_ids,
+            "attention_mask": attention_mask,
+            "pixel_values": images,
+            "image_sizes": image_sizes,
+            "image_bound": image_bounds_list,
+            "tgt_sizes": tgt_sizes,
+            "audio_bounds": audio_bounds_list,
+            "spk_bounds": spk_bounds_list,
+        }
+
+        return data
+
+    @property
+    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
+    def model_input_names(self):
+        tokenizer_input_names = self.tokenizer.model_input_names
+        image_processor_input_names = self.image_processor.model_input_names
+        feature_extractor_input_names = self.feature_extractor.model_input_names
+        return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names + feature_extractor_input_names))
+
+    def pad(self, inputs, max_length=None, padding_value=0, padding_side="left"):
+        items = []
+        if isinstance(inputs[0], list):
+            assert isinstance(inputs[0][0], torch.Tensor)
+            for it in inputs:
+                for tr in it:
+                    items.append(tr)
+        else:
+            assert isinstance(inputs[0], torch.Tensor)
+            items = inputs
+
+        batch_size = len(items)
+        shape = items[0].shape
+        dim = len(shape)
+        assert dim <= 2
+        if max_length is None:
+            max_length = 0
+        max_length = max(max_length, max(item.shape[-1] for item in items))
+        min_length = min(item.shape[-1] for item in items)
+        dtype = items[0].dtype
+
+        if dim == 0:
+            return torch.stack([item for item in items], dim=0), [0]
+        elif dim == 1:
+            if max_length == min_length:
+                return torch.stack([item for item in items], dim=0), [0] * batch_size
+            tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value
+        else:
+            tensor = torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype) + padding_value
+
+        padding_length = []
+        for i, item in enumerate(items):
+            if dim == 1:
+                if padding_side == "left":
+                    tensor[i, -len(item) :] = item.clone()
+                else:
+                    tensor[i, : len(item)] = item.clone()
+            elif dim == 2:
+                if padding_side == "left":
+                    tensor[i, -len(item) :, :] = item.clone()
+                else:
+                    tensor[i, : len(item), :] = item.clone()
+            padding_length.append(tensor.shape[-1] - len(item))
+
+        return tensor, padding_length
+
+
+class MelSpectrogramFeatures(torch.nn.Module):
+    def __init__(
+        self,
+        sample_rate=24000,
+        n_fft=1024,
+        hop_length=256,
+        n_mels=100,
+        padding: Literal["center", "same"] = "center",
+    ):
+        super().__init__()
+        if padding not in ["center", "same"]:
+            raise ValueError("Padding must be 'center' or 'same'.")
+        self.padding = padding
+        self.mel_spec = torchaudio.transforms.MelSpectrogram(
+            sample_rate=sample_rate,
+            n_fft=n_fft,
+            hop_length=hop_length,
+            n_mels=n_mels,
+            center=padding == "center",
+            power=1,
+        )
+
+    def __call__(self, audio: torch.Tensor) -> torch.Tensor:
+        """
+        audio: Tensor([num_channels, num_samples])
+        """
+        return super().__call__(audio)
+
+    def forward(self, audio: torch.Tensor) -> torch.Tensor:
+        """
+        audio: Tensor([num_channels, num_samples])
+        """
+        mel: torch.Tensor = self.mel_spec(audio)
+        features = torch.log(torch.clip(mel, min=1e-5))
+        return features
+
+
+class ChatTTSProcessor:
+    def __init__(self, text_tokenizer):
+        self.audio_processor = MelSpectrogramFeatures()
+        self.text_tokenizer = text_tokenizer
+
+    def __call__(self, text_list, audio_list):
+        assert len(text_list) == len(audio_list)
+        input_ids_varlen = []
+        for text in text_list:
+            input_ids_ = self.text_tokenizer.encode(text, return_tensors="pt", add_special_tokens=False)  # [1, seq_len]
+            input_ids_ = input_ids_.squeeze(0)  # [seq_len]
+            input_ids_varlen.append(input_ids_)
+
+        audio_features_varlen = []
+        for audio in audio_list:
+            assert audio.shape.__len__() == 1  # [seq_len]
+            try:
+                mel = self.audio_processor(audio)  # [100(num_mel_bins), seq_len_mel]
+            except Exception as e:
+                raise e
+            audio_features_varlen.append(mel)
+
+        return {
+            "tts_input_ids_varlen": input_ids_varlen,  # return List[Tensor]
+            "tts_input_features_varlen": audio_features_varlen,  # return List[Tensor]
+        }

Chat/resampler.py

@@ -0,0 +1,871 @@
+# coding=utf-8
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+from functools import partial
+from typing import Dict, Optional, Callable, List, Generator, Tuple
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn
+from torch import Tensor
+from torch.nn.functional import *
+from torch.nn.init import trunc_normal_
+from torch.nn.modules.activation import *
+from transformers.integrations import is_deepspeed_zero3_enabled
+
+
+def get_2d_sincos_pos_embed(embed_dim, image_size):
+    """
+    image_size: image_size or (image_height, image_width)
+    return:
+    pos_embed: [image_height, image_width, embed_dim]
+    """
+    if isinstance(image_size, int):
+        grid_h_size, grid_w_size = image_size, image_size
+    else:
+        grid_h_size, grid_w_size = image_size[0], image_size[1]
+
+    grid_h = np.arange(grid_h_size, dtype=np.float32)
+    grid_w = np.arange(grid_w_size, dtype=np.float32)
+    grid = np.meshgrid(grid_w, grid_h)  # here w goes first
+    grid = np.stack(grid, axis=0)
+
+    pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
+    return pos_embed
+
+
+def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
+    assert embed_dim % 2 == 0
+
+    # use half of dimensions to encode grid_h
+    emb_h = get_1d_sincos_pos_embed_from_grid_new(embed_dim // 2, grid[0])  # (H, W, D/2)
+    emb_w = get_1d_sincos_pos_embed_from_grid_new(embed_dim // 2, grid[1])  # (H, W, D/2)
+
+    emb = np.concatenate([emb_h, emb_w], axis=-1)  # (H, W, D)
+    return emb
+
+
+def get_1d_sincos_pos_embed_from_grid_new(embed_dim, pos):
+    """
+    embed_dim: output dimension for each position
+    pos: a list of positions to be encoded: size (H, W)
+    out: (H, W, D)
+    """
+    assert embed_dim % 2 == 0
+    omega = np.arange(embed_dim // 2, dtype=np.float32)
+    omega /= embed_dim / 2.0
+    omega = 1.0 / 10000**omega  # (D/2,)
+
+    out = np.einsum("hw,d->hwd", pos, omega)  # (H, W, D/2), outer product
+
+    emb_sin = np.sin(out)  # (H, W, D/2)
+    emb_cos = np.cos(out)  # (H, W, D/2)
+
+    emb = np.concatenate([emb_sin, emb_cos], axis=-1)  # (H, W, D)
+    return emb
+
+
+class Resampler(nn.Module):
+    """
+    A 2D perceiver-resampler network with one cross attention layers by
+       given learnable queries and 2d sincos pos_emb
+    Outputs:
+        A tensor with the shape of (batch_size, num_queries, embed_dim)
+    """
+
+    def __init__(
+        self,
+        num_queries,
+        embed_dim,
+        num_heads,
+        kv_dim=None,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6),
+        adaptive=False,
+        max_size=(70, 70),
+    ):
+        super().__init__()
+        self.num_queries = num_queries
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.adaptive = adaptive
+        self.max_size = max_size
+
+        self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
+
+        if kv_dim is not None and kv_dim != embed_dim:
+            self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
+        else:
+            self.kv_proj = nn.Identity()
+
+        self.attn = MultiheadAttention(embed_dim, num_heads)
+        self.ln_q = norm_layer(embed_dim)
+        self.ln_kv = norm_layer(embed_dim)
+
+        self.ln_post = norm_layer(embed_dim)
+        self.proj = nn.Parameter((embed_dim**-0.5) * torch.randn(embed_dim, embed_dim))
+
+        self._set_2d_pos_cache(self.max_size)
+
+    def _set_2d_pos_cache(self, max_size, device="cpu"):
+        if is_deepspeed_zero3_enabled():
+            device = "cuda"
+        pos_embed = torch.from_numpy(get_2d_sincos_pos_embed(self.embed_dim, max_size)).float().to(device)
+        self.register_buffer("pos_embed", pos_embed, persistent=False)
+
+    def _adjust_pos_cache(self, tgt_sizes, device):
+        max_h = torch.max(tgt_sizes[:, 0])
+        max_w = torch.max(tgt_sizes[:, 1])
+        if max_h > self.max_size[0] or max_w > self.max_size[1]:
+            self.max_size = [max(max_h, self.max_size[0]), max(max_w, self.max_size[1])]
+            self._set_2d_pos_cache(self.max_size, device)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=0.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    def _initialize_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=0.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+    def forward(self, x, tgt_sizes=None):
+        assert x.shape[0] == tgt_sizes.shape[0]
+        bs = x.shape[0]
+
+        device = x.device
+        dtype = x.dtype
+
+        patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]
+
+        self._adjust_pos_cache(tgt_sizes, device=device)
+
+        max_patch_len = torch.max(patch_len)
+        key_padding_mask = torch.zeros((bs, max_patch_len), dtype=torch.bool, device=device)
+
+        pos_embed = []
+        for i in range(bs):
+            tgt_h, tgt_w = tgt_sizes[i]
+            pos_embed.append(self.pos_embed[:tgt_h, :tgt_w, :].reshape((tgt_h * tgt_w, -1)).to(dtype))  # patches * D
+            key_padding_mask[i, patch_len[i] :] = True
+
+        pos_embed = torch.nn.utils.rnn.pad_sequence(pos_embed, batch_first=True, padding_value=0.0).permute(
+            1, 0, 2
+        )  # BLD => L * B * D
+
+        x = self.kv_proj(x)  # B * L * D
+        x = self.ln_kv(x).permute(1, 0, 2)  # L * B * D
+
+        q = self.ln_q(self.query)  # Q * D
+
+        out = self.attn(
+            self._repeat(q, bs),  # Q * B * D
+            x + pos_embed,  # L * B * D +  L * B * D
+            x,
+            key_padding_mask=key_padding_mask,
+        )[0]
+        #  out: Q * B * D
+        x = out.permute(1, 0, 2)  # B * Q * D
+
+        x = self.ln_post(x)
+        x = x @ self.proj
+        return x
+
+    def _repeat(self, query, N: int):
+        return query.unsqueeze(1).repeat(1, N, 1)
+
+
+class MultiheadAttention(nn.MultiheadAttention):
+    def __init__(
+        self,
+        embed_dim,
+        num_heads,
+        dropout=0.0,
+        bias=True,
+        add_bias_kv=False,
+        add_zero_attn=False,
+        kdim=None,
+        vdim=None,
+        batch_first=False,
+        device=None,
+        dtype=None,
+    ):
+        super().__init__(
+            embed_dim, num_heads, dropout, bias, add_bias_kv, add_zero_attn, kdim, vdim, batch_first, device, dtype
+        )
+
+        # rewrite out_proj layer，with nn.Linear
+        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias, device=device, dtype=dtype)
+
+    def forward(
+        self,
+        query: Tensor,
+        key: Tensor,
+        value: Tensor,
+        key_padding_mask: Optional[Tensor] = None,
+        need_weights: bool = True,
+        attn_mask: Optional[Tensor] = None,
+        average_attn_weights: bool = True,
+        is_causal: bool = False,
+    ) -> Tuple[Tensor, Optional[Tensor]]:
+        why_not_fast_path = ""
+        if (
+            (attn_mask is not None and torch.is_floating_point(attn_mask))
+            or (key_padding_mask is not None)
+            and torch.is_floating_point(key_padding_mask)
+        ):
+            why_not_fast_path = "floating-point masks are not supported for fast path."
+
+        is_batched = query.dim() == 3
+
+        key_padding_mask = _canonical_mask(
+            mask=key_padding_mask,
+            mask_name="key_padding_mask",
+            other_type=F._none_or_dtype(attn_mask),
+            other_name="attn_mask",
+            target_type=query.dtype,
+        )
+
+        attn_mask = _canonical_mask(
+            mask=attn_mask,
+            mask_name="attn_mask",
+            other_type=None,
+            other_name="",
+            target_type=query.dtype,
+            check_other=False,
+        )
+
+        if not is_batched:
+            why_not_fast_path = f"input not batched; expected query.dim() of 3 but got {query.dim()}"
+        elif query is not key or key is not value:
+            # When lifting this restriction, don't forget to either
+            # enforce that the dtypes all match or test cases where
+            # they don't!
+            why_not_fast_path = "non-self attention was used (query, key, and value are not the same Tensor)"
+        elif self.in_proj_bias is not None and query.dtype != self.in_proj_bias.dtype:
+            why_not_fast_path = (
+                f"dtypes of query ({query.dtype}) and self.in_proj_bias ({self.in_proj_bias.dtype}) don't match"
+            )
+        elif self.in_proj_weight is None:
+            why_not_fast_path = "in_proj_weight was None"
+        elif query.dtype != self.in_proj_weight.dtype:
+            # this case will fail anyway, but at least they'll get a useful error message.
+            why_not_fast_path = (
+                f"dtypes of query ({query.dtype}) and self.in_proj_weight ({self.in_proj_weight.dtype}) don't match"
+            )
+        elif self.training:
+            why_not_fast_path = "training is enabled"
+        elif (self.num_heads % 2) != 0:
+            why_not_fast_path = "self.num_heads is not even"
+        elif not self.batch_first:
+            why_not_fast_path = "batch_first was not True"
+        elif self.bias_k is not None:
+            why_not_fast_path = "self.bias_k was not None"
+        elif self.bias_v is not None:
+            why_not_fast_path = "self.bias_v was not None"
+        elif self.add_zero_attn:
+            why_not_fast_path = "add_zero_attn was enabled"
+        elif not self._qkv_same_embed_dim:
+            why_not_fast_path = "_qkv_same_embed_dim was not True"
+        elif query.is_nested and (key_padding_mask is not None or attn_mask is not None):
+            why_not_fast_path = "supplying both src_key_padding_mask and src_mask at the same time \
+                                 is not supported with NestedTensor input"
+        elif torch.is_autocast_enabled():
+            why_not_fast_path = "autocast is enabled"
+
+        if not why_not_fast_path:
+            tensor_args = (
+                query,
+                key,
+                value,
+                self.in_proj_weight,
+                self.in_proj_bias,
+                self.out_proj.weight,
+                self.out_proj.bias,
+            )
+            # We have to use list comprehensions below because TorchScript does not support
+            # generator expressions.
+            if torch.overrides.has_torch_function(tensor_args):
+                why_not_fast_path = "some Tensor argument has_torch_function"
+            elif _is_make_fx_tracing():
+                why_not_fast_path = "we are running make_fx tracing"
+            elif not all(_check_arg_device(x) for x in tensor_args):
+                why_not_fast_path = (
+                    "some Tensor argument's device is neither one of "
+                    f"cpu, cuda or {torch.utils.backend_registration._privateuse1_backend_name}"
+                )
+            elif torch.is_grad_enabled() and any(_arg_requires_grad(x) for x in tensor_args):
+                why_not_fast_path = (
+                    "grad is enabled and at least one of query or the "
+                    "input/output projection weights or biases requires_grad"
+                )
+            if not why_not_fast_path:
+                merged_mask, mask_type = self.merge_masks(attn_mask, key_padding_mask, query)
+
+                if self.in_proj_bias is not None and self.in_proj_weight is not None:
+                    return torch._native_multi_head_attention(
+                        query,
+                        key,
+                        value,
+                        self.embed_dim,
+                        self.num_heads,
+                        self.in_proj_weight,
+                        self.in_proj_bias,
+                        self.out_proj.weight,
+                        self.out_proj.bias,
+                        merged_mask,
+                        need_weights,
+                        average_attn_weights,
+                        mask_type,
+                    )
+
+        any_nested = query.is_nested or key.is_nested or value.is_nested
+        assert not any_nested, (
+            "MultiheadAttention does not support NestedTensor outside of its fast path. "
+            + f"The fast path was not hit because {why_not_fast_path}"
+        )
+
+        if self.batch_first and is_batched:
+            # make sure that the transpose op does not affect the "is" property
+            if key is value:
+                if query is key:
+                    query = key = value = query.transpose(1, 0)
+                else:
+                    query, key = (x.transpose(1, 0) for x in (query, key))
+                    value = key
+            else:
+                query, key, value = (x.transpose(1, 0) for x in (query, key, value))
+
+        if not self._qkv_same_embed_dim:
+            attn_output, attn_output_weights = self.multi_head_attention_forward(
+                query,
+                key,
+                value,
+                self.embed_dim,
+                self.num_heads,
+                self.in_proj_weight,
+                self.in_proj_bias,
+                self.bias_k,
+                self.bias_v,
+                self.add_zero_attn,
+                self.dropout,
+                self.out_proj.weight,
+                self.out_proj.bias,
+                training=self.training,
+                key_padding_mask=key_padding_mask,
+                need_weights=need_weights,
+                attn_mask=attn_mask,
+                use_separate_proj_weight=True,
+                q_proj_weight=self.q_proj_weight,
+                k_proj_weight=self.k_proj_weight,
+                v_proj_weight=self.v_proj_weight,
+                average_attn_weights=average_attn_weights,
+                is_causal=is_causal,
+            )
+        else:
+            attn_output, attn_output_weights = self.multi_head_attention_forward(
+                query,
+                key,
+                value,
+                self.embed_dim,
+                self.num_heads,
+                self.in_proj_weight,
+                self.in_proj_bias,
+                self.bias_k,
+                self.bias_v,
+                self.add_zero_attn,
+                self.dropout,
+                self.out_proj.weight,
+                self.out_proj.bias,
+                training=self.training,
+                key_padding_mask=key_padding_mask,
+                need_weights=need_weights,
+                attn_mask=attn_mask,
+                average_attn_weights=average_attn_weights,
+                is_causal=is_causal,
+            )
+        if self.batch_first and is_batched:
+            return attn_output.transpose(1, 0), attn_output_weights
+        else:
+            return attn_output, attn_output_weights
+
+    def multi_head_attention_forward(
+        self,
+        query: Tensor,
+        key: Tensor,
+        value: Tensor,
+        embed_dim_to_check: int,
+        num_heads: int,
+        in_proj_weight: Optional[Tensor],
+        in_proj_bias: Optional[Tensor],
+        bias_k: Optional[Tensor],
+        bias_v: Optional[Tensor],
+        add_zero_attn: bool,
+        dropout_p: float,
+        out_proj_weight: Tensor,
+        out_proj_bias: Optional[Tensor],
+        training: bool = True,
+        key_padding_mask: Optional[Tensor] = None,
+        need_weights: bool = True,
+        attn_mask: Optional[Tensor] = None,
+        use_separate_proj_weight: bool = False,
+        q_proj_weight: Optional[Tensor] = None,
+        k_proj_weight: Optional[Tensor] = None,
+        v_proj_weight: Optional[Tensor] = None,
+        static_k: Optional[Tensor] = None,
+        static_v: Optional[Tensor] = None,
+        average_attn_weights: bool = True,
+        is_causal: bool = False,
+    ) -> Tuple[Tensor, Optional[Tensor]]:
+        tens_ops = (query, key, value, in_proj_weight, in_proj_bias, bias_k, bias_v, out_proj_weight, out_proj_bias)
+
+        is_batched = _mha_shape_check(query, key, value, key_padding_mask, attn_mask, num_heads)
+
+        # For unbatched input, we unsqueeze at the expected batch-dim to pretend that the input
+        # is batched, run the computation and before returning squeeze the
+        # batch dimension so that the output doesn't carry this temporary batch dimension.
+        if not is_batched:
+            # unsqueeze if the input is unbatched
+            query = query.unsqueeze(1)
+            key = key.unsqueeze(1)
+            value = value.unsqueeze(1)
+            if key_padding_mask is not None:
+                key_padding_mask = key_padding_mask.unsqueeze(0)
+
+        # set up shape vars
+        tgt_len, bsz, embed_dim = query.shape
+        src_len, _, _ = key.shape
+
+        key_padding_mask = _canonical_mask(
+            mask=key_padding_mask,
+            mask_name="key_padding_mask",
+            other_type=F._none_or_dtype(attn_mask),
+            other_name="attn_mask",
+            target_type=query.dtype,
+        )
+
+        if is_causal and attn_mask is None:
+            raise RuntimeError(
+                "Need attn_mask if specifying the is_causal hint. "
+                "You may use the Transformer module method "
+                "`generate_square_subsequent_mask` to create this mask."
+            )
+
+        if is_causal and key_padding_mask is None and not need_weights:
+            # when we have a kpm or need weights, we need attn_mask
+            # Otherwise, we use the is_causal hint go as is_causal
+            # indicator to SDPA.
+            attn_mask = None
+        else:
+            attn_mask = _canonical_mask(
+                mask=attn_mask,
+                mask_name="attn_mask",
+                other_type=None,
+                other_name="",
+                target_type=query.dtype,
+                check_other=False,
+            )
+
+            if key_padding_mask is not None:
+                # We have the attn_mask, and use that to merge kpm into it.
+                # Turn off use of is_causal hint, as the merged mask is no
+                # longer causal.
+                is_causal = False
+
+        assert (
+            embed_dim == embed_dim_to_check
+        ), f"was expecting embedding dimension of {embed_dim_to_check}, but got {embed_dim}"
+        if isinstance(embed_dim, torch.Tensor):
+            # embed_dim can be a tensor when JIT tracing
+            head_dim = embed_dim.div(num_heads, rounding_mode="trunc")
+        else:
+            head_dim = embed_dim // num_heads
+        assert head_dim * num_heads == embed_dim, f"embed_dim {embed_dim} not divisible by num_heads {num_heads}"
+        if use_separate_proj_weight:
+            # allow MHA to have different embedding dimensions when separate projection weights are used
+            assert (
+                key.shape[:2] == value.shape[:2]
+            ), f"key's sequence and batch dims {key.shape[:2]} do not match value's {value.shape[:2]}"
+        else:
+            assert key.shape == value.shape, f"key shape {key.shape} does not match value shape {value.shape}"
+
+        #
+        # compute in-projection
+        #
+        if not use_separate_proj_weight:
+            assert in_proj_weight is not None, "use_separate_proj_weight is False but in_proj_weight is None"
+            q, k, v = _in_projection_packed(query, key, value, in_proj_weight, in_proj_bias)
+        else:
+            assert q_proj_weight is not None, "use_separate_proj_weight is True but q_proj_weight is None"
+            assert k_proj_weight is not None, "use_separate_proj_weight is True but k_proj_weight is None"
+            assert v_proj_weight is not None, "use_separate_proj_weight is True but v_proj_weight is None"
+            if in_proj_bias is None:
+                b_q = b_k = b_v = None
+            else:
+                b_q, b_k, b_v = in_proj_bias.chunk(3)
+            q, k, v = _in_projection(query, key, value, q_proj_weight, k_proj_weight, v_proj_weight, b_q, b_k, b_v)
+
+        # prep attention mask
+
+        if attn_mask is not None:
+            # ensure attn_mask's dim is 3
+            if attn_mask.dim() == 2:
+                correct_2d_size = (tgt_len, src_len)
+                if attn_mask.shape != correct_2d_size:
+                    raise RuntimeError(
+                        f"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}."
+                    )
+                attn_mask = attn_mask.unsqueeze(0)
+            elif attn_mask.dim() == 3:
+                correct_3d_size = (bsz * num_heads, tgt_len, src_len)
+                if attn_mask.shape != correct_3d_size:
+                    raise RuntimeError(
+                        f"The shape of the 3D attn_mask is {attn_mask.shape}, but should be {correct_3d_size}."
+                    )
+            else:
+                raise RuntimeError(f"attn_mask's dimension {attn_mask.dim()} is not supported")
+
+        # add bias along batch dimension (currently second)
+        if bias_k is not None and bias_v is not None:
+            assert static_k is None, "bias cannot be added to static key."
+            assert static_v is None, "bias cannot be added to static value."
+            k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
+            v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
+            if attn_mask is not None:
+                attn_mask = pad(attn_mask, (0, 1))
+            if key_padding_mask is not None:
+                key_padding_mask = pad(key_padding_mask, (0, 1))
+        else:
+            assert bias_k is None
+            assert bias_v is None
+
+        #
+        # reshape q, k, v for multihead attention and make em batch first
+        #
+        q = q.view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1)
+        if static_k is None:
+            k = k.view(k.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
+        else:
+            # TODO finish disentangling control flow so we don't do in-projections when statics are passed
+            assert (
+                static_k.size(0) == bsz * num_heads
+            ), f"expecting static_k.size(0) of {bsz * num_heads}, but got {static_k.size(0)}"
+            assert static_k.size(2) == head_dim, f"expecting static_k.size(2) of {head_dim}, but got {static_k.size(2)}"
+            k = static_k
+        if static_v is None:
+            v = v.view(v.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
+        else:
+            # TODO finish disentangling control flow so we don't do in-projections when statics are passed
+            assert (
+                static_v.size(0) == bsz * num_heads
+            ), f"expecting static_v.size(0) of {bsz * num_heads}, but got {static_v.size(0)}"
+            assert static_v.size(2) == head_dim, f"expecting static_v.size(2) of {head_dim}, but got {static_v.size(2)}"
+            v = static_v
+
+        # add zero attention along batch dimension (now first)
+        if add_zero_attn:
+            zero_attn_shape = (bsz * num_heads, 1, head_dim)
+            k = torch.cat([k, torch.zeros(zero_attn_shape, dtype=k.dtype, device=k.device)], dim=1)
+            v = torch.cat([v, torch.zeros(zero_attn_shape, dtype=v.dtype, device=v.device)], dim=1)
+            if attn_mask is not None:
+                attn_mask = pad(attn_mask, (0, 1))
+            if key_padding_mask is not None:
+                key_padding_mask = pad(key_padding_mask, (0, 1))
+
+        # update source sequence length after adjustments
+        src_len = k.size(1)
+
+        # merge key padding and attention masks
+        if key_padding_mask is not None:
+            assert key_padding_mask.shape == (
+                bsz,
+                src_len,
+            ), f"expecting key_padding_mask shape of {(bsz, src_len)}, but got {key_padding_mask.shape}"
+            key_padding_mask = (
+                key_padding_mask.view(bsz, 1, 1, src_len)
+                .expand(-1, num_heads, -1, -1)
+                .reshape(bsz * num_heads, 1, src_len)
+            )
+            if attn_mask is None:
+                attn_mask = key_padding_mask
+            else:
+                attn_mask = attn_mask + key_padding_mask
+
+        # adjust dropout probability
+        if not training:
+            dropout_p = 0.0
+
+        #
+        # (deep breath) calculate attention and out projection
+        #
+
+        if need_weights:
+            B, Nt, E = q.shape
+            q_scaled = q / math.sqrt(E)
+
+            assert not (is_causal and attn_mask is None), "FIXME: is_causal not implemented for need_weights"
+
+            if attn_mask is not None:
+                attn_output_weights = torch.baddbmm(attn_mask, q_scaled, k.transpose(-2, -1))
+            else:
+                attn_output_weights = torch.bmm(q_scaled, k.transpose(-2, -1))
+            attn_output_weights = softmax(attn_output_weights, dim=-1)
+            if dropout_p > 0.0:
+                attn_output_weights = dropout(attn_output_weights, p=dropout_p)
+
+            attn_output = torch.bmm(attn_output_weights, v)
+
+            attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len * bsz, embed_dim)
+            attn_output = self.out_proj(attn_output)
+            attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
+
+            # optionally average attention weights over heads
+            attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
+            if average_attn_weights:
+                attn_output_weights = attn_output_weights.mean(dim=1)
+
+            if not is_batched:
+                # squeeze the output if input was unbatched
+                attn_output = attn_output.squeeze(1)
+                attn_output_weights = attn_output_weights.squeeze(0)
+            return attn_output, attn_output_weights
+        else:
+            # attn_mask can be either (L,S) or (N*num_heads, L, S)
+            # if attn_mask's shape is (1, L, S) we need to unsqueeze to (1, 1, L, S)
+            # in order to match the input for SDPA of (N, num_heads, L, S)
+            if attn_mask is not None:
+                if attn_mask.size(0) == 1 and attn_mask.dim() == 3:
+                    attn_mask = attn_mask.unsqueeze(0)
+                else:
+                    attn_mask = attn_mask.view(bsz, num_heads, -1, src_len)
+
+            q = q.view(bsz, num_heads, tgt_len, head_dim)
+            k = k.view(bsz, num_heads, src_len, head_dim)
+            v = v.view(bsz, num_heads, src_len, head_dim)
+
+            attn_output = F.scaled_dot_product_attention(q, k, v, attn_mask, dropout_p, is_causal)
+            attn_output = attn_output.permute(2, 0, 1, 3).contiguous().view(bsz * tgt_len, embed_dim)
+
+            attn_output = self.out_proj(attn_output)
+            attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
+            if not is_batched:
+                # squeeze the output if input was unbatched
+                attn_output = attn_output.squeeze(1)
+            return attn_output, None
+
+
+def _mha_shape_check(
+    query: Tensor,
+    key: Tensor,
+    value: Tensor,
+    key_padding_mask: Optional[Tensor],
+    attn_mask: Optional[Tensor],
+    num_heads: int,
+):
+    # Verifies the expected shape for `query, `key`, `value`, `key_padding_mask` and `attn_mask`
+    # and returns if the input is batched or not.
+    # Raises an error if `query` is not 2-D (unbatched) or 3-D (batched) tensor.
+
+    # Shape check.
+    if query.dim() == 3:
+        # Batched Inputs
+        is_batched = True
+        assert key.dim() == 3 and value.dim() == 3, (
+            "For batched (3-D) `query`, expected `key` and `value` to be 3-D"
+            f" but found {key.dim()}-D and {value.dim()}-D tensors respectively"
+        )
+        if key_padding_mask is not None:
+            assert key_padding_mask.dim() == 2, (
+                "For batched (3-D) `query`, expected `key_padding_mask` to be `None` or 2-D"
+                f" but found {key_padding_mask.dim()}-D tensor instead"
+            )
+        if attn_mask is not None:
+            assert attn_mask.dim() in (2, 3), (
+                "For batched (3-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D"
+                f" but found {attn_mask.dim()}-D tensor instead"
+            )
+    elif query.dim() == 2:
+        # Unbatched Inputs
+        is_batched = False
+        assert key.dim() == 2 and value.dim() == 2, (
+            "For unbatched (2-D) `query`, expected `key` and `value` to be 2-D"
+            f" but found {key.dim()}-D and {value.dim()}-D tensors respectively"
+        )
+
+        if key_padding_mask is not None:
+            assert key_padding_mask.dim() == 1, (
+                "For unbatched (2-D) `query`, expected `key_padding_mask` to be `None` or 1-D"
+                f" but found {key_padding_mask.dim()}-D tensor instead"
+            )
+
+        if attn_mask is not None:
+            assert attn_mask.dim() in (2, 3), (
+                "For unbatched (2-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D"
+                f" but found {attn_mask.dim()}-D tensor instead"
+            )
+            if attn_mask.dim() == 3:
+                expected_shape = (num_heads, query.shape[0], key.shape[0])
+                assert (
+                    attn_mask.shape == expected_shape
+                ), f"Expected `attn_mask` shape to be {expected_shape} but got {attn_mask.shape}"
+    else:
+        raise AssertionError(
+            f"query should be unbatched 2D or batched 3D tensor but received {query.dim()}-D query tensor"
+        )
+
+    return is_batched
+
+
+def _canonical_mask(
+    mask: Optional[Tensor],
+    mask_name: str,
+    other_type: Optional[DType],
+    other_name: str,
+    target_type: DType,
+    check_other: bool = True,
+) -> Optional[Tensor]:
+
+    if mask is not None:
+        _mask_dtype = mask.dtype
+        _mask_is_float = torch.is_floating_point(mask)
+        if _mask_dtype != torch.bool and not _mask_is_float:
+            raise AssertionError(f"only bool and floating types of {mask_name} are supported")
+        if check_other and other_type is not None:
+            if _mask_dtype != other_type:
+                warnings.warn(
+                    f"Support for mismatched {mask_name} and {other_name} "
+                    "is deprecated. Use same type for both instead."
+                )
+        if not _mask_is_float:
+            mask = torch.zeros_like(mask, dtype=target_type).masked_fill_(mask, float("-inf"))
+    return mask
+
+
+def _in_projection_packed(
+    q: Tensor,
+    k: Tensor,
+    v: Tensor,
+    w: Tensor,
+    b: Optional[Tensor] = None,
+) -> List[Tensor]:
+    r"""
+    Performs the in-projection step of the attention operation, using packed weights.
+    Output is a triple containing projection tensors for query, key and value.
+    Args:
+        q, k, v: query, key and value tensors to be projected. For self-attention,
+            these are typically the same tensor; for encoder-decoder attention,
+            k and v are typically the same tensor. (We take advantage of these
+            identities for performance if they are present.) Regardless, q, k and v
+            must share a common embedding dimension; otherwise their shapes may vary.
+        w: projection weights for q, k and v, packed into a single tensor. Weights
+            are packed along dimension 0, in q, k, v order.
+        b: optional projection biases for q, k and v, packed into a single tensor
+            in q, k, v order.
+    Shape:
+        Inputs:
+        - q: :math:`(..., E)` where E is the embedding dimension
+        - k: :math:`(..., E)` where E is the embedding dimension
+        - v: :math:`(..., E)` where E is the embedding dimension
+        - w: :math:`(E * 3, E)` where E is the embedding dimension
+        - b: :math:`E * 3` where E is the embedding dimension
+        Output:
+        - in output list :math:`[q', k', v']`, each output tensor will have the
+            same shape as the corresponding input tensor.
+    """
+    E = q.size(-1)
+    if k is v:
+        if q is k:
+            # self-attention
+            proj = linear(q, w, b)
+            # reshape to 3, E and not E, 3 is deliberate for better memory coalescing and keeping same order as chunk()
+            proj = proj.unflatten(-1, (3, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()
+            return proj[0], proj[1], proj[2]
+        else:
+            # encoder-decoder attention
+            w_q, w_kv = w.split([E, E * 2])
+            if b is None:
+                b_q = b_kv = None
+            else:
+                b_q, b_kv = b.split([E, E * 2])
+            q_proj = linear(q, w_q, b_q)
+            kv_proj = linear(k, w_kv, b_kv)
+            # reshape to 2, E and not E, 2 is deliberate for better memory coalescing and keeping same order as chunk()
+            kv_proj = kv_proj.unflatten(-1, (2, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()
+            return (q_proj, kv_proj[0], kv_proj[1])
+    else:
+        w_q, w_k, w_v = w.chunk(3)
+        if b is None:
+            b_q = b_k = b_v = None
+        else:
+            b_q, b_k, b_v = b.chunk(3)
+        return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)
+
+
+def _in_projection(
+    q: Tensor,
+    k: Tensor,
+    v: Tensor,
+    w_q: Tensor,
+    w_k: Tensor,
+    w_v: Tensor,
+    b_q: Optional[Tensor] = None,
+    b_k: Optional[Tensor] = None,
+    b_v: Optional[Tensor] = None,
+) -> Tuple[Tensor, Tensor, Tensor]:
+    r"""
+    Performs the in-projection step of the attention operation. This is simply
+    a triple of linear projections, with shape constraints on the weights which
+    ensure embedding dimension uniformity in the projected outputs.
+    Output is a triple containing projection tensors for query, key and value.
+    Args:
+        q, k, v: query, key and value tensors to be projected.
+        w_q, w_k, w_v: weights for q, k and v, respectively.
+        b_q, b_k, b_v: optional biases for q, k and v, respectively.
+    Shape:
+        Inputs:
+        - q: :math:`(Qdims..., Eq)` where Eq is the query embedding dimension and Qdims are any
+            number of leading dimensions.
+        - k: :math:`(Kdims..., Ek)` where Ek is the key embedding dimension and Kdims are any
+            number of leading dimensions.
+        - v: :math:`(Vdims..., Ev)` where Ev is the value embedding dimension and Vdims are any
+            number of leading dimensions.
+        - w_q: :math:`(Eq, Eq)`
+        - w_k: :math:`(Eq, Ek)`
+        - w_v: :math:`(Eq, Ev)`
+        - b_q: :math:`(Eq)`
+        - b_k: :math:`(Eq)`
+        - b_v: :math:`(Eq)`
+        Output: in output triple :math:`(q', k', v')`,
+         - q': :math:`[Qdims..., Eq]`
+         - k': :math:`[Kdims..., Eq]`
+         - v': :math:`[Vdims..., Eq]`
+    """
+    Eq, Ek, Ev = q.size(-1), k.size(-1), v.size(-1)
+    assert w_q.shape == (Eq, Eq), f"expecting query weights shape of {(Eq, Eq)}, but got {w_q.shape}"
+    assert w_k.shape == (Eq, Ek), f"expecting key weights shape of {(Eq, Ek)}, but got {w_k.shape}"
+    assert w_v.shape == (Eq, Ev), f"expecting value weights shape of {(Eq, Ev)}, but got {w_v.shape}"
+    assert b_q is None or b_q.shape == (Eq,), f"expecting query bias shape of {(Eq,)}, but got {b_q.shape}"
+    assert b_k is None or b_k.shape == (Eq,), f"expecting key bias shape of {(Eq,)}, but got {b_k.shape}"
+    assert b_v is None or b_v.shape == (Eq,), f"expecting value bias shape of {(Eq,)}, but got {b_v.shape}"
+    return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)

Chat/special_tokens_map.json

@@ -0,0 +1,264 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<point>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</point>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<slice>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</slice>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<image_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</image_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<unit>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</unit>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<asr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</asr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<query>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</query>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|audio_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|audio_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|spk_bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|spk|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|spk_eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tts_bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tts_eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|listen|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|speak|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|interrupt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|vad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|vad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<reserved_43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<reserved_53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": "<unk>"
+}

Chat/tokenization_minicpmo_fast.py

@@ -0,0 +1,110 @@
+# coding=utf-8
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from transformers import Qwen2TokenizerFast
+
+
+class MiniCPMOTokenizerFast(Qwen2TokenizerFast):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        # image
+        self.im_start = "<image>"
+        self.im_end = "</image>"
+        self.ref_start = "<ref>"
+        self.ref_end = "</ref>"
+        self.box_start = "<box>"
+        self.box_end = "</box>"
+        self.quad_start = "<quad>"
+        self.quad_end = "</quad>"
+        self.slice_start = "<slice>"
+        self.slice_end = "</slice>"
+        self.im_id_start = "<image_id>"
+        self.im_id_end = "</image_id>"
+
+        # audio
+        self.audio_start = "<|audio_start|>"
+        self.audio_end = "<|audio_end|>"
+        self.spk_start = "<|spk_bos|>"
+        self.spk_end = "<|spk_eos|>"
+        self.tts_start = "<|tts_bos|>"
+        self.tts_end = "<|tts_eos|>"
+
+    @property
+    def eos_id(self):
+        return self.eos_token_id
+
+    @property
+    def bos_id(self):
+        return self.bos_token_id
+
+    @property
+    def unk_id(self):
+        return self.unk_token_id
+
+    @property
+    def im_start_id(self):
+        return self.convert_tokens_to_ids(self.im_start)
+
+    @property
+    def im_end_id(self):
+        return self.convert_tokens_to_ids(self.im_end)
+
+    @property
+    def slice_start_id(self):
+        return self.convert_tokens_to_ids(self.slice_start)
+
+    @property
+    def slice_end_id(self):
+        return self.convert_tokens_to_ids(self.slice_end)
+
+    @property
+    def im_id_start_id(self):
+        return self.convert_tokens_to_ids(self.im_id_start)
+
+    @property
+    def im_id_end_id(self):
+        return self.convert_tokens_to_ids(self.im_id_end)
+
+    @property
+    def audio_start_id(self):
+        return self.convert_tokens_to_ids(self.audio_start)
+
+    @property
+    def audio_end_id(self):
+        return self.convert_tokens_to_ids(self.audio_end)
+
+    @property
+    def spk_start_id(self):
+        return self.convert_tokens_to_ids(self.spk_start)
+
+    @property
+    def spk_end_id(self):
+        return self.convert_tokens_to_ids(self.spk_end)
+
+    @property
+    def tts_start_id(self):
+        return self.convert_tokens_to_ids(self.tts_start)
+
+    @property
+    def tts_end_id(self):
+        return self.convert_tokens_to_ids(self.tts_end)
+
+    @staticmethod
+    def escape(text: str) -> str:
+        return text
+
+    @staticmethod
+    def unescape(text: str) -> str:
+        return text

Chat/tokenizer_config.json

@@ -0,0 +1,523 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "128244": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "</image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "</ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "<box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "</box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "</quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<point>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "</point>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<slice>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151676": {
+      "content": "</slice>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151677": {
+      "content": "<image_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151678": {
+      "content": "</image_id>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151679": {
+      "content": "<unit>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151680": {
+      "content": "</unit>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151681": {
+      "content": "<asr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151682": {
+      "content": "</asr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151683": {
+      "content": "<query>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151684": {
+      "content": "</query>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151685": {
+      "content": "<|audio_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151686": {
+      "content": "<|audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151687": {
+      "content": "<|audio_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151688": {
+      "content": "<|spk_bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151689": {
+      "content": "<|spk|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151690": {
+      "content": "<|spk_eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151691": {
+      "content": "<|tts_bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151692": {
+      "content": "<|tts_eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151693": {
+      "content": "<|listen|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151694": {
+      "content": "<|speak|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151695": {
+      "content": "<|interrupt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151696": {
+      "content": "<|vad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151697": {
+      "content": "<|vad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151698": {
+      "content": "<reserved_43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151699": {
+      "content": "<reserved_53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<image>",
+    "</image>",
+    "<ref>",
+    "</ref>",
+    "<box>",
+    "</box>",
+    "<quad>",
+    "</quad>",
+    "<point>",
+    "</point>",
+    "<slice>",
+    "</slice>",
+    "<image_id>",
+    "</image_id>",
+    "<unit>",
+    "</unit>",
+    "<asr>",
+    "</asr>",
+    "<query>",
+    "</query>",
+    "<|audio_start|>",
+    "<|audio|>",
+    "<|audio_end|>",
+    "<|spk_bos|>",
+    "<|spk|>",
+    "<|spk_eos|>",
+    "<|tts_bos|>",
+    "<|tts_eos|>",
+    "<|listen|>",
+    "<|speak|>",
+    "<|interrupt|>",
+    "<|vad_start|>",
+    "<|vad_end|>",
+    "<reserved_43>",
+    "<reserved_53>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_minicpmo_fast.MiniCPMOTokenizerFast",
+      null
+    ]
+  },
+  "tokenizer_class": "MiniCPMOTokenizerFast",
+  "unk_token": "<unk>"
+}

Chat/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51fd9e358109aa6e551299abf37cc4d2352c063b82d162387970f6402a9bcbb2
+size 7800

Chat/utils.py

@@ -0,0 +1,203 @@
+# coding=utf-8
+# Copyright 2025 The OpenBMB Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+
+import librosa
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+
+def is_silent(data):
+    if np.abs(data).max() < 3e-3:
+        return True
+    else:
+        return False
+
+
+def sentence_end(txt):
+    for c in [".", "。", "!", "?", "！", "？"]:
+        if c in txt:
+            if c == ".":  # check not number before it like 1.
+                idx = txt.find(c)
+                if idx > 0:
+                    if txt[idx - 1].isdigit():
+                        continue
+            return c
+    return ""
+
+
+class NumberToTextConverter:
+    r"""
+    A helper class to ensure text-to-speech (TTS) systems read numeric digits
+    in the desired language (Chinese or English) digit-by-digit. It forcibly
+    replaces all numeric substrings in text with their language-specific
+    textual representations, thereby reducing the likelihood of TTS mistakes
+    on numbers.
+    Note: MiniCPM-o 2.6 only use this in streaming mode.
+
+    Attributes:
+        num_to_chinese (dict):
+            Mapping from digit (str) to its Chinese textual form (str).
+        num_to_english (dict):
+            Mapping from digit (str) to its English textual form (str).
+
+    Example:
+        >>> converter = NumberToTextConverter()
+        >>> converter.replace_numbers_with_text("我有2个苹果", language="chinese")
+        '我有两个苹果'
+        >>> converter.replace_numbers_with_text("I have 23 books", language="english")
+        'I have two three books'
+    """
+
+    def __init__(self):
+        self.num_to_chinese = {
+            "0": "零",
+            "1": "一",
+            "2": "二",
+            "3": "三",
+            "4": "四",
+            "5": "五",
+            "6": "六",
+            "7": "七",
+            "8": "八",
+            "9": "九",
+        }
+        self.num_to_english = {
+            "0": "zero",
+            "1": "one",
+            "2": "two",
+            "3": "three",
+            "4": "four",
+            "5": "five",
+            "6": "six",
+            "7": "seven",
+            "8": "eight",
+            "9": "nine",
+        }
+
+    def number_to_chinese_digit_by_digit(self, num_str):
+        result = ""
+        for char in num_str:
+            if char in self.num_to_chinese:
+                result += self.num_to_chinese[char]
+        return result
+
+    def number_to_english_digit_by_digit(self, num_str):
+        result = []
+        for char in num_str:
+            if char in self.num_to_english:
+                result.append(self.num_to_english[char])
+        return " ".join(result)
+
+    def detect_language(self, text):
+        chinese_count = len(re.findall(r"[\u4e00-\u9fff]", text))
+        english_count = len(re.findall(r"[a-zA-Z]", text))
+        return "chinese" if chinese_count >= english_count else "english"
+
+    def replace_numbers_with_text(self, text, language=None):
+        if language is None:
+            language = self.detect_language(text)
+        numbers = re.findall(r"\d+", text)
+
+        for num in numbers:
+            if language == "chinese":
+                replacement = self.number_to_chinese_digit_by_digit(num)
+            else:
+                replacement = self.number_to_english_digit_by_digit(num)
+            text = text.replace(num, replacement, 1)
+
+        return text
+
+
+class VoiceChecker:
+    r"""
+    A simple utility class to detect silence or low variation in consecutive audio chunks by comparing
+    the mel-spectrogram distances. It keeps track of consecutive zero-distance and low-distance chunks
+    to decide if the audio is considered "bad" (e.g., overly silent or not changing enough).
+
+    Attributes:
+        previous_mel (`np.ndarray` or `None`):
+            Holds the previously observed mel-spectrogram in decibel scale. Used to compute
+            the next distance; reset via :meth:`reset`.
+        consecutive_zeros (`int`):
+            The number of consecutive chunks that were detected as silent (distance = 0).
+        consecutive_low_distance (`int`):
+            The number of consecutive chunks whose distance was below the threshold.
+
+    Example:
+        >>> checker = VoiceChecker()
+        >>> # Suppose we have audio_wav (list or np.ndarray) and mel_spec (np.ndarray)
+        >>> # We split them into chunks and call checker.is_bad(...)
+        >>> is_audio_bad = checker.is_bad(audio_wav, mel_spec, chunk_size=2560, thresh=100.0)
+        >>> if is_audio_bad:
+        ...     print("Audio deemed bad!")
+        >>> # Reset states if needed
+        >>> checker.reset()
+    """
+
+    def __init__(self):
+        self.previous_mel = None
+        self.consecutive_zeros = 0
+        self.consecutive_low_distance = 0
+
+    def compute_distance(self, audio_chunk, mel_spec):
+        if is_silent(audio_chunk):
+            return 0.0  # 检查是否为空白片段
+
+        mel_db = librosa.power_to_db(mel_spec)
+        if self.previous_mel is None:
+            self.previous_mel = mel_db
+            return -1.0
+
+        distance = np.linalg.norm(np.mean(mel_db, axis=1) - np.mean(self.previous_mel, axis=1))
+        self.previous_mel = mel_db
+        return distance
+
+    def is_bad(self, audio_wav, mel_spec, chunk_size=2560, thresh=100.0):
+        num_chunks = len(audio_wav) // chunk_size
+        mel_chunk_size = mel_spec.shape[-1] // num_chunks
+        for i in range(num_chunks):
+            audio_chunk = audio_wav[i * chunk_size : (i + 1) * chunk_size]
+            mel_spec_chunk = mel_spec[:, i * mel_chunk_size : (i + 1) * mel_chunk_size]
+
+            distance = self.compute_distance(audio_chunk, mel_spec_chunk)
+            logger.warning(
+                f"mel dist: {distance:.1f}, zero: {self.consecutive_zeros}, low: {self.consecutive_low_distance}"
+            )
+            if distance == 0:
+                self.consecutive_low_distance = 0  # reset
+                self.consecutive_zeros += 1
+                if self.consecutive_zeros >= 12:
+                    logger.warning("VoiceChecker detected 1.2 s silent. Marking as failed.")
+                    return True
+            elif distance < thresh:
+                self.consecutive_zeros = 0
+                self.consecutive_low_distance += 1
+                if self.consecutive_low_distance >= 5:
+                    logger.warning("VoiceChecker detected 5 consecutive low distance chunks. Marking as failed.")
+                    return True
+            else:
+                self.consecutive_low_distance = 0
+                self.consecutive_zeros = 0
+
+        return False
+
+    def reset(self):
+        self.previous_mel = None
+        self.consecutive_zeros = 0
+        self.consecutive_low_distance = 0

Chat/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)