upload

config.json

@@ -0,0 +1,50 @@
+{
+  "_name_or_path": "OdysseyAgent",
+  "architectures": [
+    "QWenLMHeadModel"
+  ],
+  "attn_dropout_prob": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_qwen.QWenConfig",
+    "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel"
+  },
+  "bf16": true,
+  "emb_dropout_prob": 0.0,
+  "fp16": false,
+  "fp32": false,
+  "hidden_size": 4096,
+  "his_len": 4,
+  "initializer_range": 0.02,
+  "intermediate_size": 22016,
+  "kv_channels": 128,
+  "layer_norm_epsilon": 1e-06,
+  "max_position_embeddings": 8192,
+  "model_type": "qwen",
+  "no_bias": true,
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "onnx_safe": null,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 1.0,
+  "scale_attn_weights": true,
+  "seq_length": 2048,
+  "tie_word_embeddings": false,
+  "tokenizer_type": "QWenTokenizer",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.32.0",
+  "use_cache": true,
+  "use_dynamic_ntk": true,
+  "use_flash_attn": false,
+  "use_logn_attn": true,
+  "visual": {
+    "heads": 16,
+    "image_size": 448,
+    "image_start_id": 151857,
+    "layers": 48,
+    "mlp_ratio": 4.9231,
+    "output_dim": 4096,
+    "patch_size": 14,
+    "width": 1664
+  },
+  "vocab_size": 151936
+}

configuration_qwen.py

@@ -0,0 +1,65 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from transformers import PretrainedConfig
+
+
+class QWenConfig(PretrainedConfig):
+    model_type = "qwen"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=151936,
+        hidden_size=4096,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        emb_dropout_prob=0.0,
+        attn_dropout_prob=0.0,
+        layer_norm_epsilon=1e-6,
+        initializer_range=0.02,
+        max_position_embeddings=8192,
+        scale_attn_weights=True,
+        use_cache=True,
+        bf16=False,
+        fp16=False,
+        fp32=False,
+        kv_channels=128,
+        rotary_pct=1.0,
+        rotary_emb_base=10000,
+        use_dynamic_ntk=True,
+        use_logn_attn=True,
+        use_flash_attn="auto",
+        intermediate_size=22016,
+        no_bias=True,
+        tie_word_embeddings=False,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.emb_dropout_prob = emb_dropout_prob
+        self.attn_dropout_prob = attn_dropout_prob
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.initializer_range = initializer_range
+        self.scale_attn_weights = scale_attn_weights
+        self.use_cache = use_cache
+        self.max_position_embeddings = max_position_embeddings
+        self.bf16 = bf16
+        self.fp16 = fp16
+        self.fp32 = fp32
+        self.kv_channels = kv_channels
+        self.rotary_pct = rotary_pct
+        self.rotary_emb_base = rotary_emb_base
+        self.use_dynamic_ntk = use_dynamic_ntk
+        self.use_logn_attn = use_logn_attn
+        self.use_flash_attn = use_flash_attn
+        self.no_bias = no_bias
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs
+        )

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.32.0"
+}

modeling_qwen.py

@@ -0,0 +1,1421 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+print('OdysseyAgent')
+
+import importlib
+import math
+from typing import TYPE_CHECKING, Optional, Tuple, Union, Callable, List, Any, Generator
+import os, json
+import numpy as np
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch.cuda.amp import autocast
+
+from torch.nn import CrossEntropyLoss
+from transformers import PreTrainedTokenizer, GenerationConfig, StoppingCriteriaList
+from transformers.generation.logits_process import LogitsProcessorList
+
+if TYPE_CHECKING:
+    from transformers.generation.streamers import BaseStreamer
+from transformers.generation.utils import GenerateOutput
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPast,
+    CausalLMOutputWithPast,
+)
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import logging
+
+try:
+    from einops import rearrange
+except ImportError:
+    rearrange = None
+from torch import nn
+
+SUPPORT_CUDA = torch.cuda.is_available()
+SUPPORT_BF16 = SUPPORT_CUDA and torch.cuda.is_bf16_supported()
+SUPPORT_FP16 = SUPPORT_CUDA and torch.cuda.get_device_capability(0)[0] >= 7
+
+from torch.nn.init import trunc_normal_
+import sys
+sys.path.append('../OdysseyAgent')
+
+from configuration_qwen import QWenConfig
+from qwen_generation_utils import (
+    HistoryType,
+    make_context,
+    decode_tokens,
+    get_stop_words_ids,
+    StopWordsLogitsProcessor,
+)
+from visual import VisionTransformer
+
+IMAGE_HISTORY = '../data/his_index.json'
+
+USE_RESAMPLER = True
+
+print(IMAGE_HISTORY)
+logger = logging.get_logger(__name__)
+
+_CHECKPOINT_FOR_DOC = "qwen"
+_CONFIG_FOR_DOC = "QWenConfig"
+
+QWen_PRETRAINED_MODEL_ARCHIVE_LIST = ["qwen-7b"]
+
+_ERROR_BAD_CHAT_FORMAT = """\
+We detect you are probably using the pretrained model (rather than chat model) for chatting, since the chat_format in generation_config is not "chatml".
+If you are directly using the model downloaded from Huggingface, please make sure you are using our "Qwen/Qwen-7B-Chat" Huggingface model (rather than "Qwen/Qwen-7B") when you call model.chat().
+我们检测到您可能在使用预训练模型（而非chat模型）进行多轮chat，因为您当前在generation_config指定的chat_format，并未设置为我们在对话中所支持的"chatml"格式。
+如果您在直接使用我们从Huggingface提供的模型，请确保您在调用model.chat()时，使用的是"Qwen/Qwen-7B-Chat"模型（而非"Qwen/Qwen-7B"预训练模型）。
+"""
+
+_SENTINEL = object()
+_ERROR_STREAM_IN_CHAT = """\
+Pass argument `stream` to model.chat() is buggy, deprecated, and marked for removal. Please use model.chat_stream(...) instead of model.chat(..., stream=True).
+向model.chat()传入参数stream的用法可能存在Bug，该用法已被废弃，将在未来被移除。请使用model.chat_stream(...)代替model.chat(..., stream=True)。
+"""
+
+apply_rotary_emb_func = None
+rms_norm = None
+
+
+# Copied from transformers.models.bart.modeling_bart._make_causal_mask
+def _make_causal_mask(
+    input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
+):
+    """
+    Make causal mask used for bi-directional self-attention.
+    """
+    bsz, tgt_len = input_ids_shape
+    mask = torch.full((tgt_len, tgt_len), torch.finfo(dtype).min, device=device)
+    mask_cond = torch.arange(mask.size(-1), device=device)
+    mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
+    mask = mask.to(dtype)
+
+    if past_key_values_length > 0:
+        mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
+    return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
+
+
+# Copied from transformers.models.bart.modeling_bart._expand_mask
+def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
+    """
+    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
+    """
+    bsz, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+
+    expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
+
+    inverted_mask = 1.0 - expanded_mask
+
+    return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)
+
+def get_abs_pos(abs_pos, tgt_size):
+    # abs_pos: L, C
+    # tgt_size: M
+    # return: M, C
+    src_size = int(math.sqrt(abs_pos.size(0)))
+    tgt_size = int(math.sqrt(tgt_size))
+    dtype = abs_pos.dtype
+
+    if src_size != tgt_size:
+        return F.interpolate(
+            abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
+            size=(tgt_size, tgt_size),
+            mode="bicubic",
+            align_corners=False,
+        ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype)
+    else:
+        return abs_pos
+
+
+def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
+    """
+    grid_size: int of the grid height and width
+    return:
+    pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
+    """
+    grid_h = np.arange(grid_size, dtype=np.float32)
+    grid_w = np.arange(grid_size, dtype=np.float32)
+    grid = np.meshgrid(grid_w, grid_h)  # here w goes first
+    grid = np.stack(grid, axis=0)
+
+    grid = grid.reshape([2, 1, grid_size, grid_size])
+    pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
+    if cls_token:
+        pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
+    return pos_embed
+
+
+def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
+    assert embed_dim % 2 == 0
+
+    # use half of dimensions to encode grid_h
+    emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0])  # (H*W, D/2)
+    emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1])  # (H*W, D/2)
+
+    emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
+    return emb
+
+
+def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
+    """
+    embed_dim: output dimension for each position
+    pos: a list of positions to be encoded: size (M,)
+    out: (M, D)
+    """
+    assert embed_dim % 2 == 0
+    omega = np.arange(embed_dim // 2, dtype=np.float32)
+    omega /= embed_dim / 2.
+    omega = 1. / 10000**omega  # (D/2,)
+
+    pos = pos.reshape(-1)  # (M,)
+    out = np.einsum('m,d->md', pos, omega)  # (M, D/2), outer product
+
+    emb_sin = np.sin(out) # (M, D/2)
+    emb_cos = np.cos(out) # (M, D/2)
+
+    emb = np.concatenate([emb_sin, emb_cos], axis=1)  # (M, D)
+    return emb
+
+
+
+class HisResampler(nn.Module):
+    """
+    A 2D perceiver-resampler network with one cross attention layers by
+        (grid_size**2) learnable queries and 2d sincos pos_emb
+    Outputs:
+        A tensor with the shape of (grid_size**2, embed_dim)
+    """
+    def __init__(
+            self,
+            embed_dim=4096,
+            num_heads=32,
+            grid_size=16,
+            kv_dim=None,
+            norm_layer=nn.LayerNorm
+    ):
+        super().__init__()
+        self.num_queries = grid_size ** 2
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+
+        self.pos_embed = nn.Parameter(
+            torch.from_numpy(get_2d_sincos_pos_embed(embed_dim, grid_size)).float()
+        ).requires_grad_(False)
+
+        self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
+        trunc_normal_(self.query, std=.02)
+
+        if kv_dim is not None and kv_dim != embed_dim:
+            self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
+        else:
+            self.kv_proj = nn.Identity()
+
+        self.attn = nn.MultiheadAttention(embed_dim, num_heads)
+        self.ln_q = norm_layer(embed_dim)
+        self.ln_kv = norm_layer(embed_dim)
+        
+        self.ln_post = norm_layer(embed_dim)
+        self.proj = nn.Parameter((embed_dim** -0.5) * torch.randn(embed_dim, embed_dim))
+        
+        self.apply(self._init_weights)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    def forward(self, x, attn_mask=None):
+
+        #pos_embed = get_abs_pos(self.pos_embed, x.size(1))
+        x = self.kv_proj(x)
+        x = self.ln_kv(x).permute(1, 0, 2)
+
+        N = x.shape[1]
+        q = self.ln_q(self.query)
+        out = self.attn(
+            self._repeat(q, N),# + self.pos_embed.unsqueeze(1),
+            x, # + pos_embed.unsqueeze(1),
+            x,
+            attn_mask=attn_mask)[0]
+        out = out.permute(1, 0, 2)
+        out = self.ln_post(out)
+        out = out @ self.proj
+        return out
+
+    def _repeat(self, query, N: int):
+        return query.unsqueeze(1).repeat(1, N, 1)
+
+
+
+class QWenAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+
+        self.register_buffer("masked_bias", torch.tensor(-1e4), persistent=False)
+        self.seq_length = config.seq_length
+
+        self.hidden_size = config.hidden_size
+        self.split_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // self.num_heads
+
+        self.scale_attn_weights = True
+
+        self.projection_size = config.kv_channels * config.num_attention_heads
+
+        assert self.projection_size % config.num_attention_heads == 0
+        self.hidden_size_per_attention_head = (
+            self.projection_size // config.num_attention_heads
+        )
+
+        self.c_attn = nn.Linear(config.hidden_size, 3 * self.projection_size)
+
+        self.c_proj = nn.Linear(
+            config.hidden_size, self.projection_size, bias=not config.no_bias
+        )
+
+        self.is_fp32 = not (config.bf16 or config.fp16)
+        self.bf16 = config.bf16
+
+        self.use_dynamic_ntk = config.use_dynamic_ntk
+        self.use_logn_attn = config.use_logn_attn
+
+        logn_list = [
+            math.log(i, self.seq_length) if i > self.seq_length else 1
+            for i in range(1, 32768)
+        ]
+        self.logn_tensor = torch.tensor(logn_list)[None, :, None, None]
+
+        self.attn_dropout = nn.Dropout(config.attn_dropout_prob)
+
+    def _attn(self, query, key, value, registered_causal_mask, attention_mask=None, head_mask=None):
+        attn_weights = torch.matmul(query, key.transpose(-1, -2))
+
+        if self.scale_attn_weights:
+            attn_weights = attn_weights / torch.full(
+                [],
+                value.size(-1) ** 0.5,
+                dtype=attn_weights.dtype,
+                device=attn_weights.device,
+            )
+
+        query_length, key_length = query.size(-2), key.size(-2)
+        # causal_mask = self.bias[
+        #     :, :, key_length - query_length : key_length, :key_length
+        # ]
+        # mask_value = torch.finfo(attn_weights.dtype).min
+        # mask_value = torch.full([], mask_value, dtype=attn_weights.dtype).to(
+        #     attn_weights.device
+        # )
+        # attn_weights = torch.where(
+        #     causal_mask, attn_weights.to(attn_weights.dtype), mask_value
+        # )
+        attn_weights = attn_weights + attention_mask
+
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+
+        attn_weights = attn_weights.type(value.dtype)
+        attn_weights = self.attn_dropout(attn_weights)
+
+        if head_mask is not None:
+            attn_weights = attn_weights * head_mask
+
+        attn_output = torch.matmul(attn_weights, value)
+        attn_output = attn_output.transpose(1, 2)
+
+        return attn_output, attn_weights
+
+    def _upcast_and_reordered_attn(
+        self, query, key, value, registered_causal_mask, attention_mask=None, head_mask=None
+    ):
+        bsz, num_heads, q_seq_len, dk = query.size()
+        _, _, k_seq_len, _ = key.size()
+
+        attn_weights = torch.empty(
+            bsz * num_heads,
+            q_seq_len,
+            k_seq_len,
+            dtype=torch.float32,
+            device=query.device,
+        )
+
+        scale_factor = 1.0
+        if self.scale_attn_weights:
+            scale_factor /= float(value.size(-1)) ** 0.5
+
+        with autocast(enabled=False):
+            q, k = query.reshape(-1, q_seq_len, dk), key.transpose(-1, -2).reshape(
+                -1, dk, k_seq_len
+            )
+            attn_weights = torch.baddbmm(
+                attn_weights, q.float(), k.float(), beta=0, alpha=scale_factor
+            )
+            attn_weights = attn_weights.reshape(bsz, num_heads, q_seq_len, k_seq_len)
+
+        query_length, key_length = query.size(-2), key.size(-2)
+        causal_mask = registered_causal_mask[
+            :, :, key_length - query_length : key_length, :key_length
+        ]
+        mask_value = torch.finfo(attn_weights.dtype).min
+        mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(
+            attn_weights.device
+        )
+        attn_weights = torch.where(causal_mask, attn_weights, mask_value)
+
+        if attention_mask is not None:
+            attn_weights = attn_weights + attention_mask
+
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+
+        if attn_weights.dtype != torch.float32:
+            raise RuntimeError(
+                "Error with upcasting, attn_weights does not have dtype torch.float32"
+            )
+        attn_weights = attn_weights.type(value.dtype)
+        attn_weights = self.attn_dropout(attn_weights)
+
+        if head_mask is not None:
+            attn_weights = attn_weights * head_mask
+
+        attn_output = torch.matmul(attn_weights, value)
+
+        return attn_output, attn_weights
+
+    def _split_heads(self, tensor, num_heads, attn_head_size):
+        new_shape = tensor.size()[:-1] + (num_heads, attn_head_size)
+        tensor = tensor.view(new_shape)
+        return tensor
+
+    def _merge_heads(self, tensor, num_heads, attn_head_size):
+        tensor = tensor.contiguous()
+        new_shape = tensor.size()[:-2] + (num_heads * attn_head_size,)
+        return tensor.view(new_shape)
+
+    def forward(
+        self,
+        hidden_states: Optional[Tuple[torch.FloatTensor]],
+        rotary_pos_emb: Optional[List[torch.Tensor]] = None,
+        registered_causal_mask: Optional[torch.Tensor] = None,
+        layer_past: Optional[Tuple[torch.Tensor]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+    ):
+
+        mixed_x_layer = self.c_attn(hidden_states)
+
+        query, key, value = mixed_x_layer.split(self.split_size, dim=2)
+
+        query = self._split_heads(query, self.num_heads, self.head_dim)
+        key = self._split_heads(key, self.num_heads, self.head_dim)
+        value = self._split_heads(value, self.num_heads, self.head_dim)
+
+        if rotary_pos_emb is not None:
+            cur_len = query.shape[1]
+            rotary_pos_emb = [i[:, -cur_len:, :, :] for i in rotary_pos_emb]
+            rotary_pos_emb = (rotary_pos_emb,) * 2
+            q_pos_emb, k_pos_emb = rotary_pos_emb
+            # Slice the pos emb for current inference
+            query = apply_rotary_pos_emb(query, q_pos_emb)
+            key = apply_rotary_pos_emb(key, k_pos_emb)
+
+        if layer_past is not None:
+            past_key, past_value = layer_past[0], layer_past[1]
+            key = torch.cat((past_key, key), dim=1)
+            value = torch.cat((past_value, value), dim=1)
+
+        if use_cache:
+            present = (key, value)
+        else:
+            present = None
+
+        if self.use_logn_attn and not self.training:
+            if self.logn_tensor.device != query.device or self.logn_tensor.dtype != query.dtype:
+                self.logn_tensor = self.logn_tensor.to(query.device).type_as(query)
+            seq_start = key.size(1) - query.size(1)
+            seq_end = key.size(1)
+            logn_tensor = self.logn_tensor[:, seq_start:seq_end, :, :]
+            query = query * logn_tensor.expand_as(query)
+
+        query = query.permute(0, 2, 1, 3)
+        key = key.permute(0, 2, 1, 3)
+        value = value.permute(0, 2, 1, 3)
+        attn_output, attn_weight = self._attn(
+            query, key, value, registered_causal_mask, attention_mask, head_mask
+        )
+        context_layer = self._merge_heads(
+            attn_output, self.num_heads, self.head_dim
+        )
+
+        attn_output = self.c_proj(context_layer)
+
+        outputs = (attn_output, present)
+        if output_attentions:
+            outputs += (attn_weight,)
+
+        return outputs
+
+
+class QWenMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.w1 = nn.Linear(
+            config.hidden_size, config.intermediate_size // 2, bias=not config.no_bias
+        )
+        self.w2 = nn.Linear(
+            config.hidden_size, config.intermediate_size // 2, bias=not config.no_bias
+        )
+        ff_dim_in = config.intermediate_size // 2
+        self.c_proj = nn.Linear(ff_dim_in, config.hidden_size, bias=not config.no_bias)
+
+    def forward(self, hidden_states):
+        a1 = self.w1(hidden_states)
+        a2 = self.w2(hidden_states)
+        intermediate_parallel = a1 * F.silu(a2)
+        output = self.c_proj(intermediate_parallel)
+        return output
+
+class QWenBlock(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        hidden_size = config.hidden_size
+        self.bf16 = config.bf16
+
+        self.ln_1 = RMSNorm(
+            hidden_size,
+            eps=config.layer_norm_epsilon,
+        )
+        self.attn = QWenAttention(config)
+        self.ln_2 = RMSNorm(
+            hidden_size,
+            eps=config.layer_norm_epsilon,
+        )
+
+        self.mlp = QWenMLP(config)
+
+    def forward(
+        self,
+        hidden_states: Optional[Tuple[torch.FloatTensor]],
+        rotary_pos_emb: Optional[List[torch.Tensor]] = None,
+        registered_causal_mask: Optional[torch.Tensor] = None,
+        layer_past: Optional[Tuple[torch.Tensor]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = False,
+        output_attentions: Optional[bool] = False,
+    ):
+        layernorm_output = self.ln_1(hidden_states)
+
+        attn_outputs = self.attn(
+            layernorm_output,
+            rotary_pos_emb,
+            registered_causal_mask=registered_causal_mask,
+            layer_past=layer_past,
+            attention_mask=attention_mask,
+            head_mask=head_mask,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+        )
+        attn_output = attn_outputs[0]
+
+        outputs = attn_outputs[1:]
+
+        residual = hidden_states
+        layernorm_input = attn_output + residual
+
+        layernorm_output = self.ln_2(layernorm_input)
+
+        residual = layernorm_input
+        mlp_output = self.mlp(layernorm_output)
+        hidden_states = residual + mlp_output
+
+        if use_cache:
+            outputs = (hidden_states,) + outputs
+        else:
+            outputs = (hidden_states,) + outputs[1:]
+
+        return outputs
+
+
+class QWenPreTrainedModel(PreTrainedModel):
+    config_class = QWenConfig
+    base_model_prefix = "transformer"
+    is_parallelizable = False
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["QWenBlock"]
+
+    def __init__(self, *inputs, **kwargs):
+        super().__init__(*inputs, **kwargs)
+
+    def _init_weights(self, module):
+        """Initialize the weights."""
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+        elif isinstance(module, RMSNorm):
+            module.weight.data.fill_(1.0)
+
+        for name, p in module.named_parameters():
+            if name == "c_proj.weight":
+                p.data.normal_(
+                    mean=0.0,
+                    std=(
+                        self.config.initializer_range
+                        / math.sqrt(2 * self.config.num_hidden_layers)
+                    ),
+                )
+
+    def _set_gradient_checkpointing(self, module, value=False):
+        if isinstance(module, QWenModel):
+            module.gradient_checkpointing = value
+
+
+class QWenModel(QWenPreTrainedModel):
+    _keys_to_ignore_on_load_missing = ["attn.masked_bias"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.his_len = config.his_len
+        self.vocab_size = config.vocab_size
+        self.num_hidden_layers = config.num_hidden_layers
+        self.embed_dim = config.hidden_size
+
+        self.gradient_checkpointing = False
+        self.use_dynamic_ntk = config.use_dynamic_ntk
+        self.seq_length = config.seq_length
+
+        self.wte = nn.Embedding(self.vocab_size, self.embed_dim)
+
+        self.drop = nn.Dropout(config.emb_dropout_prob)
+
+        if config.rotary_pct == 1.0:
+            self.rotary_ndims = None
+        else:
+            assert config.rotary_pct < 1
+            self.rotary_ndims = int(
+                config.kv_channels * config.rotary_pct
+            )
+        dim = (
+            self.rotary_ndims
+            if self.rotary_ndims is not None
+            else config.kv_channels
+        )
+        self.rotary_emb = RotaryEmbedding(dim, base=config.rotary_emb_base)
+
+        self.use_flash_attn = config.use_flash_attn
+        self.is_fp32 = not (config.bf16 or config.fp16)
+        self.registered_causal_mask = None
+
+        self.h = nn.ModuleList(
+            [
+                QWenBlock(
+                    config
+                )
+                for i in range(config.num_hidden_layers)
+            ]
+        )
+        self.ln_f = RMSNorm(
+            self.embed_dim,
+            eps=config.layer_norm_epsilon,
+        )
+
+        self.visual = VisionTransformer(**config.visual)
+
+        self.post_init()
+        
+        if USE_RESAMPLER:
+            print('init RESAMPLER')
+            self.his_resampler = HisResampler()
+        
+        self.imgtoken_dict = {}
+        if os.path.isdir(IMAGE_HISTORY):
+            for subdata in os.listdir(IMAGE_HISTORY):
+                sub_img_dict = json.load(open(os.path.join(IMAGE_HISTORY, subdata)))
+                self.imgtoken_dict.update(sub_img_dict)
+        else:
+            self.imgtoken_dict = json.load(open(IMAGE_HISTORY))
+
+        print('imgtoken_dict cache len:', len(self.imgtoken_dict))
+
+    def set_input_embeddings(self, new_embeddings):
+        self.wte = new_embeddings
+    
+    # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask
+    def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):
+        # create causal mask
+        # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+        combined_attention_mask = None
+        if input_shape[-1] > 1:
+            combined_attention_mask = _make_causal_mask(
+                input_shape,
+                inputs_embeds.dtype,
+                device=inputs_embeds.device,
+                past_key_values_length=past_key_values_length,
+            )
+
+        if attention_mask is not None:
+            # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
+            expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(
+                inputs_embeds.device
+            )
+            combined_attention_mask = (
+                expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask
+            )
+
+        return combined_attention_mask
+
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ):
+        device = input_ids.device if input_ids is not None else inputs_embeds.device
+        
+        if past_key_values is None and torch.any(input_ids == self.config.visual['image_start_id']):
+            bos_pos = torch.where(input_ids == self.config.visual['image_start_id'])
+            eos_pos = torch.where(input_ids == self.config.visual['image_start_id'] + 1)
+            assert (bos_pos[0] == eos_pos[0]).all()
+            img_pos = torch.stack((bos_pos[0], bos_pos[1], eos_pos[1]), dim=1)
+            now_images = []
+            his_images = []
+            C_list = []
+            images = []
+            his_idx = []
+            his_image_temp = []
+            for idx, (i, a, b) in enumerate(img_pos):
+                image = input_ids[i][a + 1 : b - 1].tolist()
+                image = image[ : image.index(self.config.visual['image_start_id'] + 2)]
+                image_path = bytes(image).decode('utf-8')
+                
+                if image_path.startswith('image-history: '):
+                    his_idx.append(idx)
+                    image_path = image_path.replace('image-history: ', '')
+                    his_list = self.imgtoken_dict[image_path][-self.his_len:] # t0 - tn-1
+                    assert len(his_list) > 0, his_list
+                    
+                    his_images.extend(his_list)
+                    his_image_temp.append(his_list)
+                    
+                else:
+                    now_images.append(image_path)
+            
+            now_images = self.visual.encode(now_images)
+            
+            if len(his_images) > 0:
+                his_images = self.visual.encode(his_images)
+                his_tkn = None
+                
+                start_pos = 0
+                for his_scr in his_image_temp:
+                    his_len = len(his_scr)
+                    his_img_feature = his_images[start_pos: start_pos + his_len] # [b, l, d]
+                    if USE_RESAMPLER:
+                        his_img_feature = his_img_feature.reshape(1, -1, his_img_feature.size(-1))
+                        his_vis_tkn = self.his_resampler(his_img_feature) # [l, d]
+                    else:
+                        raise ValueError("You cannot run without History Redsampler!")
+                    his_tkn = his_vis_tkn if his_tkn is None else torch.concat((his_tkn, his_vis_tkn), dim=0)
+                    start_pos += his_len
+                assert start_pos == len(his_images)
+                his_images = his_tkn
+            
+            now_p, his_p = 0, 0
+            for j in range(len(img_pos)):
+                if j not in his_idx:
+                    images.append(now_images[now_p])
+                    now_p += 1
+                else:
+                    images.append(his_images[his_p])
+                    his_p += 1
+            images = torch.stack(images, dim=0)
+            assert len(images) == len(img_pos) == len(now_images) + len(his_images)
+            
+            fake_images = None
+        elif self.training:
+            fake_images=torch.zeros(1,3,224,224).to(
+                dtype=self.visual.conv1.weight.dtype, device=self.visual.conv1.weight.device)
+            images = self.visual(fake_images)
+        else:
+            fake_images = None
+            images = None
+
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError(
+                "You cannot specify both input_ids and inputs_embeds at the same time"
+            )
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            input_ids = input_ids.view(-1, input_shape[-1])
+            batch_size = input_ids.shape[0]
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+            batch_size = inputs_embeds.shape[0]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        
+        if token_type_ids is not None:
+            token_type_ids = token_type_ids.view(-1, input_shape[-1])
+        if position_ids is not None:
+            position_ids = position_ids.view(-1, input_shape[-1])
+
+        if past_key_values is None:
+            past_length = 0
+            past_key_values = tuple([None] * len(self.h))
+        else:
+            past_length = past_key_values[0][0].size(-2)
+
+        if position_ids is None:
+            position_ids = torch.arange(
+                past_length,
+                input_shape[-1] + past_length,
+                dtype=torch.long,
+                device=device,
+            )
+            position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
+
+        encoder_attention_mask = None
+        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
+
+        if inputs_embeds is None:
+            inputs_embeds = self.wte(input_ids)
+
+        if batch_size <= 0:
+            raise ValueError("batch_size has to be defined and > 0")
+        attention_mask = self._prepare_decoder_attention_mask(
+            attention_mask, input_shape, inputs_embeds, past_length
+        )
+        
+        hidden_states = inputs_embeds
+
+        kv_seq_len = hidden_states.size()[1]
+        if past_key_values[0] is not None:
+            # past key values[0][0] shape: bs * seq_len * head_num * dim
+            kv_seq_len += past_key_values[0][0].shape[1]
+        if (
+            self.use_dynamic_ntk
+            and kv_seq_len == hidden_states.size()[1]
+            and not self.training
+        ):
+            context_value = math.log(kv_seq_len / self.seq_length, 2) + 1
+            ntk_alpha = 2 ** math.ceil(context_value) - 1
+            ntk_alpha = max(ntk_alpha, 1)
+        else:
+            ntk_alpha = self.rotary_emb._ntk_alpha_cached
+
+        rotary_pos_emb = self.rotary_emb(kv_seq_len, ntk_alpha=ntk_alpha)
+        for idx in range(len(rotary_pos_emb)):
+            rotary_pos_emb[idx] = rotary_pos_emb[idx].to(hidden_states.device)
+        hidden_states = self.drop(hidden_states).clone()
+        
+        if fake_images is not None:
+            hidden_states = hidden_states + images.mean()*0
+        elif images is not None:
+            for idx, (i, a, b) in enumerate(img_pos):
+                hidden_states[i][a + 1 : b] = images[idx]
+        output_shape = input_shape + (hidden_states.size(-1),)
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+
+        presents = () if use_cache else None
+        all_self_attentions = () if output_attentions else None
+        all_hidden_states = () if output_hidden_states else None
+        for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
+
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+
+                def create_custom_forward(module):
+                    def custom_forward(*inputs):
+                        # None for past_key_value
+                        return module(*inputs, use_cache, output_attentions)
+
+                    return custom_forward
+
+                outputs = torch.utils.checkpoint.checkpoint(
+                    create_custom_forward(block),
+                    hidden_states,
+                    rotary_pos_emb,
+                    self.registered_causal_mask,
+                    None,
+                    attention_mask,
+                    head_mask[i],
+                    encoder_hidden_states,
+                    encoder_attention_mask,
+                )
+            else:
+                outputs = block(
+                    hidden_states,
+                    layer_past=layer_past,
+                    rotary_pos_emb=rotary_pos_emb,
+                    registered_causal_mask=self.registered_causal_mask,
+                    attention_mask=attention_mask,
+                    head_mask=head_mask[i],
+                    encoder_hidden_states=encoder_hidden_states,
+                    encoder_attention_mask=encoder_attention_mask,
+                    use_cache=use_cache,
+                    output_attentions=output_attentions,
+                )
+
+            hidden_states = outputs[0]
+            if use_cache is True:
+                presents = presents + (outputs[1],)
+
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
+
+        hidden_states = self.ln_f(hidden_states)
+        hidden_states = hidden_states.view(output_shape)
+        # Add last hidden state
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v for v in [hidden_states, presents, all_hidden_states] if v is not None
+            )
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=presents,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+        )
+
+
+class QWenLMHeadModel(QWenPreTrainedModel):
+    _keys_to_ignore_on_load_missing = [r"h\.\d+\.attn\.rotary_emb\.inv_freq"]
+    _keys_to_ignore_on_load_unexpected = [r"h\.\d+\.attn\.masked_bias"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        assert (
+            config.bf16 + config.fp16 + config.fp32 <= 1
+        ), "Only one of \"bf16\", \"fp16\", \"fp32\" can be true"
+
+        autoset_precision = config.bf16 + config.fp16 + config.fp32 == 0
+
+        if autoset_precision:
+            if SUPPORT_BF16:
+                logger.warn(
+                    "The model is automatically converting to bf16 for faster inference. "
+                    "If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to \"AutoModelForCausalLM.from_pretrained\"."
+                )
+                config.bf16 = True
+            elif SUPPORT_FP16:
+                logger.warn(
+                    "The model is automatically converting to fp16 for faster inference. "
+                    "If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to \"AutoModelForCausalLM.from_pretrained\"."
+                )
+                config.fp16 = True
+            else:
+                config.fp32 = True
+
+        if config.bf16 and SUPPORT_CUDA and not SUPPORT_BF16:
+            logger.warn("Your device does NOT seem to support bf16, you can switch to fp16 or fp32 by by passing fp16/fp32=True in \"AutoModelForCausalLM.from_pretrained\".")
+        if config.fp16 and SUPPORT_CUDA and not SUPPORT_FP16:
+            logger.warn("Your device does NOT support faster inference with fp16, please switch to fp32 which is likely to be faster")
+        if config.fp32:
+            if SUPPORT_BF16:
+                logger.warn("Your device support faster inference by passing bf16=True in \"AutoModelForCausalLM.from_pretrained\".")
+            elif SUPPORT_FP16:
+                logger.warn("Your device support faster inference by passing fp16=True in \"AutoModelForCausalLM.from_pretrained\".")
+
+        self.transformer = QWenModel(config)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        if config.bf16:
+            self.transformer.bfloat16()
+            self.lm_head.bfloat16()
+        if config.fp16:
+            self.transformer.half()
+            self.lm_head.half()
+        self.post_init()
+        
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs
+    ):
+        token_type_ids = kwargs.get("token_type_ids", None)
+        if past_key_values:
+            input_ids = input_ids[:, -1].unsqueeze(-1)
+            if token_type_ids is not None:
+                token_type_ids = token_type_ids[:, -1].unsqueeze(-1)
+
+        attention_mask = kwargs.get("attention_mask", None)
+        position_ids = kwargs.get("position_ids", None)
+
+        if attention_mask is not None and position_ids is None:
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -1].unsqueeze(-1)
+        else:
+            position_ids = None
+
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "position_ids": position_ids,
+                "attention_mask": attention_mask,
+                "token_type_ids": token_type_ids,
+            }
+        )
+        return model_inputs
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        head_mask: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+
+        transformer_outputs = self.transformer(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        hidden_states = transformer_outputs[0]
+
+        lm_logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            labels = labels.to(lm_logits.device)
+            shift_logits = lm_logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss_fct = CrossEntropyLoss()
+            loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)
+            )
+
+        if not return_dict:
+            output = (lm_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=lm_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )
+
+    @staticmethod
+    def _reorder_cache(
+        past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
+    ) -> Tuple[Tuple[torch.Tensor]]:
+
+        return tuple(
+            tuple(
+                past_state.index_select(0, beam_idx.to(past_state.device))
+                for past_state in layer_past
+            )
+            for layer_past in past_key_values
+        )
+
+    def chat(
+        self,
+        tokenizer: PreTrainedTokenizer,
+        query: str,
+        history: Optional[HistoryType],
+        system: str = "You are a helpful assistant.",
+        append_history: bool = True,
+        stream: Optional[bool] = _SENTINEL,
+        stop_words_ids: Optional[List[List[int]]] = None,
+        generation_config: Optional[GenerationConfig] = None,
+        **kwargs,
+    ) -> Tuple[str, HistoryType]:
+        generation_config = generation_config if generation_config is not None else self.generation_config
+
+        assert stream is _SENTINEL, _ERROR_STREAM_IN_CHAT
+        assert generation_config.chat_format == 'chatml', _ERROR_BAD_CHAT_FORMAT
+        if history is None:
+            history = []
+        if stop_words_ids is None:
+            stop_words_ids = []
+
+        max_window_size = kwargs.get('max_window_size', None)
+        if max_window_size is None:
+            max_window_size = generation_config.max_window_size
+        raw_text, context_tokens = make_context(
+            tokenizer,
+            query,
+            history=history,
+            system=system,
+            max_window_size=max_window_size,
+            chat_format=generation_config.chat_format,
+        )
+
+        stop_words_ids.extend(get_stop_words_ids(
+            generation_config.chat_format, tokenizer
+        ))
+        input_ids = torch.tensor([context_tokens]).to(self.device)
+        outputs = self.generate(
+                    input_ids,
+                    stop_words_ids=stop_words_ids,
+                    return_dict_in_generate=False,
+                    generation_config=generation_config,
+                    **kwargs,
+                )
+
+        response = decode_tokens(
+            outputs[0],
+            tokenizer,
+            raw_text_len=len(raw_text),
+            context_length=len(context_tokens),
+            chat_format=generation_config.chat_format,
+            verbose=False,
+            errors='replace'
+        )
+
+        if append_history:
+            history.append((query, response))
+
+        return response, history
+
+    def chat_stream(
+            self,
+            tokenizer: PreTrainedTokenizer,
+            query: str,
+            history: Optional[HistoryType],
+            system: str = "You are a helpful assistant.",
+            stop_words_ids: Optional[List[List[int]]] = None,
+            logits_processor: Optional[LogitsProcessorList] = None,
+            generation_config: Optional[GenerationConfig] = None,
+            **kwargs,
+    ) -> Generator[str, Any, None]:
+        generation_config = generation_config if generation_config is not None else self.generation_config
+        assert generation_config.chat_format == 'chatml', _ERROR_BAD_CHAT_FORMAT
+        if history is None:
+            history = []
+        if stop_words_ids is None:
+            stop_words_ids = []
+
+        max_window_size = kwargs.get('max_window_size', None)
+        if max_window_size is None:
+            max_window_size = generation_config.max_window_size
+        raw_text, context_tokens = make_context(
+            tokenizer,
+            query,
+            history=history,
+            system=system,
+            max_window_size=max_window_size,
+            chat_format=generation_config.chat_format,
+        )
+
+        stop_words_ids.extend(get_stop_words_ids(
+            generation_config.chat_format, tokenizer
+        ))
+        if stop_words_ids is not None:
+            stop_words_logits_processor = StopWordsLogitsProcessor(
+                stop_words_ids=stop_words_ids,
+                eos_token_id=generation_config.eos_token_id,
+            )
+            if logits_processor is None:
+                logits_processor = LogitsProcessorList([stop_words_logits_processor])
+            else:
+                logits_processor.append(stop_words_logits_processor)
+        input_ids = torch.tensor([context_tokens]).to(self.device)
+
+        from transformers_stream_generator.main import NewGenerationMixin, StreamGenerationConfig
+        self.__class__.generate_stream = NewGenerationMixin.generate
+        self.__class__.sample_stream = NewGenerationMixin.sample_stream
+        stream_config = StreamGenerationConfig(**generation_config.to_dict(), do_stream=True)
+
+        def stream_generator():
+            outputs = []
+            for token in self.generate_stream(
+                    input_ids,
+                    return_dict_in_generate=False,
+                    generation_config=stream_config,
+                    logits_processor=logits_processor,
+                    seed=-1,
+                    **kwargs):
+                outputs.append(token.item())
+                yield tokenizer.decode(outputs, skip_special_tokens=True, errors='ignore', keep_image_special=True)
+
+        return stream_generator()
+
+    def generate(
+        self,
+        inputs: Optional[torch.Tensor] = None,
+        generation_config: Optional[GenerationConfig] = None,
+        logits_processor: Optional[LogitsProcessorList] = None,
+        stopping_criteria: Optional[StoppingCriteriaList] = None,
+        prefix_allowed_tokens_fn: Optional[
+            Callable[[int, torch.Tensor], List[int]]
+        ] = None,
+        synced_gpus: Optional[bool] = None,
+        assistant_model: Optional["PreTrainedModel"] = None,
+        streamer: Optional["BaseStreamer"] = None,
+        **kwargs,
+    ) -> Union[GenerateOutput, torch.LongTensor]:
+        generation_config = generation_config if generation_config is not None else self.generation_config
+
+        # Process stop_words_ids.
+        stop_words_ids = kwargs.pop("stop_words_ids", None)
+        if stop_words_ids is None and generation_config is not None:
+            stop_words_ids = getattr(generation_config, "stop_words_ids", None)
+        if stop_words_ids is None:
+            stop_words_ids = getattr(generation_config, "stop_words_ids", None)
+
+        if stop_words_ids is not None:
+            stop_words_logits_processor = StopWordsLogitsProcessor(
+                stop_words_ids=stop_words_ids,
+                eos_token_id=generation_config.eos_token_id,
+            )
+            if logits_processor is None:
+                logits_processor = LogitsProcessorList([stop_words_logits_processor])
+            else:
+                logits_processor.append(stop_words_logits_processor)
+
+        return super().generate(
+            inputs,
+            generation_config=generation_config,
+            logits_processor=logits_processor,
+            stopping_criteria=stopping_criteria,
+            prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
+            synced_gpus=synced_gpus,
+            assistant_model=assistant_model,
+            streamer=streamer,
+            **kwargs,
+        )
+
+
+class RotaryEmbedding(torch.nn.Module):
+    def __init__(self, dim, base=10000):
+        super().__init__()
+        self.dim = dim
+        self.base = base
+        self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
+        if importlib.util.find_spec("einops") is None:
+            raise RuntimeError("einops is required for Rotary Embedding")
+
+        self._rotary_pos_emb_cache = None
+        self._seq_len_cached = 0
+        self._ntk_alpha_cached = 1.0
+
+    def update_rotary_pos_emb_cache(self, max_seq_len, offset=0, ntk_alpha=1.0):
+        seqlen = max_seq_len + offset
+        if seqlen > self._seq_len_cached or ntk_alpha != self._ntk_alpha_cached:
+            base = self.base * ntk_alpha ** (self.dim / (self.dim - 2))
+            self.inv_freq = 1.0 / (
+                base
+                ** (
+                    torch.arange(0, self.dim, 2, device=self.inv_freq.device).float()
+                    / self.dim
+                )
+            )
+            self._seq_len_cached = max(2 * seqlen, 16)
+            self._ntk_alpha_cached = ntk_alpha
+            seq = torch.arange(self._seq_len_cached, device=self.inv_freq.device)
+            freqs = torch.outer(seq.type_as(self.inv_freq), self.inv_freq)
+            
+            emb = torch.cat((freqs, freqs), dim=-1)
+            from einops import rearrange
+
+            emb = rearrange(emb, "n d -> 1 n 1 d")
+
+            cos, sin = emb.cos(), emb.sin()
+            self._rotary_pos_emb_cache = [cos, sin]
+
+    def forward(self, max_seq_len, offset=0, ntk_alpha=1.0):
+        self.update_rotary_pos_emb_cache(max_seq_len, offset, ntk_alpha)
+        cos, sin = self._rotary_pos_emb_cache
+        return [cos[:, offset : offset + max_seq_len], sin[:, offset : offset + max_seq_len]]
+
+
+def _rotate_half(x):
+    from einops import rearrange
+
+    x = rearrange(x, "... (j d) -> ... j d", j=2)
+    x1, x2 = x.unbind(dim=-2)
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(t, freqs):
+    cos, sin = freqs
+    if apply_rotary_emb_func is not None and t.is_cuda:
+        t_ = t.float()
+        cos = cos.squeeze(0).squeeze(1)[:, : cos.shape[-1] // 2]
+        sin = sin.squeeze(0).squeeze(1)[:, : sin.shape[-1] // 2]
+        output = apply_rotary_emb_func(t_, cos, sin).type_as(t)
+        return output
+    else:
+        rot_dim = freqs[0].shape[-1]
+        cos, sin = freqs
+        t_, t_pass_ = t[..., :rot_dim], t[..., rot_dim:]
+        t_ = t_.float()
+        t_pass_ = t_pass_.float()
+        t_ = (t_ * cos) + (_rotate_half(t_) * sin)
+        return torch.cat((t_, t_pass_), dim=-1).type_as(t)
+
+
+class RMSNorm(torch.nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-6):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+
+    def _norm(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+
+    def forward(self, x):
+        if rms_norm is not None and x.is_cuda:
+            return rms_norm(x, self.weight, self.eps)
+        else:
+            output = self._norm(x.float()).type_as(x)
+            return output * self.weight
+
+
+if __name__ == '__main__':
+    import json
+    from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
+    model_path = '/mnt/petrelfs/luquanfeng/Gui-Copilot/Qwenvl-chat'
+    tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+    tokenizer.padding_side = 'left'
+    tokenizer.pad_token_id = tokenizer.eod_id
+    qs = ["Picture 1: <img>/mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/2971401471600307458_9.png</img>\nPlease show the next action. Goal: What's the price of the new iPhone on eBay?\nPrevious screenshots: <img>image-history: /mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/2971401471600307458_9.png</img>\nPrevious Actions: TYPE: n ebay\nCLICK: (393, 113)\nSCROLL: UP\nSCROLL: UP\nCLICK: (356, 599)\nCLICK: (454, 914)", 
+          "Picture 1: <img>/mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_3.png</img>\nPlease show the next action. Goal: What's on the menu at Chipotle?\nPrevious screenshots: <img>image-history: /mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_3.png</img>\nPrevious Actions: CLICK: (451, 891)",
+          "Picture 1: <img>/mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_2.png</img>\nPlease show the next action. Goal: What's on the menu at Chipotle?\nPrevious screenshots: <img>image-history: /mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_2.png</img>\nPrevious Actions: CLICK: (451, 891)",
+          "Picture 1: <img>/mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_1.png</img>\nPlease show the next action. Goal: What's on the menu at Chipotle?\nPrevious screenshots: <img>image-history: /mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_1.png</img>\nPrevious Actions: CLICK: (451, 891)",
+          "Picture 1: <img>/mnt/petrelfs/luquanfeng/gui_data/aitw_pt/general/img/7681036113908451219_0.png</img>\nPlease show the next action. Goal: What's on the menu at Chipotle?"]
+    device = 'cuda'
+    
+    batch_raw_text = []
+    for q in qs:
+        raw_text, _ = make_context(tokenizer, q, system="You are a helpful assistant.", max_window_size=6144, chat_format='chatml')
+        batch_raw_text.append(raw_text)
+
+    batch_input = tokenizer(batch_raw_text, return_tensors='pt', padding='longest')
+    batch_input_ids = batch_input['input_ids'].to(device)
+    batch_input_attention_mask = batch_input['attention_mask'].to(device)
+    print(batch_input_ids.shape)
+    
+    model = AutoModelForCausalLM.from_pretrained(model_path, device_map=device, trust_remote_code=True).eval()
+    import pdb; pdb.set_trace()
+    batch_out_ids = model.generate(
+                input_ids=batch_input_ids,
+                attention_mask=batch_input_attention_mask,
+                do_sample=False,
+                #top_k=0,
+                #top_p=0.5,
+                num_beams=1,
+                length_penalty=1,
+                num_return_sequences=1,
+                use_cache=True,
+                pad_token_id=tokenizer.eod_id,
+                eos_token_id=tokenizer.eod_id,
+                min_new_tokens=1,
+                max_new_tokens=10,
+                )
+    print(batch_out_ids.shape)
+    print(batch_out_ids)
+    print('-----------------------------------------------')
+    
+    padding_lens = [batch_input_ids[i].eq(tokenizer.pad_token_id).sum().item() for i in range(batch_input_ids.size(0))]
+
+    batch_response = [
+        decode_tokens(
+            batch_out_ids[i][padding_lens[i]:],
+            tokenizer,
+            raw_text_len=len(batch_raw_text[i]),
+            context_length=(batch_input_ids[i].size(0)-padding_lens[i]),
+            chat_format="chatml",
+            verbose=False,
+            errors='replace'
+        ) for i in range(len(qs))
+    ]
+    print(batch_input_ids.shape, batch_out_ids.shape)
+    print(batch_response)

pytorch_model-00001-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:577ac170fa6eefba90aef7a761ae1aa8d9e8fa53c505270ed59b5b34126fd654
+size 9969772092

pytorch_model-00002-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9a89a7b0c3f3acf02341152c8262b81d90ac457fc2561a6d09c4d2628961e16
+size 9516452100

pytorch_model.bin.index.json

@@ -0,0 +1,873 @@
+{
+  "metadata": {
+    "total_size": 19485918720
+  },
+  "weight_map": {
+    "lm_head.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.0.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.0.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.1.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.10.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.11.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.12.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.13.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.14.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.15.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.16.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.17.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.18.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.19.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.2.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.20.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.21.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.21.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.22.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.22.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.22.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.22.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.22.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.22.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.22.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.22.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.23.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.23.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.23.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.23.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.23.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.23.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.23.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.23.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.24.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.24.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.24.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.24.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.24.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.24.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.24.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.24.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.25.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.25.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.25.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.25.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.25.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.25.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.25.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.25.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.26.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.27.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.28.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.29.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.3.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.3.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.30.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.30.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_attn.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_attn.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.attn.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.31.mlp.w2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.h.4.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.4.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.5.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.6.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.7.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.8.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_attn.bias": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_attn.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.attn.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w1.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.h.9.mlp.w2.weight": "pytorch_model-00001-of-00002.bin",
+    "transformer.his_resampler.attn.in_proj_bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.attn.in_proj_weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.ln_kv.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.ln_kv.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.ln_post.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.ln_post.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.ln_q.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.ln_q.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.pos_embed": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.proj": "pytorch_model-00002-of-00002.bin",
+    "transformer.his_resampler.query": "pytorch_model-00002-of-00002.bin",
+    "transformer.ln_f.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.attn_pool.attn.in_proj_bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.attn_pool.attn.in_proj_weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.attn_pool.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.attn_pool.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.attn_pool.kv_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.attn_pool.ln_kv.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.attn_pool.ln_kv.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.attn_pool.ln_q.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.attn_pool.ln_q.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.attn_pool.pos_embed": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.attn_pool.query": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.conv1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.ln_post.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.ln_post.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.ln_pre.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.ln_pre.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.positional_embedding": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.proj": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.0.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.1.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.10.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.11.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.12.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.13.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.14.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.15.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.16.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.17.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.18.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.19.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.2.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.20.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.21.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.22.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.23.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.24.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.25.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.26.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.27.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.28.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.29.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.3.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.30.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.31.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.32.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.33.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.34.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.35.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.36.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.37.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.38.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.39.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.4.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.40.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.41.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.42.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.43.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.44.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.45.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.46.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.47.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.5.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.6.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.7.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.8.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.attn.in_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.attn.in_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.attn.out_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.attn.out_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.ln_1.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.ln_1.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.ln_2.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.ln_2.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.mlp.c_fc.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.mlp.c_fc.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.mlp.c_proj.bias": "pytorch_model-00002-of-00002.bin",
+    "transformer.visual.transformer.resblocks.9.mlp.c_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "transformer.wte.weight": "pytorch_model-00001-of-00002.bin"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,3 @@
+{
+  "pad_token": "<|endoftext|>"
+}

tokenization_qwen.py

@@ -0,0 +1,598 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Tokenization classes for QWen."""
+
+import base64
+import logging
+import os
+import requests
+import unicodedata
+from typing import Collection, Dict, List, Set, Tuple, Union, Any, Callable, Optional
+
+import tiktoken
+import numpy as np
+from PIL import Image
+from PIL import ImageFont
+from PIL import ImageDraw
+from transformers import PreTrainedTokenizer, AddedToken
+from transformers.utils import try_to_load_from_cache
+
+import matplotlib.colors as mcolors
+from matplotlib.font_manager import FontProperties
+
+logger = logging.getLogger(__name__)
+
+
+VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken", "ttf": "SimSun.ttf"}
+FONT_PATH = try_to_load_from_cache("Qwen/Qwen-VL-Chat", "SimSun.ttf")
+if FONT_PATH is None:
+    if not os.path.exists("SimSun.ttf"):
+        ttf = requests.get("https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/SimSun.ttf")
+        open("SimSun.ttf", "wb").write(ttf.content)
+    FONT_PATH = "SimSun.ttf"
+
+PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
+ENDOFTEXT = "<|endoftext|>"
+IMSTART = "<|im_start|>"
+IMEND = "<|im_end|>"
+# as the default behavior is changed to allow special tokens in
+# regular texts, the surface forms of special tokens need to be
+# as different as possible to minimize the impact
+EXTRAS = tuple((f"<|extra_{i}|>" for i in range(205)))
+SPECIAL_TOKENS = (
+    ENDOFTEXT,
+    IMSTART,
+    IMEND,
+) + EXTRAS
+IMG_TOKEN_SPAN = 256
+
+
+def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
+    with open(tiktoken_bpe_file, "rb") as f:
+        contents = f.read()
+    return {
+        base64.b64decode(token): int(rank)
+        for token, rank in (line.split() for line in contents.splitlines() if line)
+    }
+
+def _list_find(
+    input_list: List[Any],
+    candidates: Tuple[Any],
+    start: int = 0,
+):
+    for i in range(start, len(input_list)):
+        if input_list[i] in candidates:
+            return i
+    return -1
+
+def _replace_closed_tag(
+    input_tokens: List[Any],
+    start_tags: Union[Any, Tuple[Any]],
+    end_tags: Union[Any, Tuple[Any]],
+    inclusive_replace_func: Callable,
+    exclusive_replace_func: Callable = lambda x: x,
+):
+    if isinstance(start_tags, (str, int)):
+        start_tags = (start_tags,)
+    if isinstance(end_tags, (str, int)):
+        end_tags = (end_tags,)
+    assert len(start_tags) == len(end_tags)
+
+    output_tokens = []
+    end = 0
+    while True:
+        start = _list_find(input_tokens, start_tags, end)
+        if start == -1:
+            break
+        output_tokens.extend(exclusive_replace_func(input_tokens[end : start]))
+        tag_idx = start_tags.index(input_tokens[start])
+        end = _list_find(input_tokens, (end_tags[tag_idx],), start)
+        if end == -1:
+            raise ValueError("Unclosed image token")
+        output_tokens.extend(inclusive_replace_func(input_tokens[start : end + 1]))
+        end += 1
+    output_tokens.extend(exclusive_replace_func(input_tokens[end : ]))
+    return output_tokens
+
+class QWenTokenizer(PreTrainedTokenizer):
+    """QWen tokenizer."""
+
+    vocab_files_names = VOCAB_FILES_NAMES
+
+    def __init__(
+        self,
+        vocab_file,
+        errors="replace",
+        image_start_tag='<img>',
+        image_end_tag='</img>',
+        image_pad_tag='<imgpad>',
+        ref_start_tag='<ref>',
+        ref_end_tag='</ref>',
+        box_start_tag='<box>',
+        box_end_tag='</box>',
+        quad_start_tag='<quad>',
+        quad_end_tag='</quad>',
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.image_start_tag = image_start_tag
+        self.image_end_tag = image_end_tag
+        self.image_pad_tag = image_pad_tag
+        self.ref_start_tag = ref_start_tag
+        self.ref_end_tag = ref_end_tag
+        self.box_start_tag = box_start_tag
+        self.box_end_tag = box_end_tag
+        self.quad_start_tag = quad_start_tag
+        self.quad_end_tag = quad_end_tag
+        self.IMAGE_ST = (
+            ref_start_tag, ref_end_tag,
+            box_start_tag, box_end_tag,
+            quad_start_tag, quad_end_tag,
+            image_start_tag, image_end_tag,
+            image_pad_tag
+        )
+
+        self.errors = errors  # how to handle errors in decoding
+
+        self.mergeable_ranks = _load_tiktoken_bpe(vocab_file)  # type: dict[bytes, int]
+        self.special_tokens = {
+            token: index
+            for index, token in enumerate(
+                SPECIAL_TOKENS + self.IMAGE_ST, start=len(self.mergeable_ranks)
+            )
+        }
+        self.img_start_id = self.special_tokens[self.image_start_tag]
+        self.img_end_id = self.special_tokens[self.image_end_tag]
+        self.img_pad_id = self.special_tokens[self.image_pad_tag]
+        self.ref_start_id = self.special_tokens[self.ref_start_tag]
+        self.ref_end_id = self.special_tokens[self.ref_end_tag]
+        self.box_start_id = self.special_tokens[self.box_start_tag]
+        self.box_end_id = self.special_tokens[self.box_end_tag]
+        self.quad_start_id = self.special_tokens[self.quad_start_tag]
+        self.quad_end_id = self.special_tokens[self.quad_end_tag]
+        self.image_special_tokens = set([
+            self.ref_start_id, self.ref_end_id, self.box_start_id, self.box_end_id,
+            self.quad_start_id, self.quad_end_id,
+        ])
+
+        enc = tiktoken.Encoding(
+            "Qwen",
+            pat_str=PAT_STR,
+            mergeable_ranks=self.mergeable_ranks,
+            special_tokens=self.special_tokens,
+        )
+        assert (
+            len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
+        ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
+
+        self.decoder = {
+            v: k for k, v in self.mergeable_ranks.items()
+        }  # type: dict[int, bytes|str]
+        self.decoder.update({v: k for k, v in self.special_tokens.items()})
+
+        self.tokenizer = enc  # type: tiktoken.Encoding
+
+        self.eod_id = self.tokenizer.eot_token
+        self.im_start_id = self.special_tokens[IMSTART]
+        self.im_end_id = self.special_tokens[IMEND]
+
+    def __getstate__(self):
+        # for pickle lovers
+        state = self.__dict__.copy()
+        del state['tokenizer']
+        return state
+
+    def __setstate__(self, state):
+        # tokenizer is not python native; don't pass it; rebuild it
+        self.__dict__.update(state)
+        enc = tiktoken.Encoding(
+            "Qwen",
+            pat_str=PAT_STR,
+            mergeable_ranks=self.mergeable_ranks,
+            special_tokens=self.special_tokens,
+        )
+        self.tokenizer = enc
+
+
+    def __len__(self) -> int:
+        return self.tokenizer.n_vocab
+
+    def get_vocab(self) -> Dict[bytes, int]:
+        return self.mergeable_ranks
+
+    def convert_tokens_to_ids(
+        self, tokens: Union[bytes, str, List[Union[bytes, str]]]
+    ) -> List[int]:
+        ids = []
+        if isinstance(tokens, (str, bytes)):
+            if tokens in self.special_tokens:
+                return self.special_tokens[tokens]
+            else:
+                return self.mergeable_ranks.get(tokens)
+        for token in tokens:
+            if token in self.special_tokens:
+                ids.append(self.special_tokens[token])
+            else:
+                ids.append(self.mergeable_ranks.get(token))
+        return ids
+
+    def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
+        if not special_tokens and new_tokens:
+            raise ValueError('Adding regular tokens is not supported')
+        for token in new_tokens:
+            surface_form = token.content if isinstance(token, AddedToken) else token
+            if surface_form not in SPECIAL_TOKENS + self.IMAGE_ST:
+                raise ValueError('Adding unknown special tokens is not supported')
+        return 0
+
+    def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
+        """
+        Save only the vocabulary of the tokenizer (vocabulary).
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        file_path = os.path.join(save_directory, "qwen.tiktoken")
+        with open(file_path, "w", encoding="utf8") as w:
+            for k, v in self.mergeable_ranks.items():
+                line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
+                w.write(line)
+        return (file_path,)
+
+    def tokenize(
+        self,
+        text: str,
+        allowed_special: Union[Set, str] = "all",
+        disallowed_special: Union[Collection, str] = (),
+        **kwargs,
+    ) -> List[Union[bytes, str]]:
+        """
+        Converts a string in a sequence of tokens.
+
+        Args:
+            text (`str`):
+                The sequence to be encoded.
+            allowed_special (`Literal["all"]` or `set`):
+                The surface forms of the tokens to be encoded as special tokens in regular texts.
+                Default to "all".
+            disallowed_special (`Literal["all"]` or `Collection`):
+                The surface forms of the tokens that should not be in regular texts and trigger errors.
+                Default to an empty tuple.
+
+            kwargs (additional keyword arguments, *optional*):
+                Will be passed to the underlying model specific encode method.
+
+        Returns:
+            `List[bytes|str]`: The list of tokens.
+        """
+        tokens = []
+        text = unicodedata.normalize("NFC", text)
+
+        # this implementation takes a detour: text -> token id -> token surface forms
+        for t in self.tokenizer.encode(
+            text, allowed_special=allowed_special, disallowed_special=disallowed_special
+        ):
+            tokens.append(self.decoder[t])
+
+        def _encode_imgurl(img_tokens):
+            assert img_tokens[0] == self.image_start_tag and img_tokens[-1] == self.image_end_tag
+            img_tokens = img_tokens[1:-1]
+            img_url = b''.join(img_tokens)
+            out_img_tokens = list(map(self.decoder.get, img_url))
+            if len(out_img_tokens) > IMG_TOKEN_SPAN:
+                raise ValueError("The content in {}..{} is too long".format(
+                    self.image_start_tag, self.image_end_tag))
+            out_img_tokens.extend([self.image_pad_tag] * (IMG_TOKEN_SPAN - len(out_img_tokens)))
+            out_img_tokens = [self.image_start_tag] + out_img_tokens + [self.image_end_tag]
+            return out_img_tokens
+
+        return _replace_closed_tag(tokens, self.image_start_tag, self.image_end_tag, _encode_imgurl)
+
+    def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
+        """
+        Converts a sequence of tokens in a single string.
+        """
+        text = ""
+        temp = b""
+        for t in tokens:
+            if isinstance(t, str):
+                if temp:
+                    text += temp.decode("utf-8", errors=self.errors)
+                    temp = b""
+                text += t
+            elif isinstance(t, bytes):
+                temp += t
+            else:
+                raise TypeError("token should only be of type types or str")
+        if temp:
+            text += temp.decode("utf-8", errors=self.errors)
+        return text
+
+    @property
+    def vocab_size(self):
+        return self.tokenizer.n_vocab
+
+    def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
+        """Converts an id to a token, special tokens included"""
+        if index in self.decoder:
+            return self.decoder[index]
+        raise ValueError("unknown ids")
+
+    def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
+        """Converts a token to an id using the vocab, special tokens included"""
+        if token in self.special_tokens:
+            return self.special_tokens[token]
+        if token in self.mergeable_ranks:
+            return self.mergeable_ranks[token]
+        raise ValueError("unknown token")
+
+    def _tokenize(self, text: str, **kwargs):
+        """
+        Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
+        vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
+
+        Do NOT take care of added tokens.
+        """
+        raise NotImplementedError
+
+    def _decode(
+        self,
+        token_ids: Union[int, List[int]],
+        skip_special_tokens: bool = False,
+        errors: str = None,
+        **kwargs,
+    ) -> str:
+        if isinstance(token_ids, int):
+            token_ids = [token_ids]
+
+        def _decode_imgurl(img_token_ids):
+            assert img_token_ids[0] == self.img_start_id and img_token_ids[-1] == self.img_end_id
+            img_token_ids = img_token_ids[1:-1]
+            img_token_ids = img_token_ids[ : img_token_ids.index(self.img_pad_id)]
+            img_url = bytes(img_token_ids).decode('utf-8')
+            return [self.img_start_id] + self.tokenizer.encode(img_url) + [self.img_end_id]
+
+        token_ids = _replace_closed_tag(token_ids, self.img_start_id, self.img_end_id, _decode_imgurl)
+
+        if skip_special_tokens:
+            if kwargs.get('keep_image_special', False):
+                token_ids = [i for i in token_ids if i < self.eod_id 
+                    or i in self.image_special_tokens]
+            else:
+                token_ids = [i for i in token_ids if i < self.eod_id]
+        return self.tokenizer.decode(token_ids, errors=errors or self.errors)
+
+    def to_list_format(self, text: str):
+        text = unicodedata.normalize("NFC", text)
+        token_ids = self.tokenizer.encode(
+            text, allowed_special=set(self.IMAGE_ST + (ENDOFTEXT,)))
+
+        def _encode_vl_info(tokens):
+            if len(tokens) == 0:
+                return []
+            if tokens[0] == self.img_start_id and tokens[-1] == self.img_end_id:
+                key = 'image'
+            elif tokens[0] == self.ref_start_id and tokens[-1] == self.ref_end_id:
+                key = 'ref'
+            elif tokens[0] == self.box_start_id and tokens[-1] == self.box_end_id:
+                key = 'box'
+            elif tokens[0] == self.quad_start_id and tokens[-1] == self.quad_end_id:
+                key = 'quad'
+            else:
+                _tobytes = lambda x: x.encode('utf-8') if isinstance(x, str) else x
+                return [{'text': b''.join(map(_tobytes, map(self.decoder.get, tokens))).decode('utf-8')}]
+            _tobytes = lambda x: x.encode('utf-8') if isinstance(x, str) else x
+            val = b''.join(map(_tobytes, map(self.decoder.get, tokens[1:-1]))).decode('utf-8')
+            return [{key: val}]
+
+        return _replace_closed_tag(
+            token_ids,
+            (self.img_start_id, self.ref_start_id, self.box_start_id, self.quad_start_id),
+            (self.img_end_id, self.ref_end_id, self.box_end_id, self.quad_end_id),
+            _encode_vl_info,
+            _encode_vl_info,
+        )
+
+    def from_list_format(self, list_format: List[Dict]):
+        text = ''
+        num_images = 0
+        for ele in list_format:
+            if 'image' in ele:
+                num_images += 1
+                text += f'Picture {num_images}: '
+                text += self.image_start_tag + ele['image'] + self.image_end_tag
+                text += '\n'
+            elif 'text' in ele:
+                text += ele['text']
+            elif 'box' in ele:
+                if 'ref' in ele:
+                    text += self.ref_start_tag + ele['ref'] + self.ref_end_tag
+                for box in ele['box']:
+                    text += self.box_start_tag + '(%d,%d),(%d,%d)' % (box[0], box[1], box[2], box[3]) + self.box_end_tag
+            else:
+                raise ValueError("Unsupport element: " + str(ele))
+        return text
+
+    def _fetch_latest_picture(self, response, history):
+        if history is None:
+            history = []
+        _history = history + [(response, None)]
+        for q, r in _history[::-1]:
+            for ele in self.to_list_format(q)[::-1]:
+                if 'image' in ele:
+                    return ele['image']
+        return None
+
+    def _fetch_all_box_with_ref(self, text):
+        list_format = self.to_list_format(text)
+        output = []
+        for i, ele in enumerate(list_format):
+            if 'box' in ele:
+                bbox = tuple(map(int, ele['box'].replace('(', '').replace(')', '').split(',')))
+                assert len(bbox) == 4
+                output.append({'box': bbox})
+                if i > 0 and 'ref' in list_format[i-1]:
+                    output[-1]['ref'] = list_format[i-1]['ref'].strip()
+        return output
+
+    def draw_bbox_on_latest_picture(
+        self,
+        response,
+        history=None,
+    ) -> Optional[Image.Image]:
+        image = self._fetch_latest_picture(response, history)
+        if image is None:
+            return None
+        if image.startswith("http://") or image.startswith("https://"):
+            image = Image.open(requests.get(image, stream=True).raw).convert("RGB")
+            h, w = image.height, image.width
+        else:
+            image = np.asarray(Image.open(image).convert("RGB"))
+            h, w = image.shape[0], image.shape[1]
+        visualizer = Visualizer(image)
+
+        boxes = self._fetch_all_box_with_ref(response)
+        if not boxes:
+            return None
+        color = random.choice([_ for _ in mcolors.TABLEAU_COLORS.keys()]) # init color
+        for box in boxes:
+            if 'ref' in box: # random new color for new refexps
+                color = random.choice([_ for _ in mcolors.TABLEAU_COLORS.keys()])
+            x1, y1, x2, y2 = box['box']
+            x1, y1, x2, y2 = (int(x1 / 1000 * w), int(y1 / 1000 * h), int(x2 / 1000 * w), int(y2 / 1000 * h))
+            visualizer.draw_box((x1, y1, x2, y2), alpha=1, edge_color=color)
+            if 'ref' in box:
+                visualizer.draw_text(box['ref'], (x1, y1), color=color, horizontal_alignment="left")
+        return visualizer.output
+
+
+import colorsys
+import logging
+import math
+import numpy as np
+import matplotlib as mpl
+import matplotlib.colors as mplc
+import matplotlib.figure as mplfigure
+import torch
+from matplotlib.backends.backend_agg import FigureCanvasAgg
+from PIL import Image
+import random
+
+logger = logging.getLogger(__name__)
+
+
+class VisImage:
+    def __init__(self, img, scale=1.0):
+        self.img = img
+        self.scale = scale
+        self.width, self.height = img.shape[1], img.shape[0]
+        self._setup_figure(img)
+
+    def _setup_figure(self, img):
+        fig = mplfigure.Figure(frameon=False)
+        self.dpi = fig.get_dpi()
+        # add a small 1e-2 to avoid precision lost due to matplotlib's truncation
+        # (https://github.com/matplotlib/matplotlib/issues/15363)
+        fig.set_size_inches(
+            (self.width * self.scale + 1e-2) / self.dpi,
+            (self.height * self.scale + 1e-2) / self.dpi,
+        )
+        self.canvas = FigureCanvasAgg(fig)
+        # self.canvas = mpl.backends.backend_cairo.FigureCanvasCairo(fig)
+        ax = fig.add_axes([0.0, 0.0, 1.0, 1.0])
+        ax.axis("off")
+        self.fig = fig
+        self.ax = ax
+        self.reset_image(img)
+
+    def reset_image(self, img):
+        img = img.astype("uint8")
+        self.ax.imshow(img, extent=(0, self.width, self.height, 0), interpolation="nearest")
+
+    def save(self, filepath):
+        self.fig.savefig(filepath)
+
+    def get_image(self):
+        canvas = self.canvas
+        s, (width, height) = canvas.print_to_buffer()
+
+        buffer = np.frombuffer(s, dtype="uint8")
+
+        img_rgba = buffer.reshape(height, width, 4)
+        rgb, alpha = np.split(img_rgba, [3], axis=2)
+        return rgb.astype("uint8")
+
+
+class Visualizer:
+    def __init__(self, img_rgb, metadata=None, scale=1.0):
+        self.img = np.asarray(img_rgb).clip(0, 255).astype(np.uint8)
+        self.font_path = FONT_PATH
+        self.output = VisImage(self.img, scale=scale)
+        self.cpu_device = torch.device("cpu")
+
+        # too small texts are useless, therefore clamp to 14
+        self._default_font_size = max(
+            np.sqrt(self.output.height * self.output.width) // 30, 15 // scale
+        )
+
+    def draw_text(
+        self,
+        text,
+        position,
+        *,
+        font_size=None,
+        color="g",
+        horizontal_alignment="center",
+        rotation=0,
+    ):
+        if not font_size:
+            font_size = self._default_font_size
+
+        # since the text background is dark, we don't want the text to be dark
+        color = np.maximum(list(mplc.to_rgb(color)), 0.2)
+        color[np.argmax(color)] = max(0.8, np.max(color))
+
+        x, y = position
+        self.output.ax.text(
+            x,
+            y,
+            text,
+            size=font_size * self.output.scale,
+            fontproperties=FontProperties(fname=self.font_path),
+            bbox={"facecolor": "black", "alpha": 0.8, "pad": 0.7, "edgecolor": "none"},
+            verticalalignment="top",
+            horizontalalignment=horizontal_alignment,
+            color=color,
+            zorder=10,
+            rotation=rotation,
+        )
+        return self.output
+
+    def draw_box(self, box_coord, alpha=0.5, edge_color="g", line_style="-"):
+        
+        x0, y0, x1, y1 = box_coord
+        width = x1 - x0
+        height = y1 - y0
+
+        linewidth = max(self._default_font_size / 4, 1)
+
+        self.output.ax.add_patch(
+            mpl.patches.Rectangle(
+                (x0, y0),
+                width,
+                height,
+                fill=False,
+                edgecolor=edge_color,
+                linewidth=linewidth * self.output.scale,
+                alpha=alpha,
+                linestyle=line_style,
+            )
+        )
+        return self.output
+
+    def get_output(self):
+        
+        return self.output

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_qwen.QWenTokenizer",
+      null
+    ]
+  },
+  "clean_up_tokenization_spaces": true,
+  "model_max_length": 800,
+  "padding_side": "right",
+  "tokenizer_class": "QWenTokenizer"
+}

zero_to_fp32.py

@@ -0,0 +1,587 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel()
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag)