Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/neg_1.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/neg_2.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/positive_1.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/positive_2.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/query_1.png filter=lfs diff=lfs merge=lfs -text
+assets/query_2.png filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,50 @@
+{
+  "architectures": [
+    "Qwen2VLForConditionalGeneration"
+  ],
+  "auto_map": {
+    "AutoModel": "modeling_qwen2vl_for_embedding.Qwen2VLForEmbedding"
+  },
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "image_token_id": 151655,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2_vl",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "mrope_section": [
+      16,
+      24,
+      24
+    ],
+    "rope_type": "default",
+    "type": "default"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float16",
+  "transformers_version": "4.45.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "hidden_size": 1536,
+    "in_chans": 3,
+    "model_type": "qwen2_vl",
+    "spatial_patch_size": 14
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "temperature": 0.01,
+  "top_k": 1,
+  "top_p": 0.001,
+  "transformers_version": "4.45.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95c4ede4b0d96a2b43f843410c45fc975783085920b905c358b6a25e903af83a
+size 1995539504

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8433fb9079a5de6c73f1b0ae98598a948d78a0a828428c8752a57213c0a97232
+size 1993062176

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6d04211aee4bf2e9815af5b9354c85156c9d7bc72bbeaeea1b27ab3faff36c5
+size 429447968

model.safetensors.index.json

@@ -0,0 +1,736 @@
+{
+  "metadata": {
+    "total_size": 4417971200
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors",
+    "visual.blocks.0.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.0.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.1.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.10.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.11.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.12.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.13.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.14.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.15.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.16.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.17.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.18.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.19.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.2.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.20.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.21.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.22.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.23.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.24.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.25.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.26.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.27.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.28.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.29.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.3.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.30.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.31.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.4.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.5.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.6.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.7.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.8.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.attn.proj.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.attn.proj.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.attn.qkv.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.attn.qkv.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.norm1.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.norm1.weight": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.norm2.bias": "model-00001-of-00003.safetensors",
+    "visual.blocks.9.norm2.weight": "model-00001-of-00003.safetensors",
+    "visual.merger.ln_q.bias": "model-00001-of-00003.safetensors",
+    "visual.merger.ln_q.weight": "model-00001-of-00003.safetensors",
+    "visual.merger.mlp.0.bias": "model-00001-of-00003.safetensors",
+    "visual.merger.mlp.0.weight": "model-00001-of-00003.safetensors",
+    "visual.merger.mlp.2.bias": "model-00001-of-00003.safetensors",
+    "visual.merger.mlp.2.weight": "model-00001-of-00003.safetensors",
+    "visual.patch_embed.proj.weight": "model-00001-of-00003.safetensors"
+  }
+}

modeling_qwen2vl_for_embedding.py

@@ -0,0 +1,164 @@
+import logging
+from transformers import AutoProcessor, Qwen2VLForConditionalGeneration
+import torch
+from PIL import Image
+from typing import List, Optional, Tuple, Union, cast
+import numpy as np
+from tqdm import tqdm
+import sys
+import os
+from torch.utils.data import DataLoader
+from torch import nn
+
+
+logger = logging.getLogger(__name__)
+
+class Qwen2VLForEmbedding(Qwen2VLForConditionalGeneration):
+    def __init__(self, config):
+        super().__init__(config)
+    
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        pixel_values: Optional[torch.Tensor] = None,
+        pixel_values_videos: Optional[torch.FloatTensor] = None,
+        image_grid_thw: Optional[torch.LongTensor] = None,
+        video_grid_thw: Optional[torch.LongTensor] = None,
+        rope_deltas: Optional[torch.LongTensor] = None,
+    ): 
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if inputs_embeds is None:
+            inputs_embeds = self.model.embed_tokens(input_ids)
+            if pixel_values is not None:
+                pixel_values = pixel_values.type(self.visual.get_dtype())
+                image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw)
+                image_mask = (input_ids == self.config.image_token_id).unsqueeze(-1).expand_as(inputs_embeds)
+                image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+                inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
+
+            if pixel_values_videos is not None:
+                pixel_values_videos = pixel_values_videos.type(self.visual.get_dtype())
+                video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw)
+                video_mask = (input_ids == self.config.video_token_id).unsqueeze(-1).expand_as(inputs_embeds)
+                video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+                inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)
+
+            if attention_mask is not None:
+                attention_mask = attention_mask.to(inputs_embeds.device)
+
+        outputs = self.model(
+            input_ids=None,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = outputs[0] 
+        embeddings = hidden_states[:, -1, :]
+        embeddings = torch.nn.functional.normalize(embeddings, dim=-1)
+        return embeddings
+
+    def set_processor(self, model_name_or_path, max_len=3072, eos_token_id=151643, min_image_token=64, max_image_token=2500):
+        self.max_len = max_len
+        self.eos_token_id = eos_token_id
+        self.processor = AutoProcessor.from_pretrained(
+            model_name_or_path,
+            min_pixels=min_image_token * 28 * 28,
+            max_pixels=max_image_token * 28 * 28
+        )
+        assert self.processor.tokenizer.padding_side == 'left'
+    
+    def prepare_text_input(self, image=None, text=None, q_or_c=None, task_instruction=None):
+        assert q_or_c in ["query", "candidate", "q", "c"]
+        
+        prompt_template = "<|im_start|>system\n{}<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n<|endoftext|>"
+        
+        if "q" in q_or_c:
+            if task_instruction is None:
+                system_prompt = "You are a helpful assistant."
+                task_instruction_example_csr = "Represent the given image with the given query."
+                print(f"""Warning: For optimal performance, UniSE-MLLM requires the task instruction to be specified in the query. For example, for the Composed Screenshot Retrieval task, you might use a specific instruction like: {task_instruction_example_csr}.""")
+            else:
+                system_prompt = task_instruction
+
+            if image is None:
+                user_prompt = text
+            else:
+                if text is not None:
+                    user_prompt = f"Query:{text}<|vision_start|><|image_pad|><|vision_end|>"
+                else:
+                    user_prompt = "<|vision_start|><|image_pad|><|vision_end|>"
+            text_input = prompt_template.format(system_prompt, user_prompt)
+        else:
+            if text is not None:
+                system_prompt = "Represent the given text."
+                user_prompt = f"{text}"
+            if image is not None:
+                system_prompt = "Represent the given text-rich image, focusing on extracting and interpreting both its rich text content and visual features."
+                user_prompt = f"<|vision_start|><|image_pad|><|vision_end|>"
+            text_input = prompt_template.format(system_prompt, user_prompt)
+        return text_input
+
+    def data_process(self, images=None, text=None, q_or_c=None, task_instruction=None):
+        if images is not None:
+            _is_list = isinstance(images, list)
+        elif text is not None:
+            _is_list = isinstance(text, list)
+        else:
+            raise ValueError("images and text cannot be both None.")
+        
+        assert q_or_c in ["query", "candidate", "q", "c"]
+
+        if not _is_list :
+            text_input = self.prepare_text_input(images, text, q_or_c, task_instruction)
+            text_input = [text_input]
+            
+
+            if images is not None:
+                images = Image.open(images).convert("RGB")
+                images = [images]
+                inputs = self.processor(images=images, text=text_input, return_tensors="pt", padding=True, truncation=True,  max_length=self.max_len)
+            else:
+                inputs = self.processor(text=text_input, return_tensors="pt", padding=True, truncation=True,  max_length=self.max_len)
+            if inputs.input_ids.size(-1) == self.max_len:
+                inputs.input_ids[:, -1] = self.eos_token_id
+            assert (inputs.input_ids[:, -1] == self.eos_token_id).all()
+            assert (inputs.attention_mask[:, -1] == 1).all()
+
+        else:
+            if text is None:
+                text = [None] * len(images)
+            text_input = [self.prepare_text_input(_image, _text, q_or_c, task_instruction) for _image, _text in zip(images, text)]
+            
+            if images is not None:
+                images = [Image.open(_image).convert("RGB") for _image in images]
+                inputs = self.processor(images=images, text=text_input, return_tensors="pt", padding=True, truncation=True,  max_length=self.max_len)
+            else:
+                inputs = self.processor(text=text_input, return_tensors="pt", padding=True, truncation=True, max_length=self.max_len)
+            if inputs.input_ids.size(-1) == self.max_len:
+                inputs.input_ids[:, -1] = self.eos_token_id
+            assert (inputs.input_ids[:, -1] == self.eos_token_id).all()
+            assert (inputs.attention_mask[:, -1] == 1).all()
+
+        inputs = inputs.to(self.device)
+
+        return inputs

preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "min_pixels": 3136,
+  "max_pixels": 12845056,
+  "patch_size": 14,
+  "temporal_patch_size": 2,
+  "merge_size": 2,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "processor_class": "Qwen2VLProcessor"
+}

tokenizer_config.json

@@ -0,0 +1,129 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": ["<|im_start|>", "<|im_end|>", "<|object_ref_start|>","<|object_ref_end|>","<|box_start|>","<|box_end|>","<|quad_start|>","<|quad_end|>","<|vision_start|>","<|vision_end|>","<|vision_pad|>","<|image_pad|>","<|video_pad|>"],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "padding_side": "left",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}