|
|
--- |
|
|
library_name: transformers |
|
|
pipeline_tag: image-text-to-text |
|
|
inference: true |
|
|
widget: |
|
|
- text: Hello! |
|
|
example_title: Hello world |
|
|
group: Python |
|
|
base_model: |
|
|
- Qwen/Qwen3-VL-235B-A22B-Instruct |
|
|
--- |
|
|
|
|
|
This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [Qwen/Qwen3-VL-235B-A22B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct). |
|
|
|
|
|
### Example usage: |
|
|
|
|
|
```python |
|
|
import numpy as np |
|
|
import torch |
|
|
import transformers |
|
|
from PIL import Image |
|
|
from transformers import ( |
|
|
AutoModel, |
|
|
AutoModelForCausalLM, |
|
|
AutoProcessor, |
|
|
AutoTokenizer, |
|
|
Qwen3VLMoeForConditionalGeneration, |
|
|
) |
|
|
|
|
|
model_id = "yujiepan/qwen3-vl-moe-tiny-random" |
|
|
model = Qwen3VLMoeForConditionalGeneration.from_pretrained( |
|
|
model_id, dtype=torch.bfloat16, device_map="cuda", |
|
|
# attn_implementation="flash_attention_2", |
|
|
) |
|
|
processor = AutoProcessor.from_pretrained(model_id) |
|
|
messages = [ |
|
|
{ |
|
|
"role": "user", |
|
|
"content": [ |
|
|
{ |
|
|
"type": "image", |
|
|
"image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg", |
|
|
}, |
|
|
{"type": "text", "text": "Describe this image."}, |
|
|
], |
|
|
} |
|
|
] |
|
|
|
|
|
# Preparation for inference |
|
|
inputs = processor.apply_chat_template( |
|
|
messages, |
|
|
tokenize=True, |
|
|
add_generation_prompt=True, |
|
|
return_dict=True, |
|
|
return_tensors="pt" |
|
|
).to(model.device) |
|
|
|
|
|
# Inference: Generation of the output |
|
|
generated_ids = model.generate(**inputs, max_new_tokens=32) |
|
|
generated_ids_trimmed = [ |
|
|
out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) |
|
|
] |
|
|
output_text = processor.batch_decode( |
|
|
generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False |
|
|
) |
|
|
print(output_text) |
|
|
``` |
|
|
|
|
|
### Codes to create this repo: |
|
|
|
|
|
```python |
|
|
import json |
|
|
from pathlib import Path |
|
|
|
|
|
import accelerate |
|
|
import torch |
|
|
from huggingface_hub import file_exists, hf_hub_download |
|
|
from transformers import ( |
|
|
AutoConfig, |
|
|
AutoModelForCausalLM, |
|
|
AutoProcessor, |
|
|
GenerationConfig, |
|
|
Qwen3VLMoeForConditionalGeneration, |
|
|
set_seed, |
|
|
) |
|
|
|
|
|
source_model_id = "Qwen/Qwen3-VL-235B-A22B-Instruct" |
|
|
save_folder = "/tmp/yujiepan/qwen3-vl-moe-tiny-random" |
|
|
|
|
|
processor = AutoProcessor.from_pretrained(source_model_id, trust_remote_code=True) |
|
|
processor.save_pretrained(save_folder) |
|
|
|
|
|
with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f: |
|
|
config_json = json.load(f) |
|
|
|
|
|
config_json['text_config'].update({ |
|
|
'head_dim': 32, |
|
|
'hidden_size': 8, |
|
|
'intermediate_size': 64, |
|
|
'moe_intermediate_size': 64, |
|
|
'num_hidden_layers': 2, |
|
|
'num_attention_heads': 8, |
|
|
'num_key_value_heads': 4, |
|
|
'num_experts': 16, |
|
|
# 'decoder_sparse_step': 2, |
|
|
}) |
|
|
config_json['text_config']['rope_scaling']['mrope_section'] = [8, 4, 4] |
|
|
config_json['vision_config'].update( |
|
|
{ |
|
|
'hidden_size': 64, |
|
|
'intermediate_size': 64, |
|
|
'num_heads': 2, |
|
|
'out_hidden_size': 8, |
|
|
'depth': 6, |
|
|
'deepstack_visual_indexes': [1, 3, 5], |
|
|
} |
|
|
) |
|
|
with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f: |
|
|
json.dump(config_json, f, indent=2) |
|
|
|
|
|
config = AutoConfig.from_pretrained( |
|
|
save_folder, |
|
|
trust_remote_code=True, |
|
|
) |
|
|
print(config) |
|
|
torch.set_default_dtype(torch.bfloat16) |
|
|
model = Qwen3VLMoeForConditionalGeneration(config) |
|
|
torch.set_default_dtype(torch.float32) |
|
|
if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'): |
|
|
model.generation_config = GenerationConfig.from_pretrained( |
|
|
source_model_id, trust_remote_code=True, |
|
|
) |
|
|
model.generation_config.do_sample = True |
|
|
print(model.generation_config) |
|
|
model = model.cpu() |
|
|
with torch.no_grad(): |
|
|
for name, p in sorted(model.named_parameters()): |
|
|
torch.nn.init.normal_(p, 0, 0.1) |
|
|
print(name, p.shape) |
|
|
model.save_pretrained(save_folder) |
|
|
``` |
|
|
|
|
|
### Printing the model: |
|
|
|
|
|
```text |
|
|
Qwen3VLMoeForConditionalGeneration( |
|
|
(model): Qwen3VLMoeModel( |
|
|
(visual): Qwen3VLMoeVisionModel( |
|
|
(patch_embed): Qwen3VLMoeVisionPatchEmbed( |
|
|
(proj): Conv3d(3, 64, kernel_size=(2, 16, 16), stride=(2, 16, 16)) |
|
|
) |
|
|
(pos_embed): Embedding(2304, 64) |
|
|
(rotary_pos_emb): Qwen3VLMoeVisionRotaryEmbedding() |
|
|
(blocks): ModuleList( |
|
|
(0-5): 6 x Qwen3VLMoeVisionBlock( |
|
|
(norm1): LayerNorm((64,), eps=1e-06, elementwise_affine=True) |
|
|
(norm2): LayerNorm((64,), eps=1e-06, elementwise_affine=True) |
|
|
(attn): Qwen3VLMoeVisionAttention( |
|
|
(qkv): Linear(in_features=64, out_features=192, bias=True) |
|
|
(proj): Linear(in_features=64, out_features=64, bias=True) |
|
|
) |
|
|
(mlp): Qwen3VLMoeVisionMLP( |
|
|
(linear_fc1): Linear(in_features=64, out_features=64, bias=True) |
|
|
(linear_fc2): Linear(in_features=64, out_features=64, bias=True) |
|
|
(act_fn): PytorchGELUTanh() |
|
|
) |
|
|
) |
|
|
) |
|
|
(merger): Qwen3VLMoeVisionPatchMerger( |
|
|
(norm): LayerNorm((64,), eps=1e-06, elementwise_affine=True) |
|
|
(linear_fc1): Linear(in_features=256, out_features=256, bias=True) |
|
|
(act_fn): GELU(approximate='none') |
|
|
(linear_fc2): Linear(in_features=256, out_features=8, bias=True) |
|
|
) |
|
|
(deepstack_merger_list): ModuleList( |
|
|
(0-2): 3 x Qwen3VLMoeVisionPatchMerger( |
|
|
(norm): LayerNorm((256,), eps=1e-06, elementwise_affine=True) |
|
|
(linear_fc1): Linear(in_features=256, out_features=256, bias=True) |
|
|
(act_fn): GELU(approximate='none') |
|
|
(linear_fc2): Linear(in_features=256, out_features=8, bias=True) |
|
|
) |
|
|
) |
|
|
) |
|
|
(language_model): Qwen3VLMoeTextModel( |
|
|
(embed_tokens): Embedding(151936, 8) |
|
|
(layers): ModuleList( |
|
|
(0-1): 2 x Qwen3VLMoeTextDecoderLayer( |
|
|
(self_attn): Qwen3VLMoeTextAttention( |
|
|
(q_proj): Linear(in_features=8, out_features=256, bias=False) |
|
|
(k_proj): Linear(in_features=8, out_features=128, bias=False) |
|
|
(v_proj): Linear(in_features=8, out_features=128, bias=False) |
|
|
(o_proj): Linear(in_features=256, out_features=8, bias=False) |
|
|
(q_norm): Qwen3VLMoeTextRMSNorm((32,), eps=1e-06) |
|
|
(k_norm): Qwen3VLMoeTextRMSNorm((32,), eps=1e-06) |
|
|
) |
|
|
(mlp): Qwen3VLMoeTextSparseMoeBlock( |
|
|
(gate): Qwen3VLMoeTextRouter(in_features=8, out_features=16, bias=False) |
|
|
(experts): Qwen3VLMoeTextExperts( |
|
|
(act_fn): SiLU() |
|
|
) |
|
|
) |
|
|
(input_layernorm): Qwen3VLMoeTextRMSNorm((8,), eps=1e-06) |
|
|
(post_attention_layernorm): Qwen3VLMoeTextRMSNorm((8,), eps=1e-06) |
|
|
) |
|
|
) |
|
|
(norm): Qwen3VLMoeTextRMSNorm((8,), eps=1e-06) |
|
|
(rotary_emb): Qwen3VLMoeTextRotaryEmbedding() |
|
|
) |
|
|
) |
|
|
(lm_head): Linear(in_features=8, out_features=151936, bias=False) |
|
|
) |
|
|
``` |
