Upload folder using huggingface_hub

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

config.json

@@ -0,0 +1,555 @@
+{
+  "_name_or_path": "/mnt/bn/tiktok-mm-4/aiic/public/model/OV-Qwen2-7B-AM9",
+  "add_time_token": true,
+  "architectures": [
+    "LlavaAVQwenForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "image_aspect_ratio": "anyres_max_9",
+  "image_crop_resolution": null,
+  "image_grid_pinpoints": [
+    [
+      384,
+      384
+    ],
+    [
+      384,
+      768
+    ],
+    [
+      384,
+      1152
+    ],
+    [
+      384,
+      1536
+    ],
+    [
+      768,
+      768
+    ],
+    [
+      384,
+      1920
+    ],
+    [
+      768,
+      768
+    ],
+    [
+      384,
+      2304
+    ],
+    [
+      768,
+      1152
+    ],
+    [
+      384,
+      2688
+    ],
+    [
+      768,
+      1152
+    ],
+    [
+      384,
+      3072
+    ],
+    [
+      768,
+      1536
+    ],
+    [
+      384,
+      3456
+    ],
+    [
+      768,
+      1536
+    ],
+    [
+      1152,
+      1152
+    ],
+    [
+      768,
+      1920
+    ],
+    [
+      1152,
+      1152
+    ],
+    [
+      768,
+      1920
+    ],
+    [
+      1152,
+      1152
+    ],
+    [
+      768,
+      2304
+    ],
+    [
+      1152,
+      1536
+    ],
+    [
+      768,
+      2304
+    ],
+    [
+      1152,
+      1536
+    ],
+    [
+      768,
+      2688
+    ],
+    [
+      1152,
+      1536
+    ],
+    [
+      768,
+      2688
+    ],
+    [
+      1152,
+      1920
+    ],
+    [
+      768,
+      3072
+    ],
+    [
+      1152,
+      1920
+    ],
+    [
+      1536,
+      1536
+    ],
+    [
+      768,
+      3072
+    ],
+    [
+      1152,
+      1920
+    ],
+    [
+      1536,
+      1536
+    ],
+    [
+      768,
+      3456
+    ],
+    [
+      1152,
+      2304
+    ],
+    [
+      1536,
+      1536
+    ],
+    [
+      768,
+      3456
+    ],
+    [
+      1152,
+      2304
+    ],
+    [
+      1536,
+      1536
+    ],
+    [
+      768,
+      3840
+    ],
+    [
+      1152,
+      2304
+    ],
+    [
+      1536,
+      1920
+    ],
+    [
+      768,
+      3840
+    ],
+    [
+      1152,
+      2688
+    ],
+    [
+      1536,
+      1920
+    ],
+    [
+      768,
+      4224
+    ],
+    [
+      1152,
+      2688
+    ],
+    [
+      1536,
+      1920
+    ],
+    [
+      768,
+      4224
+    ],
+    [
+      1152,
+      2688
+    ],
+    [
+      1536,
+      1920
+    ],
+    [
+      768,
+      4608
+    ],
+    [
+      1152,
+      3072
+    ],
+    [
+      1536,
+      2304
+    ],
+    [
+      768,
+      4608
+    ],
+    [
+      1152,
+      3072
+    ],
+    [
+      1536,
+      2304
+    ],
+    [
+      1920,
+      1920
+    ],
+    [
+      768,
+      4992
+    ],
+    [
+      1152,
+      3072
+    ],
+    [
+      1536,
+      2304
+    ],
+    [
+      1920,
+      1920
+    ],
+    [
+      768,
+      4992
+    ],
+    [
+      1152,
+      3456
+    ],
+    [
+      1536,
+      2304
+    ],
+    [
+      1920,
+      1920
+    ],
+    [
+      768,
+      5376
+    ],
+    [
+      1152,
+      3456
+    ],
+    [
+      1536,
+      2688
+    ],
+    [
+      1920,
+      1920
+    ],
+    [
+      768,
+      5376
+    ],
+    [
+      1152,
+      3456
+    ],
+    [
+      1536,
+      2688
+    ],
+    [
+      1920,
+      1920
+    ],
+    [
+      768,
+      5760
+    ],
+    [
+      1152,
+      3840
+    ],
+    [
+      1536,
+      2688
+    ],
+    [
+      1920,
+      2304
+    ],
+    [
+      768,
+      5760
+    ],
+    [
+      1152,
+      3840
+    ],
+    [
+      1536,
+      2688
+    ],
+    [
+      1920,
+      2304
+    ],
+    [
+      768,
+      6144
+    ],
+    [
+      1152,
+      3840
+    ],
+    [
+      1536,
+      3072
+    ],
+    [
+      1920,
+      2304
+    ],
+    [
+      768,
+      6144
+    ],
+    [
+      1152,
+      4224
+    ],
+    [
+      1536,
+      3072
+    ],
+    [
+      1920,
+      2304
+    ],
+    [
+      768,
+      6528
+    ],
+    [
+      1152,
+      4224
+    ],
+    [
+      1536,
+      3072
+    ],
+    [
+      1920,
+      2304
+    ],
+    [
+      768,
+      6528
+    ],
+    [
+      1152,
+      4224
+    ],
+    [
+      1536,
+      3072
+    ],
+    [
+      1920,
+      2688
+    ],
+    [
+      768,
+      6912
+    ],
+    [
+      1152,
+      4608
+    ],
+    [
+      1536,
+      3456
+    ],
+    [
+      1920,
+      2688
+    ],
+    [
+      2304,
+      2304
+    ]
+  ],
+  "image_split_resolution": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "mf_split_init": false,
+  "mm_hidden_size": 1152,
+  "mm_patch_merge_type": "spatial_unpad",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_tunable_parts": "mm_vision_tower,mm_mlp_adapter,mm_language_model",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "mm_vision_tower": "google/siglip-so400m-patch14-384",
+  "mm_vision_tower_lr": 2e-06,
+  "model_args": {
+    "add_time_token": true,
+    "audio_visual": true,
+    "beats_path": null,
+    "do_rag": false,
+    "flash_attn": false,
+    "fps": 2,
+    "freeze_backbone": true,
+    "freeze_beats": true,
+    "freeze_final_linear": true,
+    "freeze_speech_QFormer": true,
+    "freeze_whisper": true,
+    "from_stage_1_5": false,
+    "image_processor": "google/siglip-so400m-patch14-384",
+    "lora_alpha": 256,
+    "lora_dropout": 0.05,
+    "lora_enable": true,
+    "lora_r": 64,
+    "mf_split_init": false,
+    "mm_mask_drop_mode": "fixed",
+    "mm_mask_drop_ratio": 0.25,
+    "mm_mask_drop_ratio_lower": null,
+    "mm_mask_drop_ratio_upper": null,
+    "mm_mask_drop_skip_percentage": 0.0,
+    "mm_newline_position": "grid",
+    "mm_patch_merge_type": "spatial_unpad",
+    "mm_perceiver_depth": 3,
+    "mm_perceiver_ff_mult": 4,
+    "mm_perceiver_latents": 32,
+    "mm_perceiver_pretrained": null,
+    "mm_pooling_position": "before",
+    "mm_projector_type": "mlp2x_gelu",
+    "mm_qformer_depth": 3,
+    "mm_qformer_latents": 32,
+    "mm_qformer_pretrained": null,
+    "mm_resampler_type": "spatial_pool",
+    "mm_spatial_pool_mode": "max",
+    "mm_spatial_pool_out_channels": 1152,
+    "mm_spatial_pool_stride": 2,
+    "mm_use_im_patch_token": false,
+    "mm_use_im_start_end": false,
+    "mm_vision_select_feature": "patch",
+    "mm_vision_select_layer": -2,
+    "mm_vlmattention_bert_type": "qformer_pretrain",
+    "mm_vlmattention_compress_type": null,
+    "mm_vlmattention_num_query": 32,
+    "mm_vlmattention_pretrained": null,
+    "modality_max_length": "None",
+    "model_name_or_path": "/mnt/bn/tiktok-mm-4/aiic/users/guangzhisun/llava-video/output/models/sft_finevideo_with_caption_qa_reasoning/checkpoint_93000_qwen",
+    "multi_frame_num": 30,
+    "multi_frame_projector": false,
+    "niv_cnn_params": "[(10, 5), (3, 2), (3, 2), (3, 2), (3, 2), (2, 2), (2, 2)]",
+    "niv_in_channels": 4,
+    "niv_out_channels": 401,
+    "num_speech_query_token": 1,
+    "patchify_video_feature": false,
+    "pretrain_mm_mlp_adapter": null,
+    "rag_input_frames": 1,
+    "rag_topk": 5,
+    "rag_type": "direct",
+    "salmonn_path": null,
+    "second_per_window": 0.2,
+    "second_stride": 0.2,
+    "segmentation": -1,
+    "spt_projector": false,
+    "tune_mm_mlp_adapter": false,
+    "tune_mm_vision_resampler": false,
+    "unfreeze_mm_vision_tower": false,
+    "use_final_linear": true,
+    "use_flash_tower": true,
+    "use_mfcnn": false,
+    "use_mftrans": false,
+    "use_niv": false,
+    "use_speech_Qformer": true,
+    "version": "qwen_1_5",
+    "vision_tower": "google/siglip-so400m-patch14-384",
+    "whisper_lora": false,
+    "whisper_path": "openai/whisper-large-v3",
+    "window_level_Qformer": true
+  },
+  "model_type": "qwen2",
+  "multi_frame_num": 30,
+  "multi_frame_projector": false,
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "pos_skipping_range": 4096,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "tokenizer_model_max_length": 32768,
+  "tokenizer_padding_side": "right",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.2",
+  "use_cache": true,
+  "use_flash_tower": true,
+  "use_mfcnn": false,
+  "use_mftrans": false,
+  "use_mm_proj": true,
+  "use_pos_skipping": false,
+  "use_sliding_window": false,
+  "vision_tower_pretrained": null,
+  "vocab_size": 152064
+}

generation_config.json

@@ -0,0 +1,32 @@
+{
+  "beats_path": null,
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "freeze_beats": true,
+  "freeze_speech_QFormer": true,
+  "freeze_whisper": true,
+  "niv_cnn_params": "[(10, 5), (3, 2), (3, 2), (3, 2), (3, 2), (2, 2), (2, 2)]",
+  "niv_in_channels": 4,
+  "niv_out_channels": 401,
+  "num_speech_query_token": 1,
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "salmonn_path": null,
+  "second_per_window": 0.2,
+  "second_stride": 0.2,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "train_orm": "onlineormtraingrpoprime",
+  "transformers_version": "4.39.2",
+  "use_final_linear": true,
+  "use_niv": false,
+  "use_speech_Qformer": true,
+  "video_fps": 2,
+  "whisper_path": "openai/whisper-large-v3",
+  "window_level_Qformer": true
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55b48adee0f23054120634b7146af6240ef26ffcd1b32b6100cd3f872ec36291
+size 4877669664

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4881e407d5cc3395d5f2150ef3d761a76564ffc52beb45e930e93502ca8dd649
+size 4932751008

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac9f33fd16b0f0561bf7f7d39d916a6d5f9887f700e04f825d13d407911a0c2e
+size 4998052768

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df807325f2a614b1e970223a19f233b50debcdda8c17729b02224dfe67963049
+size 2624207888

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}