New Processor, Model for vLLM

README.md

@@ -77,6 +77,7 @@ Although HyperCLOVAX-SEED-Vision-Instruct-3B is a lightweight model, it is capab
 - [decord](https://github.com/dmlc/decord)
 
 ## Example
+**(code & benchmark score) checked with transformers 4.52.4**
 
 ```python
 
@@ -87,6 +88,112 @@ model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
 preprocessor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 
+# LLM Example
+# It is recommended to use the chat template with HyperCLOVAX models.
+# Using the chat template allows you to easily format your input in ChatML style.
+llm_chat = [
+        {"role": "system", "content": [{"type": "text", "text": "you are helpful assistant!"}]},
+        {
+                "role": "user", 
+                "content": [
+                        {"type": "text", "text": "Hello, how are you?"},
+                        {"type": "text", "text": "I said. Hello, how are you today?"},
+                ]
+        },
+        {"role": "assistant", "content": [{"type": "text", "text": "I'm doing great. How can I help you today?"}]},
+        {"role": "user", "content": [{"type": "text", "text": "I'd like to show off how chat templating works!"}]},
+]
+model_inputs = processor.apply_chat_template(
+        llm_chat, tokenize=True, return_dict=True, return_tensors="pt", add_generation_prompt=True
+)
+model_inputs = model_inputs.to(device=DEVICE)
+
+# Please adjust parameters like top_p appropriately for your use case.
+output_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=64,
+        do_sample=True,
+        top_p=0.6,
+        temperature=0.5,
+        repetition_penalty=1.0,
+)
+print("=" * 80)
+print("LLM EXAMPLE")
+print(processor.batch_decode(output_ids)[0])
+print("=" * 80)
+
+# VLM Example
+# For images and videos, you can use url, local_path, base64, or bytes as input sources.
+vlm_chat = [
+        {"role": "system", "content": [{"text": "System Prompt", "type": "text"}]},
+        {"role": "user", "content": [{"text": "User Text Prompt 1", "type": "text"}]},
+        {
+                "role": "user",
+                "content": [{
+                        "filename": "tradeoff_sota.png",
+                        "image": "https://github.com/naver-ai/rdnet/blob/main/resources/images/tradeoff_sota.png?raw=true",
+                        "lens_keywords": "Gucci Ophidia, cross bag, Ophidia small, GG, Supreme shoulder bag",
+                        "lens_local_keywords": "[0.07, 0.21, 0.92, 0.90] Gucci Ophidia",
+                        "ocr": "List the words in the image in raster order. Even if the word order feels unnatural for reading, the model will handle it as long as it follows raster order.",                        "type": "image",
+                }],
+        },
+        {
+                "role": "user",
+                "content": [{
+                        "filename": "tradeoff.png",
+                        "image": "https://github.com/naver-ai/rdnet/blob/main/resources/images/tradeoff.png?raw=true",
+                        "type": "image",
+                }],
+        },
+        {"role": "assistant", "content": [{"text": "Assistant Text Prompt 1", "type": "text"}]},
+        {"role": "user", "content": [{"text": "User Text Prompt 2", "type": "text"}]},
+        {
+                "role": "user",
+                "content": [
+                        {
+                                "type": "video",
+                                "video": "freenaturestock-rolling-mist-clouds.mp4",
+                                "lens_keywords": "Prada re-edition, nylon bag, mini cross bag, logo strap, essential shoulder bag",
+                                "lens_local_keywords": "[0.12, 0.34, 0.85, 0.76] Prada re-edition",
+                                "speech_to_text": "Please enter the dialogue, voice, sound, lines, and words in the video in text format.",
+                        },
+                        {"text": "User Text Prompt 3", "type": "text"},
+                ]
+        },
+]
+
+model_inputs = processor.apply_chat_template(   
+        vlm_chat, tokenize=True, return_dict=True, return_tensors="pt", add_generation_prompt=True,
+)
+model_inputs = model_inputs.to(device=DEVICE)
+        output_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=64,
+        do_sample=True,
+        top_p=0.6,
+        temperature=0.5,
+        repetition_penalty=1.0,
+)
+print("=" * 80)
+print("VLM EXAMPLE")
+print(processor.batch_decode(output_ids)[0])
+print("=" * 80)
+
+```
+
+## Example for v0.1.0
+**(code & benchmark score) checked with transformers 4.45.0**
+
+```python
+
+from transformers import AutoModelForCausalLM, AutoProcessor, AutoTokenizer
+
+model_name = "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B"
+revision="v0.1.0"
+model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, revision=revision).to(device="cuda")
+preprocessor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True, revision=revision)
+tokenizer = AutoTokenizer.from_pretrained(model_name, revision=revision)
+
 # LLM Example
 # It is recommended to use the chat template with HyperCLOVAX models.
 # Using the chat template allows you to easily format your input in ChatML style.
@@ -165,7 +272,10 @@ output_ids = model.generate(
         repetition_penalty=1.0,
         **preprocessed,
 )
+print("=" * 80)
+print("VLM EXAMPLE")
 print(tokenizer.batch_decode(output_ids)[0])
+print("=" * 80)
 ```
 
 - To ensure the highest level of image understanding performance, it is recommended to include additional information such as Optical Character Recognition (OCR) results and entity recognition (Lens). The provided usage examples are written under the assumption that OCR and Lens results are available. If you input data in this format, you can expect significantly improved output quality.

added_tokens.json

@@ -0,0 +1,35 @@
+{
+  "<EMAIL>": 110521,
+  "<KEY>": 110522,
+  "<NAME>": 110520,
+  "<PASSWORD>": 110523,
+  "<code_to_intermediate>": 110502,
+  "<empty_output>": 110501,
+  "<file_sep>": 110492,
+  "<intermediate_to_code>": 110503,
+  "<issue_closed>": 110495,
+  "<issue_comment>": 110494,
+  "<issue_start>": 110493,
+  "<jupyter_code>": 110498,
+  "<jupyter_output>": 110499,
+  "<jupyter_script>": 110500,
+  "<jupyter_start>": 110496,
+  "<jupyter_text>": 110497,
+  "<pr>": 110504,
+  "<pr_base>": 110507,
+  "<pr_base_code>": 110509,
+  "<pr_comment>": 110512,
+  "<pr_diff>": 110510,
+  "<pr_diff_hunk>": 110511,
+  "<pr_diff_hunk_comment_line>": 110519,
+  "<pr_event_id>": 110513,
+  "<pr_file>": 110508,
+  "<pr_in_reply_to_comment_id>": 110518,
+  "<pr_in_reply_to_review_id>": 110517,
+  "<pr_is_merged>": 110506,
+  "<pr_review>": 110514,
+  "<pr_review_comment>": 110516,
+  "<pr_review_state>": 110515,
+  "<pr_status>": 110505,
+  "<repo_name>": 110491
+}

chat_template.jinja

@@ -0,0 +1,65 @@
+<|im_start|>tool_list
+<|im_end|>
+{% for message in messages %}
+{% set content = message['content'] %}
+{% set role = message['role'] %}
+{% if loop.first and role != 'system' %}
+<|im_start|>system
+You are a helpful assistant.<|im_end|>
+{% endif %}
+{% if message['content'] is string %}
+<|im_start|>{{ role }}
+{{ message['content'] }}<|im_end|>
+{% elif message['content'] is mapping %}
+{% if content['type'] == 'image' %}
+<|im_start|>{{ role }} (mime)
+{"type": "image/jpeg", "filename": "{{ content['filename'] }}"}<|im_end|>
+<|im_start|>{{ role }} (vector)
+<|dummy3|><|im_end|>
+<|im_start|>image/aux
+다음 중 ocr은 사진에서 검출된 글자이고, lens_keyword는 사진에서 추출된 keyword와 bbox 위치입니다. bbox는 0~1 사이로 정규화된 [x1, y1, x2, y2]의 형태입니다. 참고하여 답변하세요. {"ocr": "{{ content['ocr'] or '' }}", "lens_keywords": "{{ content['lens_keywords'] or '' }}", "lens_local_keywords": "{{ content['lens_local_keywords'] or '' }}"}<|im_end|>
+{% elif content['type'] == 'video' %}
+<|im_start|>{{ role }} (mime)
+{"type": "video/mp4", "filename": "{{ content['filename'] }}"}<|im_end|>
+<|im_start|>{{ role }} (vector)
+<|_unuse_missing_100270|><|im_end|>
+<|im_start|>image/aux
+{% if content.get('is_final_grid') %}
+다음 중 lens_keyword는 사진에서 추출된 keyword와 bbox 위치입니다. bbox는 0~1 사이로 정규화된 [x1, y1, x2, y2]의 형태입니다. video_time_stamp는 비디오에서 해당 구간의 시간 정보입니다. speech_to_text는 비디오 속에서의 대화, 음성, 소리, 대사, 그리고 말을 전부 글로 받아 적은 것 입니다. 참고하여 답변하세요. {"video_time_stamp": "{{ content['video_time_stamp'] }}", "lens_keywords": "{{ content.get('lens_keywords', '') }}", "lens_local_keywords": "{{ content.get('lens_local_keywords', '') }}", "speech_to_text": "{{ content.get('speech_to_text', '') }}"}
+{% else %}
+다음 중 video_time_stamp는 비디오에서 해당 구간의 시간 정보입니다. 참고하여 답변하세요. {"video_time_stamp": "{{ content['video_time_stamp'] }}"}
+{% endif %}<|im_end|>
+{% elif content['type'] == 'text' %}
+<|im_start|>{{ role }}
+{{ content['text'] }}<|im_end|>
+{% endif %}
+{% elif message['content'] is sequence %}
+{% for content in message['content'] %}
+{% if content['type'] == 'image' %}
+<|im_start|>{{ role }} (mime)
+{"type": "image/jpeg", "filename": "{{ content['filename'] }}"}<|im_end|>
+<|im_start|>{{ role }} (vector)
+<|dummy3|><|im_end|>
+<|im_start|>image/aux
+다음 중 ocr은 사진에서 검출된 글자이고, lens_keyword는 사진에서 추출된 keyword와 bbox 위치입니다. bbox는 0~1 사이로 정규화된 [x1, y1, x2, y2]의 형태입니다. 참고하여 답변하세요. {"ocr": "{{ content['ocr'] or '' }}", "lens_keywords": "{{ content['lens_keywords'] or '' }}", "lens_local_keywords": "{{ content['lens_local_keywords'] or '' }}"}<|im_end|>
+{% elif content['type'] == 'video' %}
+<|im_start|>{{ role }} (mime)
+{"type": "video/mp4", "filename": "{{ content['filename'] }}"}<|im_end|>
+<|im_start|>{{ role }} (vector)
+<|_unuse_missing_100270|><|im_end|>
+<|im_start|>image/aux
+{% if content.get('is_final_grid') %}
+다음 중 lens_keyword는 사진에서 추출된 keyword와 bbox 위치입니다. bbox는 0~1 사이로 정규화된 [x1, y1, x2, y2]의 형태입니다. video_time_stamp는 비디오에서 해당 구간의 시간 정보입니다. speech_to_text는 비디오 속에서의 대화, 음성, 소리, 대사, 그리고 말을 전부 글로 받아 적은 것 입니다. 참고하여 답변하세요. {"video_time_stamp": "{{ content['video_time_stamp'] }}", "lens_keywords": "{{ content.get('lens_keywords', '') }}", "lens_local_keywords": "{{ content.get('lens_local_keywords', '') }}", "speech_to_text": "{{ content.get('speech_to_text', '') }}"}
+{% else %}
+다음 중 video_time_stamp는 비디오에서 해당 구간의 시간 정보입니다. 참고하여 답변하세요. {"video_time_stamp": "{{ content['video_time_stamp'] }}"}
+{% endif %}<|im_end|>
+{% elif content['type'] == 'text' %}
+<|im_start|>{{ role }}
+{{ content['text'] }}<|im_end|>
+{% endif %}
+{% endfor %}
+{% endif %}
+{% endfor %}
+{% if add_generation_prompt %}
+<|im_start|>assistant
+{% endif %}

config.json

@@ -13,8 +13,10 @@
   "freeze_mm_projector": false,
   "hidden_size": 3072,
   "ignore_index": -100,
-  "img_start_id": 100271,
-  "language_config": {
+  "video_token_id": 100270,
+  "image_token_id": 100271,
+  "mm_projector_type": "cabstractor",
+  "text_config": {
     "_attn_implementation_autoset": true,
     "_name_or_path": "",
     "add_cross_attention": false,
@@ -96,7 +98,7 @@
     "top_p": 1.0,
     "torch_dtype": "bfloat16",
     "torchscript": false,
-    "transformers_version": "4.48.2",
+    "transformers_version": "4.52.4",
     "typical_p": 1.0,
     "use_bfloat16": false,
     "use_cache": true,
@@ -105,12 +107,15 @@
   "max_image_cnt": 12,
   "max_num_grids": 9,
   "model_type": "hyperclovax_vlm",
-  "num_queries_vis_abstractor": 81,
+  "num_queries_vis_abstractor_image": 81,
+  "num_queries_vis_abstractor_video_slow": 81,
+  "num_queries_vis_abstractor_video_fast": 9,
+  "first_last_frames_slow": false,
   "proj_pos_emb": true,
   "proj_prenorm": false,
   "q_former_model_name_or_path": null,
-  "torch_dtype": "float32",
-  "transformers_version": "4.48.2",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.4",
   "unpad": true,
   "use_1x1_grid": true,
   "use_nth_layer": -2,
@@ -118,7 +123,6 @@
     "_attn_implementation_autoset": true,
     "_name_or_path": "",
     "add_cross_attention": false,
-    "anyres": true,
     "architectures": [
       "SiglipVisionModel"
     ],
@@ -191,8 +195,8 @@
     "top_p": 1.0,
     "torch_dtype": "bfloat16",
     "torchscript": false,
-    "transformers_version": "4.48.2",
+    "transformers_version": "4.52.4",
     "typical_p": 1.0,
     "use_bfloat16": true
   }
-}
+}

configuration_hyperclovax.py

@@ -1,3 +1,4 @@
+from transformers import AutoConfig
 from transformers.configuration_utils import PretrainedConfig
 from transformers.utils import logging
 
@@ -9,7 +10,7 @@ class HCXVisionConfig(PretrainedConfig):
     keys_to_ignore_at_inference = ["past_key_values"]
 
     # The `gpt2` class has a different name, so it needs to be updated accordingly.
-    language_config_attribute_map = {
+    text_config_attribute_map = {
         "n_embd": "hidden_size",
         "n_positions": "max_position_embeddings",
         "n_head": "num_attention_heads",
@@ -18,7 +19,7 @@ class HCXVisionConfig(PretrainedConfig):
 
     def __init__(
         self,
-        language_config=None,
+        text_config=None,
         vision_config=None,
         use_nth_layer=-2,
         img_start_id=100009,  # <|dummy3|>
@@ -33,18 +34,20 @@ class HCXVisionConfig(PretrainedConfig):
         use_1x1_grid=False,
         **kwargs,
     ):
-        for key, val in self.language_config_attribute_map.items():
-            if language_config is not None and key in language_config:
-                language_config[val] = language_config.pop(key)
+        for key, val in self.text_config_attribute_map.items():
+            if text_config is not None and key in text_config:
+                text_config[val] = text_config.pop(key)
 
-        self.language_config = language_config
-        self.vision_config = vision_config
+        if text_config is not None:
+            _text_config = AutoConfig.for_model(text_config["model_type"])
+            self.text_config = _text_config.from_dict(text_config)
 
-        if language_config is not None:
             # In DeepSpeed ZeRO-3, the memory size is automatically determined based on the `hidden_size` specified in the config.
-            self.hidden_size = (
-                language_config["hidden_size"] if "hidden_size" in language_config else language_config["n_embd"]
-            )
+            self.hidden_size = text_config["hidden_size"] if "hidden_size" in text_config else text_config["n_embd"]
+        if vision_config is not None:
+            _vision_config = AutoConfig.for_model(vision_config["model_type"])
+            self.vision_config = _vision_config.from_dict(vision_config)
+
         # add VLM configs
         self.use_nth_layer = use_nth_layer
         self.decoder_max_length = decoder_max_length
@@ -58,3 +61,6 @@ class HCXVisionConfig(PretrainedConfig):
         self.proj_prenorm = proj_prenorm
         self.use_1x1_grid = use_1x1_grid
         super().__init__(**kwargs)
+
+    def get_text_config(self, decoder=False):
+        return self.text_config

generation_config.json

@@ -1,4 +0,0 @@
-{
-  "_from_model_config": true,
-  "transformers_version": "4.48.2"
-}

preprocessor.py → image_processing_hyperclovax.py

@@ -1,22 +1,14 @@
-import base64
 import copy
-import io
 import math
 import os
-import uuid
 from typing import Dict, List, Optional, Union
-from urllib.parse import urlparse
 
-import av
-import cv2
 import numpy as np
-import requests
 import torch
-from decord import VideoReader, cpu
-from PIL import Image, UnidentifiedImageError
+from PIL import Image
+from transformers.feature_extraction_utils import BatchFeature
 from transformers.image_processing_utils import (
     BaseImageProcessor,
-    BatchFeature,
     get_size_dict,
 )
 from transformers.image_transforms import (
@@ -43,401 +35,16 @@ from transformers.utils import TensorType, logging
 logger = logging.get_logger(__name__)
 
 
-def determine_possible_resolutions(anyres: bool, max_num_grids: int, grid_size: int, use_1x1_grid: bool = False):
-    """
-    Finds and returns possible resolution combinations with a total number of grids less than or equal to max_num_grids.
-
-    For example, if max_num_grids is 4, the possible grid combinations are:
-    [1x1, 1x2, 1x3, 1x4, 2x1, 2x2, 3x1, 4x1], and the resolutions are calculated accordingly.
-
-    Example:
-        >>> possible_resolutions = determine_possible_resolutions(anyres=True, max_num_grids=4, grid_size=336)
-        >>> print(possible_resolutions)
-        [[336, 336], [336, 672], [336, 1008], [336, 1344], [672, 336], [672, 672], [1008, 336], [1344, 336]]
-
-    Args:
-        anyres (bool): Whether to allow any resolution combinations up to the maximum grid count.
-        max_num_grids (int): The maximum number of grids allowed (height x width must be ≤ this value).
-        grid_size (int): The size of each grid in pixels (e.g., 336).
-        use_1x1_grid (bool, optional): Whether to include the 1x1 grid as a valid resolution. Defaults to False.
-
-    Returns:
-        List[List[int]]: A list of possible [height, width] resolution pairs.
-    """
-    possible_resolutions = []
-    if anyres:
-        assert max_num_grids > 0
-        for i in range(1, max_num_grids + 1):
-            for j in range(1, max_num_grids + 1):
-                if i == 1 and j == 1 and not use_1x1_grid:
-                    continue
-                if i * j <= max_num_grids:
-                    possible_resolutions.append([i, j])
-
-        possible_resolutions = [[ys * grid_size, xs * grid_size] for ys, xs in possible_resolutions]
-
-    return possible_resolutions
-
-
-def divide_to_grids(image: np.array, grid_size: int, input_data_format=None) -> List[np.array]:
-    """
-    Divides a local image into grids of size (grid_size x grid_size).
-
-    Args:
-        image (np.array): Input image as a NumPy array.
-        grid_size (int): The size (in pixels) of each square grid.
-        input_data_format (optional): Optional format specifier (e.g., "channels_first" or "channels_last").
-
-    Returns:
-        List[np.array]: A list of image patches, each of size (grid_size x grid_size).
-    """
-    grids = []
-    height, width = get_image_size(image, channel_dim=input_data_format)
-    for i in range(0, height, grid_size):
-        for j in range(0, width, grid_size):
-            if input_data_format == ChannelDimension.LAST:
-                grid = image[i : i + grid_size, j : j + grid_size]
-            else:
-                grid = image[:, i : i + grid_size, j : j + grid_size]
-            grids.append(grid)
-
-    return grids
-
-
-def pad(
-    image: np.array,
-    target_size: tuple,
-    background_color=(127, 127, 127),
-    input_data_format=None,
-) -> np.array:
-    """
-    Pads the input image on the sides (top/bottom and left/right) to match the target height and width.
-
-    Args:
-        image (np.array): Input image as a NumPy array.
-        target_size (tuple): Target size as (target_height, target_width).
-        background_color (tuple, optional): RGB color value used for padding. Defaults to (127, 127, 127).
-        input_data_format (optional): Optional format specifier (e.g., "channels_first" or "channels_last").
-
-    Returns:
-        np.array: The padded image with the specified target size.
-    """
-    target_height, target_width = target_size
-    height, width = get_image_size(image, channel_dim=input_data_format)
-
-    # result = np.ones((target_height, target_width, image.shape[2]), dtype=image.dtype) * background_color
-    result = np.empty((target_height, target_width, image.shape[2]), dtype=image.dtype)
-    for i in range(image.shape[2]):
-        result[..., i].fill(background_color[i])
-
-    paste_x = (target_width - width) // 2
-    paste_y = (target_height - height) // 2
-
-    result[paste_y : paste_y + height, paste_x : paste_x + width, :] = image
-
-    return result
-
-
-def expand2square(
-    image: np.array,
-    bboxes_dict=None,
-    background_color=(127, 127, 127),
-    input_data_format=None,
-) -> np.array:
-    """
-    Expands the input image to a square shape by placing it at the center of a new square canvas,
-    with padding added to the shorter side (either top/bottom or left/right).
-
-    The image is always centered on the new canvas, and padding is applied symmetrically.
-
-    Args:
-        image (np.array): Input image as a NumPy array.
-        bboxes_dict (dict, optional): A dictionary of bounding boxes, where each value is an NDArray of shape (N, 4, 2)
-            with box coordinates in the format [[xtl, ytl], [xtr, ytr], [xbr, ybr], [xbl, ybl]].
-            Supports multiple categories (e.g., "ocr", "html") simultaneously.
-        background_color (tuple, optional): RGB color to fill the padding area. Defaults to (127, 127, 127).
-        input_data_format (optional): Optional format specifier for image data (e.g., "channels_first" or "channels_last").
-
-    Returns:
-        np.array: A square-shaped image with the original image centered and padded as needed.
-
-    Example:
-        >>> _img = np.ones((80, 100), dtype=np.uint8) * 100
-        >>> _bboxes_dict = {"words": np.array([[[10, 10], [20, 10], [20, 20], [10, 20]],
-        ...                                    [[30, 30], [40, 30], [40, 40], [30, 40]]])}
-        >>> _img, _bboxes_dict = expand2square(_img, _bboxes_dict, (255, 255, 255))
-        >>> _img.shape
-        (100, 100)
-        >>> guessed_ocr_bboxes = np.array([[[20, 10], [30, 10], [30, 20], [20, 20]],
-        ...                                [[40, 30], [50, 30], [50, 40], [40, 40]]])
-        >>> np.testing.assert_array_almost_equal(_bboxes_dict["words"], guessed_ocr_bboxes) is None
-        True
-    """
-    height, width = get_image_size(image, channel_dim=input_data_format)
-    if width == height:
-        return image, bboxes_dict
-    elif width > height:
-        # result = np.ones((width, width, image.shape[2]), dtype=image.dtype) * background_color
-        result = np.empty((width, width, image.shape[2]), dtype=image.dtype)
-        for i in range(image.shape[2]):
-            result[..., i].fill(background_color[i])
-
-        result[(width - height) // 2 : (width - height) // 2 + height, :] = image
-        if bboxes_dict is not None:
-            for key in bboxes_dict:
-                bboxes_dict[key][:, :, 1] += (width - height) // 2
-        return result, bboxes_dict
-    else:
-        # result = np.ones((height, height, image.shape[2]), dtype=image.dtype) * background_color
-        result = np.empty((height, height, image.shape[2]), dtype=image.dtype)
-        for i in range(image.shape[2]):
-            result[..., i].fill(background_color[i])
-
-        result[:, (height - width) // 2 : (height - width) // 2 + width] = image
-        if bboxes_dict is not None:
-            for key in bboxes_dict:
-                bboxes_dict[key][:, :, 0] += (height - width) // 2
-        return result, bboxes_dict
-
-
-def resize_longside(
-    image: np.array,
-    size: int,
-    resample: PILImageResampling = PILImageResampling.BICUBIC,
-    data_format: Optional[Union[str, ChannelDimension]] = None,
-    input_data_format: Optional[Union[str, ChannelDimension]] = None,
-):
-    """
-    Resizes the image so that its longer side matches the specified size, maintaining the original aspect ratio.
-
-    Args:
-        image (np.array): Input image as a NumPy array.
-        size (int): Target size for the longer side of the image.
-        resample (PILImageResampling, optional): Resampling method to use during resizing. Defaults to BICUBIC.
-        data_format (str or ChannelDimension, optional): Output data format (e.g., "channels_first" or "channels_last").
-        input_data_format (str or ChannelDimension, optional): Input data format of the image.
-
-    Returns:
-        np.array: The resized image with its aspect ratio preserved.
-    """
-    height, width = get_image_size(image, channel_dim=input_data_format)
-
-    if width == height:
-        target_height, target_width = size, size
-    elif width > height:
-        target_width = size
-        target_height = math.ceil(height / width * size)
-    else:
-        target_width = math.ceil(width / height * size)
-        target_height = size
-
-    return resize(
-        image,
-        size=(target_height, target_width),
-        resample=resample,
-        data_format=data_format,
-        input_data_format=input_data_format,
-    )
-
-
-def select_best_resolution(original_size: tuple, possible_resolutions: list) -> tuple:
-    """   
-    Selects the best-fit resolution from a list of possible resolutions based on the original image size.
-    This function evaluates each resolution by computing its effective and wasted area compared to the original size.
-    The optimal resolution is the one that maximizes the effective area while minimizing unused (wasted) space.
-    
-    Args:
-        original_size (tuple): The original image size in the format (height, width).
-        possible_resolutions (list): A list of candidate resolutions in the format [(height1, width1), (height2, width2), ...].
-        
-    Returns:
-        tuple: The best-fit resolution in the format (height, width).
-
-    This function includes code adapted from the file image_processing_llava_next.py in the LLaVA-Next 
-    project(https://github.com/huggingface/transformers/blob/v4.40.2/src/transformers/models/llava_next/image_processing_llava_next.py),
-    which is licensed under apache-2.0.
-    """
-    original_height, original_width = original_size
-    best_fit = None
-    max_effective_resolution = 0
-    min_wasted_resolution = float("inf")
-
-    for height, width in possible_resolutions:
-        scale = min(width / original_width, height / original_height)
-        downscaled_width, downscaled_height = int(original_width * scale), int(original_height * scale)
-        effective_resolution = min(downscaled_width * downscaled_height, original_width * original_height)
-        wasted_resolution = (width * height) - effective_resolution
-
-        if effective_resolution > max_effective_resolution or (
-            effective_resolution == max_effective_resolution and wasted_resolution < min_wasted_resolution
-        ):
-            max_effective_resolution = effective_resolution
-            min_wasted_resolution = wasted_resolution
-            best_fit = (height, width)
-
-    return best_fit
-
-
-def _get_local_grids_output_size(image: np.array, target_resolution: tuple, input_data_format=None):
-    """
-    Computes the number of local grids (patches) along the height and width when resizing an image
-    to the target resolution.
-
-    Args:
-        image (np.array): Input image as a NumPy array.
-        target_resolution (tuple): Target resolution in the format (target_height, target_width).
-        input_data_format (optional): Optional format specifier (e.g., "channels_first" or "channels_last").
-
-    Returns:
-        tuple: A tuple (grid_h, grid_w) representing the number of grids along the height and width.
-    """
-    original_height, original_width = get_image_size(image, channel_dim=input_data_format)
-    target_height, target_width = target_resolution
-
-    scale_w = target_width / original_width
-    scale_h = target_height / original_height
-
-    if scale_w < scale_h:
-        new_width = target_width
-        new_height = min(math.ceil(original_height * scale_w), target_height)
-    else:
-        new_height = target_height
-        new_width = min(math.ceil(original_width * scale_h), target_width)
-
-    return new_height, new_width
-
-
-def determine_anyres_num_vision_patches(
-    num_grids,
-    image_size,
-    grid_size,
-    patch_size,
-    possible_resolutions,
-    anyres=False,
-    unpad=True,
-    num_queries_vis_abstractor=0,
-    num_queries_vis_abstractor_slow=0,
-    is_video=False,
-    first_last_frames_slow=False,  # sample-wise option
-    is_first_or_last_frames=False,  # grid-wise option
-):
-    """
-    Computes the number of visual tokens (patches) based on image resolution, grid configuration, and patch size.
-
-    This function supports both fixed-size and any-resolution settings, as well as video-specific configurations
-    such as handling slow frames and frame position flags.
-
-    Args:
-        num_grids (int): Number of grids per image (e.g., 1 for 1x1, 4 for 2x2, etc.).
-        image_size (tuple): The original image size as (height, width).
-        grid_size (int): Size of each grid in pixels (e.g., 336).
-        patch_size (int): Size of each vision patch (e.g., 14 for ViT models).
-        possible_resolutions (list): List of possible resolution tuples [(h1, w1), (h2, w2), ...].
-        anyres (bool, optional): Whether to use any-resolution mode. Defaults to False.
-        unpad (bool, optional): Whether to unpad the image before computing patches. Defaults to True.
-        num_queries_vis_abstractor (int, optional): Number of query tokens for vision abstractor (fast path).
-        num_queries_vis_abstractor_slow (int, optional): Number of query tokens for vision abstractor (slow path).
-        is_video (bool, optional): Whether the input is a video. Defaults to False.
-        first_last_frames_slow (bool, optional): Whether to treat first/last video frames as "slow". Defaults to False.
-        is_first_or_last_frames (bool, optional): Whether current grid corresponds to first/last frame. Defaults to False.
-
-    Returns:
-        int: Total number of visual tokens (patches) after processing.
-    """
-    if not anyres:
-        return num_queries_vis_abstractor if num_queries_vis_abstractor > 0 else (grid_size // patch_size) ** 2
-
-    if num_queries_vis_abstractor > 0:
-        num_patch_per_grid = int(num_queries_vis_abstractor**0.5)
-    else:
-        num_patch_per_grid = grid_size // patch_size
-
-    num_global_per_grid = num_patch_per_grid
-
-    # In anyres mode, a global image is included, so there are always at least 2 grids.
-    # However, for video inputs, there is no global image, so it's possible to have only 1 grid.
-    # Therefore, the assertion below is commented out:
-    # assert num_grids > 1
-
-    # Compute the number of vision patches.
-    height, width = select_best_resolution(image_size, possible_resolutions)
-
-    num_patch_height = (height // grid_size) * num_patch_per_grid
-    num_patch_width = (width // grid_size) * num_patch_per_grid
-
-    # local images
-    if unpad:
-        original_height, original_width = image_size
-
-        original_aspect_ratio = original_width / original_height
-        current_aspect_ratio = num_patch_width / num_patch_height
-
-        if original_aspect_ratio > current_aspect_ratio:
-            scale_factor = num_patch_width / original_width
-            new_height = int(original_height * scale_factor)
-            padding = (num_patch_height - new_height) // 2
-            num_patch_height = num_patch_height - padding * 2
-        else:
-            scale_factor = num_patch_height / original_height
-            new_width = int(original_width * scale_factor)
-            padding = (num_patch_width - new_width) // 2
-            num_patch_width = num_patch_width - padding * 2
-
-        num_patches = num_patch_width * num_patch_height + num_patch_height
-    else:
-        num_patches = num_patch_width * num_patch_height
-
-    # In the "slow" strategy, when applying to first and last frames only, it is applied exclusively to those two frames.
-    if num_queries_vis_abstractor_slow > 0:
-        if first_last_frames_slow:
-            if is_first_or_last_frames:
-                num_patches += num_queries_vis_abstractor_slow - num_queries_vis_abstractor
-        else:
-            num_patches += num_queries_vis_abstractor_slow - num_queries_vis_abstractor
-        # The slowfast feature is only applicable when unpad is set to False.
-        assert unpad is False
-
-    # Global image is not included for video inputs.
-    if not is_video:
-        num_patches += num_global_per_grid**2
-
-    return num_patches
-
-
-class HCXVisionProcessor(BaseImageProcessor):
+class HCXImageProcessor(BaseImageProcessor):
     r"""
-    Constructs a VLM image processor.
-
-    This processor is based on [`CLIPImageProcessor`] and incorporates additional techniques
-    for handling high-resolution images, such as flexible resolution support (`anyres`), unpadding,
-    square padding, and multi-grid patching strategies.
-
+    Constructs a VLM image processor. Based on [`CLIPImageProcessor`] with incorporation of additional techniques for processing high resolution images.
     Args:
-        do_resize (bool): Whether to resize the image.
-        size (Dict[str, int], optional): Target size for resizing, typically with keys `"height"` and `"width"`.
-        anyres (bool): Whether to enable the any-resolution (`anyres`) feature, which allows flexible resolution handling via grid division.
-        unpad (bool): When `anyres` is enabled, whether to remove visual tokens corresponding to pure padding regions.
-        max_num_grids (int): Maximum number of grids allowed per image.
-        max_image_cnt (int): Maximum number of images that can be processed at once (used for batching).
-        num_queries_vis_abstractor (int): Number of visual query tokens per grid when using a visual resampler (e.g., Perceiver).
-        num_queries_vis_abstractor_video_fast (int): Number of visual queries for fast-path video frames.
-        num_queries_vis_abstractor_video_slow (int): Number of visual queries for slow-path video frames (e.g., first/last).
-        possible_resolutions (List): List of allowed resolution pairs when `anyres` is enabled. Example: [[336, 336], [336, 672], [672, 336]].
-        patch_size (int): Patch size for the Vision Transformer (ViT).
-        pad_to_square (bool): Whether to pad images to a square shape. If `False`, a center crop is applied to fit ViT input.
-        resample (PILImageResampling): Resampling method to use for resizing. Default is `BICUBIC`.
-        do_center_crop (bool): Whether to apply center cropping.
-        crop_size (Dict[str, int], optional): Size for center cropping.
-        do_rescale (bool): Whether to rescale pixel values.
-        rescale_factor (float or int): Factor to use for rescaling pixel values (typically `1/255`).
-        do_normalize (bool): Whether to normalize pixel values using `image_mean` and `image_std`.
-        image_mean (float or List[float], optional): Mean values for normalization. Can be a single float or list of floats per channel.
-        image_std (float or List[float], optional): Standard deviation values for normalization. Can be a single float or list of floats per channel.
-        do_convert_rgb (bool): Whether to convert the input image to RGB.
-        first_last_frames_slow (bool): Whether to treat the first and last frames of a video as “slow path” (processed differently).
-
-    Attributes:
-        model_input_names (List[str]): Names of the expected model inputs. Defaults to `["pixel_values"]`.
+        anyres: (bool) anyres 기능을 사용할지 안할지
+        unpad: (bool) anyres 사용시, unpad 기능 (순수 pad 영역에 해당하는 visual tokens 은 LLM input 에서 제거) 을 사용할지 안할지
+        num_queries_vis_abstractor: (int) 각 grid 에 대해서 resampler 를 사용하는 경우, visual query 수
+        possible_resolutions: (List) anyres 기능 사용시, 가능한 resolution 조합, 예: [[336, 336], [336, 672], [672, 336]]
+        patch_size: (int) ViT patch size
+        pad_to_square: (bool) 정사각형으로 padding 을 수행할지, 안할지를 결정. False 이면 정사각형이 아니기 때문에 center crop 을 거쳐 ViT 의 입력으로 들어감
     """
 
     model_input_names = ["pixel_values"]
@@ -448,11 +55,10 @@ class HCXVisionProcessor(BaseImageProcessor):
         size: Dict[str, int] = None,
         anyres: bool = False,
         unpad: bool = False,
-        max_num_grids: int = 9,
-        max_image_cnt: int = 12,
-        num_queries_vis_abstractor: int = 0,
-        num_queries_vis_abstractor_video_fast: int = 0,
-        num_queries_vis_abstractor_video_slow: int = 0,
+        num_queries_vis_abstractor_image: int = 81,
+        num_queries_vis_abstractor_video_slow: int = 81,
+        num_queries_vis_abstractor_video_fast: int = 9,
+        first_last_frames_slow_video: bool = False,
         possible_resolutions: List = [],
         patch_size: int = 14,
         pad_to_square: bool = True,
@@ -465,24 +71,22 @@ class HCXVisionProcessor(BaseImageProcessor):
         image_mean: Optional[Union[float, List[float]]] = None,
         image_std: Optional[Union[float, List[float]]] = None,
         do_convert_rgb: bool = True,
-        first_last_frames_slow: bool = False,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
-        size = size if size is not None else {"shortest_edge": 512}
+        size = size if size is not None else {"shortest_edge": 336}
         size = get_size_dict(size, default_to_square=False)
-        crop_size = crop_size if crop_size is not None else {"height": 512, "width": 512}
+        crop_size = crop_size if crop_size is not None else {"height": 336, "width": 336}
         crop_size = get_size_dict(crop_size, default_to_square=True, param_name="crop_size")
 
         self.do_resize = do_resize
         self.size = size
         self.anyres = anyres
         self.unpad = unpad
-        self.max_num_grids = max_num_grids
-        self.max_image_cnt = max_image_cnt
-        self.num_queries_vis_abstractor = num_queries_vis_abstractor
-        self.num_queries_vis_abstractor_video_fast = num_queries_vis_abstractor_video_fast
+        self.num_queries_vis_abstractor_image = num_queries_vis_abstractor_image
         self.num_queries_vis_abstractor_video_slow = num_queries_vis_abstractor_video_slow
+        self.num_queries_vis_abstractor_video_fast = num_queries_vis_abstractor_video_fast
+        self.first_last_frames_slow_video = first_last_frames_slow_video
         self.possible_resolutions = [_resolution for _resolution in possible_resolutions]
         self.patch_size = patch_size
         self.pad_to_square = pad_to_square
@@ -495,9 +99,6 @@ class HCXVisionProcessor(BaseImageProcessor):
         self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
         self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
         self.do_convert_rgb = do_convert_rgb
-        self.first_last_frames_slow = first_last_frames_slow
-
-        assert self.crop_size["height"] == self.crop_size["width"]
 
     def resize(
         self,
@@ -508,20 +109,6 @@ class HCXVisionProcessor(BaseImageProcessor):
         input_data_format: Optional[Union[str, ChannelDimension]] = None,
         **kwargs,
     ) -> np.ndarray:
-        """
-        Resizes the input image to the specified target size.
-
-        Args:
-            image (np.ndarray): The input image to resize.
-            size (Dict[str, int]): A dictionary specifying the target size with keys `"height"` and `"width"`.
-            resample (PILImageResampling, optional): The resampling filter to use. Defaults to `BICUBIC`.
-            data_format (str or ChannelDimension, optional): The desired output data format (e.g., "channels_last").
-            input_data_format (str or ChannelDimension, optional): The input data format of the image.
-            **kwargs: Additional keyword arguments, if any.
-
-        Returns:
-            np.ndarray: The resized image as a NumPy array.
-        """
         default_to_square = True
         if "shortest_edge" in size:
             size = size["shortest_edge"]
@@ -563,40 +150,11 @@ class HCXVisionProcessor(BaseImageProcessor):
         data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
         input_data_format: Optional[Union[str, ChannelDimension]] = None,
     ) -> Image.Image:
-        """
-        Applies a sequence of preprocessing operations to the input image(s), including resizing, cropping, rescaling,
-        normalization, and format conversion.
-
-        This method is typically used internally to prepare images for model input.
-
-        Args:
-            images (ImageInput): A single image or a batch of images to preprocess.
-            do_resize (bool, optional): Whether to resize the image(s).
-            size (Dict[str, int], optional): Target size for resizing, with keys `"height"` and `"width"`.
-            resample (PILImageResampling, optional): Resampling method to use for resizing.
-            do_center_crop (bool, optional): Whether to apply center cropping.
-            crop_size (int, optional): Size of the center crop (applied to both height and width).
-            do_rescale (bool, optional): Whether to rescale the image pixel values.
-            rescale_factor (float, optional): Factor to use when rescaling pixel values (e.g., 1/255).
-            do_normalize (bool, optional): Whether to normalize the image using `image_mean` and `image_std`.
-            image_mean (float or List[float], optional): Mean value(s) used for normalization.
-            image_std (float or List[float], optional): Standard deviation value(s) used for normalization.
-            data_format (ChannelDimension, optional): The desired output data format (e.g., `ChannelDimension.FIRST`).
-            input_data_format (str or ChannelDimension, optional): The format of the input image(s).
-
-        Returns:
-            Image.Image: The preprocessed image or batch of images, ready for model input.
-        """
         images = make_list_of_images(images)
 
         if do_resize:
             images = [
-                self.resize(
-                    image=image,
-                    size=size,
-                    resample=resample,
-                    input_data_format=input_data_format,
-                )
+                self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format)
                 for image in images
             ]
 
@@ -607,22 +165,12 @@ class HCXVisionProcessor(BaseImageProcessor):
 
         if do_rescale:
             images = [
-                self.rescale(
-                    image=image,
-                    scale=rescale_factor,
-                    input_data_format=input_data_format,
-                )
-                for image in images
+                self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) for image in images
             ]
 
         if do_normalize:
             images = [
-                self.normalize(
-                    image=image,
-                    mean=image_mean,
-                    std=image_std,
-                    input_data_format=input_data_format,
-                )
+                self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format)
                 for image in images
             ]
 
@@ -633,59 +181,20 @@ class HCXVisionProcessor(BaseImageProcessor):
         return images
 
     def _resize_for_local_grids(
-        self,
-        image: np.array,
-        target_resolution: tuple,
-        resample,
-        input_data_format: ChannelDimension,
+        self, image: np.array, target_resolution: tuple, resample, input_data_format: ChannelDimension
     ) -> np.array:
-        """
-        Resizes the image to the given target resolution for use in local grid processing.
-
-        This function ensures that the image is properly resized to match the (height, width) specified
-        in `target_resolution`, using the provided resampling method. It supports channel-first and
-        channel-last formats based on `input_data_format`.
-
-        Args:
-            image (np.array): Input image as a NumPy array.
-            target_resolution (tuple): Target resolution as (height, width) for resizing.
-            resample: Resampling method to use (e.g., `PILImageResampling.BICUBIC`).
-            input_data_format (ChannelDimension): Format of the input image (e.g., `ChannelDimension.FIRST` or `LAST`).
-
-        Returns:
-            np.array: The resized image in NumPy array format.
-        """
         new_height, new_width = _get_local_grids_output_size(image, target_resolution, input_data_format)
 
         # Resize the image
-        resized_image = resize(
-            image,
-            (new_height, new_width),
-            resample=resample,
-            input_data_format=input_data_format,
-        )
+        resized_image = resize(image, (new_height, new_width), resample=resample, input_data_format=input_data_format)
 
         return resized_image
 
     def _pad_for_patching(
-        self,
-        image: np.array,
-        target_resolution: tuple,
-        input_data_format: ChannelDimension,
+        self, image: np.array, target_resolution: tuple, input_data_format: ChannelDimension
     ) -> np.array:
         """
-        Pads the image to match the target resolution, ensuring compatibility with patch-based models.
-
-        This is typically used to make sure the image dimensions are divisible by the patch size or to
-        meet specific model input requirements. Padding is applied symmetrically where needed.
-
-        Args:
-            image (np.array): Input image as a NumPy array.
-            target_resolution (tuple): The desired resolution after padding, in the format (height, width).
-            input_data_format (ChannelDimension): Format of the input image (e.g., `ChannelDimension.FIRST` or `LAST`).
-
-        Returns:
-            np.array: The padded image as a NumPy array.
+        Pad an image to a target resolution while maintaining aspect ratio.
         """
         target_height, target_width = target_resolution
 
@@ -708,34 +217,13 @@ class HCXVisionProcessor(BaseImageProcessor):
         data_format: ChannelDimension,
         input_data_format: ChannelDimension,
     ) -> List[np.array]:
-        """
-        Splits the input image into multiple local grids based on possible resolutions and grid size.
-
-        The function selects the best resolution from the provided list, resizes the image accordingly,
-        and divides it into non-overlapping grid patches of size (grid_size x grid_size). It is commonly
-        used for any-resolution (anyres) visual processing.
-
-        Args:
-            image (np.array): Input image as a NumPy array.
-            possible_resolutions (List[Tuple[int, int]]): List of allowed resolutions to choose from.
-            grid_size (int): The size of each grid patch (e.g., 336 pixels).
-            resample (PILImageResampling): Resampling method used during resizing.
-            data_format (ChannelDimension): Output data format (e.g., `ChannelDimension.FIRST`).
-            input_data_format (ChannelDimension): Input data format of the image.
-
-        Returns:
-            List[np.array]: A list of grid image patches as NumPy arrays.
-        """
         if not isinstance(possible_resolutions, list):
             raise ValueError("possible_resolutions must be a list of possible resolutions.")
 
         image_size = get_image_size(image, channel_dim=input_data_format)
         best_resolution = select_best_resolution(image_size, possible_resolutions)
         resized_image = self._resize_for_local_grids(
-            image,
-            best_resolution,
-            resample=resample,
-            input_data_format=input_data_format,
+            image, best_resolution, resample=resample, input_data_format=input_data_format
         )
         padded_image = self._pad_for_patching(resized_image, best_resolution, input_data_format=input_data_format)
         local_grids = divide_to_grids(padded_image, grid_size=grid_size, input_data_format=input_data_format)
@@ -755,7 +243,11 @@ class HCXVisionProcessor(BaseImageProcessor):
         size: Dict[str, int] = None,
         anyres: bool = None,
         unpad: bool = None,
-        is_video_list: List[bool] = None,
+        is_video: bool = False,
+        num_queries_vis_abstractor_image: int = None,
+        num_queries_vis_abstractor_video_slow: int = None,
+        num_queries_vis_abstractor_video_fast: int = None,
+        first_last_frames_slow_video: bool = None,
         possible_resolutions: List = None,
         patch_size: int = None,
         pad_to_square: bool = None,
@@ -771,52 +263,43 @@ class HCXVisionProcessor(BaseImageProcessor):
         return_tensors: Optional[Union[str, TensorType]] = None,
         data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
         input_data_format: Optional[Union[str, ChannelDimension]] = None,
-        is_first_or_last_frames: List[bool] = False,
+        return_dummy_image: bool = False,
+        first_last_frames_slow: bool = False,
+        is_first_or_last_frames: bool = False,
+        **kwargs,
     ):
         """
-        Preprocesses images using HCXVisionProcessor.
-
-        This method prepares images for visual language models by applying resizing, padding, cropping,
-        normalization, and tokenization into visual patches. In video mode, each frame is converted to
-        a 1D sequence of patches. The `unpad` option is disabled when processing videos.
-
-        Args:
-            images (ImageInput): A single image or a batch of images (PIL, NumPy, or tensor format).
-            do_resize (bool, optional): Whether to resize the image(s).
-            size (Dict[str, int], optional): Resize target with keys `"height"` and `"width"`.
-            anyres (bool, optional): Whether to use any-resolution processing with grid splitting.
-            unpad (bool, optional): Whether to remove visual tokens that belong to padding areas (only in non-video mode).
-            is_video_list (List[bool], optional): A list indicating which inputs are video frames.
-            possible_resolutions (List, optional): List of resolution pairs allowed in `anyres` mode.
-            patch_size (int, optional): Patch size for the Vision Transformer (ViT).
-            pad_to_square (bool, optional): Whether to pad the image to a square.
-            resample (PILImageResampling, optional): Resampling method to use for resizing.
-            do_center_crop (bool, optional): Whether to apply center cropping.
-            crop_size (int, optional): Target crop size for center cropping.
-            do_rescale (bool, optional): Whether to rescale image pixel values.
-            rescale_factor (float, optional): Factor for pixel rescaling, e.g., `1/255`.
-            do_normalize (bool, optional): Whether to normalize using mean and std.
-            image_mean (float or List[float], optional): Mean value(s) for normalization.
-            image_std (float or List[float], optional): Standard deviation(s) for normalization.
-            do_convert_rgb (bool, optional): Whether to convert the image to RGB.
-            return_tensors (str or TensorType, optional): Desired output tensor type (e.g., "pt" for PyTorch).
-            data_format (ChannelDimension, optional): Output data format (e.g., `ChannelDimension.FIRST`).
-            input_data_format (str or ChannelDimension, optional): Format of the input image.
-            is_first_or_last_frames (List[bool], optional): Flags indicating whether each image is a first/last video frame.
-
-        Returns:
-            Tuple:
-                pixel_values (List[torch.Tensor]): A list of 4D image tensors ready for model input.
-                image_sizes (List[List[int]]): A list of list containing the original width and height [width, height]
-                    of each image, e.g., `[[width, height], ...]`.
-                vision_query_lengths (List[int]): A list of integers representing the number of visual tokens
-                    each image contributes to the LLM input.
+        HCXVisionImageProcessor 로 image tensor, original image size (width, height), visual tokens
+        :return pixel_values: List of 4D tensor 로 image tensor
+        :return image_sizes: List of Dict 로 image width, height [{"width": image 1 의 width, "height": image 1 의 height}, {"width": image 2 의 width, "height": image 2 의 height}, ...]
+        :return vision_query_lengths: List of int 로 각 image 가 LLM 입력으로 전달될때 변환되는 visual token 수
         """
+
         do_resize = do_resize if do_resize is not None else self.do_resize
         size = size if size is not None else self.size
         size = get_size_dict(size, param_name="size", default_to_square=False)
         anyres = anyres if anyres is not None else self.anyres
         unpad = unpad if unpad is not None else self.unpad
+        num_queries_vis_abstractor_image = (
+            num_queries_vis_abstractor_image
+            if num_queries_vis_abstractor_image is not None
+            else self.num_queries_vis_abstractor_image
+        )
+        num_queries_vis_abstractor_video_slow = (
+            num_queries_vis_abstractor_video_slow
+            if num_queries_vis_abstractor_video_slow is not None
+            else self.num_queries_vis_abstractor_video_slow
+        )
+        num_queries_vis_abstractor_video_fast = (
+            num_queries_vis_abstractor_video_fast
+            if num_queries_vis_abstractor_video_fast is not None
+            else self.num_queries_vis_abstractor_video_fast
+        )
+        first_last_frames_slow_video = (
+            first_last_frames_slow_video
+            if first_last_frames_slow_video is not None
+            else self.first_last_frames_slow_video
+        )
         possible_resolutions = possible_resolutions if possible_resolutions is not None else self.possible_resolutions
         patch_size = patch_size if patch_size is not None else self.patch_size
         pad_to_square = pad_to_square if pad_to_square is not None else self.pad_to_square
@@ -831,6 +314,17 @@ class HCXVisionProcessor(BaseImageProcessor):
         image_std = image_std if image_std is not None else self.image_std
         do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
 
+        if is_video:
+            num_queries_vis_abstractor = num_queries_vis_abstractor_video_fast
+            num_queries_vis_abstractor_slow = num_queries_vis_abstractor_video_slow
+            unpad = False
+        else:
+            num_queries_vis_abstractor = num_queries_vis_abstractor_image
+            num_queries_vis_abstractor_slow = 0
+
+        if return_dummy_image:
+            images = Image.new("RGB", (224, 224), (0, 0, 0))
+
         images = make_list_of_images(images)
 
         if not valid_images(images):
@@ -861,38 +355,25 @@ class HCXVisionProcessor(BaseImageProcessor):
 
         assert crop_size["height"] == crop_size["width"]
 
-        # Padding operations for the global image can become a bottleneck when the original image width or height is large.
-        # To mitigate this, the image is first resized such that the longest side is scaled proportionally based on size["shortest_edge"],
-        # and then padding is applied to reach the target dimensions.
+        # global image 의 padding 연산은, image original width, height 가 클 때 bottleneck 이 될 수 있음
+        # 장축의 길이를 size["shortest_edge"] 로 resize 를 먼저 한 뒤에, padding
         if anyres:
             anyres_global_images = copy.deepcopy(images)
             if pad_to_square:
                 background_color = tuple(int(x * 255) for x in self.image_mean)
                 anyres_global_images = [
-                    resize_longside(
-                        copy.deepcopy(image),
-                        size["shortest_edge"],
-                        resample,
-                        input_data_format,
-                    )
+                    resize_longside(copy.deepcopy(image), size["shortest_edge"], resample, input_data_format)
                     for image in anyres_global_images
                 ]
                 anyres_global_images = [
-                    expand2square(
-                        image,
-                        background_color=background_color,
-                        input_data_format=input_data_format,
-                    )[0]
+                    expand2square(image, background_color=background_color, input_data_format=input_data_format)[0]
                     for image in anyres_global_images
                 ]
             else:
                 anyres_global_images = [
                     self.resize(
                         image=image,
-                        size={
-                            "height": size["shortest_edge"],
-                            "width": size["shortest_edge"],
-                        },
+                        size={"height": size["shortest_edge"], "width": size["shortest_edge"]},
                         resample=resample,
                         input_data_format=input_data_format,
                     )
@@ -906,32 +387,11 @@ class HCXVisionProcessor(BaseImageProcessor):
                     resize_longside(image, size["shortest_edge"], resample, input_data_format) for image in images
                 ]
                 images = [
-                    expand2square(
-                        image,
-                        background_color=background_color,
-                        input_data_format=input_data_format,
-                    )[0]
+                    expand2square(image, background_color=background_color, input_data_format=input_data_format)[0]
                     for image in images
                 ]
 
-        num_queries_vis_abstractors = []
-        num_queries_vis_abstractors_slow = []
-        first_last_frames_slows = []
-
-        for image, is_video, anyres_global_image, image_size in zip(
-            images, is_video_list, anyres_global_images, image_sizes
-        ):
-            if is_video:
-                num_queries_vis_abstractor = self.num_queries_vis_abstractor_video_fast
-                num_queries_vis_abstractor_slow = self.num_queries_vis_abstractor_video_slow
-            else:
-                num_queries_vis_abstractor = self.num_queries_vis_abstractor
-                num_queries_vis_abstractor_slow = 0
-
-            num_queries_vis_abstractors.append(num_queries_vis_abstractor)
-            num_queries_vis_abstractors_slow.append(num_queries_vis_abstractor_slow)
-            first_last_frames_slows.append(self.first_last_frames_slow)
-
+        for image, anyres_global_image, image_size in zip(images, anyres_global_images, image_sizes):
             if anyres:
                 # convert image into a list of grids
                 # we intentially use the same data format as the input data format
@@ -943,7 +403,7 @@ class HCXVisionProcessor(BaseImageProcessor):
                     data_format=input_data_format,
                     input_data_format=input_data_format,
                 )
-                # Global image (thumbnail) is not used for video inputs.
+                # video 에 대해서는 global image (thumbnail) 를 사용하지 않음
                 if not is_video:
                     image_grids = [anyres_global_image] + image_grids
             else:
@@ -968,617 +428,362 @@ class HCXVisionProcessor(BaseImageProcessor):
             pixel_values = np.array(pixel_values)
             new_images.append(pixel_values)
 
-            num_grids = pixel_values.shape[0]
-
             vision_query_length = determine_anyres_num_vision_patches(
-                num_grids=num_grids,
                 image_size=image_size,
                 grid_size=crop_size["height"],
                 patch_size=patch_size,
                 possible_resolutions=possible_resolutions,
                 anyres=anyres,
-                unpad=False if is_video else unpad,
+                unpad=unpad,
                 num_queries_vis_abstractor=num_queries_vis_abstractor,
                 num_queries_vis_abstractor_slow=num_queries_vis_abstractor_slow,
                 is_video=is_video,
-                first_last_frames_slow=self.first_last_frames_slow,
-                is_first_or_last_frames=self.first_last_frames_slow,
+                first_last_frames_slow=first_last_frames_slow,
+                is_first_or_last_frames=is_first_or_last_frames,
             )
 
             vision_query_lengths.append(vision_query_length)
 
+        if return_dummy_image:
+            vision_query_lengths = []
+
         data = {
-            "pixel_values": [[torch.tensor(new_image) for new_image in new_images]],
-            "image_sizes": [[[image_size[1], image_size[0]] for image_size in image_sizes]],
-            "vision_query_lengths": [vision_query_lengths],
-            "is_videos": [is_video_list],
-            "num_queries_vis_abstractors": [num_queries_vis_abstractors],
-            "num_queries_vis_abstractors_slow": [num_queries_vis_abstractors_slow],
-            "first_last_frames_slows": [first_last_frames_slows],
+            "pixel_values": [torch.tensor(new_image) for new_image in new_images],
+            "image_sizes": [{"width": image_size[1], "height": image_size[0]} for image_size in image_sizes],
+            "vision_query_lengths": vision_query_lengths,
         }
 
-        return BatchFeature(data=data)
+        return BatchFeature(data=data, tensor_type=return_tensors)
+
+    def save_pretrained(
+        self,
+        save_directory: Union[str, os.PathLike],
+        *args,
+        **kwargs,
+    ):
+        self.register_for_auto_class()
+        super().save_pretrained(save_directory, *args, **kwargs)
 
-    def load_images_videos(self, vlm_chat):
-        """
-        Loads and prepares images or video frames from a VLM chat input.
 
-        This function parses the input `vlm_chat` object, extracts image or video sources,
-        and loads them into memory as PIL or NumPy images, ready for preprocessing.
+def determine_anyres_num_vision_patches(
+    image_size,
+    grid_size,
+    patch_size,
+    possible_resolutions,
+    anyres=False,
+    unpad=True,
+    num_queries_vis_abstractor=0,
+    num_queries_vis_abstractor_slow=0,
+    is_video=False,
+    first_last_frames_slow=False,  # sample-wise option
+    is_first_or_last_frames=False,  # grid-wise option
+):
+    """
+    Computes the number of visual tokens (patches) based on image resolution, grid configuration, and patch size.
+
+    This function supports both fixed-size and any-resolution settings, as well as video-specific configurations
+    such as handling slow frames and frame position flags.
+
+    Args:
+        num_grids (int): Number of grids per image (e.g., 1 for 1x1, 4 for 2x2, etc.).
+        image_size (tuple): The original image size as (height, width).
+        grid_size (int): Size of each grid in pixels (e.g., 336).
+        patch_size (int): Size of each vision patch (e.g., 14 for ViT models).
+        possible_resolutions (list): List of possible resolution tuples [(h1, w1), (h2, w2), ...].
+        anyres (bool, optional): Whether to use any-resolution mode. Defaults to False.
+        unpad (bool, optional): Whether to unpad the image before computing patches. Defaults to True.
+        num_queries_vis_abstractor (int, optional): Number of query tokens for vision abstractor (fast path).
+        num_queries_vis_abstractor_slow (int, optional): Number of query tokens for vision abstractor (slow path).
+        is_video (bool, optional): Whether the input is a video. Defaults to False.
+        first_last_frames_slow (bool, optional): Whether to treat first/last video frames as "slow". Defaults to False.
+        is_first_or_last_frames (bool, optional): Whether current grid corresponds to first/last frame. Defaults to False.
 
-        Args:
-            vlm_chat: A VLM chat input structure containing multimodal elements
-                    (e.g., images, videos, URLs, or file paths). The format is typically a list of messages
-                    with associated media fields.
+    Returns:
+        int: Total number of visual tokens (patches) after processing.
+    """
 
-        Returns:
-            List[Union[PIL.Image.Image, List[PIL.Image.Image]]]:
-                A list of loaded images. For video entries, a list of frames is returned instead of a single image.
-        """
-        vlm_chat = copy.deepcopy(vlm_chat)
-
-        new_vlm_chat = []
-        all_images = []  # images + images_from_videos
-        is_video_list = []
-
-        for line in vlm_chat:
-            if "content" in line:
-                content = line["content"]
-
-                if "image" in content:
-                    if "filename" not in content:
-                        content["filename"] = f"{uuid.uuid4().hex}.jpg"
-                    image_pil = load_image(content["image"])
-                    all_images.append(image_pil)
-                    is_video_list.append(False)
-                    new_vlm_chat.append(line)
-
-                elif "video" in content:
-                    video_bytesio = load_video_to_bytesio(content["video"])
-                    pil_img_frames, video_time_stamp = process_video(
-                        video_bytesio, self.max_num_grids, self.max_image_cnt, self.crop_size["width"]
-                    )
-                    all_images.extend(pil_img_frames)
-                    is_video_list.extend([True] * len(pil_img_frames))
+    if not anyres:
+        return num_queries_vis_abstractor if num_queries_vis_abstractor > 0 else (grid_size // patch_size) ** 2
 
-                    if "filename" not in content:
-                        content["filename"] = f"{uuid.uuid4().hex}.mp4"
+    if num_queries_vis_abstractor > 0:
+        num_patch_per_grid = int(num_queries_vis_abstractor**0.5)
+    else:
+        num_patch_per_grid = grid_size // patch_size
 
-                    for i, image_time_stamp in enumerate(video_time_stamp):
-                        new_line = copy.deepcopy(line)
-                        basename, ext = os.path.splitext(content["filename"])
-                        new_line["content"]["filename"] = f"{basename}-{i}{ext}"
-                        new_line["content"]["video_time_stamp"] = image_time_stamp
+    num_global_per_grid = num_patch_per_grid
 
-                        if i == len(video_time_stamp) - 1:
-                            new_line["content"]["is_final_grid"] = True
+    # In anyres mode, a global image is included, so there are always at least 2 grids.
+    # However, for video inputs, there is no global image, so it's possible to have only 1 grid.
+    # Therefore, the assertion below is commented out:
+    # assert num_grids > 1
 
-                            for last_frame_target_key in ["lens_keywords", "lens_local_keywords", "speech_to_text"]:
-                                if last_frame_target_key in content:
-                                    new_line["content"][last_frame_target_key] = content[last_frame_target_key]
+    # Compute the number of vision patches.
+    height, width = select_best_resolution(image_size, possible_resolutions)
 
-                        new_vlm_chat.append(new_line)
-                else:
-                    new_vlm_chat.append(line)
+    num_patch_height = (height // grid_size) * num_patch_per_grid
+    num_patch_width = (width // grid_size) * num_patch_per_grid
 
-        return new_vlm_chat, all_images, is_video_list
+    # local images
+    if unpad:
+        original_height, original_width = image_size
 
+        original_aspect_ratio = original_width / original_height
+        current_aspect_ratio = num_patch_width / num_patch_height
 
-def process_video(video_bytesio, max_num_grids, max_image_cnt, vit_input_size):
-    """
-    Processes a video file and extracts frames suitable for vision transformer (ViT) input.
+        if original_aspect_ratio > current_aspect_ratio:
+            scale_factor = num_patch_width / original_width
+            new_height = int(original_height * scale_factor)
+            padding = (num_patch_height - new_height) // 2
+            num_patch_height = num_patch_height - padding * 2
+        else:
+            scale_factor = num_patch_height / original_height
+            new_width = int(original_width * scale_factor)
+            padding = (num_patch_width - new_width) // 2
+            num_patch_width = num_patch_width - padding * 2
 
-    The function reads video data from a BytesIO object, extracts a limited number of frames
-    based on `max_num_grids` and `max_image_cnt`, and resizes them to the appropriate ViT input size.
+        num_patches = num_patch_width * num_patch_height + num_patch_height
+    else:
+        num_patches = num_patch_width * num_patch_height
 
-    Args:
-        video_bytesio (io.BytesIO): A BytesIO object containing the raw video file data.
-        max_num_grids (int): The maximum number of grids allowed (e.g., for tiling or patching).
-        max_image_cnt (int): The maximum number of frames to extract from the video.
-        vit_input_size (int): The desired input size (height and width) for the ViT model.
+    # In the "slow" strategy, when applying to first and last frames only, it is applied exclusively to those two frames.
+    if num_queries_vis_abstractor_slow > 0:
+        if first_last_frames_slow:
+            if is_first_or_last_frames:
+                num_patches += num_queries_vis_abstractor_slow - num_queries_vis_abstractor
+        else:
+            num_patches += num_queries_vis_abstractor_slow - num_queries_vis_abstractor
+        # The slowfast feature is only applicable when unpad is set to False.
+        assert unpad is False
 
-    Returns:
-        List[np.ndarray]: A list of processed video frames as NumPy arrays, each resized to (vit_input_size, vit_input_size).
-    """
-    frames, time_interval = video_decoder(
-        video_bytesio, max_num_grids=max_num_grids, max_image_cnt=max_image_cnt, default_interval=0.4
-    )
-    pil_img_frames, video_time_stamp = combine_frames_into_images(
-        frames, time_interval, max_grid_shape=(max_num_grids, 1), vit_input_size=vit_input_size
-    )
+    # Global image is not included for video inputs.
+    if not is_video:
+        num_patches += num_global_per_grid**2
 
-    return pil_img_frames, video_time_stamp
+    return num_patches
 
 
-def load_image(image_src):
+def divide_to_grids(image: np.array, grid_size: int, input_data_format=None) -> List[np.array]:
     """
-    Loads an image from various sources (file path, URL, base64 string, or raw bytes)
-    and returns it as a PIL Image object.
+    Divides a local image into grids of size (grid_size x grid_size).
 
     Args:
-        image_src (str or bytes): The image source. It can be:
-            - A local file path
-            - A URL
-            - A base64-encoded string
-            - Raw image bytes
+        image (np.array): Input image as a NumPy array.
+        grid_size (int): The size (in pixels) of each square grid.
+        input_data_format (optional): Optional format specifier (e.g., "channels_first" or "channels_last").
 
     Returns:
-        PIL.Image.Image: The loaded image as a PIL Image object.
-
-    Raises:
-        ValueError: If the image cannot be loaded or the format is unsupported.
-        TypeError: If the input is not of type str or bytes.
+        List[np.array]: A list of image patches, each of size (grid_size x grid_size).
     """
-    try:
-        # 1. If input is bytes type
-        if isinstance(image_src, bytes):
-            return Image.open(io.BytesIO(image_src))
-
-        # 2. If input is str type (path, URL, base64)
-        if isinstance(image_src, str):
-            # 2a. Check if it's a Base64 data URI format ('data:image/...')
-            if image_src.startswith("data:image"):
-                try:
-                    # Remove the 'data:image/...;base64,' part and decode
-                    header, encoded = image_src.split(",", 1)
-                    image_bytes = base64.b64decode(encoded)
-                    return Image.open(io.BytesIO(image_bytes))
-                except (ValueError, base64.binascii.Error) as e:
-                    raise ValueError(f"Invalid base64 data URI format: {e}") from e
-
-            # 2b. Check if it's a URL format ('http://' or 'https://')
-            elif image_src.startswith("http://") or image_src.startswith("https://"):
-                try:
-                    response = requests.get(image_src, stream=True, timeout=10)
-                    response.raise_for_status()  # Raise an exception for HTTP errors
-                    image_bytes = response.content
-                    return Image.open(io.BytesIO(image_bytes))
-                except requests.exceptions.RequestException as e:
-                    raise ValueError(f"Error loading image from URL '{image_src}': {e}") from e
-
-            # 2c. Assume it's a local file path
+    grids = []
+    height, width = get_image_size(image, channel_dim=input_data_format)
+    for i in range(0, height, grid_size):
+        for j in range(0, width, grid_size):
+            if input_data_format == ChannelDimension.LAST:
+                grid = image[i : i + grid_size, j : j + grid_size]
             else:
-                return Image.open(image_src)
-
-        else:
-            raise TypeError(f"Unsupported image_src type: {type(image_src)}")
+                grid = image[:, i : i + grid_size, j : j + grid_size]
+            grids.append(grid)
 
-    # Common exception handling
-    except FileNotFoundError:
-        raise ValueError(f"Image loading error: File not found '{image_src}'")
-    except UnidentifiedImageError:
-        raise ValueError("Image loading error: Cannot identify image file format.")
-    except IOError as e:
-        raise ValueError(f"Image loading error (I/O): {e}") from e
-    except Exception as e:
-        raise ValueError(f"Unexpected error during image loading: {e}") from e
+    return grids
 
 
-def load_video_to_bytesio(video_src):
+def pad(
+    image: np.array,
+    target_size: tuple,
+    background_color=(127, 127, 127),
+    input_data_format=None,
+) -> np.array:
     """
-    Loads video data from various sources (file path, URL, base64 string, or raw bytes)
-    and returns an `io.BytesIO` object containing the raw video content.
+    Pads the input image on the sides (top/bottom and left/right) to match the target height and width.
 
     Args:
-        video_src (str or bytes): The video source. Supported formats include:
-            - Local file path
-            - URL
-            - Base64-encoded data URI string
-            - Raw video bytes
+        image (np.array): Input image as a NumPy array.
+        target_size (tuple): Target size as (target_height, target_width).
+        background_color (tuple, optional): RGB color value used for padding. Defaults to (127, 127, 127).
+        input_data_format (optional): Optional format specifier (e.g., "channels_first" or "channels_last").
 
     Returns:
-        io.BytesIO: A `BytesIO` object containing the loaded video data.
-
-    Raises:
-        ValueError: If the video cannot be loaded due to issues such as an invalid path,
-                    URL failure, malformed base64 string, or unsupported format.
-        TypeError: If the input is not a `str` or `bytes` object.
+        np.array: The padded image with the specified target size.
     """
-    video_bytes = None
-    try:
-        # 1. If input is bytes type
-        if isinstance(video_src, bytes):
-            video_bytes = video_src
-
-        # 2. If input is str type (path, URL, base64)
-        elif isinstance(video_src, str):
-            # 2a. Check if it's a Base64 data URI format ('data:video/...')
-            if video_src.startswith("data:video"):
-                try:
-                    # Remove the 'data:video/...;base64,' part and decode
-                    header, encoded = video_src.split(",", 1)
-                    video_bytes = base64.b64decode(encoded)
-                except (ValueError, base64.binascii.Error) as e:
-                    raise ValueError(f"Invalid base64 data URI format: {e}") from e
-
-            # 2b. Check if it looks like a URL
-            elif urlparse(video_src).scheme in ("http", "https"):
-                try:
-                    response = requests.get(
-                        video_src, stream=True, timeout=30
-                    )  # Increased timeout for potentially large videos
-                    response.raise_for_status()  # Raise an exception for HTTP errors (4xx or 5xx)
-                    # Read all content from the stream into bytes
-                    video_bytes = response.content
-                except requests.exceptions.MissingSchema:
-                    # If urlparse thinks it's a scheme but requests disagrees (e.g., "http:/example.com")
-                    # Treat it as a potential file path below.
-                    pass
-                except requests.exceptions.RequestException as e:
-                    raise ValueError(f"Error loading video from URL '{video_src}': {e}") from e
-
-            # 2c. Assume it's a local file path if not base64 or confirmed URL
-            if video_bytes is None:  # Only attempt file read if not already loaded as base64 or URL failed gracefully
-                # Check if it could potentially be a file path
-                # Note: This check is basic. A string like "http:/path/file" might incorrectly be treated as a path here
-                # if the requests call failed due to MissingSchema. More robust path validation could be added.
-                if (
-                    os.path.exists(video_src) or "/" in video_src or "\\" in video_src
-                ):  # Basic check if it resembles a path
-                    try:
-                        with open(video_src, "rb") as f:
-                            video_bytes = f.read()
-                    except FileNotFoundError:
-                        raise ValueError(f"Video loading error: File not found at path '{video_src}'")
-                    except IsADirectoryError:
-                        raise ValueError(f"Video loading error: Path '{video_src}' is a directory, not a file.")
-                    except IOError as e:
-                        raise ValueError(f"Video loading error (I/O) for path '{video_src}': {e}") from e
-                else:
-                    # If it's not base64, not a valid downloadable URL, and doesn't look like a path/doesn't exist
-                    raise ValueError(f"Unsupported string input format or resource not found: '{video_src}'")
-
-        # 3. If the type is unsupported
-        else:
-            raise TypeError(f"Unsupported video_src type: {type(video_src)}")
-
-        # Final check if video_bytes was successfully obtained
-        if video_bytes is None:
-            raise ValueError(f"Could not load video data from the provided source: {video_src}")
+    target_height, target_width = target_size
+    height, width = get_image_size(image, channel_dim=input_data_format)
 
-        # Return the bytes wrapped in BytesIO
-        return io.BytesIO(video_bytes)
+    # result = np.ones((target_height, target_width, image.shape[2]), dtype=image.dtype) * background_color
+    result = np.empty((target_height, target_width, image.shape[2]), dtype=image.dtype)
+    for i in range(image.shape[2]):
+        result[..., i].fill(background_color[i])
 
-    # Catch specific exceptions first for better error reporting
-    except FileNotFoundError as e:  # Should be caught above, but as a safeguard
-        raise ValueError(f"Video loading error: File not found '{video_src}'") from e
-    except requests.exceptions.RequestException as e:  # Already handled, but for clarity
-        raise ValueError(f"Video loading error (Network): {e}") from e
-    except (ValueError, TypeError) as e:  # Re-raise ValueErrors/TypeErrors raised intentionally within the try block
-        raise e
-    except Exception as e:
-        # Catch any other unexpected errors during processing
-        raise ValueError(f"Unexpected error during video loading from source '{video_src}': {e}") from e
+    paste_x = (target_width - width) // 2
+    paste_y = (target_height - height) // 2
 
+    result[paste_y : paste_y + height, paste_x : paste_x + width, :] = image
 
-def video_decoder(video_bytesio, max_num_grids, max_image_cnt, default_interval=0.4):
-    """
-    Decodes video data from a BytesIO object and returns a list of extracted frames.
+    return result
 
-    Args:
-        video_bytesio (io.BytesIO): A BytesIO object containing the raw video data.
-        max_num_grids (int): Maximum number of grids allowed per image. Used to determine how many frames to extract.
-        max_image_cnt (int): Maximum number of frames to extract from the video.
-        default_interval (float, optional): Default time interval (in seconds) between frames. Used when frame rate info is unavailable. TODO: make configurable.
 
-    Returns:
-        Tuple:
-            frames (List[PIL.Image.Image]): A list of extracted frames as PIL Images.
-            time_interval (float): Time interval (in seconds) between selected frames.
-    """
-    error_messages = []
-    frames = []
-
-    # 1. Try decoding the video using Decord.
-    try:
-        vr = VideoReader(video_bytesio, ctx=cpu(0), num_threads=8)
-        fps = vr.get_avg_fps()
-        play_time = len(vr) / fps
-        total_frames = len(vr)
-        frame_indices, time_interval = extract_frame_indices(
-            play_time, total_frames, fps, max_num_grids, max_image_cnt, default_interval=default_interval
-        )  # Sample every 0.4 seconds; if the video is too long, apply uniform sampling instead.
-        if frame_indices is None:
-            frame_indices = range(len(vr))  # Convert all frames.
-        batch_frames = vr.get_batch(frame_indices).asnumpy()
-        frames = [Image.fromarray(frame).convert("RGB") for frame in batch_frames]
-        return frames, time_interval
-    except Exception as e:
-        print("error with decord")
-        error_messages.append(f"Decord 실패: {e}")
-
-    # 2. Fallback: Try decoding the video using PyAV.
-    try:
-        container = av.open(video_bytesio)
-        fps = container.streams.video[0].average_rate
-        play_time = len(container) / fps
-        total_frames = len(container)
-        frame_indices, time_interval = extract_frame_indices(
-            play_time, total_frames, fps, max_num_grids, max_image_cnt, default_interval=default_interval
-        )  # Sample frames every 0.4 seconds. If the video is long, use uniform sampling to limit the number of frames.
-        # Even if frame_indices were assigned using Decord, reprocess them to be compatible with PyAV.
-        target_indices = None if frame_indices is None else set(frame_indices)
-        frames = []
-        for i, frame in enumerate(container.decode(video=0)):
-            if target_indices is not None and i not in target_indices:
-                continue  # Skip frames that are not in the required indices.
-            pil_frame = Image.fromarray(frame.to_ndarray(format="rgb24")).convert("RGB")
-            frames.append(pil_frame)
-        if frames:
-            return frames, time_interval
-        else:
-            raise Exception("Decoding with PyAV succeeded, but no frames were extracted.")
-    except Exception as e:
-        error_messages.append(f"PyAV failed: {e}")
-
-    # 3. Fallback: Try decoding the video using OpenCV.
-    try:
-        byte_data = np.frombuffer(video_bytesio.getvalue(), dtype=np.uint8)
-        video = cv2.imdecode(byte_data, cv2.IMREAD_UNCHANGED)
-
-        cap = cv2.VideoCapture(video)
-        total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
-        fps = cap.get(cv2.CAP_PROP_FPS)
-        play_time = total_frames / fps
-        frame_indices, time_interval = extract_frame_indices(
-            play_time, total_frames, fps, max_num_grids, max_image_cnt, default_interval=default_interval
-        )  # Sample frames every 0.4 seconds; if the video is too long, apply uniform sampling to limit the total number of frames.
-        if frame_indices is None:
-            frame_indices = range(total_frames)  # Convert all frames.
-
-        index_set = set(frame_indices)  # Convert to a set for faster lookup.
-        current_index = 0
-
-        while cap.isOpened():
-            ret, frame = cap.read()
-            if not ret:
-                break
-            if current_index in index_set:
-                frames.append(Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)).convert("RGB"))
-            current_index += 1
-            if current_index > max(index_set):  # Stop processing once all required indices have been handled.
-                break
-
-        cap.release()
-        if frames:
-            return frames, time_interval
-    except Exception as e:
-        error_messages.append(f"OpenCV failed: {e}")
-
-    if error_messages:
-        raise Exception(f"All decoding attempts have failed.: {error_messages}")
-
-
-def convert_format_for_multi_image(img, json, convert_key_list=["words", "text", "objects", "entities"]):
+def expand2square(
+    image: np.array,
+    bboxes_dict=None,
+    background_color=(127, 127, 127),
+    input_data_format=None,
+) -> np.array:
     """
-    Converts the format of image and annotation data from a single-image dataset to a multi-image dataset format.
+    Expands the input image to a square shape by placing it at the center of a new square canvas,
+    with padding added to the shorter side (either top/bottom or left/right).
 
-    Single-image datasets typically return a single image and its associated annotation as individual objects.
-    This function wraps them in a dictionary format used by multi-image datasets.
+    The image is always centered on the new canvas, and padding is applied symmetrically.
 
     Args:
-        img: The input image (e.g., a PIL Image or NumPy array).
-        json: The annotation data associated with the image.
-        convert_key_list (List[str], optional): A list of keys to extract and convert from the original JSON.
-            Defaults to ["words", "text", "objects", "entities"].
+        image (np.array): Input image as a NumPy array.
+        bboxes_dict (dict, optional): A dictionary of bounding boxes, where each value is an NDArray of shape (N, 4, 2)
+            with box coordinates in the format [[xtl, ytl], [xtr, ytr], [xbr, ybr], [xbl, ybl]].
+            Supports multiple categories (e.g., "ocr", "html") simultaneously.
+        background_color (tuple, optional): RGB color to fill the padding area. Defaults to (127, 127, 127).
+        input_data_format (optional): Optional format specifier for image data (e.g., "channels_first" or "channels_last").
 
     Returns:
-        Tuple[Dict, Dict]:
-            - A dictionary mapping image IDs to images (e.g., {"image_0": img}).
-            - A dictionary mapping image IDs to corresponding annotation JSONs (with filtered keys).
-    """
-    is_multi_image_dataset = isinstance(img, dict)
-    if not is_multi_image_dataset:
-        img = {"00": img}
-
-        for convert_key in convert_key_list:
-            if convert_key in json:
-                json[convert_key] = {"00": json[convert_key]}
-
-        for json_key in json:
-            if "region" in json_key:
-                json[json_key] = {"00": json[json_key]}
-
-    return is_multi_image_dataset, img, json
-
+        np.array: A square-shaped image with the original image centered and padded as needed.
 
-def convert_tags_for_video(img, json):
+    Example:
+        >>> _img = np.ones((80, 100), dtype=np.uint8) * 100
+        >>> _bboxes_dict = {"words": np.array([[[10, 10], [20, 10], [20, 20], [10, 20]],
+        ...                                    [[30, 30], [40, 30], [40, 40], [30, 40]]])}
+        >>> _img, _bboxes_dict = expand2square(_img, _bboxes_dict, (255, 255, 255))
+        >>> _img.shape
+        (100, 100)
+        >>> guessed_ocr_bboxes = np.array([[[20, 10], [30, 10], [30, 20], [20, 20]],
+        ...                                [[40, 30], [50, 30], [50, 40], [40, 40]]])
+        >>> np.testing.assert_array_almost_equal(_bboxes_dict["words"], guessed_ocr_bboxes) is None
+        True
     """
-    Converts <video_00> tags to <image_xx> tags based on the number of video frames.
+    height, width = get_image_size(image, channel_dim=input_data_format)
+    if width == height:
+        return image, bboxes_dict
+    elif width > height:
+        # result = np.ones((width, width, image.shape[2]), dtype=image.dtype) * background_color
+        result = np.empty((width, width, image.shape[2]), dtype=image.dtype)
+        for i in range(image.shape[2]):
+            result[..., i].fill(background_color[i])
 
-    In video datasets, annotations often use a generic <video_00> tag. This function replaces that tag
-    with frame-specific tags such as <image_00>, <image_01>, ..., <image_NN> based on the number of frames in `img`.
+        result[(width - height) // 2 : (width - height) // 2 + height, :] = image
+        if bboxes_dict is not None:
+            for key in bboxes_dict:
+                bboxes_dict[key][:, :, 1] += (width - height) // 2
+        return result, bboxes_dict
+    else:
+        # result = np.ones((height, height, image.shape[2]), dtype=image.dtype) * background_color
+        result = np.empty((height, height, image.shape[2]), dtype=image.dtype)
+        for i in range(image.shape[2]):
+            result[..., i].fill(background_color[i])
 
-    Args:
-        img: A list of video frames (e.g., list of PIL Images or NumPy arrays).
-        json: The annotation data containing <video_00> tags to be replaced.
+        result[:, (height - width) // 2 : (height - width) // 2 + width] = image
+        if bboxes_dict is not None:
+            for key in bboxes_dict:
+                bboxes_dict[key][:, :, 0] += (height - width) // 2
+        return result, bboxes_dict
 
-    Returns:
-        Dict: The updated annotation JSON with frame-specific <image_xx> tags.
-    """
-    image_tag = "".join([f"<image_{idx:02d}>" for idx in range(len(img))])
-    # image_tag = "<image_00>"  # Use this format to construct and insert image-specific tags.
-    for json_key in json:
-        if "qa_pairs" in json_key:
-            new_qa_pairs = []
-            for qa_pair in json[json_key]:
-                question = qa_pair[0]
-                # Replace <video_00> tags with corresponding <image_xx> tags.
-                question = question.replace("<video_00>", image_tag)
-                new_qa_pairs.append([question, qa_pair[1]])
-            json[json_key] = new_qa_pairs
-
-    return img, json
-
-
-def split_list(input_list, split_value):
-    """
-    Splits a list into sublists using a specified delimiter value.
 
-    Each time `split_value` is encountered in `input_list`, a new sublist is started.
-    The delimiter itself is not included in the output.
+def resize_longside(
+    image: np.array,
+    size: int,
+    resample: PILImageResampling = PILImageResampling.BICUBIC,  # type: ignore
+    data_format: Optional[Union[str, ChannelDimension]] = None,
+    input_data_format: Optional[Union[str, ChannelDimension]] = None,
+):
+    """
+    Resizes the image so that its longer side matches the specified size, maintaining the original aspect ratio.
 
     Args:
-        input_list (List[Any]): The input list to split.
-        split_value (Any): The value used as the delimiter for splitting.
+        image (np.array): Input image as a NumPy array.
+        size (int): Target size for the longer side of the image.
+        resample (PILImageResampling, optional): Resampling method to use during resizing. Defaults to BICUBIC.
+        data_format (str or ChannelDimension, optional): Output data format (e.g., "channels_first" or "channels_last").
+        input_data_format (str or ChannelDimension, optional): Input data format of the image.
 
     Returns:
-        List[List[Any]]: A list of sublists, split by the specified delimiter.
-
-    Example:
-        >>> split_list(["a", "b", "|", "c", "d", "|", "e"], "|")
-        [['a', 'b'], ['c', 'd'], ['e']]
+        np.array: The resized image with its aspect ratio preserved.
     """
-    temp_list = []
-    result = []
+    height, width = get_image_size(image, channel_dim=input_data_format)
 
-    for value in input_list:
-        if value == split_value:
-            result.append(temp_list)
-            temp_list = []
-        else:
-            temp_list.append(value)
-    result.append(temp_list)
+    if width == height:
+        target_height, target_width = size, size
+    elif width > height:
+        target_width = size
+        target_height = math.ceil(height / width * size)
+    else:
+        target_width = math.ceil(width / height * size)
+        target_height = size
 
-    return result
+    return resize(
+        image,
+        size=(target_height, target_width),
+        resample=resample,
+        data_format=data_format,
+        input_data_format=input_data_format,
+    )
 
 
-def combine_frames_into_images(frames, time_interval, max_grid_shape=(3, 3), vit_input_size=378):
+def _get_local_grids_output_size(image: np.array, target_resolution: tuple, input_data_format=None):
     """
-    Combines a sequence of video frames into grid-based images and generates corresponding time range labels.
-
-    Frames are grouped and arranged into a grid (e.g., 3x3) such that each combined image contains up to
-    `max_grid_shape[0] * max_grid_shape[1]` frames. Each combined image is resized to the given ViT input size.
+    Computes the number of local grids (patches) along the height and width when resizing an image
+    to the target resolution.
 
     Args:
-        frames (List[PIL.Image.Image]): A list of frames extracted from a video.
-        time_interval (float): Time interval (in seconds) between consecutive frames.
-        max_grid_shape (Tuple[int, int], optional): The maximum grid shape as (rows, cols). Defaults to (3, 3).
-        vit_input_size (int, optional): The target size (height and width) for the Vision Transformer input. Defaults to 378.
+        image (np.array): Input image as a NumPy array.
+        target_resolution (tuple): Target resolution in the format (target_height, target_width).
+        input_data_format (optional): Optional format specifier (e.g., "channels_first" or "channels_last").
 
     Returns:
-        Tuple:
-            image_list (List[PIL.Image.Image]): A list of grid-combined images.
-            image_time_stamps (List[str]): A list of time span labels for each combined image,
-                e.g., ["0.00s~1.50s", "1.50s~3.00s", ...].
+        tuple: A tuple (grid_h, grid_w) representing the number of grids along the height and width.
     """
-    # grid_size = int(np.sqrt(max_num_grids))
-    # assert grid_size**2 == max_num_grids, "max_num_grids must be a perfect square."
-    max_num_grids = max_grid_shape[0] * max_grid_shape[1]
-    assert (
-        max_grid_shape[1] == 1
-    ), f"For video processing, decided to concatenate frames horizontally into a wide image."
-
-    # List to store the resulting combined images.
-    image_list = []
-
-    # Calculate the number of canvases needed.
-    num_frames = len(frames)
-    num_canvases = num_frames // max_num_grids
-    leftover_frames = num_frames % max_num_grids
-
-    time_stamp = 0  # second
-    image_time_stamps = []
-
-    for canvas_idx in range(num_canvases):
-        # Initialize the current canvas.
-        combined_image = Image.new(
-            "RGB", (vit_input_size * max_grid_shape[0], vit_input_size * max_grid_shape[1]), color=(0, 0, 0)
-        )
-
-        # Determine the frames to fill in the current canvas.
-        start_idx = canvas_idx * max_num_grids
-        end_idx = min(start_idx + max_num_grids, num_frames)
-
-        for idx in range(start_idx, end_idx):
-            img = frames[idx]
-
-            # Resize each frame to a square shape.
-            img_resized = img.resize((vit_input_size, vit_input_size))
-
-            # Calculate the (row, column) position to place the frame within the grid layout.
-            local_idx = idx - start_idx
-            x_offset = (local_idx % max_grid_shape[0]) * vit_input_size
-            y_offset = (local_idx // max_grid_shape[0]) * vit_input_size
-
-            # Calculate the position to place the frame in the grid.
-            combined_image.paste(img_resized, (x_offset, y_offset))
-
-        # Append the current canvas to the result list.
-        image_list.append(combined_image)
-        frame_cnt = end_idx - start_idx
-        image_time_stamps.append(f"{time_stamp:.2f}s~{time_stamp + frame_cnt * time_interval:.2f}s")
-        time_stamp += frame_cnt * time_interval
-
-    if leftover_frames > 0:
-        # canvas_idx might be undefined; default to 0 if not previously assigned to avoid "referenced before assignment" error.
-        canvas_idx = num_canvases
-        # Add the remaining frames to the final canvas.
-        combined_image = Image.new("RGB", (vit_input_size * leftover_frames, vit_input_size * 1), color=(0, 0, 0))
-
-        for idx in range(leftover_frames):
-            img = frames[num_canvases * max_num_grids + idx]
-
-            # Resize the frame to a square (equal width and height).
-            img_resized = img.resize((vit_input_size, vit_input_size))
-
-            # Calculate the (row, column) position to place the frame within the grid layout.
-            x_offset = (idx % leftover_frames) * vit_input_size
-            y_offset = (idx // leftover_frames) * vit_input_size
+    original_height, original_width = get_image_size(image, channel_dim=input_data_format)
+    target_height, target_width = target_resolution
 
-            # Calculate the position to place the frame within the grid layout.
-            combined_image.paste(img_resized, (x_offset, y_offset))
+    scale_w = target_width / original_width
+    scale_h = target_height / original_height
 
-        # Add the current canvas to the list of combined images.
-        image_list.append(combined_image)
-        frame_cnt = leftover_frames
-        image_time_stamps.append(f"{time_stamp:.2f}s~{time_stamp + frame_cnt * time_interval:.2f}s")
-        time_stamp += frame_cnt * time_interval
+    if scale_w < scale_h:
+        new_width = target_width
+        new_height = min(math.ceil(original_height * scale_w), target_height)
+    else:
+        new_height = target_height
+        new_width = min(math.ceil(original_width * scale_h), target_width)
 
-    return image_list, image_time_stamps
+    return new_height, new_width
 
 
-def extract_frame_indices(play_time, total_frames, fps, max_num_grids, max_image_cnt, default_interval=0.4):
+def select_best_resolution(original_size: tuple, possible_resolutions: list) -> tuple:
     """
-    Extracts specific frame indices from a video based on duration, frame count, and sampling strategy.
+    Selects the best-fit resolution from a list of possible resolutions based on the original image size.
 
-    The function determines which frames to extract given the video duration (`play_time`),
-    total frame count, and frame rate. It samples frames at regular intervals (default: 0.4s),
-    but if the number of frames exceeds the limit defined by `max_num_grids * max_image_cnt`,
-    it performs uniform sampling to stay within that limit.
+    This function, adapted from LLaVA-Next
+    (https://github.com/huggingface/transformers/blob/v4.40.2/src/transformers/models/llava_next/image_processing_llava_next.py),
+    evaluates each resolution by computing its effective and wasted area compared to the original size.
+    The optimal resolution is the one that maximizes the effective area while minimizing unused (wasted) space.
 
     Args:
-        play_time (float): Total play time of the video in seconds.
-        total_frames (int): Total number of frames in the video.
-        fps (float): Frames per second of the video.
-        max_num_grids (int): Maximum number of grids to display.
-        max_image_cnt (int): Maximum number of images per grid.
-        default_interval (float, optional): Interval in seconds between frame samples. Defaults to 0.4.
+        original_size (tuple): The original image size in the format (height, width).
+        possible_resolutions (list): A list of candidate resolutions in the format [(height1, width1), (height2, width2), ...].
 
     Returns:
-        Tuple:
-            frame_indices (List[int]): A list of selected frame indices.
-            time_interval (float): Time interval between selected frames (in seconds).
+        tuple: The best-fit resolution in the format (height, width).
     """
+    original_height, original_width = original_size
+    best_fit = None
+    max_effective_resolution = 0
+    min_wasted_resolution = float("inf")
 
-    # Calculate how many frames to extract with the default interval
-    default_frame_count = int(play_time / default_interval)
-
-    # Maximum frames allowed based on max_num_grids and max_image_cnt
-    max_frames_allowed = max_num_grids * max_image_cnt
-
-    # Determine whether we can use the default interval or need uniform sampling
-    if default_frame_count <= max_frames_allowed:
-        # Default interval is sufficient, extract frames every 0.4 seconds
-        frame_interval = int(total_frames / default_frame_count)
-    else:
-        # Use uniform sampling to fit within max_frames_allowed
-        frame_interval = int(total_frames / max_frames_allowed)
-
-    # Extract frame indices at the calculated interval
-    selected_indices = list(range(0, total_frames, frame_interval))
+    for height, width in possible_resolutions:
+        scale = min(width / original_width, height / original_height)
+        downscaled_width, downscaled_height = int(original_width * scale), int(original_height * scale)
+        effective_resolution = min(downscaled_width * downscaled_height, original_width * original_height)
+        wasted_resolution = (width * height) - effective_resolution
 
-    time_interval = frame_interval / fps
+        if effective_resolution > max_effective_resolution or (
+            effective_resolution == max_effective_resolution and wasted_resolution < min_wasted_resolution
+        ):
+            max_effective_resolution = effective_resolution
+            min_wasted_resolution = wasted_resolution
+            best_fit = (height, width)
 
-    # Ensure the number of selected indices does not exceed max_frames_allowed
-    return selected_indices[:max_frames_allowed], time_interval
+    return best_fit

modeling_hyperclovax.py

@@ -2,7 +2,6 @@ import ast
 import contextlib
 import gc
 import json
-import math
 import os
 from dataclasses import dataclass
 from functools import partial
@@ -33,10 +32,11 @@ from transformers.models.auto import CONFIG_MAPPING
 from transformers.utils import ModelOutput
 
 from .configuration_hyperclovax import HCXVisionConfig
-from .preprocessor import select_best_resolution
+from .image_processing_hyperclovax import select_best_resolution
 
 EOT = "<|endofturn|>"
-IMG_LOC = "<|dummy3|>"
+IMAGE_LOC = "<|dummy3|>"
+VIDEO_LOC = "<|_unuse_missing_100270|>"
 
 
 def get_rank():
@@ -220,11 +220,9 @@ def reshape_and_unpad_image_features(
 
 
 def anyres_postprocessing(
-    image_forward_outs: torch.FloatTensor,
-    split_sizes: List[int],
+    image_forward_outs: List[torch.FloatTensor],
     image_sizes: List[List[int]],
     possible_resolutions: List[Tuple[int, int]],
-    is_videos: List[bool],
     patch_size: int,
     grid_size: int,
     image_newline: torch.FloatTensor,
@@ -247,8 +245,6 @@ def anyres_postprocessing(
             dimensions of the corresponding image sample. Used for unpadding.
         possible_resolutions (List[Tuple[int, int]]): A list of supported resolution tuples `(height, width)` used by
             `reshape_and_unpad_image_features` for spatial reconstruction, especially during unpadding.
-        is_videos (List[bool]): A list of boolean flags indicating whether each corresponding sample in the batch is a
-            video [`True`] or an image [`False`].
         patch_size (int): The spatial dimension (height and width) of the square patches the image was divided into.
         grid_size (int): The spatial dimension (height and width) of the square grid onto which patches are mapped.
             `grid_size` should be divisible by `patch_size`.
@@ -274,102 +270,28 @@ def anyres_postprocessing(
         assert (num_queries_vis_abstractor**0.5).is_integer(), "n_queries must be square number"
         height = width = int(num_queries_vis_abstractor**0.5)
 
-    image_features = torch.split(image_forward_outs, split_sizes, dim=0)
-
     # post-processing (unpad, add newline)
     new_image_features = []
-    for image_idx, (image_feature, is_video) in enumerate(zip(image_features, is_videos)):
+    for image_idx, image_feature in enumerate(image_forward_outs):
         if image_feature.shape[0] > 1:
-            if not is_video:
-                image_feature = reshape_and_unpad_image_features(
-                    image_feature=image_feature,
-                    height=height,
-                    width=width,
-                    image_size=image_sizes[image_idx],
-                    possible_resolutions=possible_resolutions,
-                    grid_size=grid_size,  # Pass grid info if needed by helper
-                    unpad=unpad,
-                    image_newline=image_newline,
-                )
-            else:
-                image_feature = image_feature.flatten(0, 1)
+            image_feature = reshape_and_unpad_image_features(
+                image_feature=image_feature,
+                height=height,
+                width=width,
+                image_size=image_sizes[image_idx],
+                possible_resolutions=possible_resolutions,
+                grid_size=grid_size,  # Pass grid info if needed by helper
+                unpad=unpad,
+                image_newline=image_newline,
+            )
         else:
             image_feature = image_feature[0]
-            if unpad and not is_video:
-                image_feature = torch.cat((image_feature, image_newline[None].to(image_feature.device)), dim=0)
+            image_feature = torch.cat((image_feature, image_newline[None].to(image_feature.device)), dim=0)
         new_image_features.append(image_feature)
     image_features = new_image_features
     return image_features
 
 
-def adaptive_anyres_postprocessing(
-    image_forward_outs: torch.FloatTensor,
-    image_sizes: List[List[int]],
-    possible_resolutions: List[Tuple[int, int]],
-    is_videos: List[bool],
-    group_ids: List[List[int]],
-    num_queries_vis_abstractors: List[List[int]],
-    grid_size: int,
-    image_newline: torch.FloatTensor,
-    unpad: bool = False,
-) -> List[torch.FloatTensor]:
-    """Adaptive AnyRes postprocessing for multi-group feature aggregation.
-
-    Processes 2D visual features into 1D sequences with group-wise adaptive processing. Each image can belong to
-    multiple processing groups with different query configurations. Features are processed per group and aggregated
-    according to group_ids.
-
-    Args:
-        image_forward_outs (List[torch.FloatTensor]): List of input tensors with shape
-            (number_of_images_in_grid, total_patches, feature_dim) containing visual features.
-        image_sizes (List[List[int]]): Original image dimensions for each sample. [[width, height], ... ]
-        possible_resolutions (List[Tuple[int, int]]): Supported resolutions. [[height, width], ... ]
-        is_videos (List[bool]): Flags indicating video inputs
-        group_ids (List[List[int]]): Group indices for feature aggregation. Each group means a single grid.
-        num_queries_vis_abstractors (List[List[int]]): Query numbers per group
-        grid_size (int): Total grid size for spatial processing
-        image_newline (torch.FloatTensor): Sample-wise config. Newline embedding tensor
-        unpad (bool, optional): Sample-wise config. Enable padding removal. Defaults to False.
-
-    Returns:
-        List[torch.FloatTensor]: Aggregated features per group
-
-    Raises:
-        AssertionError: If num_queries is not square number in any group
-    """
-    # post-processing (unpad, add newline)
-    new_image_features = []
-    for image_idx, (image_feature, is_video) in enumerate(zip(image_forward_outs, is_videos)):
-        num_queries_vis_abstractor = num_queries_vis_abstractors[image_idx]
-        assert (num_queries_vis_abstractor**0.5).is_integer(), "n_queries must be square number"
-        height = width = int(num_queries_vis_abstractor**0.5)
-
-        if image_feature.shape[0] > 1:
-            if not is_video:
-                image_feature = reshape_and_unpad_image_features(
-                    image_feature=image_feature,
-                    height=height,
-                    width=width,
-                    image_size=image_sizes[image_idx],
-                    possible_resolutions=possible_resolutions,
-                    grid_size=grid_size,
-                    unpad=unpad,
-                    image_newline=image_newline,
-                )
-            else:
-                image_feature = image_feature.flatten(0, 1)
-        else:
-            image_feature = image_feature[0]
-            if unpad and not is_video:
-                image_feature = torch.cat((image_feature, image_newline[None].to(image_feature.device)), dim=0)
-        new_image_features.append(image_feature)
-
-    image_features = [
-        torch.cat([new_image_features[group_id] for group_id in group_ids_list], dim=0) for group_ids_list in group_ids
-    ]
-    return image_features
-
-
 @dataclass
 class HCXVisionOutput(ModelOutput):
     """Output class for vision models, containing various computation results.
@@ -413,9 +335,11 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
 
     config_class = HCXVisionConfig
     vision_model_name = "vision_model"
-    _no_split_modules = ["CLIPAttention", "SiglipVisionModel"]
+    _no_split_modules = ["SiglipEncoderLayer", "LlamaDecoderLayer", "HyperCLOVAXDecoderLayer"]
     supports_gradient_checkpointing = True
     _skip_keys_device_placement = "past_key_values"
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
 
     def __init__(
         self,
@@ -434,98 +358,57 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
                 - is_safetensor_save: Whether to save model using safetensors format.
 
         Raises:
-            ValueError: If vision_config is not defined or if language_config is not defined.
+            ValueError: If vision_config is not defined or if text_config is not defined.
         """
-        super().__init__(config)
+        super().__init__(config)  # self.config = config
 
-        self.flag_changed_max_position_embeddings = False
+        # init configs
+        text_config = self._init_text_config(config)
+        vision_config = self._init_vision_config(config)
+        
+        ## possible_resolution should be matched with preprocessor_config.json
+        config.possible_resolutions = self._init_possible_resolutions(config, vision_config)
 
-        vision_model_type = config.vision_config["model_type"]
-        if vision_model_type in CONFIG_MAPPING:
-            vision_config = CONFIG_MAPPING[vision_model_type](**config.vision_config)
-            vision_config.auto_map = {}
-        else:
-            if config.vision_model_name_or_path is not None:
-                vision_config = AutoConfig.from_pretrained(config.vision_model_name_or_path, trust_remote_code=True)
-            elif config.vision_config["_name_or_path"] is not None:
-                vision_config = AutoConfig.from_pretrained(
-                    config.vision_config["_name_or_path"], trust_remote_code=True
-                )
-            else:
-                raise ValueError("vision_config is not defined")
+        # init models & parameters
+        with no_init_weights():  # weight will be loaded in from_pretrained
+            self.vision_model = AutoModel.from_config(vision_config, trust_remote_code=True)
 
-        self.use_liger = kwargs.pop("use_liger", False)
-        self.use_fused_ce = kwargs.pop("use_fused_ce", False)
-        self.reduction = "sum" if kwargs.pop("use_sum_loss", False) else "mean"
+        self.mm_projector = self._init_mm_projector(config, text_config, vision_config)
 
-        self.vision_config = vision_config
-        vision_config.anyres = config.anyres
-        vision_config.max_num_grids = config.max_num_grids
+        self.language_model = AutoModelForCausalLM.from_config(text_config)
+        self.lm_head_vocab_size = getattr(text_config, "padded_vocab_size", text_config.vocab_size)
+        self.language_model.lm_head = nn.Linear(text_config.hidden_size, self.lm_head_vocab_size, bias=False)
 
-        possible_resolutions = []
         if config.anyres:
-            assert config.max_num_grids > 0
-            for i in range(1, config.max_num_grids + 1):
-                for j in range(1, config.max_num_grids + 1):
-                    if i == 1 and j == 1 and not config.use_1x1_grid:
-                        continue
-                    if i * j <= config.max_num_grids:
-                        possible_resolutions.append([i, j])
-
-            possible_resolutions = [
-                [ys * vision_config.image_size, xs * vision_config.image_size] for ys, xs in possible_resolutions
-            ]
-
-        self.possible_resolutions = possible_resolutions
+            self.image_newline = nn.Parameter(torch.empty(text_config.hidden_size, dtype=self.dtype))
 
-        with no_init_weights():
-            self.vision_model = AutoModel.from_config(
-                vision_config, trust_remote_code=True
-            )  # weight will be loaded in from_pretrained
+        # modify configs or model settings
+        if text_config.model_type in ["llama", "hyperclovax", "gpt2"]:
+            self.language_model.gradient_checkpointing_enable()
+        if text_config.model_type == "hyperclovax" and self.use_liger:
+            self.language_model._get_apply_liger_kernel_converter()(model=self.language_model)
 
-        assert config.language_config["model_type"] == "llama"
-        language_config = CONFIG_MAPPING["llama"](**config.language_config)
-        language_config._attn_implementation = kwargs.get("attn_implementation", "sdpa")  # activate flash attention
-        language_config.logits_scaling = 1.0
+        # update configs
+        self.vision_config = vision_config = self.vision_model.config
+        self.text_config = text_config = self.language_model.config
+        config.update({"vision_config": vision_config})
+        config.update({"text_config": text_config})
 
-        self.language_config = language_config
-        self.language_model = AutoModelForCausalLM.from_config(language_config)
-
-        self.language_model.gradient_checkpointing_enable()
-        self.num_queries_vis_abstractor = config.num_queries_vis_abstractor
+        # etc
+        self.use_liger = kwargs.pop("use_liger", False)
+        self.use_fused_ce = kwargs.pop("use_fused_ce", False)
+        self.use_meansum_loss = kwargs.pop("use_meansum_loss", False)
+        self.freeze_before_sampler = kwargs.pop("freeze_before_sampler", False)
+        self.use_turnmeansum_loss = kwargs.pop("use_turnmeansum_loss", False)
+        self.vision_input_chunk_size = kwargs.pop("vision_input_chunk_size", None)
+        self.is_safetensor_save = kwargs.get("is_safetensor_save", True)
 
-        # mm_projctor(==connector); vision_model_hidden_size -> LLM embedding size
-        input_hidden_size = vision_config.hidden_size
-        self.mm_projector = HCXVisionCAbstractor(
-            num_queries=self.num_queries_vis_abstractor,
-            num_input_tokens=(self.vision_config.image_size // self.vision_config.patch_size) ** 2,
-            encoder_hidden_size=input_hidden_size,
-            hidden_size=input_hidden_size,
-            output_hidden_size=language_config.hidden_size,
-            pos_emb=config.proj_pos_emb,
-            prenorm=config.proj_prenorm,
-        )
-        self.use_nth_layer = config.use_nth_layer
-        self.config.update({"vision_config": self.vision_model.config.to_dict()})
-        self.config.update({"language_config": self.language_model.config.to_dict()})
-        self.lm_head_vocab_size = (
-            language_config.padded_vocab_size
-            if hasattr(language_config, "padded_vocab_size")
-            else language_config.vocab_size
-        )
-        self.language_model.lm_head = nn.Linear(language_config.hidden_size, self.lm_head_vocab_size, bias=False)
-        self.model_parallel = False
-        self.device_map = None
-        self.use_no_grad = None
-        self.decoder_max_length = config.decoder_max_length
+        use_sum_loss = True if kwargs.pop("use_sum_loss", False) else False
+        self.reduction = self._init_reduction_type(use_sum_loss)
 
-        self.anyres = config.anyres
-        self.unpad = config.unpad
-        if self.anyres:
-            self.image_newline = nn.Parameter(torch.empty(language_config.hidden_size, dtype=self.dtype))
+        self.vision_model_use_no_grad = None  # forward 시 체크 및 할당
 
-        self.is_safetensor_save = kwargs.get("is_safetensor_save", True)
-        self._backward_compatibility_gradient_checkpointing()
+        self._backward_compatibility_gradient_checkpointing()  # self.post_init() 에 포함되어 있는 gc 가능한지 확인하고 켜주는 함수
 
     def _init_weights(self, module):
         # copies from https://github.com/kakaobrain/honeybee/blob/main/honeybee/common_layers.py#L55
@@ -545,10 +428,96 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
             embed_std = 1 / torch.sqrt(torch.tensor(module.size(0), dtype=torch.float)).to(module.dtype)
             module.data.normal_(mean=0.0, std=embed_std)
 
+    def _init_reduction_type(self, use_sum_loss):
+        assert not (
+            self.use_meansum_loss and self.use_turnmeansum_loss
+        ), "use_meansum_loss and use_turnmeansum_loss cannot both be True; only one or neither may be True."
+        if self.use_meansum_loss or self.use_turnmeansum_loss:
+            reduction = "none"
+        elif use_sum_loss:
+            reduction = "sum"
+        else:
+            reduction = "mean"
+        return reduction
+
+    def _init_vision_config(self, config):
+        vision_model_type = config.vision_config.model_type
+        if vision_model_type in CONFIG_MAPPING:
+            vision_config = CONFIG_MAPPING[vision_model_type](**config.vision_config.to_dict())
+            vision_config.auto_map = {}
+        else:
+            if config.vision_model_name_or_path is not None:
+                vision_config = AutoConfig.from_pretrained(config.vision_model_name_or_path, trust_remote_code=True)
+            elif config.vision_config._name_or_path is not None:
+                vision_config = AutoConfig.from_pretrained(config.vision_config._name_or_path, trust_remote_code=True)
+            else:
+                raise ValueError("vision_config is not defined")
+
+        vision_config.anyres = config.anyres
+        vision_config.max_num_grids = config.max_num_grids
+        return vision_config
+
+    def _init_text_config(self, config):
+        if hasattr(config, "text_config") and config.text_config is not None:
+            model_type = config.text_config.model_type
+            text_config = CONFIG_MAPPING[model_type](**config.text_config.to_dict())
+        else:
+            raise ValueError("text_config is not defined")
+        text_config._attn_implementation = config._attn_implementation
+        if text_config.model_type != "hyperclovax":
+            text_config.logits_scaling = 1.0
+        return text_config
+
+    def _init_possible_resolutions(self, config, vision_config):
+        """possible_resolution should be matched with preprocessor_config.json"""
+        if not getattr(config, "possible_resolutions", []):
+            possible_resolutions = []
+            if config.anyres:
+                assert config.max_num_grids > 0
+                for i in range(1, config.max_num_grids + 1):
+                    for j in range(1, config.max_num_grids + 1):
+                        if i == 1 and j == 1 and not config.use_1x1_grid:
+                            continue
+                        if i * j <= config.max_num_grids:
+                            possible_resolutions.append([i, j])
+
+                possible_resolutions = [
+                    [ys * vision_config.image_size, xs * vision_config.image_size] for ys, xs in possible_resolutions
+                ]
+            return possible_resolutions
+        else:
+            return config.possible_resolutions
+
+    def _init_mm_projector(self, config, text_config, vision_config):
+        input_hidden_size = vision_config.hidden_size
+        if config.mm_projector_type == "linear":
+            mm_projector = nn.Linear(input_hidden_size, text_config.hidden_size)
+            mm_projector.dtype = next(mm_projector.parameters()).dtype
+        elif config.mm_projector_type == "cabstractor":
+            mm_projector = HCXVisionCAbstractor(
+                num_queries=config.num_queries_vis_abstractor_image,
+                num_input_tokens=(vision_config.image_size // vision_config.patch_size) ** 2,
+                encoder_hidden_size=input_hidden_size,
+                hidden_size=input_hidden_size,
+                output_hidden_size=text_config.hidden_size,
+                pos_emb=config.proj_pos_emb,
+                prenorm=config.proj_prenorm,
+            )
+        else:
+            mm_projector = HCXVisionMlp(
+                config.mm_projector_type,
+                input_hidden_size,
+                hidden_features=input_hidden_size,  # TODO: llava 처럼 hidden_size 를 input_hidden_size 가 아니라 LLM embedding size 로 바꿔주기
+                out_features=self.text_config.hidden_size,
+            )
+        return mm_projector
+
     def forward(
         self,
         input_ids: Optional[torch.LongTensor] = None,
-        pixel_values: Optional[List[List[torch.FloatTensor]]] = None,
+        pixel_values_images: Optional[List[List[torch.FloatTensor]]] = None,
+        image_sizes_images: Optional[List[List[Tuple[int, int]]]] = None,
+        pixel_values_videos: Optional[List[List[torch.FloatTensor]]] = None,
         past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
         attention_mask: Optional[torch.FloatTensor] = None,
         inputs_embeds: Optional[torch.FloatTensor] = None,
@@ -557,14 +526,6 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
-        image_sizes: Optional[List[List[List[int]]]] = None,
-        vision_query_lengths: Optional[List[List[int]]] = None,
-        non_vision_query_lengths: Optional[List[int]] = None,
-        img_start_ids_list: Optional[List[List[int]]] = None,
-        num_queries_vis_abstractors: Optional[List[List[int]]] = None,
-        num_queries_vis_abstractors_slow: Optional[List[List[int]]] = None,
-        first_last_frames_slows: Optional[List[bool]] = None,
-        is_video_list: Optional[List[bool]] = None,
         **kwargs,
     ) -> Union[Tuple, HCXVisionOutput]:
         """Forward pass of the model.
@@ -608,33 +569,28 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
             If return_dict=False, returns a tuple containing the above items except loss_per_sample.
         """
         output_attentions = (
-            output_attentions if output_attentions is not None else self.config.vision_config["output_attentions"]
+            output_attentions if output_attentions is not None else self.config.vision_config.output_attentions
         )
         output_hidden_states = (
-            output_hidden_states
-            if output_hidden_states is not None
-            else self.config.vision_config["output_hidden_states"]
+            output_hidden_states if output_hidden_states is not None else self.config.vision_config.output_hidden_states
         )
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
         if inputs_embeds is None and past_key_values is None:
-            inputs_embeds = self.extract_inputs_embeds(
-                input_ids=input_ids,
-                pixel_values=pixel_values,
-                past_key_values=past_key_values,
-                image_sizes=image_sizes,
-                vision_query_lengths=vision_query_lengths,
-                non_vision_query_lengths=non_vision_query_lengths,
-                img_start_ids_list=img_start_ids_list,
-                num_queries_vis_abstractors=num_queries_vis_abstractors,
-                num_queries_vis_abstractors_slow=num_queries_vis_abstractors_slow,
-                first_last_frames_slows=first_last_frames_slows,
-                is_videos=is_video_list,
-            )
+            if pixel_values_images is not None or pixel_values_videos is not None:
+                inputs_embeds = self.extract_inputs_embeds(
+                    input_ids=input_ids,
+                    pixel_values_images=pixel_values_images,
+                    image_sizes_images=image_sizes_images,
+                    pixel_values_videos=pixel_values_videos,
+                )
+            else:
+                inputs_embeds = self.get_input_embeddings()(input_ids)
 
         if inputs_embeds is not None:
             input_ids = None
 
+        ################################
         # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
         outputs = self.language_model.base_model(
             input_ids=input_ids,
@@ -648,7 +604,7 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
         )
 
         hidden_states = outputs[0]
-        hidden_states = hidden_states * self.language_config.logits_scaling
+        hidden_states = hidden_states * self.text_config.logits_scaling
 
         loss = None
         loss_per_sample = None
@@ -657,10 +613,12 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
             # Shift so that tokens < n predict n
             shift_logits = logits[..., :-1, :].contiguous()
             shift_labels = labels[..., 1:].contiguous()
+
             # Flatten the tokens
             loss_fct = CrossEntropyLoss(reduction="none")  # ignore IGNORE_INDEX(-100)
             shift_logits = shift_logits.view(-1, self.lm_head_vocab_size)
             shift_labels = shift_labels.view(-1)
+
             # Enable model/pipeline parallelism
             shift_labels = shift_labels.to(shift_logits.device)
             loss = loss_fct(shift_logits, shift_labels)
@@ -682,66 +640,6 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
             attentions=outputs.attentions,
         )
 
-    def determine_non_vision_query_lengths(
-        self, input_ids: torch.LongTensor, pad_id: int, img_start_id: int
-    ) -> List[int]:
-        """Calculate the lengths of non-vision query parts in the input.
-
-        This method calculates the length of text tokens (excluding visual tokens) for each sample.
-        When input_ids are collated, they are padded with pad_id on the right, so this method finds
-        these values by identifying pad tokens and img_start_id tokens.
-
-        Args:
-            input_ids: Input token IDs with img_start_id markers for image positions.
-            pad_id: Token ID used for padding.
-            img_start_id: Token ID marking the start of image data.
-
-        Returns:
-            List of lengths of non-vision query parts for each sample in the batch.
-        """
-        non_vision_query_lengths = []
-        batch_size, len_seq = input_ids.size(0), input_ids.size(1)
-
-        for i in range(batch_size):
-            temp_idx = (input_ids[i] == pad_id).nonzero()
-            eos_idx = temp_idx[0, 0].item() if len(temp_idx) > 0 else len_seq
-            num_imgs = (input_ids[i] == img_start_id).sum().item()
-            non_vision_query_lengths.append(eos_idx - num_imgs)
-
-        if all([pad_id in input_id for input_id in input_ids.tolist()]):
-            non_vision_query_lengths = [
-                non_vision_query_length + 1 for non_vision_query_length in non_vision_query_lengths
-            ]
-
-        return non_vision_query_lengths
-
-    def determine_vision_query_lengths(
-        self, image_features: List[List[torch.Tensor]], image_cnts: List[int]
-    ) -> List[List[int]]:
-        """Calculate the lengths of vision query parts in the input.
-
-        This method calculates the lengths of visual tokens for each image in each sample based on
-        the shapes of image feature tensors. For samples without any images, a dummy image is included
-        but then converted to an empty list.
-
-        Args:
-            image_features: List of lists of image features tensors.
-            image_cnts: List of counts of images for each sample in the batch.
-
-        Returns:
-            List of lists of lengths of visual tokens for each image in each sample.
-        """
-        vision_query_lengths = [
-            [image_feature.size(0) for image_feature in image_feature_list] for image_feature_list in image_features
-        ]
-
-        for i, image_cnt in enumerate(image_cnts):
-            if image_cnt == 0:
-                assert len(vision_query_lengths[i]) == 1  # 현재 검정 이미지 1개 들어가있음
-                vision_query_lengths[i] = []  # 빈 list 로 변환
-
-        return vision_query_lengths
-
     # Copied from transformers.models.llava.modeling_llava.LlavaForConditionalGeneration.get_input_embeddings
     def get_input_embeddings(self):
         return self.language_model.get_input_embeddings()
@@ -780,16 +678,9 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
     def extract_inputs_embeds(
         self,
         input_ids: Optional[torch.LongTensor] = None,
-        pixel_values: Optional[List[List[torch.FloatTensor]]] = None,  # list of list of 4D tensors
-        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
-        image_sizes: Optional[List[List[List[int]]]] = None,
-        vision_query_lengths: Optional[List[List[int]]] = None,
-        non_vision_query_lengths: Optional[List[int]] = None,
-        img_start_ids_list: Optional[List[List[int]]] = None,
-        num_queries_vis_abstractors: Optional[List[List[int]]] = None,
-        num_queries_vis_abstractors_slow: Optional[List[List[int]]] = None,
-        first_last_frames_slows: Optional[List[bool]] = None,
-        is_videos: Optional[List[str]] = None,
+        pixel_values_images: Optional[List[List[torch.FloatTensor]]] = None,
+        image_sizes_images: Optional[List[List[Tuple[int, int]]]] = None,
+        pixel_values_videos: Optional[List[List[torch.FloatTensor]]] = None,
     ):
         """Extract input embeddings by processing text tokens and visual features.
 
@@ -805,9 +696,6 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
             vision_query_lengths: List of lists of lengths when each image is converted to visual tokens.
             non_vision_query_lengths: List of lengths of text tokens (excluding visual tokens) for each sample.
             img_start_ids_list: List of lists containing indices of img_start_id tokens for each sample.
-            num_queries_vis_abstractors: List of lists containing number of visual tokens for each image grid.
-            num_queries_vis_abstractors_slow: List of lists containing number of visual tokens for
-                the slow part when applying the slowfast algorithm to video frames.
             first_last_frames_slows: List of booleans indicating whether the slowfast algorithm is
                 applied to the first or last frames of the video.
             is_videos: List of booleans indicating which inputs are videos.
@@ -815,241 +703,193 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
         Returns:
             Combined embeddings of text tokens and visual features.
         """
-        inputs_embeds = None
-        if past_key_values:
-            pass
-        else:
-            # Flatten CLIP and connector for feature encoding, then convert back to List of List format
-            len_pixel_values = [len(pixel_value) for pixel_value in pixel_values]
-            concat_pixel_values = torch.cat(list(chain(*pixel_values)), dim=0)  # list of list of 4D Tensor
-            visual_token_idx = 0 if "siglip" in self.vision_config.model_type else 1
-            # Check if all parameters of the model require_grad=False
-            if self.use_no_grad is None:
-                self.use_no_grad = all(not p.requires_grad for p in self.vision_model.vision_model.encoder.parameters())
-            context = torch.no_grad() if self.use_no_grad else contextlib.nullcontext()
-            with context:
-                if self.use_no_grad:
-                    # Fixed number of for-loop iterations to 10.
-                    # Currently no memory effect observed, so proceeding without chunking.
-                    n_chunks = 1
-                else:
-                    n_chunks = 1
-                total_len = concat_pixel_values.size(0)
-                # Calculate the size of each chunk based on total data length (divided into 10 chunks)
-                chunk_size = math.ceil(total_len / n_chunks) if total_len > 0 else 1
-                image_forward_outs_chunks = []
-
-                for i in range(n_chunks):
-                    start = i * chunk_size
-                    end = (i + 1) * chunk_size
-                    # Current chunk slice (could be an empty tensor if there's no data)
-                    chunk = concat_pixel_values[start:end].to(self.vision_model.dtype)
-                    # If the current chunk size is smaller than chunk_size, pad with dummy data
-                    if chunk.size(0) < chunk_size:
-                        # print(f"chunk.size(0): {chunk.size(0)}, chunk_size: {chunk_size}")
-                        pad_size = chunk_size - chunk.size(0)
-                        # Create dummy tensor based on concat_pixel_values shape
-                        dummy_shape = (pad_size,) + tuple(concat_pixel_values.shape[1:])
-                        dummy = torch.zeros(
-                            dummy_shape,
-                            dtype=concat_pixel_values.dtype,
-                            device=concat_pixel_values.device,
-                        )
-                        chunk = torch.cat([chunk, dummy], dim=0)
-
-                    # Pass the chunk through the vision model (processed according to use_nth_layer)
-                    if self.use_nth_layer == -1:
-                        # Replace post_layernorm of the last layer with Identity
-                        self.vision_model.vision_model.post_layernorm = nn.Identity()
-                        outs = self.vision_model(chunk)
-                        outs = outs.last_hidden_state[:, visual_token_idx:]
-                    else:
-                        outs = self.vision_model(chunk, output_hidden_states=True)
-                        outs = outs.hidden_states[self.use_nth_layer][:, visual_token_idx:]
-                    image_forward_outs_chunks.append(outs)
-
-                # Concatenate results from all chunks
-                image_forward_outs = torch.cat(image_forward_outs_chunks, dim=0).to(image_forward_outs_chunks[0].dtype)
-
-            if num_queries_vis_abstractors is None:
-                assert num_queries_vis_abstractors_slow is None
-                image_sizes = list(chain(*image_sizes))
-                if is_videos is not None:
-                    is_videos = list(chain(*is_videos))
-                group_ids = None
-                image_forward_outs = image_forward_outs.to(dtype=self.mm_projector.dtype)
-                image_forward_outs = self.mm_projector(image_forward_outs)
-            else:
-                # adaptive anyres is only implemented in HCXVisionCAbstractor
-                assert isinstance(self.mm_projector, HCXVisionCAbstractor)
-
-                (
-                    num_queries_vis_abstractors,
-                    num_grids,
-                    image_sizes,
-                    is_videos,
-                    group_ids,
-                ) = self.compute_adaptive_params(
-                    pixel_values,
-                    num_queries_vis_abstractors,
-                    num_queries_vis_abstractors_slow,
-                    image_sizes,
-                    is_videos,
-                    first_last_frames_slows,
-                )
+        # for convert back to List of List format
+        len_pixel_values_images = [len(pixel_value) for pixel_value in pixel_values_images] if pixel_values_images else []
+        len_pixel_values_videos = [len(pixel_value) for pixel_value in pixel_values_videos] if pixel_values_videos else []
 
-                image_forward_outs = image_forward_outs.to(dtype=self.mm_projector.dtype)
-                image_forward_outs = self.mm_projector(
-                    image_forward_outs,
-                    num_queries_vis_abstractors=num_queries_vis_abstractors,
-                    num_grids=num_grids,
-                )
+        if sum(len_pixel_values_images) + sum(len_pixel_values_videos) == 0:
+            return None
+
+        inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        if sum(len_pixel_values_images) > 0:
+            image_features_batch = self.forward_images(
+                pixel_values_images, image_sizes_images, len_pixel_values_images
+            )
+            for i, image_features in enumerate(image_features_batch):
+                if len(image_features) > 0:
+                    image_token_indices = (input_ids[i] == self.config.image_token_id).nonzero().squeeze()
+                    inputs_embeds[i][image_token_indices] = torch.cat(image_features).to(inputs_embeds.dtype)
+
+        if sum(len_pixel_values_videos) > 0:
+            video_features_batch = self.forward_videos(pixel_values_videos, len_pixel_values_videos)
+            for i, video_features in enumerate(video_features_batch):
+                if len(video_features) > 0:
+                    video_token_indices = (input_ids[i] == self.config.video_token_id).nonzero().squeeze()
+                    inputs_embeds[i][video_token_indices] = torch.cat(video_features).to(inputs_embeds.dtype)
+
+        return inputs_embeds
 
-            if self.anyres:
-                split_sizes = [pixel_value.shape[0] for pixel_value in chain(*pixel_values)]
-
-                if num_queries_vis_abstractors is None:
-                    image_features = anyres_postprocessing(
-                        image_forward_outs=image_forward_outs,
-                        split_sizes=split_sizes,
-                        image_sizes=image_sizes,
-                        num_queries_vis_abstractor=self.num_queries_vis_abstractor,
-                        unpad=self.unpad,
-                        is_videos=is_videos,
-                        patch_size=self.vision_model.config.patch_size,
-                        grid_size=self.vision_model.config.image_size,
-                        image_newline=self.image_newline,
-                        possible_resolutions=self.possible_resolutions,
-                    )
-                else:
-                    image_features = adaptive_anyres_postprocessing(
-                        image_forward_outs=image_forward_outs,
-                        image_sizes=image_sizes,
-                        num_queries_vis_abstractors=num_queries_vis_abstractors,
-                        unpad=self.unpad,
-                        is_videos=is_videos,
-                        grid_size=self.vision_model.config.image_size,
-                        image_newline=self.image_newline,
-                        possible_resolutions=self.possible_resolutions,
-                        group_ids=group_ids,
-                    )
+    def forward_images(
+        self,
+        pixel_values_images: List[List[torch.FloatTensor]],
+        image_sizes_images: List[List[Tuple[int, int]]],
+        len_pixel_values_images: List[int],
+    ) -> List[List[torch.Tensor]]:
+        if sum(len_pixel_values_images) == 0:
+            return None
+
+        concat_pixel_values_images = torch.cat(list(chain(*pixel_values_images)), dim=0)
+
+        visual_token_idx = 0 if "siglip" in self.vision_config.model_type else 1
+        context_vision_model = torch.no_grad() if self.vision_model_use_no_grad else contextlib.nullcontext()
+        with context_vision_model:
+            if self.config.use_nth_layer == -1:
+                # Replace post_layernorm of the last layer with Identity
+                self.vision_model.vision_model.post_layernorm = nn.Identity()
+                image_forward_outs = self.vision_model(concat_pixel_values_images)
+                image_forward_outs = image_forward_outs.last_hidden_state[:, visual_token_idx:]
             else:
-                if num_queries_vis_abstractors is None:
-                    image_features = [image_forward_out for image_forward_out in image_forward_outs]
-                else:
-                    image_features = [image_forward_out.unsqueeze(0) for image_forward_out in image_forward_outs]
-
-            # print(f"BEFORE GROUPING: len(image_features): {len(image_features)}")
-            image_features = [
-                image_features[sum(len_pixel_values[:i]) : sum(len_pixel_values[: i + 1])]
-                for i in range(len(len_pixel_values))
-            ]
+                image_forward_outs = self.vision_model(concat_pixel_values_images, output_hidden_states=True)
+                image_forward_outs = image_forward_outs.hidden_states[self.config.use_nth_layer][:, visual_token_idx:]
+
+        image_forward_outs = image_forward_outs.to(dtype=self.mm_projector.dtype)
+        image_forward_outs = self.mm_projector(image_forward_outs)  # b (h w) d
+
+        # feature 를 분할. e.g. torch.Size([18, 81, 3072]) -> [torch.Size([9, 81, 3072]), torch.Size([9, 81, 3072])]
+        split_sizes = [pixel_value.shape[0] for pixel_value in chain(*pixel_values_images)]
+        image_forward_outs = torch.split(image_forward_outs, split_sizes, dim=0)
+
+        # newline 붙여주기 (anyres postprocessing)
+        image_features = anyres_postprocessing(
+            image_forward_outs=image_forward_outs,
+            image_sizes=[image_size for image_sizes in image_sizes_images for image_size in image_sizes],
+            num_queries_vis_abstractor=self.config.num_queries_vis_abstractor_image,
+            unpad=self.config.unpad,
+            patch_size=self.vision_config.patch_size,
+            grid_size=self.vision_config.image_size,
+            image_newline=self.image_newline,
+            possible_resolutions=self.config.possible_resolutions,
+        )
+
+        # 원래 pixel_values_images 형태로 복원
+        image_features = [
+            image_features[sum(len_pixel_values_images[:i]) : sum(len_pixel_values_images[: i + 1])]
+            for i in range(len(len_pixel_values_images))
+        ]
 
-            batch_size = input_ids.size(0)
-            image_feature_dim = image_features[0][0].size(1)
-            image_feature_dtype = image_features[0][0].dtype
+        return image_features
 
-            if img_start_ids_list is None:
-                image_cnts = (input_ids == self.config.img_start_id).sum(dim=1).tolist()
+    def forward_videos(
+        self,
+        pixel_values_videos: List[List[torch.FloatTensor]],
+        len_pixel_values_videos: List[int],
+    ) -> List[torch.Tensor]:
+
+        len_video_grids = sum(len_pixel_values_videos)
+        if len_video_grids == 0:
+            return None
+
+        # Run Vision Model
+        concat_pixel_values_videos = torch.cat(list(chain(*pixel_values_videos)), dim=0)
+
+        visual_token_idx = 0 if "siglip" in self.vision_config.model_type else 1
+        context_vision_model = torch.no_grad() if self.vision_model_use_no_grad else contextlib.nullcontext()
+        with context_vision_model:
+            if self.config.use_nth_layer == -1:
+                # Replace post_layernorm of the last layer with Identity
+                self.vision_model.vision_model.post_layernorm = nn.Identity()
+                video_forward_outs = self.vision_model(concat_pixel_values_videos)
+                video_forward_outs = video_forward_outs.last_hidden_state[:, visual_token_idx:]
             else:
-                image_cnts = [len(img_start_ids) for img_start_ids in img_start_ids_list]
+                video_forward_outs = self.vision_model(concat_pixel_values_videos, output_hidden_states=True)
+                video_forward_outs = video_forward_outs.hidden_states[self.config.use_nth_layer][:, visual_token_idx:]
+
+        video_forward_outs = video_forward_outs.to(dtype=self.mm_projector.dtype)
+
+        # Run MM-Projector
+        # len(num_grids) == len(num_queries_vis_abstractors) + 1
+        grid_idx = 0
+        num_grids = [grid_idx]  # e.g. [0, 9, 18, 19, 27, 28, 36, 37, 45, 46, 54, 55, 56]
+        num_queries_vis_abstractors = []  # e.g. [81, 81, 81, 9, 81, 9, 81, 9, 81, 9, 81, 9]
+        len_total_frames = video_forward_outs.shape[0]
+
+        if self.config.first_last_frames_slow:
+            # TODO: 동작 확인 안 했음. 해야 함.
+            # slowfast (first_last_frames_slow)
+            assert len_total_frames != 0
+            if len_total_frames <= 2:
+                num_queries_vis_abstractors.append(self.config.num_queries_vis_abstractor_video_slow)
+                grid_idx += len_total_frames
+                num_grids.append(grid_idx)
+            else:
+                num_queries_vis_abstractors.append(self.config.num_queries_vis_abstractor_video_slow)
+                grid_idx += 1
+                num_grids.append(grid_idx)
 
-            if non_vision_query_lengths is None:
-                non_vision_query_lengths = self.determine_non_vision_query_lengths(
-                    input_ids, self.tokenizer.pad_token_id, self.config.img_start_id
-                )
+                num_queries_vis_abstractors.append(self.config.num_queries_vis_abstractor_video_fast)
+                grid_idx += len_total_frames - 2
+                num_grids.append(grid_idx)
 
-            if vision_query_lengths is None:
-                vision_query_lengths = self.determine_vision_query_lengths(image_features, image_cnts)
+                num_queries_vis_abstractors.append(self.config.num_queries_vis_abstractor_video_slow)
+                grid_idx += 1
+                num_grids.append(grid_idx)
+        else:
+            # slowfast
+            for pixel_values_frames in pixel_values_videos:
+                for pixel_values_frame in pixel_values_frames:
+                    if len(pixel_values_frame) > 0:
+                        num_queries_vis_abstractors.append(self.config.num_queries_vis_abstractor_video_slow)
+                        grid_idx += 1
+                        num_grids.append(grid_idx)
+                        num_queries_vis_abstractors.append(self.config.num_queries_vis_abstractor_video_fast)
+                        grid_idx = grid_idx + len(pixel_values_frame) - 1
+                        num_grids.append(grid_idx)
+
+        video_forward_outs = self.mm_projector(video_forward_outs, num_queries_vis_abstractors, num_grids)
+
+        # video_group 별로 concat 처리.
+        # 예를 들어, 3x3 grid 를 사용했을 경우, 총 9개의 feature 가 모일 때까지, grouped_features 에 리스트를 모아주고, concat 처리.
+        video_features = []  # what we want to return
+        target_features = []
+        target_group_size = 0
+        group_counter = 0
+        video_groups = [
+            len(frame) for frames in pixel_values_videos for frame in frames
+        ]  # for concat video features after projector
+
+        for forward_out in video_forward_outs:
+            target_group_size += len(forward_out)
+            target_features.append(forward_out.flatten(0, 1))
+
+            video_group_size = video_groups[group_counter]
+            if video_group_size == target_group_size:
+                video_features.append(torch.cat(target_features, dim=0))
+                target_features = []
+                group_counter += 1
+                target_group_size = 0
+
+            elif video_group_size < target_group_size:
+                raise RuntimeError(f"video_group_size < target_group_size!! [{video_group_size} < {target_group_size}]")
+
+        assert len(target_features) == 0, f"target_features is not empty!! {target_features}"
+        assert len(video_groups) == len(video_features)
+
+        # 원래 pixel_values_videos 형태로 복원
+        video_features = [
+            video_features[sum(len_pixel_values_videos[:i]) : sum(len_pixel_values_videos[: i + 1])]
+            for i in range(len(len_pixel_values_videos))
+        ]
 
-            # Slicing is faster than concatenation
-            len_inputs_embeds = max(
-                [
-                    sum(vision_query_length) + non_vision_query_length
-                    for non_vision_query_length, vision_query_length in zip(
-                        non_vision_query_lengths, vision_query_lengths
-                    )
-                ]
-            )
-            len_inputs_embeds = min(self.decoder_max_length, len_inputs_embeds)
-
-            inputs_embeds = torch.zeros(
-                [batch_size, len_inputs_embeds, image_feature_dim],
-                dtype=image_feature_dtype,
-                device=self.device,
-                requires_grad=True,
-            ).clone()
-            # temp_embeds : torch.bfloat16 : [batchsize, 174, 3072]
-            temp_embeds = self.get_input_embeddings()(input_ids)
-
-            # The complete format is <PROMPT><USER_PREFIX><VISION_QUERIES>Sentence
-            for batch_idx, sample in enumerate(input_ids):
-                # Concatenate with visual tokens and then slice
-                non_vision_query_length = non_vision_query_lengths[batch_idx]
-                # Safely concatenate with visual tokens and then slice
-                sample = sample[: non_vision_query_length + image_cnts[batch_idx]]
-
-                if image_cnts[batch_idx] == 0:  # Text instruction data doesn't insert image features
-                    temp_idx = 0
-                    # Reference: https://github.com/haotian-liu/LLaVA/commit/44e0562f9497fb79f042427307472a87d266d90a#diff-4477387d506ccb1897a13972cba26c9da3fad4d3e1c32ec4b8bd8ff7acd3f292
-                    # https://github.com/intel/intel-extension-for-transformers/issues/1201#issuecomment-1915875119
-                    inputs_embeds[batch_idx, :non_vision_query_length] = temp_embeds[batch_idx][
-                        :non_vision_query_length
-                    ]
-                    inputs_embeds[batch_idx, temp_idx:temp_idx] = image_features[batch_idx][0][
-                        0:0
-                    ]  # First image of batch_idx sample (dummy image)
-                else:
-                    if img_start_ids_list is None:
-                        img_start_ids = (sample == self.config.img_start_id).nonzero()
-                    else:
-                        img_start_ids = img_start_ids_list[batch_idx]
-                    assert len(img_start_ids) == image_cnts[batch_idx] == len(image_features[batch_idx])
-                    # Initialize starting points for input embeddings and temporary embeddings
-                    input_start, temp_start = 0, 0
-
-                    # Iterate through each image starting point in the batch
-                    for multi_img_idx, img_start_idx in enumerate(img_start_ids):
-                        # Calculate token length up to the current image starting point
-                        token_len = img_start_idx - temp_start
-
-                        # Copy tokens to inputs_embeds
-                        inputs_embeds[batch_idx, input_start : input_start + token_len] = temp_embeds[
-                            batch_idx, temp_start : temp_start + token_len
-                        ]
-
-                        inputs_embeds[
-                            batch_idx,
-                            input_start
-                            + token_len : input_start
-                            + token_len
-                            + vision_query_lengths[batch_idx][multi_img_idx],
-                        ] = image_features[batch_idx][multi_img_idx]
-
-                        # Update starting points for next token processing
-                        input_start += token_len + vision_query_lengths[batch_idx][multi_img_idx]
-                        temp_start += token_len + 1  # Increase by 1 to skip the image start token
-
-                    # Process tokens after the last image end token
-                    token_len = min(sample[temp_start:].size(0), inputs_embeds.size(1) - input_start)
-                    inputs_embeds[batch_idx, input_start : input_start + token_len] = temp_embeds[
-                        batch_idx, temp_start : temp_start + token_len
-                    ]
-        return inputs_embeds
+        return video_features
 
     @torch.no_grad()
     def generate(
         self,
         input_ids: Optional[torch.LongTensor] = None,
-        pixel_values: Optional[List[List[torch.FloatTensor]]] = None,
-        image_sizes: Optional[List[List[List[int]]]] = None,
-        vision_query_lengths: Optional[List[List[int]]] = None,
-        non_vision_query_lengths: Optional[List[int]] = None,
-        num_queries_vis_abstractors: Optional[List[List[int]]] = None,
-        num_queries_vis_abstractors_slow: Optional[List[List[int]]] = None,
-        first_last_frames_slows: Optional[List[bool]] = None,
-        is_videos: Optional[List[bool]] = None,
-        img_start_ids_list: Optional[List[List[int]]] = None,
+        pixel_values_images: Optional[List[List[torch.FloatTensor]]] = None,
+        image_sizes_images: Optional[List[List[Tuple[int, int]]]] = None,
+        pixel_values_videos: Optional[List[List[torch.FloatTensor]]] = None,
         pad_token_id: Optional[int] = None,
         eos_token_id: Optional[int] = None,
         bad_words_ids: Optional[List[List[int]]] = None,
@@ -1063,6 +903,7 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
         repetition_penalty: float = 1.0,
         length_penalty: int = 1,
         use_cache: bool = True,
+        verbose: bool = False,
         **kwargs,
     ) -> torch.LongTensor:
         """Generate text based on input tokens and images.
@@ -1109,29 +950,27 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
         if bad_words_ids is None:
             bad_words_ids = [
                 [
-                    self.config.language_config["bos_token_id"],
+                    self.config.text_config.bos_token_id,
                 ],
                 [
-                    self.config.language_config["eos_token_id"],
+                    self.config.text_config.eos_token_id,
                 ],
             ]
 
-        if pixel_values is None:
+        if (pixel_values_images is None or all(len(pixel_values) == 0 for pixel_values in pixel_values_images)) and (
+            pixel_values_videos is None or all(len(pixel_values) == 0 for pixel_values in pixel_values_videos)
+        ):
             return self.language_model.generate(
                 input_ids, pad_token_id=pad_token_id, eos_token_id=eos_token_id, bad_words_ids=bad_words_ids, **kwargs
             )
+
         inputs_embeds = self.extract_inputs_embeds(
             input_ids=input_ids,
-            pixel_values=self.to_vision_model_device(pixel_values),
-            image_sizes=image_sizes,
-            vision_query_lengths=vision_query_lengths,
-            non_vision_query_lengths=non_vision_query_lengths,
-            img_start_ids_list=img_start_ids_list,
-            num_queries_vis_abstractors=num_queries_vis_abstractors,
-            num_queries_vis_abstractors_slow=num_queries_vis_abstractors_slow,
-            first_last_frames_slows=first_last_frames_slows,
-            is_videos=is_videos,
+            pixel_values_images=pixel_values_images,
+            image_sizes_images=image_sizes_images,
+            pixel_values_videos=pixel_values_videos,
         )
+
         inputs_embeds = inputs_embeds.to(device=self.language_model.device, dtype=self.language_model.dtype)
 
         # pred : torch.int64 : [batchsize, generated token_length]
@@ -1140,7 +979,7 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
             pad_token_id=pad_token_id,
             eos_token_id=eos_token_id,
             bad_words_ids=bad_words_ids,
-            max_length=max_length,
+            max_new_tokens=max_length,
             min_length=min_length,
             num_beams=num_beams,
             do_sample=(False if temperature == 0.0 else do_sample),  # set do_sample=False if invalid temperature
@@ -1151,9 +990,26 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
             length_penalty=length_penalty,
             early_stopping=(False if num_beams <= 1 else True),  # set early_stopping=False when not beam_search
             use_cache=use_cache,
-            **kwargs,
         )
 
+        if verbose:
+            llm_query = self.tokenizer.batch_decode(
+                [
+                    [token_id for token_id in input_ids_row if token_id != self.tokenizer.pad_token_id]
+                    for input_ids_row in input_ids.detach().cpu().tolist()
+                ],
+                skip_special_tokens=False,
+            )[0]
+            llm_pred = self.tokenizer.batch_decode(
+                [
+                    [token_id for token_id in pred_row if token_id != self.tokenizer.pad_token_id]
+                    for pred_row in pred.detach().cpu().tolist()
+                ],
+                skip_special_tokens=False,
+            )[0]
+            print(f"# [info] llm_query: {llm_query}")
+            print(f"# [info] llm_pred: {llm_pred}")
+
         return pred
 
     def to_vision_model_device(self, input_tensor: Union[torch.Tensor, List]) -> Union[torch.Tensor, List]:
@@ -1240,11 +1096,17 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
             model: HCXVisionForCausalLM = super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
             model.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path)
 
-        img_start_id = model.tokenizer.encode(IMG_LOC, add_special_tokens=False)
+        image_token_id = model.tokenizer.encode(IMAGE_LOC, add_special_tokens=False)
+        assert (
+            len(image_token_id) == 1
+        ), f'"<|dummy3|>" was not encoded into a single special token. Encoding result: {image_token_id}'
+        model.config.image_token_id = image_token_id[0]
+        
+        video_token_id = model.tokenizer.encode(VIDEO_LOC, add_special_tokens=False)
         assert (
-            len(img_start_id) == 1
-        ), f'"<|dummy3|>" was not encoded into a single special token. Encoding result: {img_start_id}'
-        model.config.img_start_id = img_start_id[0]
+            len(video_token_id) == 1
+        ), f'"<|_unuse_missing_100270|>" was not encoded into a single special token. Encoding result: {video_token_id}'
+        model.config.video_token_id = video_token_id[0]
 
         model.save_only_vision = save_only_vision
         model.save_only_qformer = save_only_qformer
@@ -1293,212 +1155,37 @@ class HCXVisionForCausalLM(PreTrainedModel, GenerationMixin):
 
         return state_dict
 
-    def compute_adaptive_params(
-        self,
-        pixel_values: Optional[List[List[torch.FloatTensor]]] = None,
-        num_queries_vis_abstractors: Optional[List[List[int]]] = None,
-        num_queries_vis_abstractors_slow: Optional[List[List[int]]] = None,
-        image_sizes: Optional[List[List[List[int]]]] = None,
-        is_videos: Optional[List[bool]] = None,
-        first_last_frames_slows: Optional[List[bool]] = None,
-    ) -> Tuple[List[int], List[int], List[List[int]], List[bool], List[List[int]]]:
-        """Compute adaptive parameters for processing different image and video inputs.
-
-        This method calculates parameters needed for adaptive processing, especially when handling
-        variable resolutions or applying the slowfast algorithm to video frames. It flattens
-        batch-level inputs (lists of lists) into single lists representing all images/frames
-        in the batch. Based on slowfast configuration, it may split video frames into 'slow'
-        and 'fast' components, adjusting query counts and grid indices accordingly.
-
-        Args:
-            pixel_values: List of lists of image tensors (per sample). Used to determine the initial number of grids per
-                image/frame.
-            num_queries_vis_abstractors: List of lists (per sample) containing the base number of visual tokens
-                generated by the visual abstractor for each image grid
-                (e.g., 81 for a full grid, 9 for a subsampled/fast grid).
-            num_queries_vis_abstractors_slow: List of lists (per sample) containing the number of visual tokens for the
-                'slow' path when applying slowfast. Non-zero values here trigger the slowfast processing logic.
-            image_sizes: List of lists (per sample) of original image dimensions ([width, height]).
-            is_videos: List of lists (per sample) of booleans indicating if each input item is part of a video sequence.
-            first_last_frames_slows: List (per sample) of booleans. If True, slowfast logic
-                (if active based on `num_queries_vis_abstractors_slow`) is applied only to the first or last frame(s)
-                within each video sequence.
 
-        Returns:
-            Tuple containing:
-                - num_queries_vis_abstractors: Flattened list of final query counts per processed grid.
-                  Values might be adjusted based on slow/fast splitting
-                  (e.g., using values from `num_queries_vis_abstractors_slow` for slow frames).
-                  Example: [81, 81, 81, 9, 81, 9, ...] (Image, Image, Vid_Slow, Vid_Fast, Vid_Slow, Vid_Fast...)
-                - num_grids: Flattened list representing cumulative grid counts, acting as end indices for slicing the
-                  flattened `image_forward_outs`. Adjusted for slow/fast splits.
-                  Example: [0, 1, 9, 10, 18, 19, 27, ...] (Indices after Grid0_Slow(1),
-                  Grid1_Fast(8), Grid2_Slow(1), Grid3_Fast(8)...).
-                - image_sizes: Flattened list of image dimensions ([width, height]), potentially duplicated if slow/fast
-                  splitting occurred.
-                - is_videos: Flattened list of booleans indicating video status, potentially duplicated for
-                  slow/fast splits. Example: [False, False, True, True, True, True, ...]
-                  (Image1, Image2, Vid_grid1_slow, Vid_grid1_fast, Vid_grid2_slow, Vid_grid2_fast...)
-                - group_ids: List of lists, grouping indices that correspond to the same original image or frame.
-                  If a frame is split into slow/fast, its group will contain multiple indices.
-                  Example: [[0], [1], [2, 3], [4, 5], ...]
-                  (Group for Image1, Group for Image2, Group for Vid1_Slow+Fast, Group for Vid2_Slow+Fast...).
 
-        Raises:
-            AssertionError: If input validation fails (e.g., negative query counts).
-            Exception: If an unexpected case is encountered during slowfast processing.
-        """
-
-        # Check if all elements are integers greater than or equal to 0
-        assert all(
-            all(isinstance(value, int) and value >= 0 for value in sublist) for sublist in num_queries_vis_abstractors
-        ), "All values in num_queries_vis_abstractors must be integers >= 0."
-
-        assert all(
-            all(isinstance(value, int) and value >= 0 for value in sublist)
-            for sublist in num_queries_vis_abstractors_slow
-        ), "All values in num_queries_vis_abstractors_slow must be integers >= 0."
-
-        assert is_videos is not None
-
-        # Is it the first or last image? (for applying slowfast to video processing)
-        is_first_images = []
-        is_last_images = []
-        for is_video in is_videos:
-            for idx, is_video_item in enumerate(is_video):
-                if idx == 0:
-                    is_first_images.append(True)
-                else:
-                    is_first_images.append(False)
-                if idx == len(is_video) - 1:
-                    is_last_images.append(True)
-                else:
-                    is_last_images.append(False)
-
-        num_queries_vis_abstractors = list(chain(*num_queries_vis_abstractors))
-        num_queries_vis_abstractors_slow = list(chain(*num_queries_vis_abstractors_slow))
-        image_sizes = list(chain(*image_sizes))
-        is_videos = list(chain(*is_videos))
-        first_last_frames_slows = list(chain(*first_last_frames_slows))
-
-        # Use slowfast mode if there's at least one visual token count greater than 0 in num_queries_vis_abstractors_slow
-        use_slowfast = any([num_query > 0 for num_query in num_queries_vis_abstractors_slow])
-        num_grids = [pixel_value.shape[0] for pixel_value in chain(*pixel_values)]
-        num_grids = [0] + num_grids
-        group_ids = []
-
-        if use_slowfast:
-            new_num_grids = [num_grids[0]]
-            new_num_queries = []
-            new_image_sizes = []
-            new_is_videos = []
-
-            # When using slowfast, split more finely
-            # 0th local grid is slow frame, remaining local grids are fast frames
-            for (
-                num_query,
-                num_query_slow,
-                num_grid,
-                image_size,
-                is_video,
-                first_last_frames_slow,
-                is_first_image,
-                is_last_image,
-            ) in zip(
-                num_queries_vis_abstractors,
-                num_queries_vis_abstractors_slow,
-                num_grids[1:],
-                image_sizes,
-                is_videos,
-                first_last_frames_slows,
-                is_first_images,
-                is_last_images,
-            ):
-
-                if not first_last_frames_slow and num_query_slow > 0:  # Process all image in slowfast mode
-                    assert is_video  # slowfast mode is only applied to videos
-
-                    this_group_ids = [group_ids[-1][-1] + 1 if group_ids else 0]
-
-                    # slow frame (first grid)
-                    new_num_grids.append(new_num_grids[-1] + 1)
-                    new_num_queries.append(num_query_slow)
-                    new_image_sizes.append(image_size)
-                    new_is_videos.append(is_video)
-
-                    if num_grid >= 2:
-                        # fast frames
-                        new_num_grids.append(new_num_grids[-1] + num_grid - 1)
-                        new_num_queries.append(num_query)
-                        new_image_sizes.append(image_size)
-                        new_is_videos.append(is_video)
-                        this_group_ids.append(this_group_ids[-1] + 1)
-
-                    group_ids.append(this_group_ids)
-                elif (
-                    first_last_frames_slow and num_query_slow > 0 and (is_first_image or is_last_image)
-                ):  # Process only first/last image in slowfast mode
-                    # Case for special treatment of first/last frames in slow mode
-                    assert is_video  # slowfast mode is only applied to videos
-
-                    this_group_ids = [group_ids[-1][-1] + 1 if group_ids else 0]
-
-                    if num_grid == 1:
-                        # Simply process with slow since there's only one grid
-                        new_num_grids.append(new_num_grids[-1] + 1)
-                        new_num_queries.append(num_query_slow)
-                        new_image_sizes.append(image_size)
-                        new_is_videos.append(is_video)
-
-                    if num_grid >= 2:
-                        # Special treatment for first or last grid depending on is_first_image or is_last_image
-
-                        if is_first_image:  # includes both first and last
-                            # slow frame (first grid)
-                            new_num_grids.append(new_num_grids[-1] + 1)
-                            new_num_queries.append(num_query_slow)
-                            new_image_sizes.append(image_size)
-                            new_is_videos.append(is_video)
-                            # fast frames
-                            new_num_grids.append(new_num_grids[-1] + num_grid - 1)
-                            new_num_queries.append(num_query)
-                            new_image_sizes.append(image_size)
-                            new_is_videos.append(is_video)
-                            this_group_ids.append(this_group_ids[-1] + 1)
-                        elif is_last_image:
-                            # fast frames
-                            new_num_grids.append(new_num_grids[-1] + num_grid - 1)
-                            new_num_queries.append(num_query)
-                            new_image_sizes.append(image_size)
-                            new_is_videos.append(is_video)
-                            # slow frame (last grid)
-                            new_num_grids.append(new_num_grids[-1] + 1)
-                            new_num_queries.append(num_query_slow)
-                            new_image_sizes.append(image_size)
-                            new_is_videos.append(is_video)
-                            this_group_ids.append(this_group_ids[-1] + 1)
-                        else:
-                            raise Exception("This case should not be reached.")
-                    group_ids.append(this_group_ids)
-                else:
-                    # Not in slowfast mode, so reduce all by num_query (fast)
-                    new_num_grids.append(new_num_grids[-1] + num_grid)
-                    new_num_queries.append(num_query)
-                    new_image_sizes.append(image_size)
-                    new_is_videos.append(is_video)
-
-                    start_group_id = group_ids[-1][-1] + 1 if group_ids else 0
-                    group_ids.append([start_group_id])
-
-            num_grids = new_num_grids
-            num_queries_vis_abstractors = new_num_queries
-            image_sizes = new_image_sizes
-            is_videos = new_is_videos
+class HCXVisionMlp(nn.Module):
+    def __init__(
+        self,
+        mm_projector_type,
+        in_features,
+        hidden_features=None,
+        out_features=None,
+        act_layer=nn.GELU,
+    ):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.mm_projector_type = mm_projector_type
+        if self.mm_projector_type == "mlp":
+            self.fc1 = nn.Linear(in_features, hidden_features)
+            self.act = act_layer()
+            self.fc2 = nn.Linear(hidden_features, out_features)
+        elif self.mm_projector_type == "inverted_mlp":
+            self.fc1 = nn.Linear(in_features, 2 * hidden_features)
+            self.act = act_layer()
+            self.fc2 = nn.Linear(2 * hidden_features, out_features)
         else:
-            num_grids = [sum(num_grids[:i]) for i in range(1, len(num_grids) + 1)]
-            group_ids = [[group_id] for group_id in range(len(is_videos))]
+            raise NotImplementedError("{} is not implemented".format(self.mm_projector_type))
 
-        return num_queries_vis_abstractors, num_grids, image_sizes, is_videos, group_ids
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.fc2(x)
+        return x
 
 
 class HCXVisionCAbstractor(nn.Module):
@@ -1570,7 +1257,7 @@ class HCXVisionCAbstractor(nn.Module):
     ) -> torch.Tensor:
         # x: [B, L, dim]
         B, L, dim = x.shape
-        hw = int(L ** 0.5)
+        hw = int(L**0.5)
         x = rearrange(x, "b (h w) d -> b d h w", h=hw, w=hw)
 
         if num_queries_vis_abstractors is not None:
@@ -1596,7 +1283,7 @@ class HCXVisionCAbstractor(nn.Module):
         for i, num_queries in enumerate(num_queries_vis_abstractors):
             hw = int(num_queries**0.5)
             sampler = nn.AdaptiveAvgPool2d((hw, hw))
-            out = sampler(x[num_grids[i]:num_grids[i + 1], :])
+            out = sampler(x[num_grids[i] : num_grids[i + 1], :])
             out = self.net[2](out)  # s2
 
             out = rearrange(out, "b d h w -> b (h w) d")
@@ -1614,8 +1301,8 @@ class HCXVisionCAbstractor(nn.Module):
         depth: int = 3,
         mlp_depth: int = 2,
     ):
-        assert (n_queries ** 0.5).is_integer(), f"n_queries must be square number. n_queries: {n_queries}"
-        hw = int(n_queries ** 0.5)
+        assert (n_queries**0.5).is_integer(), f"n_queries must be square number. n_queries: {n_queries}"
+        hw = int(n_queries**0.5)
 
         # RegBlock = ResBlock + SE
         RegBlock = partial(
@@ -1653,89 +1340,3 @@ class HCXVisionCAbstractor(nn.Module):
             layers.append(nn.Linear(output_hidden_size, output_hidden_size))
         return nn.Sequential(*layers)
 
-def load_sharded_checkpoint(
-    model, folder, pick_prefix="", replace_prefix_list=[], replace_prefix_dict={}, print_info=True
-):
-    if folder is None:
-        return {}
-
-    files = os.listdir(folder)
-
-    # find relevant files
-    pytorch_bin_files = [file for file in files if file.startswith("pytorch_model") and file.endswith(".bin")]
-    safetensor_files = [file for file in files if file.endswith(".safetensors")]
-    shard_index_file = [file for file in files if file.endswith(".index.json")]
-
-    # check if sharded
-    index_present = len(shard_index_file) > 0
-    index_file = os.path.join(folder, shard_index_file[0]) if index_present else []
-
-    # check if safetensor
-    is_safetensor = len(safetensor_files) > 0
-
-    model_keys = model.state_dict().keys()
-
-    if is_safetensor:
-        from safetensors.torch import load_file
-
-        load_function = load_file
-        shard_files = safetensor_files
-    else:
-        load_function = partial(torch.load, map_location="cpu")
-        shard_files = pytorch_bin_files
-
-    # sharded case
-    if index_present:
-        with open(index_file, "r", encoding="utf-8") as f:
-            index = json.load(f)
-        loaded_keys = index["weight_map"].keys()
-        if pick_prefix:
-            loaded_keys = [k[len(pick_prefix) :] for k in loaded_keys if k.startswith(pick_prefix)]
-        if replace_prefix_list:
-            for rep_prefix in replace_prefix_list:
-                loaded_keys = [k[len(rep_prefix) :] if k.startswith(rep_prefix) else k for k in loaded_keys]
-        if replace_prefix_dict:
-            for rep_prefix in replace_prefix_dict:
-                loaded_keys = [
-                    k.replace(rep_prefix, replace_prefix_dict[rep_prefix]) if k.startswith(rep_prefix) else k
-                    for k in loaded_keys
-                ]
-
-    for i, shard_file in enumerate(shard_files):
-        state_dict = load_function(os.path.join(folder, shard_file))
-
-        # if pick_prefix, use only pick
-        if pick_prefix:
-            state_dict = {k[len(pick_prefix) :]: v for k, v in state_dict.items() if k.startswith(pick_prefix)}
-
-        for rep_prefix in replace_prefix_list:
-            state_dict = {k[len(rep_prefix) :] if k.startswith(rep_prefix) else k: v for k, v in state_dict.items()}
-
-        for rep_prefix in replace_prefix_dict:
-            state_dict = {
-                k.replace(rep_prefix, replace_prefix_dict[rep_prefix]) if k.startswith(rep_prefix) else k: v
-                for k, v in state_dict.items()
-            }
-
-        if is_fsdp_enabled():
-            if is_local_dist_rank_0():
-                model.load_state_dict(state_dict, strict=False)
-        else:
-            model.load_state_dict(state_dict, strict=False)
-        # Make sure memory is freed before we load the next state dict.
-
-        if not index_present:
-            loaded_keys = state_dict.keys()
-
-        del state_dict
-        gc.collect()
-
-    # missing keys
-    missing_keys = [key for key in model_keys if key not in loaded_keys]
-    unexpected_keys = [key for key in loaded_keys if key not in model_keys]
-
-    if get_rank() == 0 and print_info:
-        print(f"[info] missing_keys: {missing_keys}")
-        print(f"[info] unexpected_keys: {unexpected_keys}")
-
-    return {"missing_keys": missing_keys, "unexpected_keys": unexpected_keys}

preprocessor_config.json

@@ -1,10 +1,9 @@
 {
-  "processor_class": "HCXVisionProcessor",
+  "anyres": true,
   "auto_map": {
-    "AutoProcessor": "preprocessor.HCXVisionProcessor",
-    "AutoImageProcessor": "preprocessor.HCXVisionProcessor"
+    "AutoImageProcessor": "image_processing_hyperclovax.HCXImageProcessor",
+    "AutoProcessor": "processing_hyperclovax.HCXProcessor"
   },
-  "anyres": true,
   "crop_size": {
     "height": 378,
     "width": 378
@@ -14,23 +13,22 @@
   "do_normalize": true,
   "do_rescale": true,
   "do_resize": true,
-  "max_num_grids": 9,
-  "max_image_cnt": 12,
-  "num_queries_vis_abstractor": 81,
-  "num_queries_vis_abstractor_video_fast": 9,
-  "num_queries_vis_abstractor_video_slow": 81,
-  "first_last_frames_slow": false,
   "image_mean": [
     0.5,
     0.5,
     0.5
   ],
-  "image_processor_type": "HCXVisionProcessor",
+  "image_processor_class": "AutoImageProcessor",
+  "image_processor_type": "HCXImageProcessor",
   "image_std": [
     0.5,
     0.5,
     0.5
   ],
+  "num_queries_vis_abstractor_image": 81,
+  "num_queries_vis_abstractor_video_slow": 81,
+  "num_queries_vis_abstractor_video_fast": 9,
+  "first_last_frames_slow_video": false,
   "pad_to_square": true,
   "patch_size": 14,
   "possible_resolutions": [
@@ -127,6 +125,7 @@
       378
     ]
   ],
+  "processor_class": "HCXProcessor",
   "resample": 2,
   "rescale_factor": 0.00392156862745098,
   "size": {

processing_hyperclovax.py

@@ -0,0 +1,710 @@
+import copy
+import os
+import re
+import uuid
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+import PIL
+from PIL import Image
+import torch
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_utils import ImageInput
+from transformers.processing_utils import (
+    AllKwargsForChatTemplate,
+    ChatTemplateLoadKwargs,
+    ProcessingKwargs,
+    ProcessorMixin,
+    TextInput,
+    Unpack,
+    VideoInput,
+)
+from transformers.tokenization_utils_base import AudioInput
+from transformers.utils import (
+    is_torch_device,
+    is_torch_dtype,
+    logging,
+    requires_backends,
+)
+from transformers.video_utils import VideoInput, VideoMetadata, load_video
+
+logger = logging.get_logger(__name__)
+
+
+class HCXBatchFeature(BatchFeature):
+    def to(self, *args, **kwargs) -> "BatchFeature":
+        """
+        Send all values to device by calling `v.to(*args, **kwargs)` (PyTorch only). This should support casting in
+        different `dtypes` and sending the `BatchFeature` to a different `device`.
+
+        Args:
+            args (`Tuple`):
+                Will be passed to the `to(...)` function of the tensors.
+            kwargs (`Dict`, *optional*):
+                Will be passed to the `to(...)` function of the tensors.
+                To enable asynchronous data transfer, set the `non_blocking` flag in `kwargs` (defaults to `False`).
+
+        Returns:
+            [`BatchFeature`]: The same instance after modification.
+        """
+        requires_backends(self, ["torch"])
+        import torch  # noqa
+
+        new_data = {}
+        device = kwargs.get("device")
+        non_blocking = kwargs.get("non_blocking", False)
+        # Check if the args are a device or a dtype
+        if device is None and len(args) > 0:
+            # device should be always the first argument
+            arg = args[0]
+            if is_torch_dtype(arg):
+                # The first argument is a dtype
+                pass
+            elif isinstance(arg, str) or is_torch_device(arg) or isinstance(arg, int):
+                device = arg
+            else:
+                # it's something else
+                raise ValueError(f"Attempting to cast a BatchFeature to type {str(arg)}. This is not supported.")
+        # We cast only floating point tensors to avoid issues with tokenizers casting `LongTensor` to `FloatTensor`
+        for k, v in self.items():
+            # check if v is a floating point
+            if isinstance(v, torch.Tensor) and torch.is_floating_point(v):
+                # cast and send to device
+                new_data[k] = v.to(*args, **kwargs)
+            elif isinstance(v, torch.Tensor) and device is not None:
+                new_data[k] = v.to(device=device, non_blocking=non_blocking)
+            elif "pixel_values" in k:
+                new_pixel_values_batch = []
+                for _v in v:
+                    pixel_values = [pixel_value.to(device=device, non_blocking=non_blocking) for pixel_value in _v]
+                    new_pixel_values_batch.append(pixel_values)
+                new_data[k] = new_pixel_values_batch
+            else:
+                new_data[k] = v
+        self.data = new_data
+        return self
+
+
+class HCXProcessorKwargs(ProcessingKwargs, total=False):
+    _defaults = {
+        "text_kwargs": {
+            "return_tensors": "pt",
+            "calc_non_vision_query_lengths": False,
+        },
+        "images_kwargs": {},
+        "audio_kwargs": {},
+        "videos_kwargs": {
+            "max_image_cnt": 12,
+            "max_num_grids": 9,
+        },
+    }
+
+
+class HCXProcessor(ProcessorMixin):
+    attributes = ["image_processor", "tokenizer"]
+    valid_kwargs = ["chat_template"]
+
+    image_processor_class = "AutoImageProcessor"
+    tokenizer_class = ("GPT2Tokenizer", "GPT2TokenizerFast")
+
+    def __init__(self, image_processor=None, tokenizer=None, chat_template=None, **kwargs):
+        self.image_token = "<|dummy3|>"
+        self.video_token = "<|_unuse_missing_100270|>"
+        self.image_token_pattern = re.compile(r"<\|dummy3\|>")
+        self.video_token_pattern = re.compile(r"<\|_unuse_missing_100270\|>")
+        self.image_video_token_pattern = re.compile(r"<\|dummy3\|>|<\|_unuse_missing_100270\|>")
+        self.image_token_id = (
+            tokenizer.image_token_id
+            if getattr(tokenizer, "image_token_id", None)
+            else tokenizer.convert_tokens_to_ids(self.image_token)
+        )
+        self.video_token_id = (
+            tokenizer.video_token_id
+            if getattr(tokenizer, "video_token_id", None)
+            else tokenizer.convert_tokens_to_ids(self.video_token)
+        )
+        super().__init__(image_processor, tokenizer, chat_template=chat_template)
+
+    def apply_chat_template(
+        self,
+        conversation: Union[list[dict[str, str]], list[list[dict[str, str]]]],
+        chat_template: Optional[str] = None,
+        **kwargs: Unpack[AllKwargsForChatTemplate],
+    ) -> str:
+        model_inputs = super().apply_chat_template(conversation, chat_template, **kwargs)
+
+        # vllm needs vision_query_lengths, but we don't need it
+        del model_inputs["vision_query_lengths_images"]
+        del model_inputs["vision_query_lengths_videos"]
+
+        # # vllm 호환성을 위해 이곳에서 token 을 vision_query_length만큼 늘리기 처리
+        # if "input_ids" in model_inputs:
+        #     # self.image_token 모두 찾기
+        #     input_ids = self.repeat_dummy_tokens(
+        #         model_inputs["input_ids"], self.image_token_id, model_inputs["vision_query_lengths_images"]
+        #     )
+        #     input_ids = self.repeat_dummy_tokens(
+        #         input_ids, self.video_token_id, model_inputs["vision_query_lengths_videos"]
+        #     )
+        #     model_inputs["input_ids"] = input_ids
+            
+        return model_inputs
+    
+    def repeat_dummy_tokens(self, input_ids, target_token_id, vision_query_lengths):
+        input_ids = input_ids.clone().detach()
+        batch_indices, target_indices = torch.where(input_ids == target_token_id)
+        batch_size = input_ids.shape[0]
+
+        new_input_ids = [[] for _ in range(batch_size)]
+        start_indices = [0 for _ in range(batch_size)]
+        counter = [0 for _ in range(batch_size)]
+        for batch_idx, target_idx in zip(batch_indices, target_indices):
+            start_idx = start_indices[batch_idx]
+            new_input_ids[batch_idx].append(input_ids[batch_idx][start_idx:target_idx])
+            query_length = vision_query_lengths[batch_idx][counter[batch_idx]]
+            new_input_ids[batch_idx].append(input_ids[batch_idx][target_idx].repeat(query_length))
+            start_indices[batch_idx] = target_idx + 1
+            counter[batch_idx] += 1
+
+        for batch_idx in range(batch_size):
+            start_idx = start_indices[batch_idx]
+            new_input_ids[batch_idx].append(input_ids[batch_idx][start_idx:])  # append remaining tokens
+            new_input_ids[batch_idx] = torch.cat(new_input_ids[batch_idx], dim=0)
+
+        new_input_ids = torch.stack(new_input_ids)
+        return new_input_ids
+
+    def _load_video_for_model(
+        self,
+        video: str,
+        num_frames: Optional[int] = None,
+        fps: Optional[int] = None,
+        backend: str = "opencv",
+        **kwargs: Unpack[HCXProcessorKwargs],
+    ) -> List[ImageInput]:
+        """
+        Overrided function.
+
+        Loads `video` to a List[PIL.Image] (llava style)
+
+        Args:
+            video (`str`):
+                The video to convert to the numpy array format. Can be a link to video or local path.
+            num_frames (`int`, *optional*):
+                Number of frames to sample uniformly. If not passed, the whole video is loaded.
+            fps (`int`, *optional*):
+                Number of frames to sample per second. Should be passed only when `num_frames=None`.
+                If not specified and `num_frames==None`, all frames are sampled.
+            backend (`str`, *optional*, defaults to `"opencv"`):
+                The backend to use when loading the video. Can be any of ["decord", "pyav", "opencv", "torchvision"]. Defaults to "opencv".
+
+        Returns:
+            Tuple[`np.array`, Dict]: A tuple containing:
+                - List[PIL.Image] of frames in RGB.
+                - Metadata dictionary.
+        """
+        output_kwargs = self._merge_kwargs(
+            HCXProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+
+        logger.warning_once(f"num_frames control via argument is not supported yet. Ignored num_frames: {num_frames}.")
+        logger.warning_once(f"fps control via argument is not supported yet. Ignored fps: {fps}.")
+        logger.warning_once(f"backend control via argument is not supported yet. Ignored backend: {backend}.")
+
+        def _hcx_sample_indices_fn(metadata: VideoMetadata, num_frames=None, fps=None, **kwargs):
+            max_num_grids = output_kwargs["videos_kwargs"]["max_num_grids"]
+            max_image_cnt = output_kwargs["videos_kwargs"]["max_image_cnt"]
+            frame_indices, time_interval = extract_frame_indices(
+                metadata.duration,
+                metadata.total_num_frames,
+                metadata.fps,
+                max_num_grids,
+                max_image_cnt,
+                default_interval=0.4,
+            )
+            metadata.time_interval = time_interval
+            return np.array(frame_indices)
+
+        video_loaded, video_metadata = None, None
+        for backend in ["decord", "pyav", "opencv", "torchvision"]:
+            try:
+                video_loaded, video_metadata = load_video(
+                    video, sample_indices_fn=_hcx_sample_indices_fn, backend=backend
+                )
+                break
+            except Exception as e:
+                logger.error(f"Error loading video with {backend} backend: {e}")
+                continue
+
+        assert video_loaded is not None, "Failed to load video with any backend"
+
+        return video_loaded, video_metadata
+
+    def _process_messages_for_chat_template(
+        self,
+        conversation: List[List[Dict[str, str]]],
+        batch_images: List[List[ImageInput]],
+        batch_videos: List[List[VideoInput]],
+        batch_video_metadata: List[List[Dict[str, any]]],
+        **mm_load_kwargs: Unpack[ChatTemplateLoadKwargs],
+    ):
+        """
+        Overrided function.
+        Used within `apply_chat_template` when a model has a special way to process conversation history. For example,
+        video models might want to specify in the prompt the duration of video or which frame indices at which timestamps
+        were sampled. This information cannot be accessed before the video is loaded.
+
+        For most models it is a no-op, and must be overridden by model processors which require special processing.
+
+        Args:
+            conversation (`List[Dict, str, str]`):
+                The conversation to process. Always comes in batched format.
+            batch_images (`List[List[ImageInput]]`):
+                Batch of images that were loaded from url/path defined in the conversation. The images
+                are ordered in the same way as in the conversation. Comes in nested list format, one list of `PIL` images
+                per batch.
+            batch_videos (`List[List[ImageInput]]`):
+                Batch of videos that were loaded from url/path defined in the conversation. The videos
+                are ordered in the same way as in the conversation. Comes in nested list format, one list of `PIL.Image`
+                per batch.
+            batch_video_metadata (`List[List[Dict[[str, any]]]]`):
+                Batch of metadata returned from loading videos. That includes video fps, duration and total number of framer in original video.
+                Metadata are ordered in the same way as `batch_videos`. Comes in nested list format, one list of `Dict`
+                per batch.
+        """
+
+        is_video_in_conversation = False
+        for batch_idx, messages in enumerate(conversation):
+            is_video_in_messages = False
+            is_image_in_messages = False
+            for message in messages:
+                for content in message["content"]:
+                    if content["type"] == "video":
+                        is_video_in_messages = True
+                    elif content["type"] == "image":
+                        is_image_in_messages = True
+            if not is_video_in_messages:
+                batch_videos.insert(batch_idx, [])
+                batch_video_metadata.insert(batch_idx, [])
+            if not is_image_in_messages:
+                batch_images.insert(batch_idx, [])
+
+            is_video_in_conversation = is_video_in_conversation or is_video_in_messages
+
+        if not is_video_in_conversation:
+            return conversation
+
+        # conversation processing
+        new_conversation = []
+        for batch_idx, messages in enumerate(conversation):
+            video_counter = 0
+            new_messages = []
+
+            for message in messages:
+                new_message = {
+                    "role": message["role"],
+                    "content": [],
+                }
+                for content in message["content"]:
+                    if content["type"] == "video":
+                        video = batch_videos[batch_idx][video_counter]
+                        video_meta = batch_video_metadata[batch_idx][video_counter]
+
+                        time_stamps = calc_timestamp_video_grids(video, video_meta.time_interval, max_grid_shape=(3, 3))
+                        video_counter += 1
+
+                        if "filename" in content:
+                            filename = content["filename"]
+                        else:
+                            filename = content["video"].split("/")[-1]
+                            if len(filename) > 50:
+                                filename = f"{uuid.uuid4().hex}.mp4"
+                        basename, ext = os.path.splitext(filename)
+                        if ext == "":
+                            ext = ".mp4"
+
+                        for frame_idx, time_stamp in enumerate(time_stamps):
+                            if frame_idx == len(video) - 1:
+                                # final_grid
+                                new_content = {
+                                    "filename": f"{basename}-{frame_idx}{ext}",
+                                    "video": content["video"],
+                                    "type": "video",
+                                    "video_time_stamp": time_stamp,
+                                    "lens_keywords": content["lens_keywords"],
+                                    "lens_local_keywords": content["lens_local_keywords"],
+                                    "speech_to_text": content["speech_to_text"],
+                                    "is_final_grid": True,
+                                }
+                                new_message["content"].append(new_content)
+                            else:
+                                new_content = {
+                                    "filename": f"{basename}-{frame_idx}{ext}",
+                                    "video": content["video"],
+                                    "type": "video",
+                                    "video_time_stamp": time_stamp,
+                                }
+                                new_message["content"].append(new_content)
+                    else:
+                        new_message["content"].append(copy.deepcopy(content))
+                new_messages.append(new_message)
+            new_conversation.append(new_messages)
+
+        return new_conversation
+
+    def __call__(
+        self,
+        text: TextInput = None,
+        images: List[List[ImageInput]] = None,
+        videos: List[List[VideoInput]] = None,
+        audio: AudioInput = None,
+        **kwargs: Unpack[HCXProcessorKwargs],
+    ):
+        output_kwargs = self._merge_kwargs(
+            HCXProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+
+        # prepare model inputs
+        mm_inputs = {
+            "pixel_values_images": [],
+            "image_sizes_images": [],
+            "vision_query_lengths_images": [],
+            "pixel_values_videos": [],
+            # "image_sizes_videos": [],
+            "vision_query_lengths_videos": [],
+        }
+        calc_non_vision_query_lengths = output_kwargs["text_kwargs"].pop("calc_non_vision_query_lengths")
+        if calc_non_vision_query_lengths:
+            mm_inputs["non_vision_query_lengths"] = []
+
+        # video processing
+        if videos is not None:
+            vit_input_size = self.image_processor.crop_size["width"]
+
+            video_kwargs = copy.deepcopy(output_kwargs["videos_kwargs"])
+
+            for videos_in_single_conversation in videos:
+                pixel_values_videos = []
+                vision_query_lengths_videos = []
+
+                for video_frames in videos_in_single_conversation:
+                    if len(video_frames) == 0:
+                        mm_inputs["pixel_values_videos"].append([])
+                        mm_inputs["vision_query_lengths_videos"].append([])
+                        continue
+                    video_frames_combined = combine_frames_into_images(
+                        video_frames, max_grid_shape=(3, 3), vit_input_size=vit_input_size
+                    )
+                    video_kwargs["is_video"] = True
+                    video_kwargs["return_tensors"] = None
+
+                    frames_processed = self.image_processor(images=video_frames_combined, **video_kwargs)
+                    sizes = [(size["width"], size["height"]) for size in frames_processed["image_sizes"]]
+
+                    pixel_values_videos.extend(frames_processed["pixel_values"])
+                    vision_query_lengths_videos.extend(frames_processed["vision_query_lengths"])
+
+                mm_inputs["pixel_values_videos"].append(pixel_values_videos)
+                mm_inputs["vision_query_lengths_videos"].append(vision_query_lengths_videos)
+
+        # image processing
+        if images is not None:
+            image_kwargs = copy.deepcopy(output_kwargs["images_kwargs"])
+            image_kwargs["is_video"] = False
+            image_kwargs["return_tensors"] = None
+
+            for images_in_single_conversation in images:
+                if isinstance(images_in_single_conversation, PIL.Image.Image): # single item to batch
+                    images_in_single_conversation = [images_in_single_conversation, ]
+                if len(images_in_single_conversation) == 0:
+                    mm_inputs["pixel_values_images"].append([])
+                    mm_inputs["image_sizes_images"].append([])
+                    mm_inputs["vision_query_lengths_images"].append([])
+                    continue
+                images_processed = self.image_processor(images=images_in_single_conversation, **image_kwargs)
+                sizes = [(size["width"], size["height"]) for size in images_processed["image_sizes"]]
+
+                mm_inputs["pixel_values_images"].append(images_processed["pixel_values"])
+                mm_inputs["image_sizes_images"].append(sizes)
+                mm_inputs["vision_query_lengths_images"].append(images_processed["vision_query_lengths"])
+
+        # text processing
+        def _create_replacer(_target_token, _replacements):
+            _iterator = iter(_replacements)
+
+            def _replacer(match_obj):
+                # return self.image_token
+                num_query_tokens = next(_iterator)
+                return "".join(
+                    [_target_token for _ in range(num_query_tokens)]
+                )  # vision_query_legnth 만큼 image token 을 복제
+
+            return _replacer
+
+        text_inputs = {}
+        if text is not None:
+            if not isinstance(text, list):
+                text = [text]
+
+            if images is not None:
+                new_texts = []
+                for batch_idx, text_in_single_conversation in enumerate(text):
+                    new_text = self.image_token_pattern.sub(
+                        _create_replacer(self.image_token, mm_inputs["vision_query_lengths_images"][batch_idx]),
+                        text_in_single_conversation,
+                    )
+                    new_texts.append(new_text)
+                text = new_texts
+
+            if videos is not None:
+                new_texts = []
+                for batch_idx, text_in_single_conversation in enumerate(text):
+                    new_text = self.video_token_pattern.sub(
+                        _create_replacer(self.video_token, mm_inputs["vision_query_lengths_videos"][batch_idx]),
+                        text_in_single_conversation,
+                    )
+                    new_texts.append(new_text)
+                text = new_texts
+
+            text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
+
+        # audio processing
+        if audio is not None:
+            raise NotImplementedError("Audio processing is not supported yet.")
+
+        return HCXBatchFeature(data={**text_inputs, **mm_inputs})
+
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to Siglip2Tokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)
+
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to Siglip2Tokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def post_process_image_text_to_text(
+        self, generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False, **kwargs
+    ):
+        """
+        Post-process the output of the model to decode the text.
+
+        Args:
+            generated_outputs (`torch.Tensor` or `np.ndarray`):
+                The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
+                or `(sequence_length,)`.
+            skip_special_tokens (`bool`, *optional*, defaults to `True`):
+                Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method.
+            Clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
+                Whether or not to clean up the tokenization spaces. Argument passed to the tokenizer's `batch_decode` method.
+            **kwargs:
+                Additional arguments to be passed to the tokenizer's `batch_decode method`.
+
+        Returns:
+            `List[str]`: The decoded text.
+        """
+        return self.tokenizer.batch_decode(
+            generated_outputs,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs,
+        )
+
+    @property
+    def model_input_names(self):
+        tokenizer_input_names = self.tokenizer.model_input_names
+        image_processor_input_names = self.image_processor.model_input_names
+        names_from_processor = list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
+        return names_from_processor + []
+
+
+def extract_frame_indices(play_time, total_frames, fps, max_num_grids, max_image_cnt, default_interval=0.4):
+    """
+    Extracts specific frame indices from a video based on duration, frame count, and sampling strategy.
+
+    The function determines which frames to extract given the video duration (`play_time`),
+    total frame count, and frame rate. It samples frames at regular intervals (default: 0.4s),
+    but if the number of frames exceeds the limit defined by `max_num_grids * max_image_cnt`,
+    it performs uniform sampling to stay within that limit.
+
+    Args:
+        play_time (float): Total play time of the video in seconds.
+        total_frames (int): Total number of frames in the video.
+        fps (float): Frames per second of the video.
+        max_num_grids (int): Maximum number of grids to display.
+        max_image_cnt (int): Maximum number of images per grid.
+        default_interval (float, optional): Interval in seconds between frame samples. Defaults to 0.4.
+
+    Returns:
+        Tuple:
+            frame_indices (List[int]): A list of selected frame indices.
+            time_interval (float): Time interval between selected frames (in seconds).
+    """
+
+    # Calculate how many frames to extract with the default interval
+    default_frame_count = int(play_time / default_interval)
+
+    # Maximum frames allowed based on max_num_grids and max_image_cnt
+    max_frames_allowed = max_num_grids * max_image_cnt
+
+    # Determine whether we can use the default interval or need uniform sampling
+    if default_frame_count <= max_frames_allowed:
+        # Default interval is sufficient, extract frames every 0.4 seconds
+        frame_interval = int(total_frames / default_frame_count)
+    else:
+        # Use uniform sampling to fit within max_frames_allowed
+        frame_interval = int(total_frames / max_frames_allowed)
+
+    # Extract frame indices at the calculated interval
+    selected_indices = list(range(0, total_frames, frame_interval))
+
+    time_interval = frame_interval / fps
+
+    # Ensure the number of selected indices does not exceed max_frames_allowed
+    return selected_indices[:max_frames_allowed], time_interval
+
+
+def calc_timestamp_video_grids(frames, time_interval, max_grid_shape=(3, 3)):
+    """
+    Calculates the time range labels for each grid in a video.
+
+    Args:
+        frames (List[PIL.Image.Image]): A list of frames extracted from a video.
+        time_interval (float): Time interval (in seconds) between consecutive frames.
+        max_grid_shape (Tuple[int, int], optional): The maximum grid shape as (rows, cols). Defaults to (3, 3).
+        vit_input_size (int, optional): The target size (height and width) for the Vision Transformer input. Defaults to 378.
+
+    Returns:
+        Tuple:
+            image_time_stamps (List[str]): A list of time span labels for each combined image,
+                e.g., ["0.00s~1.50s", "1.50s~3.00s", ...].
+    """
+    max_num_grids = max_grid_shape[0] * max_grid_shape[1]
+    # assert (
+    #     max_grid_shape[1] == 1
+    # ), f"For video processing, decided to concatenate frames horizontally into a wide image."
+
+    # Calculate the number of canvases needed.
+    num_frames = len(frames)
+    num_canvases = num_frames // max_num_grids
+    leftover_frames = num_frames % max_num_grids
+
+    time_stamp = 0  # second
+    image_time_stamps = []
+
+    for canvas_idx in range(num_canvases):
+        # Determine the frames to fill in the current canvas.
+        start_idx = canvas_idx * max_num_grids
+        end_idx = min(start_idx + max_num_grids, num_frames)
+
+        # Append the current canvas to the result list.
+        frame_cnt = end_idx - start_idx
+        image_time_stamps.append(f"{time_stamp:.2f}s~{time_stamp + frame_cnt * time_interval:.2f}s")
+        time_stamp += frame_cnt * time_interval
+
+    if leftover_frames > 0:
+        # Add the current canvas to the list of combined images.
+        frame_cnt = leftover_frames
+        image_time_stamps.append(f"{time_stamp:.2f}s~{time_stamp + frame_cnt * time_interval:.2f}s")
+        time_stamp += frame_cnt * time_interval
+
+    return image_time_stamps
+
+
+def combine_frames_into_images(frames, max_grid_shape=(3, 3), vit_input_size=378):
+    """
+    Combines a sequence of video frames into grid-based images and generates corresponding time range labels.
+
+    Frames are grouped and arranged into a grid (e.g., 3x3) such that each combined image contains up to
+    `max_grid_shape[0] * max_grid_shape[1]` frames. Each combined image is resized to the given ViT input size.
+
+    Args:
+        frames (NDArray): (num_frames, H, W, C) shape. A list of frames extracted from a video.
+        time_interval (float): Time interval (in seconds) between consecutive frames.
+        max_grid_shape (Tuple[int, int], optional): The maximum grid shape as (rows, cols). Defaults to (3, 3).
+        vit_input_size (int, optional): The target size (height and width) for the Vision Transformer input. Defaults to 378.
+
+    Returns:
+        Tuple:
+            image_list (List[PIL.Image.Image]): A list of grid-combined images.
+    """
+    max_num_grids = max_grid_shape[0] * max_grid_shape[1]
+    # assert (
+    #     max_grid_shape[1] == 1
+    # ), f"For video processing, decided to concatenate frames horizontally into a wide image."
+
+    # List to store the resulting combined images.
+    image_list = []
+
+    # Calculate the number of canvases needed.
+    num_frames = len(frames)
+    num_canvases = num_frames // max_num_grids
+    leftover_frames = num_frames % max_num_grids
+
+    # change frames (4d numpy tensor) to List[PIL.Image.Image]
+    frames = [Image.fromarray(frame) for frame in frames]
+
+    for canvas_idx in range(num_canvases):
+        # Initialize the current canvas.
+        combined_image = Image.new(
+            "RGB", (vit_input_size * max_grid_shape[0], vit_input_size * max_grid_shape[1]), color=(0, 0, 0)
+        )
+
+        # Determine the frames to fill in the current canvas.
+        start_idx = canvas_idx * max_num_grids
+        end_idx = min(start_idx + max_num_grids, num_frames)
+
+        for idx in range(start_idx, end_idx):
+            img = frames[idx]
+
+            # Resize each frame to a square shape.
+            img_resized = img.resize((vit_input_size, vit_input_size))
+
+            # Calculate the (row, column) position to place the frame within the grid layout.
+            local_idx = idx - start_idx
+            x_offset = (local_idx % max_grid_shape[0]) * vit_input_size
+            y_offset = (local_idx // max_grid_shape[0]) * vit_input_size
+
+            # Calculate the position to place the frame in the grid.
+            combined_image.paste(img_resized, (x_offset, y_offset))
+
+        # Append the current canvas to the result list.
+        image_list.append(combined_image)
+
+    if leftover_frames > 0:
+        # canvas_idx might be undefined; default to 0 if not previously assigned to avoid "referenced before assignment" error.
+        canvas_idx = num_canvases
+        # Add the remaining frames to the final canvas.
+        # combined_image = Image.new("RGB", (vit_input_size * leftover_frames, vit_input_size * 1), color=(0, 0, 0))  # hsk
+        combined_image = Image.new(
+            "RGB", (vit_input_size * max_grid_shape[0], vit_input_size * max_grid_shape[1]), color=(0, 0, 0)
+        )
+
+        for idx in range(leftover_frames):
+            img = frames[num_canvases * max_num_grids + idx]
+
+            # Resize the frame to a square (equal width and height).
+            img_resized = img.resize((vit_input_size, vit_input_size))
+
+            # Calculate the (row, column) position to place the frame within the grid layout.
+            # x_offset = (idx % leftover_frames) * vit_input_size  # hsk
+            # y_offset = (idx // leftover_frames) * vit_input_size  # hsk
+            x_offset = (idx % max_grid_shape[0]) * vit_input_size
+            y_offset = (idx // max_grid_shape[0]) * vit_input_size
+
+            # Calculate the position to place the frame within the grid layout.
+            combined_image.paste(img_resized, (x_offset, y_offset))
+
+        # Add the current canvas to the list of combined images.
+        image_list.append(combined_image)
+
+    return image_list

processor_config.json

@@ -0,0 +1,6 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_hyperclovax.HCXProcessor"
+  },
+  "processor_class": "HCXProcessor"
+}

special_tokens_map.json

@@ -62,7 +62,13 @@
     "rstrip": false,
     "single_word": false
   },
-  "eos_token": "<|endofturn|>",
+  "eos_token": {
+    "content": "<|endofturn|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "pad_token": {
     "content": "<|endoftext|>",
     "lstrip": false,

tokenizer_config.json

@@ -1,4 +1,5 @@
 {
+  "add_bos_token": false,
   "add_prefix_space": false,
   "added_tokens_decoder": {
     "100256": {
@@ -490,18 +491,17 @@
     "<KEY>",
     "<PASSWORD>"
   ],
+  "auto_map": {
+    "AutoProcessor": "processing_hyperclovax.HCXProcessor"
+  },
   "bos_token": "<|endoftext|>",
-  "chat_template": [
-    {
-      "name": "default",
-      "template": "<|im_start|>tool_list\n<|im_end|>\n{% for message in messages %}\n{% set content = message['content'] %}\n{% set role = message['role'] %}\n{% if loop.first and role != 'system' %}\n<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}\n{% if message['content'] is string %}\n<|im_start|>{{ role }}\n{{ message['content'] }}<|im_end|>\n{% else %}\n{% if content['type'] == 'image' %}\n<|im_start|>{{ role }} (mime)\n{\"type\": \"image/jpeg\", \"filename\": \"{{ content['filename'] }}\"}<|im_end|>\n<|im_start|>{{ role }} (vector)\n<|dummy3|><|im_end|>\n<|im_start|>image/aux\n다음 중 ocr은 사진에서 검출된 글자이고, lens_keyword는 사진에서 추출된 keyword와 bbox 위치입니다. bbox는 0~1 사이로 정규화된 [x1, y1, x2, y2]의 형태입니다. 참고하여 답변하세요. {\"ocr\": \"{{ content['ocr'] or '' }}\", \"lens_keywords\": \"{{ content['lens_keywords'] or '' }}\", \"lens_local_keywords\": \"{{ content['lens_local_keywords'] or '' }}\"}<|im_end|>\n{% elif content['type'] == 'video' %}\n<|im_start|>{{ role }} (mime)\n{\"type\": \"video/mp4\", \"filename\": \"{{ content['filename'] }}\"}<|im_end|>\n<|im_start|>{{ role }} (vector)\n<|dummy3|><|im_end|>\n<|im_start|>image/aux\n{% if content.get('is_final_grid') %}\n다음 중 lens_keyword는 사진에서 추출된 keyword와 bbox 위치입니다. bbox는 0~1 사이로 정규화된 [x1, y1, x2, y2]의 형태입니다. video_time_stamp는 비디오에서 해당 구간의 시간 정보입니다. speech_to_text는 비디오 속에서의 대화, 음성, 소리, 대사, 그리고 말을 전부 글로 받아 적은 것 입니다. 참고하여 답변하세요. {\"video_time_stamp\": \"{{ content['video_time_stamp'] }}\", \"lens_keywords\": \"{{ content.get('lens_keywords', '') }}\", \"lens_local_keywords\": \"{{ content.get('lens_local_keywords', '') }}\", \"speech_to_text\": \"{{ content.get('speech_to_text', '') }}\"}\n{% else %}\n다음 중 video_time_stamp는 비디오에서 해당 구간의 시간 정보입니다. 참고하여 답변하세요. {\"video_time_stamp\": \"{{ content['video_time_stamp'] }}\"}\n{% endif %}<|im_end|>\n{% elif content['type'] == 'text' %}\n<|im_start|>{{ role }}\n{{ content['text'] }}<|im_end|>\n{% endif %}\n{% endif %}\n{% endfor %}\n{% if add_generation_prompt %}\n<|im_start|>assistant\n{% endif %}\n"
-    }
-  ],
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|endofturn|>",
+  "errors": "replace",
   "extra_special_tokens": {},
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<|endoftext|>",
+  "processor_class": "HCXProcessor",
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "<|endoftext|>"
 }