Add files using upload-large-folder tool

PaddleMIX/comfyui/ComfyUI_ppdiffusers/README.md

@@ -0,0 +1,50 @@
+# ComfyUI_ppdiffusers
+这是一个针对ComfyUI开发的 [ppdiffusers](https://github.com/PaddlePaddle/PaddleMIX/tree/develop/ppdiffusers)扩展，支持了常见的模型加载转换、文生图、图生图、图像局部修改等任务。
+## 安装
+```bash
+git clone https://github.com/PaddlePaddle/PaddleMIX --depth 1
+cd PaddleMIX/comfyui/
+mv ComfyUI_ppdiffusers/ /path/to/your_comfyui/custom_nodes/
+```
+
+## 使用
+### 在线体验
+aistudio: https://aistudio.baidu.com/community/app/106043
+
+### 本地运行
+所有已经支持的工作流都在./workflows目录下，可以直接加载工作流json文件使用。
+原生支持加载safetensors格式模型，在线读取转换为paddle模型后，在ComfyUI中使用ppdiffusers的pipeline运行。
+
+## 已支持Node
+Stable Diffusion 1.5系列：
+- SD1.5模型加载转换
+- SD1.5文生图
+- SD1.5图生图
+- SD1.5图像局部修改
+
+Stable Diffusion XL系列：
+- SDXL模型加载转换
+- SDXL文生图
+- SDXL图生图
+- SDXL图像局部修改
+
+## 效果展示
+### SDXL
+1. 文生图  
+<img width="600" alt="image" src="https://github.com/PaddlePaddle/PaddleMIX/assets/35400185/1d9d17cd-dd1f-4e05-9c98-c1fc4fca8185">
+
+2. 图生图  
+<img width="600" alt="image" src="https://github.com/PaddlePaddle/PaddleMIX/assets/35400185/04e9cc05-9ce4-4207-88c4-3d076aaebff4">
+
+3. 局部编辑  
+<img width="600" alt="image" src="https://github.com/PaddlePaddle/PaddleMIX/assets/35400185/36ba7261-1744-41a4-b1cb-c9e99f6931f2">
+
+### SD1.5
+1. 文生图  
+<img width="600" alt="image" src="https://github.com/PaddlePaddle/PaddleMIX/assets/35400185/a939274e-a23b-4ecf-956c-56fd8343707c">
+
+2. 图生图  
+<img width="600" alt="image" src="https://github.com/PaddlePaddle/PaddleMIX/assets/35400185/b8b144bb-0b52-4235-91d9-d7bdfb44a1d8">
+
+3. 图像局部重绘  
+<img width="600" alt="image" src="https://github.com/PaddlePaddle/PaddleMIX/assets/35400185/7a75899d-48ca-4479-9fb8-18d077fc3607">

PaddleMIX/comfyui/ComfyUI_ppdiffusers/__init__.py

@@ -0,0 +1,43 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+try:
+    import paddle
+except ImportError:
+    print("Paddle is not installed. Please install it to use this node.")
+    __all__ = []
+else:
+    from .basic_nodes import NODE_CLASS_MAPPINGS as NODE_CLASS_MAPPINGS_BASIC
+    from .basic_nodes import (
+        NODE_DISPLAY_NAME_MAPPINGS as NODE_DISPLAY_NAME_MAPPINGS_BASIC,
+    )
+    from .sd_pipe_nodes import NODE_CLASS_MAPPINGS as NODE_CLASS_MAPPINGS_SD
+    from .sd_pipe_nodes import (
+        NODE_DISPLAY_NAME_MAPPINGS as NODE_DISPLAY_NAME_MAPPINGS_SD,
+    )
+    from .sdxl_pipe_nodes import NODE_CLASS_MAPPINGS as NODE_CLASS_MAPPINGS_SDXL
+    from .sdxl_pipe_nodes import (
+        NODE_DISPLAY_NAME_MAPPINGS as NODE_DISPLAY_NAME_MAPPINGS_SDXL,
+    )
+
+    NODE_CLASS_MAPPINGS = {**NODE_CLASS_MAPPINGS_BASIC, **NODE_CLASS_MAPPINGS_SD, **NODE_CLASS_MAPPINGS_SDXL}
+    NODE_DISPLAY_NAME_MAPPINGS = {
+        **NODE_DISPLAY_NAME_MAPPINGS_BASIC,
+        **NODE_DISPLAY_NAME_MAPPINGS_SD,
+        **NODE_DISPLAY_NAME_MAPPINGS_SDXL,
+    }
+    __all__ = [
+        "NODE_CLASS_MAPPINGS",
+        "NODE_DISPLAY_NAME_MAPPINGS",
+    ]

PaddleMIX/comfyui/ComfyUI_ppdiffusers/basic_nodes.py

@@ -0,0 +1,128 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import json
+import os
+from io import BytesIO
+
+import folder_paths
+import requests
+from comfy.cli_args import args
+from PIL import Image
+from PIL.PngImagePlugin import PngInfo
+
+
+class PaddleSaveImage:
+    def __init__(self):
+        self.output_dir = folder_paths.get_output_directory()
+        self.type = "output"
+        self.prefix_append = ""
+        self.compress_level = 4
+        self.serving_web_host = os.environ.get("AISTUDIO_MS_SERVING_WEB_HOST")
+        self.serving_app_token = os.environ.get("AISTUDIO_MS_AIGC_APP_JWT")
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "images": ("IMAGE",), 
+                "filename_prefix": ("STRING", {"default": "ComfyUI"}),
+                "censor": ("BOOLEAN", {"default": True})
+            },
+            "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
+        }
+
+    RETURN_TYPES = ()
+    FUNCTION = "save_images"
+
+    OUTPUT_NODE = True
+
+    CATEGORY = "🚢 paddlemix/ppdiffusers/output"
+
+    def censor_image(self, image):
+        buffered = BytesIO()
+        image.save(buffered, format="PNG")
+        img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
+        url = (
+            f"http://{self.serving_web_host}/serving/web/aigc/censor/image?serving_app_token={self.serving_app_token}"
+        )
+        data = {"image": img_str}
+        response = requests.post(url, json=data).json()
+        print(response)
+        return response["result"]["pass"]
+
+    def save_images(self, images, censor=True, filename_prefix="ComfyUI", prompt=None, extra_pnginfo=None):
+        filename_prefix += self.prefix_append
+        full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(
+            filename_prefix, self.output_dir, images[0].shape[1], images[0].shape[0]
+        )
+        results = list()
+        for (batch_number, image) in enumerate(images):
+            img = Image.fromarray(image)
+            if censor:
+                pass_censor = self.censor_image(img)
+            else:
+                pass_censor = True
+            # breakpoint()
+            if pass_censor:
+                metadata = None
+                if not args.disable_metadata:
+                    metadata = PngInfo()
+                    if prompt is not None:
+                        metadata.add_text("prompt", json.dumps(prompt))
+                    if extra_pnginfo is not None:
+                        for x in extra_pnginfo:
+                            metadata.add_text(x, json.dumps(extra_pnginfo[x]))
+
+                filename_with_batch_num = filename.replace("%batch_num%", str(batch_number))
+                file = f"{filename_with_batch_num}_{counter:05}_.png"
+                img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=self.compress_level)
+                results.append({"filename": file, "subfolder": subfolder, "type": self.type})
+            else:
+                results.append({"filename": "forbidden.png", "subfolder": "", "type": "output"})
+            counter += 1
+
+        return_dict = {"ui": {"images": results}}
+        return return_dict
+
+
+class PromptInput:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "prompt": ("STRING", {"multiline": True, "dynamicPrompts": True}),
+            }
+        }
+
+    RETURN_TYPES = ("PROMPT",)
+    RETURN_NAMES = ("prompt",)
+    FUNCTION = "encode"
+    CATEGORY = "🚢 paddlemix/ppdiffusers/input"
+
+    def encode(self, prompt):
+        # TODO: add check for prompt
+        return (prompt,)
+
+
+NODE_CLASS_MAPPINGS = {
+    "PaddleSaveImage": PaddleSaveImage,
+    "PromptInput": PromptInput,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "PromptInput": "Paddle Prompt Input",
+    "PaddleSaveImage": "Paddle Save Image",
+}

PaddleMIX/comfyui/ComfyUI_ppdiffusers/requirements.txt

@@ -0,0 +1,2 @@
+ppdiffusers
+requests

PaddleMIX/comfyui/ComfyUI_ppdiffusers/sd_pipe_nodes.py

@@ -0,0 +1,348 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import folder_paths
+import numpy as np
+import paddle
+import torch  # for convert data
+from comfy.utils import ProgressBar
+
+from ppdiffusers import (
+    StableDiffusionImg2ImgPipeline,
+    StableDiffusionInpaintPipeline,
+    StableDiffusionPipeline,
+)
+
+from .utils.schedulers import get_scheduler
+
+
+class PaddleSDCheckpointLoader:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {"required": {"ckpt_name": (folder_paths.get_filename_list("checkpoints"),)}}
+
+    RETURN_TYPES = ("PIPELINE",)
+    RETURN_NAMES = ("sd_pipe",)
+    FUNCTION = "load_checkpoint"
+    CATEGORY = "🚢 paddlemix/ppdiffusers/input"
+
+    def load_checkpoint(self, ckpt_name):
+        ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name)
+        pipe = StableDiffusionPipeline.from_single_file(ckpt_path)
+        return (pipe,)
+
+
+class PaddleSDVaeDecoder:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {"required": {"latent": ("LATENT",), "sd_pipe": ("PIPELINE",)}}
+
+    RETURN_TYPES = ("IMAGE",)
+    RETURN_NAMES = ("image",)
+    FUNCTION = "decode"
+    CATEGORY = "🚢 paddlemix/ppdiffusers/output"
+
+    def decode(self, sd_pipe, latent):
+        vae = sd_pipe.vae
+        latent = 1 / vae.config.scaling_factor * latent
+        image = vae.decode(latent, return_dict=False)[0]
+        image = (image / 2 + 0.5).clip(0, 1)
+        image = image.cast(dtype=paddle.float32).transpose([0, 2, 3, 1]).cpu().numpy()
+        image = (image * 255).astype(np.uint8)
+
+        return (image,)
+
+
+class PaddleSDText2ImagePipe:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "sd_pipe": ("PIPELINE",),
+                "prompt": ("PROMPT",),
+                "negative_prompt": ("PROMPT",),
+                "steps": (
+                    "INT",
+                    {
+                        "default": 20,
+                        "min": 1,
+                        "max": 1000,
+                    },
+                ),
+                "width": ("INT", {"default": 512, "min": 1, "max": 8192}),
+                "height": ("INT", {"default": 768, "min": 1, "max": 8192}),
+                "number": ("INT", {"default": 1, "min": 1, "max": 100}),
+                "seed": ("INT", {"default": 0, "min": 0, "max": 99999999999999999999999}),
+                "cfg": (
+                    "FLOAT",
+                    {
+                        "default": 7.5,
+                        "min": 0.0,
+                        "max": 1000.0,
+                        "step": 0.01,
+                    },
+                ),
+                "scheduler_type": (
+                    [
+                        "euler",
+                        "euler-ancestral",
+                        "pndm",
+                        "lms",
+                        "heun",
+                        "dpm-multi",
+                        "dpm-single",
+                        "kdpm2-ancestral",
+                        "kdpm2",
+                        "unipc-multi",
+                        "ddim",
+                        "ddpm",
+                        "deis-multi",
+                    ],
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("LATENT",)
+    RETURN_NAMES = ("latent",)
+    FUNCTION = "sample"
+    CATEGORY = "🚢 paddlemix/ppdiffusers/pipelines"
+
+    def sample(self, sd_pipe, prompt, negative_prompt, steps, width, height, number, seed, cfg, scheduler_type):
+
+        pipe = StableDiffusionPipeline(**sd_pipe.components)
+        pipe.scheduler = get_scheduler(scheduler_type)
+        paddle.seed(seed)
+
+        progress_bar = ProgressBar(steps)
+        latent = pipe(
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            width=width,
+            height=height,
+            num_images_per_prompt=number,
+            num_inference_steps=steps,
+            guidance_scale=cfg,
+            output_type="latent",
+            callback=lambda step, timestep, latents: progress_bar.update_absolute(
+                value=step, total=steps, preview=None
+            ),
+            callback_steps=1,
+        ).images
+
+        return (latent,)
+
+
+class PaddleSDImage2ImagePipe:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "sd_pipe": ("PIPELINE",),
+                "image": ("IMAGE",),
+                "denoise": (
+                    "FLOAT",
+                    {
+                        "default": 0.7,
+                        "min": 0.0,
+                        "max": 1.0,
+                        "step": 0.01,
+                    },
+                ),
+                "prompt": ("PROMPT",),
+                "negative_prompt": ("PROMPT",),
+                "steps": (
+                    "INT",
+                    {
+                        "default": 20,
+                        "min": 1,
+                        "max": 1000,
+                    },
+                ),
+                "number": ("INT", {"default": 1, "min": 1, "max": 100}),
+                "seed": ("INT", {"default": 0, "min": 0, "max": 99999999999999999999999}),
+                "cfg": (
+                    "FLOAT",
+                    {
+                        "default": 7.5,
+                        "min": 0.0,
+                        "max": 1000.0,
+                        "step": 0.01,
+                    },
+                ),
+                "scheduler_type": (
+                    [
+                        "euler",
+                        "euler-ancestral",
+                        "pndm",
+                        "lms",
+                        "heun",
+                        "dpm-multi",
+                        "dpm-single",
+                        "kdpm2-ancestral",
+                        "kdpm2",
+                        "unipc-multi",
+                        "ddim",
+                        "ddpm",
+                        "deis-multi",
+                    ],
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("LATENT",)
+    RETURN_NAMES = ("latent",)
+    FUNCTION = "sample"
+    CATEGORY = "🚢 paddlemix/ppdiffusers/pipelines"
+
+    def sample(self, sd_pipe, image, denoise, prompt, negative_prompt, steps, number, seed, cfg, scheduler_type):
+        # torch -> numpy
+        if isinstance(image, torch.Tensor):
+            image = image.cpu().numpy()
+
+        pipe = StableDiffusionImg2ImgPipeline(**sd_pipe.components)
+        pipe.scheduler = get_scheduler(scheduler_type)
+        paddle.seed(seed)
+
+        progress_bar = ProgressBar(steps)
+        latent = pipe(
+            image=image,
+            strength=denoise,
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=number,
+            num_inference_steps=steps,
+            guidance_scale=cfg,
+            output_type="latent",
+            callback=lambda step, timestep, latents: progress_bar.update_absolute(
+                value=step, total=steps, preview=None
+            ),
+            callback_steps=1,
+        ).images
+
+        return (latent,)
+
+
+class PaddleSDInpaintPipe:
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "sd_pipe": ("PIPELINE",),
+                "image": ("IMAGE",),
+                "mask": ("MASK",),
+                "denoise": (
+                    "FLOAT",
+                    {
+                        "default": 0.7,
+                        "min": 0.0,
+                        "max": 1.0,
+                        "step": 0.01,
+                    },
+                ),
+                "prompt": ("PROMPT",),
+                "negative_prompt": ("PROMPT",),
+                "steps": (
+                    "INT",
+                    {
+                        "default": 20,
+                        "min": 1,
+                        "max": 1000,
+                    },
+                ),
+                "number": ("INT", {"default": 1, "min": 1, "max": 100}),
+                "seed": ("INT", {"default": 0, "min": 0, "max": 99999999999999999999999}),
+                "cfg": (
+                    "FLOAT",
+                    {
+                        "default": 7.5,
+                        "min": 0.0,
+                        "max": 1000.0,
+                        "step": 0.01,
+                    },
+                ),
+                "scheduler_type": (
+                    [
+                        "euler",
+                        "euler-ancestral",
+                        "pndm",
+                        "lms",
+                        "heun",
+                        "dpm-multi",
+                        "dpm-single",
+                        "kdpm2-ancestral",
+                        "kdpm2",
+                        "unipc-multi",
+                        "ddim",
+                        "ddpm",
+                        "deis-multi",
+                    ],
+                ),
+            }
+        }
+
+    RETURN_TYPES = ("LATENT",)
+    RETURN_NAMES = ("latent",)
+    FUNCTION = "sample"
+    CATEGORY = "🚢 paddlemix/ppdiffusers/pipelines"
+
+    def sample(self, sd_pipe, image, mask, denoise, prompt, negative_prompt, steps, number, seed, cfg, scheduler_type):
+        # torch -> numpy
+        if isinstance(image, torch.Tensor):
+            image = image.cpu().numpy()
+        if isinstance(mask, torch.Tensor):
+            mask = mask.cpu().numpy()
+
+        height, width = image.shape[1] // 8 * 8, image.shape[2] // 8 * 8
+
+        pipe = StableDiffusionInpaintPipeline(**sd_pipe.components)
+        pipe.scheduler = get_scheduler(scheduler_type)
+        paddle.seed(seed)
+
+        progress_bar = ProgressBar(steps)
+        latent = pipe(
+            image=image,
+            mask_image=mask,
+            strength=denoise,
+            prompt=prompt,
+            negative_prompt=negative_prompt,
+            width=width,
+            height=height,
+            num_images_per_prompt=number,
+            num_inference_steps=steps,
+            guidance_scale=cfg,
+            output_type="latent",
+            callback=lambda step, timestep, latents: progress_bar.update_absolute(
+                value=step, total=steps, preview=None
+            ),
+            callback_steps=1,
+        ).images
+
+        return (latent,)
+
+
+NODE_CLASS_MAPPINGS = {
+    "PaddleSDCheckpointLoader": PaddleSDCheckpointLoader,
+    "PaddleSDVaeDecoder": PaddleSDVaeDecoder,
+    "PaddleSDText2ImagePipe": PaddleSDText2ImagePipe,
+    "PaddleSDImage2ImagePipe": PaddleSDImage2ImagePipe,
+    "PaddleSDInpaintPipe": PaddleSDInpaintPipe,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "PaddleSDCheckpointLoader": "Paddle SD1.5 Checkpoint Loader",
+    "PaddleSDVaeDecoder": "Paddle SD1.5 VAE Decoder",
+    "PaddleSDText2ImagePipe": "Paddle SD1.5 Text2Image Pipe",
+    "PaddleSDImage2ImagePipe": "Paddle SD1.5 Image2Image Pipe",
+    "PaddleSDInpaintPipe": "Paddle SD1.5 Inpaint Pipe",
+}

PaddleMIX/comfyui/ComfyUI_ppdiffusers/utils/schedulers.py

@@ -0,0 +1,75 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ppdiffusers.schedulers import (
+    DDIMScheduler,
+    DDPMScheduler,
+    DEISMultistepScheduler,
+    DPMSolverMultistepScheduler,
+    DPMSolverSinglestepScheduler,
+    EulerAncestralDiscreteScheduler,
+    EulerDiscreteScheduler,
+    HeunDiscreteScheduler,
+    KDPM2AncestralDiscreteScheduler,
+    KDPM2DiscreteScheduler,
+    LMSDiscreteScheduler,
+    PNDMScheduler,
+    UniPCMultistepScheduler,
+)
+
+
+def get_scheduler(scheduler_type):
+    scheduler_type = scheduler_type.lower()
+    scheduler = DDIMScheduler(
+        beta_start=0.00085,
+        beta_end=0.012,
+        beta_schedule="scaled_linear",
+        clip_sample=False,
+        set_alpha_to_one=False,
+    )
+    original_scheduler_config = scheduler.config
+    if scheduler_type == "pndm":
+        scheduler = PNDMScheduler.from_config(original_scheduler_config, skip_prk_steps=True)
+    elif scheduler_type == "lms":
+        scheduler = LMSDiscreteScheduler.from_config(original_scheduler_config)
+    elif scheduler_type == "heun":
+        scheduler = HeunDiscreteScheduler.from_config(original_scheduler_config)
+    elif scheduler_type == "euler":
+        scheduler = EulerDiscreteScheduler.from_config(original_scheduler_config)
+    elif scheduler_type == "euler-ancestral":
+        scheduler = EulerAncestralDiscreteScheduler.from_config(original_scheduler_config)
+    elif scheduler_type == "dpm-multi":
+        scheduler = DPMSolverMultistepScheduler.from_config(original_scheduler_config)
+    elif scheduler_type == "dpm-single":
+        scheduler = DPMSolverSinglestepScheduler.from_config(original_scheduler_config)
+    elif scheduler_type == "kdpm2-ancestral":
+        scheduler = KDPM2AncestralDiscreteScheduler.from_config(original_scheduler_config)
+    elif scheduler_type == "kdpm2":
+        scheduler = KDPM2DiscreteScheduler.from_config(original_scheduler_config)
+    elif scheduler_type == "unipc-multi":
+        scheduler = UniPCMultistepScheduler.from_config(original_scheduler_config)
+    elif scheduler_type == "ddim":
+        scheduler = DDIMScheduler.from_config(
+            original_scheduler_config,
+            steps_offset=1,
+            clip_sample=False,
+            set_alpha_to_one=False,
+        )
+    elif scheduler_type == "ddpm":
+        scheduler = DDPMScheduler.from_config(original_scheduler_config)
+    elif scheduler_type == "deis-multi":
+        scheduler = DEISMultistepScheduler.from_config(original_scheduler_config)
+    else:
+        raise ValueError(f"Scheduler of type {scheduler_type} doesn't exist!")
+    return scheduler

vlmeval/VLMEvalKit_old/docs/en/Contributors.md

@@ -0,0 +1,21 @@
+# Contributors
+
+## Contributors w. 3+ Major Contributions
+
+> In this section, we list all the contributors who have made significant contributions (3+) to the development of VLMEvalKit.
+
+New Qualified Contributors (2024.09):
+
+1. [amitbcp](https://github.com/amitbcp): The contributor helped support MUIRBench, Phi-3.5, Idefics3, VILA, and xGen-MM
+2. [czczup](https://github.com/czczup): The contributor helped support the InternVL Series (V1.5, Mini-InternVL, V2, etc.)
+3. [DseidLi](https://github.com/DseidLi): The contributor helped support LLaVA-OneVision, GQA, and developed the readthedocs site for VLMEvalKit
+4. [mayubo2333](https://github.com/mayubo2333): The contributor helped support MMLongBench, SlideVQA, and DUDE
+5. [sun-hailong](https://github.com/sun-hailong): The contributor helped support A-OKVQA, Parrot, MMMB, and MTL-MMBench
+6. [PhoenixZ810](https://github.com/PhoenixZ810): The contributor helped support Video-ChatGPT, Chat-UniVI, and Llama-VID
+7. [Cuiunbo](https://github.com/Cuiunbo): The contributor helped support OmniLMM-12B, MiniCPM-V Series (V1, V2, V2.5)
+
+## Full Contributor List
+
+> In this section, we list all the contributors as well as their corresponding contributions to the development of VLMEvalKit.
+
+TBD.

vlmeval/VLMEvalKit_old/docs/en/_static/css/readthedocs.css

@@ -0,0 +1,63 @@
+.header-logo {
+  background-image: url("../image/logo.svg");
+  background-size: 275px 80px;
+  height: 80px;
+  width: 275px;
+}
+
+
+@media screen and (min-width: 1100px) {
+  .header-logo {
+    top: -25px;
+  }
+}
+
+pre {
+    white-space: pre;
+}
+
+@media screen and (min-width: 2000px) {
+  .pytorch-content-left {
+    width: 1200px;
+    margin-left: 30px;
+  }
+  article.pytorch-article {
+    max-width: 1200px;
+  }
+  .pytorch-breadcrumbs-wrapper {
+    width: 1200px;
+  }
+  .pytorch-right-menu.scrolling-fixed {
+    position: fixed;
+    top: 45px;
+    left: 1580px;
+  }
+}
+
+
+article.pytorch-article section code {
+  padding: .2em .4em;
+  background-color: #f3f4f7;
+  border-radius: 5px;
+}
+
+/* Disable the change in tables */
+article.pytorch-article section table code {
+  padding: unset;
+  background-color: unset;
+  border-radius: unset;
+}
+
+table.autosummary td {
+  width: 50%
+}
+
+img.align-center {
+  display: block;
+  margin-left: auto;
+  margin-right: auto;
+}
+
+article.pytorch-article p.rubric {
+  font-weight: bold;
+}

vlmeval/VLMEvalKit_old/docs/en/_static/js/custom.js

@@ -0,0 +1,10 @@
+var collapsedSections = [];
+
+$(document).ready(function () {
+  $('.model-summary').DataTable({
+    "stateSave": false,
+    "lengthChange": false,
+    "pageLength": 20,
+    "order": []
+  });
+});

vlmeval/VLMEvalKit_old/docs/en/_templates/404.html

@@ -0,0 +1,18 @@
+{% extends "layout.html" %}
+
+{% block body %}
+
+<h1>Page Not Found</h1>
+<p>
+  The page you are looking for cannot be found.
+</p>
+<p>
+  If you just switched documentation versions, it is likely that the page you were on is moved. You can look for it in
+  the content table left, or go to <a href="{{ pathto(root_doc) }}">the homepage</a>.
+</p>
+<!-- <p>
+  If you cannot find documentation you want, please <a
+    href="">open an issue</a> to tell us!
+</p> -->
+
+{% endblock %}

vlmeval/VLMEvalKit_old/docs/en/_templates/autosummary/class.rst

@@ -0,0 +1,13 @@
+.. role:: hidden
+    :class: hidden-section
+.. currentmodule:: {{ module }}
+
+
+{{ name | underline}}
+
+.. autoclass:: {{ name }}
+    :members:
+
+..
+  autogenerated from _templates/autosummary/class.rst
+  note it does not have :inherited-members:

vlmeval/VLMEvalKit_old/docs/en/_templates/callable.rst

@@ -0,0 +1,14 @@
+.. role:: hidden
+    :class: hidden-section
+.. currentmodule:: {{ module }}
+
+
+{{ name | underline}}
+
+.. autoclass:: {{ name }}
+    :members:
+    :special-members: __call__
+
+..
+  autogenerated from _templates/callable.rst
+  note it does not have :inherited-members:

vlmeval/VLMEvalKit_old/docs/en/docutils.conf

@@ -0,0 +1,2 @@
+[html writers]
+table_style: colwidths-auto

vlmeval/VLMEvalKit_old/docs/ja/README_ja.md

@@ -0,0 +1,177 @@
+<div align="center">
+
+![LOGO](http://opencompass.openxlab.space/utils/MMLB.jpg)
+
+<b>VLMEvalKit: 大規模視覚言語モデルの評価ツールキット</b>
+
+[![][github-contributors-shield]][github-contributors-link] • [![][github-forks-shield]][github-forks-link] • [![][github-stars-shield]][github-stars-link] • [![][github-issues-shield]][github-issues-link] • [![][github-license-shield]][github-license-link]
+
+[English](/README.md) | [简体中文](/docs/zh-CN/README_zh-CN.md) | 日本語
+
+<a href="https://rank.opencompass.org.cn/leaderboard-multimodal">🏆 OpenCompass Learderboard </a> •
+<a href="#-datasets-models-and-evaluation-results">📊Datasets & Models </a> •
+<a href="#%EF%B8%8F-quickstart">🏗️Quickstart </a> •
+<a href="#%EF%B8%8F-development-guide">🛠️Development </a> •
+<a href="#-the-goal-of-vlmevalkit">🎯Goal </a> •
+<a href="#%EF%B8%8F-citation">🖊️Citation </a>
+
+<a href="https://huggingface.co/spaces/opencompass/open_vlm_leaderboard">🤗 HF Leaderboard</a> •
+<a href="https://huggingface.co/datasets/VLMEval/OpenVLMRecords">🤗 Evaluation Records</a> •
+<a href="https://discord.gg/evDT4GZmxN">🔊 Discord Channel</a> •
+<a href="https://www.arxiv.org/abs/2407.11691">📝 Technical Report</a>
+</div>
+
+**VLMEvalKit**（pythonパッケージ名は**vlmeval**）は、**大規模視覚言語モデル（LVLMs）**の**オープンソース評価ツールキット**です。このツールキットは、複数のリポジトリでのデータ準備という重労働なしに、さまざまなベンチマークでLVLMsの**ワンコマンド評価**を可能にします。VLMEvalKitでは、すべてのLVLMsに対して**生成ベースの評価**を採用し、**正確なマッチング**と**LLMベースの回答抽出**の両方で得られた評価結果を提供します。
+
+PS: 日本語の README には最新のアップデートがすべて含まれていない場合があります。英語版をご確認ください。
+
+## 📊 データセット、モデル、および評価結果
+
+**公式のマルチモーダルリーダーボードでのパフォーマンス数値は、ここからダウンロードできます！**
+
+[**OpenVLM Leaderboard**](https://huggingface.co/spaces/opencompass/open_vlm_leaderboard): [すべての詳細な結果をダウンロード](http://opencompass.openxlab.space/assets/OpenVLM.json)。
+
+**Supported Image Understanding Dataset**
+
+- デフォルトでは、すべての評価結果は[**OpenVLM Leaderboard**](https://huggingface.co/spaces/opencompass/open_vlm_leaderboard)に表示されます。
+
+| データセット                                                      | データセット名 (run.py用)                             | タスク | データセット | データセット名 (run.py用) | タスク |
+| ------------------------------------------------------------ | ------------------------------------------------------ | --------- | --------- | --------- | --------- |
+| [**MMBench シリーズ**](https://github.com/open-compass/mmbench/): <br>MMBench, MMBench-CN, CCBench | MMBench\_DEV\_[EN/CN] <br>MMBench\_TEST\_[EN/CN]<br>MMBench\_DEV\_[EN/CN]\_V11<br>MMBench\_TEST\_[EN/CN]\_V11<br>CCBench | 多肢選択問題 (MCQ) | [**MMStar**](https://github.com/MMStar-Benchmark/MMStar) | MMStar | MCQ |
+| [**MME**](https://github.com/BradyFU/Awesome-Multimodal-Large-Language-Models/tree/Evaluation) | MME | はい/いいえ (Y/N)                                         | [**SEEDBench シリーズ**](https://github.com/AILab-CVC/SEED-Bench) | SEEDBench_IMG <br>SEEDBench2 <br>SEEDBench2_Plus | MCQ                                                |
+| [**MM-Vet**](https://github.com/yuweihao/MM-Vet)             | MMVet  | VQA                                              | [**MMMU**](https://mmmu-benchmark.github.io)  | MMMU_[DEV_VAL/TEST]                      | MCQ                                |
+| [**MathVista**](https://mathvista.github.io)                 | MathVista_MINI | VQA                                         | [**ScienceQA_IMG**](https://scienceqa.github.io) | ScienceQA_[VAL/TEST]                     | MCQ                        |
+| [**COCO Caption**](https://cocodataset.org)                  | COCO_VAL | キャプション                                              | [**HallusionBench**](https://github.com/tianyi-lab/HallusionBench) | HallusionBench                                | Y/N                             |
+| [**OCRVQA**](https://ocr-vqa.github.io)*                     | OCRVQA_[TESTCORE/TEST] | VQA                                 | [**TextVQA**](https://textvqa.org)* | TextVQA_VAL                      | VQA                              |
+| [**ChartQA**](https://github.com/vis-nlp/ChartQA)*           | ChartQA_TEST | VQA                                          | [**AI2D**](https://allenai.org/data/diagrams) | AI2D_[TEST/TEST_NO_MASK]                                 | MCQ                         |
+| [**LLaVABench**](https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild) | LLaVABench | VQA                                            | [**DocVQA**](https://www.docvqa.org)+       | DocVQA_[VAL/TEST]                           | VQA                                         |
+| [**InfoVQA**](https://www.docvqa.org/datasets/infographicvqa)+ | InfoVQA_[VAL/TEST] | VQA | [**OCRBench**](https://github.com/Yuliang-Liu/MultimodalOCR) | OCRBench | VQA |
+| [**RealWorldQA**](https://x.ai/blog/grok-1.5v)            | RealWorldQA | MCQ                                          | [**POPE**](https://github.com/AoiDragon/POPE) | POPE                                           | Y/N                                            |
+| [**Core-MM**](https://github.com/core-mm/core-mm)-          | CORE_MM | VQA                                               | [**MMT-Bench**](https://mmt-bench.github.io)                 | MMT-Bench_[VAL/VAL_MI/ALL/ALL_MI]                | MCQ  |
+| [**MLLMGuard**](https://github.com/Carol-gutianle/MLLMGuard) - | MLLMGuard_DS | VQA | [**AesBench**](https://github.com/yipoh/AesBench) | AesBench_[VAL/TEST] | MCQ |
+| [**VCR-wiki**](https://huggingface.co/vcr-org/) + | VCR\_[EN/ZH]\_[EASY/HARD]_[ALL/500/100] | VQA | [**MMLongBench-Doc**](https://mayubo2333.github.io/MMLongBench-Doc/)+ | MMLongBench_DOC | VQA |
+| [**BLINK**](https://zeyofu.github.io/blink/) + | BLINK | MCQ | [**MathVision**](https://mathvision-cuhk.github.io)+ | MathVision<br>MathVision_MINI | VQA |
+| [**MT-VQA**](https://github.com/bytedance/MTVQA)+ | MTVQA_TEST | VQA | [**MMDU**](https://liuziyu77.github.io/MMDU/)+ | MMDU | VQA (multi-turn) |
+| [**Q-Bench1**](https://github.com/Q-Future/Q-Bench)+ | Q-Bench1_[VAL/TEST] | MCQ | [**A-Bench**](https://github.com/Q-Future/A-Bench)+ | A-Bench_[VAL/TEST] | MCQ |
+| [**TaskMeAnything ImageQA Random**](https://huggingface.co/datasets/weikaih/TaskMeAnything-v1-imageqa-random)+ | TaskMeAnything_v1_imageqa_random | MCQ  | | | |
+
+**\*** ゼロショット設定で合理的な結果を出せないVLMの一部の評価結果のみを提供しています
+
+**\+** 評価結果はまだ利用できません
+
+**\-** VLMEvalKitでは推論のみがサポートされています
+
+VLMEvalKitは、キーを設定すると**判定LLM**を使用して出力から回答を抽出し、それ以外の場合は**正確なマッチング**モード（出力文字列で「はい」、「いいえ」、「A」、「B」、「C」...を検索）を使用します。**正確なマッチングは、はい/いいえのタスクと多肢選択問題にのみ適用できます。**
+
+**Supported Video Understanding Dataset**
+
+| Dataset                                              | Dataset Names (for run.py) | Task | Dataset                                       | Dataset Names (for run.py) | Task |
+| ---------------------------------------------------- | -------------------------- | ---- | --------------------------------------------- | -------------------------- | ---- |
+| [**MMBench-Video**](https://mmbench-video.github.io) | MMBench-Video              | VQA  | [**Video-MME**](https://video-mme.github.io/) | Video-MME                  | MCQ  |
+
+**Supported API Models**
+
+| [**GPT-4v (20231106, 20240409)**](https://platform.openai.com/docs/guides/vision) 🎞️🚅 | [**GPT-4o**](https://openai.com/index/hello-gpt-4o/) 🎞️🚅      | [**Gemini-1.0-Pro**](https://platform.openai.com/docs/guides/vision) 🎞️🚅 | [**Gemini-1.5-Pro**](https://platform.openai.com/docs/guides/vision) 🎞️🚅 | [**Step-1V**](https://www.stepfun.com/#step1v) 🎞️🚅 |
+| ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------- |
+| [**Reka-[Edge / Flash / Core]**](https://www.reka.ai)🚅       | [**Qwen-VL-[Plus / Max]**](https://huggingface.co/spaces/Qwen/Qwen-VL-Max) 🎞️🚅 | [**Claude-3v-[Haiku / Sonnet / Opus]**](https://www.anthropic.com/news/claude-3-family) 🎞️🚅 | [**GLM-4v**](https://open.bigmodel.cn/dev/howuse/glm4v) 🚅    | [**CongRong**](https://mllm.cloudwalk.com/web) 🎞️🚅 |
+| [**Claude3.5-Sonnet**](https://www.anthropic.com/news/claude-3-5-sonnet) 🎞️🚅 | [**GPT-4o-Mini**](https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/) 🎞️🚅 | [**Yi-Vision**](https://platform.lingyiwanwu.com)🎞️🚅          | [**Hunyuan-Vision**](https://cloud.tencent.com/document/product/1729)🎞️🚅 |   [**BlueLM-V**](https://developers.vivo.com/) 🎞️🚅  |
+
+**Supported PyTorch / HF Models**
+
+| [**IDEFICS-[9B/80B/v2-8B]-Instruct**](https://huggingface.co/HuggingFaceM4/idefics-9b-instruct)🎞️🚅 | [**InstructBLIP-[7B/13B]**](https://github.com/salesforce/LAVIS/blob/main/projects/instructblip/README.md) | [**LLaVA-[v1-7B/v1.5-7B/v1.5-13B]**](https://github.com/haotian-liu/LLaVA) | [**MiniGPT-4-[v1-7B/v1-13B/v2-7B]**](https://github.com/Vision-CAIR/MiniGPT-4) |
+| ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| [**mPLUG-Owl2**](https://github.com/X-PLUG/mPLUG-Owl/tree/main/mPLUG-Owl2)🎞️ | [**OpenFlamingo-v2**](https://github.com/mlfoundations/open_flamingo)🎞️ | [**PandaGPT-13B**](https://github.com/yxuansu/PandaGPT)      | [**Qwen-VL**](https://huggingface.co/Qwen/Qwen-VL)🎞️🚅, [**Qwen-VL-Chat**](https://huggingface.co/Qwen/Qwen-VL-Chat)🎞️**🚅** |
+| [**VisualGLM-6B**](https://huggingface.co/THUDM/visualglm-6b)🚅 | [**InternLM-XComposer-[1/2]**](https://huggingface.co/internlm/internlm-xcomposer-7b)🚅 | [**ShareGPT4V-[7B/13B]**](https://sharegpt4v.github.io)🚅     | [**TransCore-M**](https://github.com/PCIResearch/TransCore-M) |
+| [**LLaVA (XTuner)**](https://huggingface.co/xtuner/llava-internlm-7b)🚅 | [**CogVLM-[Chat/Llama3]**](https://huggingface.co/THUDM/cogvlm-chat-hf)🚅 | [**ShareCaptioner**](https://huggingface.co/spaces/Lin-Chen/Share-Captioner)🚅 | [**CogVLM-Grounding-Generalist**](https://huggingface.co/THUDM/cogvlm-grounding-generalist-hf)🚅 |
+| [**Monkey**](https://github.com/Yuliang-Liu/Monkey)🚅, [**Monkey-Chat**](https://github.com/Yuliang-Liu/Monkey)🚅 | [**EMU2-Chat**](https://github.com/baaivision/Emu)🚅🎞️         | [**Yi-VL-[6B/34B]**](https://huggingface.co/01-ai/Yi-VL-6B)  | [**MMAlaya**](https://huggingface.co/DataCanvas/MMAlaya)🚅    |
+| [**InternLM-XComposer-2.5**](https://github.com/InternLM/InternLM-XComposer)🚅🎞️ | [**MiniCPM-[V1/V2/V2.5/V2.6]**](https://github.com/OpenBMB/MiniCPM-V)🚅🎞️ | [**OmniLMM-12B**](https://huggingface.co/openbmb/OmniLMM-12B) | [**InternVL-Chat-[V1-1/V1-2/V1-5/V2]**](https://github.com/OpenGVLab/InternVL)🚅🎞️, <br>[**Mini-InternVL-Chat-[2B/4B]-V1-5**](https://github.com/OpenGVLab/InternVL)🚅🎞️ |
+| [**DeepSeek-VL**](https://github.com/deepseek-ai/DeepSeek-VL/tree/main)🎞️ | [**LLaVA-NeXT**](https://llava-vl.github.io/blog/2024-01-30-llava-next/)🚅🎞️ | [**Bunny-Llama3**](https://huggingface.co/BAAI/Bunny-v1_1-Llama-3-8B-V)🚅 | [**XVERSE-V-13B**](https://github.com/xverse-ai/XVERSE-V-13B/blob/main/vxverse/models/vxverse.py) |
+| [**PaliGemma-3B**](https://huggingface.co/google/paligemma-3b-pt-448) 🚅 | [**360VL-70B**](https://huggingface.co/qihoo360/360VL-70B) 🚅 | [**Phi-3-Vision**](https://huggingface.co/microsoft/Phi-3-vision-128k-instruct)🚅 | [**WeMM**](https://github.com/scenarios/WeMM)🚅               |
+| [**GLM-4v-9B**](https://huggingface.co/THUDM/glm-4v-9b) 🚅    | [**Cambrian-[8B/13B/34B]**](https://cambrian-mllm.github.io/) | [**LLaVA-Next-[Qwen-32B]**](https://huggingface.co/lmms-lab/llava-next-qwen-32b) 🎞️ | [**Chameleon-[7B/30B]**](https://huggingface.co/facebook/chameleon-7b)🚅🎞️ |
+| [**Video-LLaVA-7B-[HF]**](https://github.com/PKU-YuanGroup/Video-LLaVA) 🎬 | [**VILA1.5-[8B/13B/40B]**](https://github.com/NVlabs/VILA/)🎞️ | [**Ovis1.5-Llama3-8B**](https://github.com/AIDC-AI/Ovis) 🚅🎞  | [**Mantis-8B-[siglip-llama3/clip-llama3/Idefics2/Fuyu]**](https://huggingface.co/TIGER-Lab/Mantis-8B-Idefics2) 🎞️ |
+
+🎞️: 複数の画像を入力としてサポートします。
+
+🚅: 追加の設定/操作なしで使用できるモデルです。
+
+🎬: 入力としてビデオをサポート。
+
+**Transformersバージョンの推奨事項:**
+
+特定のtransformerバージョンで一部のVLMが実行できない可能性があることに注意してください。各VLMを評価するために、以下の設定を推奨します：
+
+- **`transformers==4.33.0`を使用してください**: `Qwenシリーズ`, `Monkeyシリーズ`, `InternLM-XComposerシリーズ`, `mPLUG-Owl2`, `OpenFlamingo v2`, `IDEFICSシリーズ`, `VisualGLM`, `MMAlaya`, `ShareCaptioner`, `MiniGPT-4シリーズ`, `InstructBLIPシリーズ`, `PandaGPT`, `VXVERSE`, `GLM-4v-9B`.
+- **`transformers==4.37.0`を使用してください**: `LLaVAシリーズ`, `ShareGPT4Vシリーズ`, `TransCore-M`, `LLaVA (XTuner)`, `CogVLMシリーズ`, `EMU2シリーズ`, `Yi-VLシリーズ`, `MiniCPM-[V1/V2]`, `OmniLMM-12B`, `DeepSeek-VLシリーズ`, `InternVLシリーズ`, `Cambrianシリーズ`, `VILA-VLシリーズ`.
+- **`transformers==4.40.0`を使用してください**: `IDEFICS2`, `Bunny-Llama3`, `MiniCPM-Llama3-V2.5`, `360VL-70B`, `Phi-3-Vision`, `WeMM`.
+- **`transformers==latest`を使用してください**: `LLaVA-Nextシリーズ`, `PaliGemma-3B`, `Chameleon-VLシリーズ`, `Video-LLaVA-7B-HF`, `Ovis1.5シリーズ`, `Mantisシリーズ`, `MiniCPM-V2.6`.
+
+```python
+# デモ
+from vlmeval.config import supported_VLM
+model = supported_VLM['idefics_9b_instruct']()
+# 単一画像のフォワード
+ret = model.generate(['assets/apple.jpg', 'この画像には何がありますか？'])
+print(ret)  # この画像には葉がついた赤いリンゴがあります。
+# 複数画像のフォワード
+ret = model.generate(['assets/apple.jpg', 'assets/apple.jpg', '提供された画像にはリンゴが何個ありますか？'])
+print(ret)  # 提供された画像にはリンゴが2個あります。
+```
+
+## 🏗️ クイックスタート
+
+クイックスタートガイドについては、[クイックスタート](/docs/en/Quickstart.md)を参照してください。
+
+## 🛠️ 開発ガイド
+
+カスタムベンチマーク、VLMsを開発するか、単に**VLMEvalKit**に他のコードを貢献する場合は、[開発ガイド](/docs/en/Development.md)を参照してください。
+
+コミュニティからの共有を奨励し、それに応じたクレジットを共有するために、次回のレポート更新では以下のことを実施します：
+
+- 全ての貢献に対して感謝の意を示します
+- 新しいモデル、評価セット、または主要な機能への3つ以上の主要な貢献を持つ貢献者は、テクニカルレポートの著者リストに加わることができます。適格な貢献者は、issueを作成するか、または[VLM評価キット ディスコードチャンネル](https://discord.com/invite/evDT4GZmxN)で kennyutc にDMを送ることができます。私たちはそれに応じてフォローアップします。
+
+## 🎯 VLMEvalKitの目標
+
+**このコードベースは以下を目的として設計されています：**
+
+1. 研究者や開発者が既存のLVLMsを評価し、評価結果を**簡単に再現できるようにする**ための**使いやすい**、**オープンソースの評価ツールキット**を提供します。
+2. VLMの開発者が自分のモデルを簡単に評価できるようにします。複数のサポートされているベンチマークでVLMを評価するには、単一の`generate_inner()`関数を**実装するだけで**、他のすべてのワークロード（データのダウンロード、データの前処理、予測の推論、メトリックの計算）はコードベースによって処理されます。
+
+**このコードベースは以下を目的として設計されていません：**
+
+1. すべての**第三者ベンチマーク**の元の論文で報告された正確な精度数値を再現すること。その理由は2つあります：
+   1. VLMEvalKitは、すべてのVLMに対して**生成ベースの評価**を使用します（オプションで**LLMベースの回答抽出**を使用）。一方、一部のベンチマークは異なるアプローチを使用する場合があります（SEEDBenchはPPLベースの評価を使用します）。これらのベンチマークについては、対応する結果で両方のスコアを比較します。開発者には、コードベースで他の評価パラダイムをサポートすることをお勧めします。
+   2. デフォルトでは、すべてのVLMに対して同じプロンプトテンプレートを使用してベンチマークを評価します。一方、**一部のVLMには特定のプロンプトテンプレートがある**場合があります（現時点ではコードベースでカバーされていない場合があります）。VLMの開発者には、現在カバーされていない場合でも、VLMEvalKitで独自のプロンプトテンプレートを実装することをお勧めします。これにより、再現性が向上します。
+
+## 🖊️ 引用
+
+この作業が役立つ場合は、このリポジトリに**スター🌟**を付けてください。サポートありがとうございます！
+
+[![Stargazers repo roster for @open-compass/VLMEvalKit](https://reporoster.com/stars/open-compass/VLMEvalKit)](https://github.com/open-compass/VLMEvalKit/stargazers)
+
+研究でVLMEvalKitを使用する場合、または公開されたオープンソースの評価結果を参照する場合は、以下のBibTeXエントリと、使用した特定のVLM/ベンチマークに対応するBibTexエントリを使用してください。
+
+```bib
+@misc{duan2024vlmevalkit,
+      title={VLMEvalKit: An Open-Source Toolkit for Evaluating Large Multi-Modality Models},
+      author={Haodong Duan and Junming Yang and Yuxuan Qiao and Xinyu Fang and Lin Chen and Yuan Liu and Xiaoyi Dong and Yuhang Zang and Pan Zhang and Jiaqi Wang and Dahua Lin and Kai Chen},
+      year={2024},
+      eprint={2407.11691},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV},
+      url={https://arxiv.org/abs/2407.11691},
+}
+```
+
+<p align="right"><a href="#top">🔝Top に戻る</a></p>
+
+[github-contributors-link]: https://github.com/open-compass/VLMEvalKit/graphs/contributors
+[github-contributors-shield]: https://img.shields.io/github/contributors/open-compass/VLMEvalKit?color=c4f042&labelColor=black&style=flat-square
+[github-forks-link]: https://github.com/open-compass/VLMEvalKit/network/members
+[github-forks-shield]: https://img.shields.io/github/forks/open-compass/VLMEvalKit?color=8ae8ff&labelColor=black&style=flat-square
+[github-issues-link]: https://github.com/open-compass/VLMEvalKit/issues
+[github-issues-shield]: https://img.shields.io/github/issues/open-compass/VLMEvalKit?color=ff80eb&labelColor=black&style=flat-square
+[github-license-link]: https://github.com/open-compass/VLMEvalKit/blob/main/LICENSE
+[github-license-shield]: https://img.shields.io/github/license/open-compass/VLMEvalKit?color=white&labelColor=black&style=flat-square
+[github-stars-link]: https://github.com/open-compass/VLMEvalKit/stargazers
+[github-stars-shield]: https://img.shields.io/github/stars/open-compass/VLMEvalKit?color=ffcb47&labelColor=black&style=flat-square

vlmeval/VLMEvalKit_old/docs/zh-CN/ConfigSystem.md

@@ -0,0 +1,59 @@
+
+# 配置系统
+
+默认情况下，VLMEvalKit通过在`run.py`脚本中使用`--model`和`--data`参数设置模型名称（在`/vlmeval/config.py`中定义）和数据集名称（在`vlmeval/dataset/__init__.py`中定义）来启动评估。这种方法在大多数情况下简单且高效，但当用户希望使用不同设置评估多个模型/数据集时，可能不够灵活。
+
+为了解决这个问题，VLMEvalKit提供了一个更灵活的配置系统。用户可以在json文件中指定模型和数据集设置，并通过`--config`参数将配置文件的路径传递给`run.py`脚本。以下是一个示例配置json：
+
+```json
+{
+    "model": {
+        "GPT4o_20240806_T00_HIGH": {
+            "class": "GPT4V",
+            "model": "gpt-4o-2024-08-06",
+            "temperature": 0,
+            "img_detail": "high"
+        },
+        "GPT4o_20240806_T10_Low": {
+            "class": "GPT4V",
+            "model": "gpt-4o-2024-08-06",
+            "temperature": 1.0,
+            "img_detail": "low"
+        }
+    },
+    "data": {
+        "MME-RealWorld-Lite": {
+            "class": "MMERealWorld",
+            "dataset": "MME-RealWorld-Lite"
+        },
+        "MMBench_DEV_EN_V11": {
+            "class": "ImageMCQDataset",
+            "dataset": "MMBench_DEV_EN_V11"
+        }
+    }
+}
+```
+
+配置json的解释：
+
+1. 现在我们支持两个字段：`model`和`data`，每个字段都是一个字典。字典的键是模型/数据集的名称（由用户设置），值是模型/数据集的设置。
+2. 对于`model`中的项目，值是一个包含以下键的字典：
+    - `class`：模型的类名，应该是`vlmeval/vlm/__init__.py`（开源模型）或`vlmeval/api/__init__.py`（API模型）中定义的类名。
+    - 其他kwargs：其他kwargs是模型特定的参数，请参考模型类的定义以获取详细用法。例如，`model`、`temperature`、`img_detail`是`GPT4V`类的参数。值得注意的是，大多数模型类都需要`model`参数。
+3. 对于字典`data`，我们建议用户使用官方数据集名称作为键（或键的一部分），因为我们经常根据数据集名称确定后处理/判断设置。对于`data`中的项目，值是一个包含以下键的字典：
+    - `class`：数据集的类名，应该是`vlmeval/dataset/__init__.py`中定义的类名。
+    - 其他kwargs：其他kwargs是数据集特定的参数，请参考数据集类的定义以获取详细用法。通常，大多数数据集类都需要`dataset`参数。
+
+将示例配置json保存为`config.json`，您可以通过以下命令启动评估：
+
+```bash
+python run.py --config config.json
+```
+
+这将在工作目录`$WORK_DIR`下生成以下输出文件（格式为`{$WORK_DIR}/{$MODEL_NAME}/{$MODEL_NAME}_{$DATASET_NAME}_*`）：
+
+- `$WORK_DIR/GPT4o_20240806_T00_HIGH/GPT4o_20240806_T00_HIGH_MME-RealWorld-Lite*`
+- `$WORK_DIR/GPT4o_20240806_T10_Low/GPT4o_20240806_T10_Low_MME-RealWorld-Lite*`
+- `$WORK_DIR/GPT4o_20240806_T00_HIGH/GPT4o_20240806_T00_HIGH_MMBench_DEV_EN_V11*`
+- `$WORK_DIR/GPT4o_20240806_T10_Low/GPT4o_20240806_T10_Low_MMBench_DEV_EN_V11*`
+-

vlmeval/VLMEvalKit_old/docs/zh-CN/Development.md

@@ -0,0 +1,140 @@
+# 🛠️ 如何在 VLMEvalKit 中实现一个新的 Benchmark 或多模态模型（VLM）
+
+## 实现一个新的 benchmark
+
+示例 PR: **添加 Math-Vision Benchmark** ([#292](https://github.com/open-compass/VLMEvalKit/pull/292/files))
+
+目前在 VLMEvalKit 中，benchmark 以数据集类的形式呈现，当你新增一个 benchmark 时，你可以选择复用现有的数据集类 (如单选题 benchmark 可复用 `ImageMCQDataset`)，或是实现新的数据集类。你的数据集类必须支持以下两种方法 (复用父类或自行实现):
+
+- `build_prompt(self, line)`: 方法输入 `line` 类型为 int (对应数据 index) 或 `pd.Series` (对应数据原始 record)。方法输出一条 `multi-modal message` 作为多模态模型输入，`multi-modal message` 是一个图文交错的列表，如以下格式 (一图一文): `[dict(type='image', value=IMAGE_PTH), dict(type='text', value=prompt)]`。
+- `evaluate(self, eval_file, **judge_kwargs)`: 方法输入 `eval_file` 为多模态模型的预测结果 (多以 `.xlsx` 格式存在)，如 benchmark evaluation 需要大语言模型 (一般为 GPT) 辅助，则 `judge_kwargs` 传入大语言模型的参数。方法输出 benchmark 的评测结果，以 `dict` 或 `pd.DataFrame` 的形式。
+
+以下，我们简述新增数据集的通常步骤：
+
+### 1. TSV 数据文件准备 (图文评测集)
+
+目前，我们将每一个 benchmark 数据集设置为一个单独的 TSV 文件。在推理过程中，数据文件将从数据集定义的 `DATASET_URL` 链接地址自动下载到 `$LMUData` 中（如果没有明确设置的话，默认路径是 `$HOME/LMUData`）。你可以将准备好的 TSV 文件上传到一个可下载的地址（如：huggingface），或发送给我们 <[email protected]>，我们将帮助上传数据集到服务器中。此外，你也可以在环境变量中自定义设置下载路径 `LMUData=/path/to/your/data`。
+
+TSV 文件中的内容组成为：
+
+| 数据集名称 \ 字段  | index | image | image_path | question | hint | multi-choice<br>options | answer | category | l2-category | split |
+| ---------------------- | ----- | ----- | ---------- | -------- | ---- | ----------------------- | ------ | -------- | ----------- | ----- |
+| MMBench_DEV_[CN/EN]    | ✅     | ✅     |            | ✅        | ✅    | ✅                       | ✅      | ✅        | ✅           | ✅     |
+| MMBench_TEST_[CN/EN]   | ✅     | ✅     |            | ✅        | ✅    | ✅                       |        | ✅        | ✅           | ✅     |
+| CCBench                | ✅     | ✅     |            | ✅        |      | ✅                       | ✅      | ✅        |             |       |
+| SEEDBench_IMG          | ✅     | ✅     |            | ✅        |      | ✅                       | ✅      | ✅        |             |       |
+| MME                    | ✅     | ✅     |            | ✅        |      |                         | ✅      | ✅        |             |       |
+| CORE_MM                | ✅     | ✅     | ✅          | ✅        |      |                         |        | ✅        |             |       |
+| MMVet                  | ✅     | ✅     |            | ✅        |      |                         | ✅      | ✅        |             |       |
+| MMMU_DEV_VAL           | ✅     | ✅     | ✅          | ✅        |      | ✅                       | ✅      | ✅        | ✅           | ✅     |
+| COCO_VAL               | ✅     | ✅     |            |          |      |                         | ✅      |          |             |       |
+| OCRVQA_[TEST/TESTCORE] | ✅     | ✅     |            | ✅        |      |                         | ✅      |          |             |       |
+| TextVQA_VAL            | ✅     | ✅     |            | ✅        |      |                         | ✅      |          |             |       |
+| VCR_[EN/ZH]\_[EASY/HARD]_[ALL/500/100]            | ✅     | ✅     |            | ✅        |      |                         | ✅      |          |             |       |
+
+<div align="center"><b>表 1. 支持的数据集的 TSV 字段。</b></div>
+
+**TSV 中必须字段的介绍：**
+
+- **index:** 一个整数，`tsv` 中每一行的唯一标识
+- **image:** 图片的 base64 编码，你可以使用 `vlmeval/smp/vlm.py` 中实现的API进行编码和解码：
+    - 编码：`encode_image_to_base64`（对于PIL Image）/ `encode_image_file_to_base64`（对于图片文件路径）
+    - 解码：`decode_base64_to_image`（对于PIL Image）/ `decode_base64_to_image_file`（对于图片文件路径）
+- **question:** 针对图像所提取出的问题，类型为字符串
+- **answer:** 问题的答案，类型为字符串，Test 集可缺失这一字段
+
+### 2. 自定义数据集的 prompt 构建
+
+`ImageBaseDataset` 定义了默认的 prompt 格式。如果需要针对数据集添加 prompt，或给模型输入 `Interleave` 的数据格式，可以通过 `build_prompt(line)` 函数实现���该函数输入为，每次给定 TSV 文件中的一行，包含 index, image, question 等内容作为 line。该函数将返回一个多模态消息 `msg` 的字典列表 `[dict(type='image', value=IMAGE_PTH), dict(type='text', value=prompt)]`，包括图片路径和将被输入到 VLMs 的文本 prompt。对于 interleave 类型输入，可以直接将图片路径的字典放置到 image token 位置。
+
+### 3. 自定义数据集的指标实现
+
+增加对 benchmark 的评测需要自定义一个该数据集的 class 对象，从而实现数据集的指标计算。图文多模态数据集均继承自 `vlmeval/dataset/image_base.py` 中的 `ImageBaseDataset` 对象。其中 `TYPE` 定义了数据集的类型；`DATASET_URL` 为数据集的下载地址；`DATASET_MD5` 为数据集文件的 md5 一致性编码检查。
+
+在 class 中**需要实现** `evaluate(eval_file, **judge_kwargs)` 类函数，对自定义的数据集结果进行指标计算和结果输出。函数输入 `eval_file` 为模型预测结果 `{model_name}_{dataset}.xlsx` 的路径。可以通过 `load(eval_file)` 文件将其读取为 panda.DataFrames 类型，其中包含 index, question, answer, category, prediction 等字段。`judge_kwargs` 参数将传递一个评测相关的字典，如：judge 模型的名称，api 请求线程数等。**函数的返回值**为评估完成的准确度等指标，其格式为由 list 组成的字典，并组织成 panda.DataFrames 类型。
+
+## 实现一个新的模型
+
+示例 PR: **支持 LLaVA-Next-Interleave** ([#294](https://github.com/open-compass/VLMEvalKit/pull/294))
+
+**1. 支持 `generate_inner` API (必须)**
+
+现有所有的模型都在 `vlmeval/vlm` 中实现。对于一个最基本的模型，你的模型类**应该实现方法** `generate_inner(msgs, dataset=None)`。这个函数将向 VLM 输入一个多模态数据，并返回 VLM 的预测（一个字符串）。可选参数 `dataset` 可以用作模型在不同推理策略之间切换的标志。
+
+其中多模态消息 `msgs` 是一个字典列表，每个字典有两个键：类型和值：
+- `type`：我们目前支持两种类型，选项是 ["image", "text"]。
+- `value`：当类型为 `text` 时，值是文本消息（一个字符串）；当类型为 `image` 时，值可以是图像文件的本地路径，或者是图像的URL。
+
+> 目前，一个多模态消息可能包含任意交错的图像和文本。如果你的模型不支持这一点，我们推荐的做法是取第一张图像和连接的文本消息作为模型的输入。你可以在模型的 class 中设置 `INTERLEAVE = False` 并调用 `self.message_to_promptimg(message, dataset=dataset)` 函数来获取你的 prompt 和第一张图片的地址。
+
+一些多模态消息的例子:
+
+```python
+IMAGE_PTH = 'assets/apple.jpg'
+IMAGE_URL = 'https://raw.githubusercontent.com/open-compass/VLMEvalKit/main/assets/apple.jpg'
+msg1 = [
+    dict(type='image', value=IMAGE_PTH),
+    dict(type='text', value='What is in this image?')
+]
+msg2 = [
+    dict(type='image', value=IMAGE_URL),
+    dict(type='image', value=IMAGE_URL),
+    dict(type='text', value='How many apples are there in these images?')
+]
+response = model.generate(msg1)
+```
+
+为了方便起见，我们还支持接受字符串列表作为输入。在这种情况下，我们将检查一个字符串是图像路径还是图像 URL，并自动将其转换为 `list[dict]` 格式：
+
+```python
+IMAGE_PTH = 'assets/apple.jpg'
+IMAGE_URL = 'https://raw.githubusercontent.com/open-compass/VLMEvalKit/main/assets/apple.jpg'
+msg1 = [IMAGE_PTH, 'What is in this image?']
+msg2 = [IMAGE_URL, IMAGE_URL,  'How many apples are there in these images?']
+response = model.generate(msg1)
+```
+
+**2. 支持自定义提示词构建 (可选)**
+
+此外，你的模型可以通过实现两个可选方法来支持自定义提示构建：`use_custom_prompt(dataset)` 和 `build_prompt(line, dataset=None)`。
+
+- `use_custom_prompt(dataset)` 将返回一个布尔值，指示模型是否应使用自定义提示构建策略。
+- 如果`use_custom_prompt(dataset)`返回 True，`build_prompt(line, dataset)` 应该为相应的数据集返回一个自定义构建的多模态消息，line 数据是一个包含数据样本所需信息的字典。如果`use_custom_prompt(dataset)` 返回False，则将使用默认的 prompt 构建策略。
+
+**3. 支持多轮对话 (可选)**
+
+你可以通过支持 `chat_inner(message, dataset)` API 为你的模型新增多轮对话功能并兼容多轮对话评测。这个 API 输出一个字符串型回复，`message` 包含一个聊天记录的列表，格式如下：
+
+```python
+# Assume msg1, msg2, msg3, ... are multi-modal messages following the previously described format
+# `chat_inner` take the following chat history list as input:
+message = [
+    dict(role='user', content=msg1),
+    dict(role='assistant', content=msg2),
+    dict(role='user', content=msg3),
+    dict(role='assistant', content=msg4),
+	......
+    dict(role='user', content=msgn),
+]
+# `message` should contain an odd number of chat utterances, the role of utterances should be interleaved "user" and "assistant", with the role of the last utterance to be "user".
+# The chat function will call `chat_inner`
+response = model.chat(message)
+```
+
+### 示例 PRs：
+
+- 不支持交错的图像和文本，且不使用自定义提示的VLM：[[模型] 支持 glm-4v-9b](https://github.com/open-compass/VLMEvalKit/pull/221)
+- 支持交错的图像和文本及自定义提示的VLM：[添加 MiniCPM-Llama3-V-2.5](https://github.com/open-compass/VLMEvalKit/pull/205)
+- VLM API：[特征添加 glmv](https://github.com/open-compass/VLMEvalKit/pull/201)
+
+## 为 VLMEvalKit 贡献代码
+
+如果你想为 **VLMEvalKit** 贡献代码，请在提交PR之前进行预提交检查。这有助于保持代码整洁。
+
+```bash
+# 在VLMEvalKit的目录下，安装预提交 hook:
+pip install pre-commit
+pre-commit install
+pre-commit run --all-files
+# 然后提交你的代码。
+```

vlmeval/VLMEvalKit_old/docs/zh-CN/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

vlmeval/VLMEvalKit_old/docs/zh-CN/README_zh-CN.md

@@ -0,0 +1,215 @@
+<div align="center">
+
+![LOGO](http://opencompass.openxlab.space/utils/MMLB.jpg)
+
+<b>VLMEvalKit: 一种多模态大模型评测工具 </b>
+
+[![][github-contributors-shield]][github-contributors-link] • [![][github-forks-shield]][github-forks-link] • [![][github-stars-shield]][github-stars-link] • [![][github-issues-shield]][github-issues-link] • [![][github-license-shield]][github-license-link]
+
+[English](/README.md) | 简体中文 | [日本語](/docs/ja/README_ja.md)
+
+<a href="https://rank.opencompass.org.cn/leaderboard-multimodal">🏆 OpenCompass 排行榜 </a> •
+<a href="#%EF%B8%8F-quickstart">🏗️ 快速开始 </a> •
+<a href="#-datasets-models-and-evaluation-results">📊 数据集和模型 </a> •
+<a href="#%EF%B8%8F-development-guide">🛠️ 开发指南 </a> •
+<a href="#-the-goal-of-vlmevalkit">🎯 我们的目标 </a> •
+<a href="#%EF%B8%8F-citation">🖊️ 引用 </a>
+
+<a href="https://huggingface.co/spaces/opencompass/open_vlm_leaderboard">🤗 HuggingFace 排行榜 (存档全部性能) </a> •
+<a href="https://huggingface.co/datasets/VLMEval/OpenVLMRecords">🤗 原始评测记录</a> •
+<a href="https://discord.gg/evDT4GZmxN">🔊 Discord</a> •
+<a href="https://www.arxiv.org/abs/2407.11691">📝 技术报告 </a>
+</div>
+
+**VLMEvalKit** (python 包名为 **vlmeval**) 是一款专为大型视觉语言模型 (Large Vision-Language Models， LVLMs) 评测而设计的开源工具包。该工具支持在各种基准测试上对大型视觉语言模型进行**一键评估**，无需进行繁重的数据准备工作，让评估过程更加简便。在 VLMEvalKit 中，我们对所有大型视觉语言模型生成的结果进行评测，并提供基于**精确匹配**与基于 **LLM 的答案提取**两种评测结果。
+
+## 🆕 更新
+
+- **[2024-11-21]** 集成了一个新的配置系统，以实现更灵活的评估设置。查看[文档](/docs/zh-CN/ConfigSystem.md)或运行`python run.py --help`了解更多详情 🔥🔥🔥
+- **[2024-11-21]** 支持 **[QSpatial](https://andrewliao11.github.io/spatial_prompt/)**，一个用于定量空间推理的多模态基准（例如，确定大小/距离），感谢 **[andrewliao11](https://github.com/andrewliao11)** 提供官方支持 🔥🔥🔥
+- **[2024-11-21]** 支持 **[MM-Math](https://github.com/kge-sun/mm-math)**，一个包含约6K初中多模态推理数学问题的新多模态数学基准。GPT-4o-20240806在该基准上达到了22.5%的准确率 🔥🔥🔥
+- **[2024-11-16]** 支持 **[OlympiadBench](https://github.com/OpenBMB/OlympiadBench)**，一个多模态基准，包含奥林匹克级别的数学和物理问题 🔥🔥🔥
+- **[2024-11-16]** 支持 **[WildVision](https://huggingface.co/datasets/WildVision/wildvision-bench)**，一个基于多模态竞技场数据的主观多模态基准 🔥🔥🔥
+- **[2024-11-13]** 支持 **[MIA-Bench](https://arxiv.org/abs/2407.01509)**，一个多模态指令跟随基准 🔥🔥🔥
+- **[2024-11-08]** 支持 **[Aria](https://arxiv.org/abs/2410.05993)**，一个多模态原生 MoE 模型，感谢 **[teowu](https://github.com/teowu)** 🔥🔥🔥
+- **[2024-11-04]** 支持 **[WorldMedQA-V](https://www.arxiv.org/abs/2410.12722)**，该基准包含 1000 多个医学 VQA 问题，涵盖巴西、以色列、日本、西班牙等四个国家的语言，以及它们的英文翻译 🔥🔥🔥
+- **[2024-11-01]** 支持 `AUTO_SPLIT` 标志 (https://github.com/open-compass/VLMEvalKit/pull/566)，用于在低配置 GPU 上进行评估。设置后，模型将自动拆分到多个 GPU（流水线并行）以减少 GPU 内存使用（目前仅支持部分 VLMs：Qwen2-VL、Llama-3.2、LLaVA-OneVision 等） 🔥🔥🔥
+- **[2024-10-30]** 支持评估 **[MLVU](https://github.com/JUNJIE99/MLVU)** 和 **[TempCompass](https://arxiv.org/abs/2403.00476v1)**。这两个基准将很快被纳入 **[OpenVLM 视频排行榜](https://huggingface.co/spaces/opencompass/openvlm_video_leaderboard)** 🔥🔥🔥
+
+## 🏗️ 快速开始 <a id="quickstart"></a>
+
+请参阅[**快速开始**](/docs/zh-CN/Quickstart.md)获取入门指南。
+
+## 📊 评测结果，支持的数据集和模型 <a id="data-model-results"></a>
+
+### 评测结果
+
+**[OpenVLM Leaderboard](https://huggingface.co/spaces/opencompass/open_vlm_leaderboard)**: **[下载全部细粒度测试结果](http://opencompass.openxlab.space/assets/OpenVLM.json)**.
+
+### 支持的图文多模态评测集
+
+- 默认情况下，我们在 [**OpenVLM Leaderboard**](https://huggingface.co/spaces/opencompass/open_vlm_leaderboard) 提供全部测试结果
+- 使用的缩写：`MCQ`: 单项选择题; `Y/N`: 正误判断题; `MTT`: 多轮对话评测; `MTI`: 多图输入评测
+-
+- | Dataset                                                      | Dataset Names (for run.py)                                   | Task      | Dataset                                                      | Dataset Names (for run.py)                                   | Task           |
+  | ------------------------------------------------------------ | ------------------------------------------------------------ | --------- | ------------------------------------------------------------ | ------------------------------------------------------------ | -------------- |
+  | [**MMBench Series**](https://github.com/open-compass/mmbench/): <br>MMBench, MMBench-CN, CCBench | MMBench\_DEV\_[EN/CN] <br>MMBench\_TEST\_[EN/CN]<br>MMBench\_DEV\_[EN/CN]\_V11<br>MMBench\_TEST\_[EN/CN]\_V11<br>CCBench | MCQ       | [**MMStar**](https://github.com/MMStar-Benchmark/MMStar)     | MMStar                                                       | MCQ            |
+  | [**MME**](https://github.com/BradyFU/Awesome-Multimodal-Large-Language-Models/tree/Evaluation) | MME                                                          | Y/N       | [**SEEDBench Series**](https://github.com/AILab-CVC/SEED-Bench) | SEEDBench_IMG <br>SEEDBench2 <br>SEEDBench2_Plus             | MCQ            |
+  | [**MM-Vet**](https://github.com/yuweihao/MM-Vet)             | MMVet                                                        | VQA       | [**MMMU**](https://mmmu-benchmark.github.io)                 | MMMU_[DEV_VAL/TEST]                                          | MCQ            |
+  | [**MathVista**](https://mathvista.github.io)                 | MathVista_MINI                                               | VQA       | [**ScienceQA_IMG**](https://scienceqa.github.io)             | ScienceQA_[VAL/TEST]                                         | MCQ            |
+  | [**COCO Caption**](https://cocodataset.org)                  | COCO_VAL                                                     | Caption   | [**HallusionBench**](https://github.com/tianyi-lab/HallusionBench) | HallusionBench                                               | Y/N            |
+  | [**OCRVQA**](https://ocr-vqa.github.io)*                     | OCRVQA_[TESTCORE/TEST]                                       | VQA       | [**TextVQA**](https://textvqa.org)*                          | TextVQA_VAL                                                  | VQA            |
+  | [**ChartQA**](https://github.com/vis-nlp/ChartQA)*           | ChartQA_TEST                                                 | VQA       | [**AI2D**](https://allenai.org/data/diagrams)                | AI2D_[TEST/TEST_NO_MASK]                                     | MCQ            |
+  | [**LLaVABench**](https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild) | LLaVABench                                                   | VQA       | [**DocVQA**](https://www.docvqa.org)+                        | DocVQA_[VAL/TEST]                                            | VQA            |
+  | [**InfoVQA**](https://www.docvqa.org/datasets/infographicvqa)+ | InfoVQA_[VAL/TEST]                                           | VQA       | [**OCRBench**](https://github.com/Yuliang-Liu/MultimodalOCR) | OCRBench                                                     | VQA            |
+  | [**RealWorldQA**](https://x.ai/blog/grok-1.5v)               | RealWorldQA                                                  | MCQ       | [**POPE**](https://github.com/AoiDragon/POPE)                | POPE                                                         | Y/N            |
+  | [**Core-MM**](https://github.com/core-mm/core-mm)-           | CORE_MM (MTI)                                                | VQA       | [**MMT-Bench**](https://mmt-bench.github.io)                 | MMT-Bench\_[VAL/ALL]<br>MMT-Bench\_[VAL/ALL]_MI              | MCQ (MTI)      |
+  | [**MLLMGuard**](https://github.com/Carol-gutianle/MLLMGuard) - | MLLMGuard_DS                                                 | VQA       | [**AesBench**](https://github.com/yipoh/AesBench)+           | AesBench_[VAL/TEST]                                          | MCQ            |
+  | [**VCR-wiki**](https://huggingface.co/vcr-org/) +            | VCR\_[EN/ZH]\_[EASY/HARD]_[ALL/500/100]                      | VQA       | [**MMLongBench-Doc**](https://mayubo2333.github.io/MMLongBench-Doc/)+ | MMLongBench_DOC                                              | VQA (MTI)      |
+  | [**BLINK**](https://zeyofu.github.io/blink/)                 | BLINK                                                        | MCQ (MTI) | [**MathVision**](https://mathvision-cuhk.github.io)+         | MathVision<br>MathVision_MINI                                | VQA            |
+  | [**MT-VQA**](https://github.com/bytedance/MTVQA)             | MTVQA_TEST                                                   | VQA       | [**MMDU**](https://liuziyu77.github.io/MMDU/)+               | MMDU                                                         | VQA (MTT, MTI) |
+  | [**Q-Bench1**](https://github.com/Q-Future/Q-Bench)          | Q-Bench1_[VAL/TEST]                                          | MCQ       | [**A-Bench**](https://github.com/Q-Future/A-Bench)           | A-Bench_[VAL/TEST]                                           | MCQ            |
+  | [**DUDE**](https://arxiv.org/abs/2305.08455)+                | DUDE                                                         | VQA (MTI) | [**SlideVQA**](https://arxiv.org/abs/2301.04883)+            | SLIDEVQA<br>SLIDEVQA_MINI                                    | VQA (MTI)      |
+  | [**TaskMeAnything ImageQA Random**](https://huggingface.co/datasets/weikaih/TaskMeAnything-v1-imageqa-random)+ | TaskMeAnything_v1_imageqa_random                             | MCQ       | [**MMMB and Multilingual MMBench**](https://sun-hailong.github.io/projects/Parrot/)+ | MMMB\_[ar/cn/en/pt/ru/tr]<br>MMBench_dev\_[ar/cn/en/pt/ru/tr]<br>MMMB<br>MTL_MMBench_DEV<br>PS: MMMB & MTL_MMBench_DEV <br>are **all-in-one** names for 6 langs | MCQ            |
+  | [**A-OKVQA**](https://arxiv.org/abs/2206.01718)+             | A-OKVQA                                                      | MCQ       | [**MuirBench**](https://muirbench.github.io)+                | MUIRBench                                                    | MCQ            |
+  | [**GMAI-MMBench**](https://huggingface.co/papers/2408.03361)+ | GMAI-MMBench_VAL                                             | MCQ       | [**TableVQABench**](https://arxiv.org/abs/2404.19205)+       | TableVQABench                                                | VQA            |
+  | [**MME-RealWorld**](https://arxiv.org/abs/2408.13257)+       | MME-RealWorld[-CN]                                           | MCQ       | [**HRBench**](https://arxiv.org/abs/2408.15556)+             | HRBench[4K/8K]                                               | MCQ            |
+  | [**MathVerse**](https://mathverse-cuhk.github.io/)+          | MathVerse_MINI<br/>MathVerse_MINI_Vision_Only <br/>MathVerse_MINI_Vision_Dominant<br/>MathVerse_MINI_Vision_Intensive<br/>MathVerse_MINI_Text_Lite<br/>MathVerse_MINI_Text_Dominant | VQA       | [**AMBER**](https://github.com/junyangwang0410/AMBER)+       | AMBER                                                        | Y/N            |
+  | [**CRPE**](https://huggingface.co/datasets/OpenGVLab/CRPE)+  | CRPE_[EXIST/RELATION]                                        | VQA       | **[MMSearch](https://mmsearch.github.io/)**$$^1$$            | -                                                            | **-**          |
+  | **[R-Bench](https://arxiv.org/abs/2410.05474)**+             | R-Bench-[Dis/Ref]                                            | MCQ       | **[WorldMedQA-V](https://www.arxiv.org/abs/2410.12722)**+    | WorldMedQA-V                                                 | MCQ            |
+  | **[GQA](https://cs.stanford.edu/people/dorarad/gqa/about.html)**+ | GQA_TestDev_Balanced | VQA | **[MIA-Bench](https://arxiv.org/abs/2407.01509)**+ | MIA-Bench | VQA |
+  | **[WildVision](https://huggingface.co/datasets/WildVision/wildvision-bench)**+ | WildVision | VQA | **[OlympiadBench](https://github.com/OpenBMB/OlympiadBench)** | OlympiadBench | VQA |
+
+**\*** 我们只提供了部分模型上的测试结果，剩余模型无法在 zero-shot 设定下测试出合理的精度
+
+**\+** 我们尚未提供这个评测集的测试结果
+
+**\-** VLMEvalKit 仅支持这个评测集的推理，无法输出最终精度
+
+$$^1$$ VLMEvalKit 在评测集的官方代码库中被使用
+
+如果您设置了 API KEY，VLMEvalKit 将使用一个 **LLM** 从输出中提取答案进行匹配判断，否则它将使用**精确匹配**模式 (直接在输出字符串中查找“yes”，“no”，“A”，“B”，“C”等)。**精确匹配只能应用于是或否任务和多选择任务**
+
+### 支持的视频多模态评测集
+
+| Dataset                                                      | Dataset Names (for run.py) | Task                | Dataset                                       | Dataset Names (for run.py) | Task      |
+| ------------------------------------------------------------ | -------------------------- | ------------------- | --------------------------------------------- | -------------------------- | --------- |
+| [**MMBench-Video**](https://mmbench-video.github.io)         | MMBench-Video              | VQA                 | [**Video-MME**](https://video-mme.github.io/) | Video-MME                  | MCQ       |
+| [**MVBench**](https://github.com/OpenGVLab/Ask-Anything/blob/main/video_chat2/MVBENCH.md) | MVBench/MVBench_MP4        | MCQ                 | **[MLVU](https://github.com/JUNJIE99/MLVU)**  | MLVU                       | MCQ & VQA |
+| **[TempCompass](https://arxiv.org/abs/2403.00476)**          | TempCompass                | MCQ & Y/N & Caption |                                               |                            |           |
+
+### 支持的模型
+
+**API 模型**
+
+| [**GPT-4v (20231106, 20240409)**](https://platform.openai.com/docs/guides/vision) 🎞️🚅 | [**GPT-4o**](https://openai.com/index/hello-gpt-4o/) 🎞️🚅      | [**Gemini-1.0-Pro**](https://platform.openai.com/docs/guides/vision) 🎞️🚅 | [**Gemini-1.5-Pro**](https://platform.openai.com/docs/guides/vision) 🎞️🚅 | [**Step-1V**](https://www.stepfun.com/#step1v) 🎞️🚅 |
+| ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------- |
+| [**Reka-[Edge / Flash / Core]**](https://www.reka.ai)🚅       | [**Qwen-VL-[Plus / Max]**](https://huggingface.co/spaces/Qwen/Qwen-VL-Max) 🎞️🚅<br>[**Qwen-VL-[Plus / Max]-0809**](https://huggingface.co/spaces/Qwen/Qwen-VL-Max) 🎞️🚅 | [**Claude3-[Haiku / Sonnet / Opus]**](https://www.anthropic.com/news/claude-3-family) 🎞️🚅 | [**GLM-4v**](https://open.bigmodel.cn/dev/howuse/glm4v) 🚅    | [**CongRong**](https://mllm.cloudwalk.com/web) 🎞️🚅 |
+| [**Claude3.5-Sonnet (20240620, 20241022)**](https://www.anthropic.com/news/claude-3-5-sonnet) 🎞️🚅 | [**GPT-4o-Mini**](https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence/) 🎞️🚅 | [**Yi-Vision**](https://platform.lingyiwanwu.com)🎞️🚅          | [**Hunyuan-Vision**](https://cloud.tencent.com/document/product/1729)🎞️🚅 | [**BlueLM-V**](https://developers.vivo.com/) 🎞️🚅   |
+
+**基于 PyTorch / HF 的开源模型**
+
+| [**IDEFICS-[9B/80B/v2-8B/v3-8B]-Instruct**](https://huggingface.co/HuggingFaceM4/idefics-9b-instruct)🚅🎞️ | [**InstructBLIP-[7B/13B]**](https://github.com/salesforce/LAVIS/blob/main/projects/instructblip/README.md) | [**LLaVA-[v1-7B/v1.5-7B/v1.5-13B]**](https://github.com/haotian-liu/LLaVA) | [**MiniGPT-4-[v1-7B/v1-13B/v2-7B]**](https://github.com/Vision-CAIR/MiniGPT-4) |
+| ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| [**mPLUG-Owl[2/3]**](https://github.com/X-PLUG/mPLUG-Owl/tree/main/mPLUG-Owl2)🎞️ | [**OpenFlamingo-v2**](https://github.com/mlfoundations/open_flamingo)🎞️ | [**PandaGPT-13B**](https://github.com/yxuansu/PandaGPT)      | [**Qwen-VL**](https://huggingface.co/Qwen/Qwen-VL)🚅🎞️ <br>[**Qwen-VL-Chat**](https://huggingface.co/Qwen/Qwen-VL-Chat)🚅🎞️ |
+| [**VisualGLM-6B**](https://huggingface.co/THUDM/visualglm-6b)🚅 | [**InternLM-XComposer-[1/2]**](https://huggingface.co/internlm/internlm-xcomposer-7b)🚅 | [**ShareGPT4V-[7B/13B]**](https://sharegpt4v.github.io)🚅     | [**TransCore-M**](https://github.com/PCIResearch/TransCore-M) |
+| [**LLaVA (XTuner)**](https://huggingface.co/xtuner/llava-internlm-7b)🚅 | [**CogVLM-[Chat/Llama3]**](https://huggingface.co/THUDM/cogvlm-chat-hf)🚅 | [**ShareCaptioner**](https://huggingface.co/spaces/Lin-Chen/Share-Captioner)🚅 | [**CogVLM-Grounding-Generalist**](https://huggingface.co/THUDM/cogvlm-grounding-generalist-hf)🚅 |
+| [**Monkey**](https://github.com/Yuliang-Liu/Monkey)🚅<br>[**Monkey-Chat**](https://github.com/Yuliang-Liu/Monkey)🚅 | [**EMU2-Chat**](https://github.com/baaivision/Emu)🚅🎞️         | [**Yi-VL-[6B/34B]**](https://huggingface.co/01-ai/Yi-VL-6B)  | [**MMAlaya**](https://huggingface.co/DataCanvas/MMAlaya)🚅    |
+| [**InternLM-XComposer-2.5**](https://github.com/InternLM/InternLM-XComposer)🚅🎞️ | [**MiniCPM-[V1/V2/V2.5/V2.6]**](https://github.com/OpenBMB/MiniCPM-V)🚅🎞️ | [**OmniLMM-12B**](https://huggingface.co/openbmb/OmniLMM-12B) | [**InternVL-Chat-[V1-1/V1-2/V1-5/V2]**](https://github.com/OpenGVLab/InternVL)🚅🎞️ |
+| [**DeepSeek-VL**](https://github.com/deepseek-ai/DeepSeek-VL/tree/main)🎞️ | [**LLaVA-NeXT**](https://llava-vl.github.io/blog/2024-01-30-llava-next/)🚅🎞️ | [**Bunny-Llama3**](https://huggingface.co/BAAI/Bunny-v1_1-Llama-3-8B-V)🚅 | [**XVERSE-V-13B**](https://github.com/xverse-ai/XVERSE-V-13B/blob/main/vxverse/models/vxverse.py) |
+| [**PaliGemma-3B**](https://huggingface.co/google/paligemma-3b-pt-448) 🚅 | [**360VL-70B**](https://huggingface.co/qihoo360/360VL-70B) 🚅 | [**Phi-3-Vision**](https://huggingface.co/microsoft/Phi-3-vision-128k-instruct)🚅🎞️<br>[**Phi-3.5-Vision**](https://huggingface.co/microsoft/Phi-3.5-vision-instruct)🚅🎞️ | [**WeMM**](https://github.com/scenarios/WeMM)🚅               |
+| [**GLM-4v-9B**](https://huggingface.co/THUDM/glm-4v-9b) 🚅    | [**Cambrian-[8B/13B/34B]**](https://cambrian-mllm.github.io/) | [**LLaVA-Next-[Qwen-32B]**](https://huggingface.co/lmms-lab/llava-next-qwen-32b) 🎞️ | [**Chameleon-[7B/30B]**](https://huggingface.co/facebook/chameleon-7b)🚅🎞️ |
+| [**Video-LLaVA-7B-[HF]**](https://github.com/PKU-YuanGroup/Video-LLaVA) 🎬 | [**VILA1.5-[3B/8B/13B/40B]**](https://github.com/NVlabs/VILA/)🎞️ | [**Ovis[1.5-Llama3-8B/1.5-Gemma2-9B/1.6-Gemma2-9B/1.6-Llama3.2-3B]**](https://github.com/AIDC-AI/Ovis) 🚅🎞️ | [**Mantis-8B-[siglip-llama3/clip-llama3/Idefics2/Fuyu]**](https://huggingface.co/TIGER-Lab/Mantis-8B-Idefics2) 🎞️ |
+| [**Llama-3-MixSenseV1_1**](https://huggingface.co/Zero-Vision/Llama-3-MixSenseV1_1)🚅 | [**Parrot-7B**](https://github.com/AIDC-AI/Parrot) 🚅         | [**OmChat-v2.0-13B-sinlge-beta**](https://huggingface.co/omlab/omchat-v2.0-13B-single-beta_hf)  🚅 | [**Video-ChatGPT**](https://github.com/mbzuai-oryx/Video-ChatGPT) 🎬 |
+| [**Chat-UniVi-7B[-v1.5]**](https://github.com/PKU-YuanGroup/Chat-UniVi) 🎬 | [**LLaMA-VID-7B**](https://github.com/dvlab-research/LLaMA-VID) 🎬 | [**VideoChat2-HD**](https://huggingface.co/OpenGVLab/VideoChat2_HD_stage4_Mistral_7B) 🎬 | [**PLLaVA-[7B/13B/34B]**](https://huggingface.co/ermu2001/pllava-7b) 🎬 |
+| [**RBDash_72b**](https://github.com/RBDash-Team/RBDash) 🚅🎞️   | [**xgen-mm-phi3-[interleave/dpo]-r-v1.5**](https://huggingface.co/Salesforce/xgen-mm-phi3-mini-instruct-interleave-r-v1.5) 🚅🎞️ | [**Qwen2-VL-[2B/7B/72B]**](https://github.com/QwenLM/Qwen2-VL)🚅🎞️ | [**slime_[7b/8b/13b]**](https://github.com/yfzhang114/SliME)🎞️ |
+| [**Eagle-X4-[8B/13B]**](https://github.com/NVlabs/EAGLE)🚅🎞️, <br>[**Eagle-X5-[7B/13B/34B]**](https://github.com/NVlabs/EAGLE)🚅🎞️ | [**Moondream1**](https://github.com/vikhyat/moondream)🚅, <br>[**Moondream2**](https://github.com/vikhyat/moondream)🚅 | [**XinYuan-VL-2B-Instruct**](https://huggingface.co/Cylingo/Xinyuan-VL-2B)🚅🎞️ | [**Llama-3.2-[11B/90B]-Vision-Instruct**](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct)🚅 |
+| [**Kosmos2**](https://huggingface.co/microsoft/kosmos-2-patch14-224)🚅 | [**H2OVL-Mississippi-[0.8B/2B]**](https://huggingface.co/h2oai/h2ovl-mississippi-2b)🚅🎞️ | **[Pixtral-12B](https://huggingface.co/mistralai/Pixtral-12B-2409)**🎞️ | **[Falcon2-VLM-11B](https://huggingface.co/tiiuae/falcon-11B-vlm)**🚅 |
+| **[MiniMonkey](https://huggingface.co/mx262/MiniMonkey)**🚅🎞️  | **[LLaVA-OneVision](https://huggingface.co/lmms-lab/llava-onevision-qwen2-72b-ov-sft)**🚅🎞️ | **[LLaVA-Video](https://huggingface.co/collections/lmms-lab/llava-video-661e86f5e8dabc3ff793c944)**🚅🎞️ | **[Aquila-VL-2B](https://huggingface.co/BAAI/Aquila-VL-2B-llava-qwen)**🚅🎞️ |
+| [**Mini-InternVL-Chat-[2B/4B]-V1-5**](https://github.com/OpenGVLab/InternVL)🚅🎞️ | **[InternVL2 Series](https://huggingface.co/OpenGVLab/InternVL2-8B)** 🚅🎞️ | **[Janus-1.3B](https://huggingface.co/deepseek-ai/Janus-1.3B)**🚅🎞️ | **[molmoE-1B/molmo-7B/molmo-72B](https://huggingface.co/allenai/Molmo-7B-D-0924)**🚅 |
+| **[Points-[Yi-1.5-9B/Qwen-2.5-7B]](https://huggingface.co/WePOINTS/POINTS-Yi-1-5-9B-Chat)**🚅 | **[NVLM](https://huggingface.co/nvidia/NVLM-D-72B)**🚅        | **[VIntern](https://huggingface.co/5CD-AI/Vintern-3B-beta)**🚅🎞️ | **[Aria](https://huggingface.co/rhymes-ai/Aria)**🚅🎞️ |
+
+🎞️ 表示支持多图片输入。
+
+🚅 表示模型可以被直接使用，不需任何额外的配置。
+
+🎬 表示支持视频输入。
+
+### 其他
+
+**Transformers 的版本推荐:**
+
+**请注意**，某些 VLM 可能无法在某些特定的 transformers 版本下运行，我们建议使用以下设置来评估对应的VLM:
+
+- **请用** `transformers==4.33.0` **来运行**: `Qwen series`, `Monkey series`, `InternLM-XComposer Series`, `mPLUG-Owl2`, `OpenFlamingo v2`, `IDEFICS series`, `VisualGLM`, `MMAlaya`, `ShareCaptioner`, `MiniGPT-4 series`, `InstructBLIP series`, `PandaGPT`, `VXVERSE`.
+- **请用** `transformers==4.37.0 ` **来运行**: `LLaVA series`, `ShareGPT4V series`, `TransCore-M`, `LLaVA (XTuner)`, `CogVLM Series`, `EMU2 Series`, `Yi-VL Series`, `MiniCPM-[V1/V2]`, `OmniLMM-12B`, `DeepSeek-VL series`, `InternVL series`, `Cambrian Series`, `VILA Series`, `Llama-3-MixSenseV1_1`, `Parrot-7B`, `PLLaVA Series`.
+- **请用** `transformers==4.40.0 ` **来运行**: `IDEFICS2`, `Bunny-Llama3`, `MiniCPM-Llama3-V2.5`, `360VL-70B`, `Phi-3-Vision`, `WeMM`.
+- **请用** `transformers==latest` **来运行**: `LLaVA-Next series`, `PaliGemma-3B`, `Chameleon series`, `Video-LLaVA-7B-HF`, `Ovis series`, `Mantis series`, `MiniCPM-V2.6`, `OmChat-v2.0-13B-sinlge-beta`, `Idefics-3`, `GLM-4v-9B`, `VideoChat2-HD`.
+
+**如何测试一个 VLM 是否可以正常运行:**
+
+```python
+from vlmeval.config import supported_VLM
+model = supported_VLM['idefics_9b_instruct']()
+# 前向单张图片
+ret = model.generate(['assets/apple.jpg', 'What is in this image?'])
+print(ret)  # 这张图片上有一个带叶子的红苹果
+# 前向多张图片
+ret = model.generate(['assets/apple.jpg', 'assets/apple.jpg', 'How many apples are there in the provided images? '])
+print(ret)  # 提供的图片中有两个苹果
+```
+
+## 🛠️ 开发指南 <a id="development"></a>
+
+要开发自定义评测数据集，支持其他 VLMs，或为 VLMEvalKit 贡献代码，请参阅[**开发指南**](/docs/zh-CN/Development_zh-CN.md)。
+
+为激励来自社区的共享并分享相应的 credit，在下一次 report 更新中，我们将：
+
+- 致谢所有的 contribution
+- 具备三个或以上主要贡献 (支持新模型、评测集、或是主要特性) 的贡献者将可以加入技术报告的作者列表 。合条件的贡献者可以创建 issue 或是在 [VLMEvalKit Discord Channel](https://discord.com/invite/evDT4GZmxN) 私信 kennyutc，我们将进行跟进
+
+## 🎯 VLMEvalKit 的目标 <a id="goal-of-vlmevalkit"></a>
+
+**该代码库的设计目标是：**
+
+1. 提供一个**易于使用**的**开源评估工具包**，方便研究人员和开发人员评测现有的多模态大模型，并使评测结果**易于复现**。
+2. 使 VLM 开发人员能够轻松地评测自己的模型。在多个支持的基准测试上评估 VLM，只需实现一个 `generate_inner()` 函数，所有其他工作负载（数据下载、数据预处理、预测推理、度量计算）都由代码库处理。
+
+**该代码库的设计目标不是:**
+
+复现所有**第三方基准测试**原始论文中报告的准确数字。有两个相关的原因:
+1. VLMEvalKit 对所有 VLMs 使用基于生成的评估（可选使用基于 LLM 的答案提取）。同时，一些基准测试可能官方使用不同的方法（*例如，SEEDBench 使用基于 PPL 的评估*）。对于这些基准测试，我们在相应的结果中比较两个得分。我们鼓励开发人员在代码库中支持其他评估范式。
+2. 默认情况下，我们对所有多模态模型使用相同的提示模板来评估基准测试。同时，**一些多模态模型可能有他们特定的提示模板**（目前可能未在代码库中涵盖）。我们鼓励 VLM 的开发人员在 VLMEvalKit 中实现自己的提示模板，如果目前未覆盖。这将有助于提高可复现性。
+
+## 🖊️ 引用 <a id="citation"></a>
+
+如果我们的工作对您有所帮助，请考虑 **star🌟** VLMEvalKit。感谢支持！
+
+[![Stargazers repo roster for @open-compass/VLMEvalKit](https://reporoster.com/stars/open-compass/VLMEvalKit)](https://github.com/open-compass/VLMEvalKit/stargazers)
+
+如果您在研究中使用了 VLMEvalKit，或希望参考已发布的开源评估结果，请使用以下 BibTeX 条目以及与您使用的特定 VLM / 基准测试相对应的 BibTex 条目。
+
+```bib
+@misc{duan2024vlmevalkit,
+      title={VLMEvalKit: An Open-Source Toolkit for Evaluating Large Multi-Modality Models},
+      author={Haodong Duan and Junming Yang and Yuxuan Qiao and Xinyu Fang and Lin Chen and Yuan Liu and Xiaoyi Dong and Yuhang Zang and Pan Zhang and Jiaqi Wang and Dahua Lin and Kai Chen},
+      year={2024},
+      eprint={2407.11691},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV},
+      url={https://arxiv.org/abs/2407.11691},
+}
+```
+
+<p align="right"><a href="#top">🔝回到顶部</a></p>
+
+[github-contributors-link]: https://github.com/open-compass/VLMEvalKit/graphs/contributors
+[github-contributors-shield]: https://img.shields.io/github/contributors/open-compass/VLMEvalKit?color=c4f042&labelColor=black&style=flat-square
+[github-forks-link]: https://github.com/open-compass/VLMEvalKit/network/members
+[github-forks-shield]: https://img.shields.io/github/forks/open-compass/VLMEvalKit?color=8ae8ff&labelColor=black&style=flat-square
+[github-issues-link]: https://github.com/open-compass/VLMEvalKit/issues
+[github-issues-shield]: https://img.shields.io/github/issues/open-compass/VLMEvalKit?color=ff80eb&labelColor=black&style=flat-square
+[github-license-link]: https://github.com/open-compass/VLMEvalKit/blob/main/LICENSE
+[github-license-shield]: https://img.shields.io/github/license/open-compass/VLMEvalKit?color=white&labelColor=black&style=flat-square
+[github-stars-link]: https://github.com/open-compass/VLMEvalKit/stargazers
+[github-stars-shield]: https://img.shields.io/github/stars/open-compass/VLMEvalKit?color=ffcb47&labelColor=black&style=flat-square

vlmeval/VLMEvalKit_old/docs/zh-CN/_static/js/custom.js

@@ -0,0 +1,10 @@
+var collapsedSections = [];
+
+$(document).ready(function () {
+  $('.model-summary').DataTable({
+    "stateSave": false,
+    "lengthChange": false,
+    "pageLength": 20,
+    "order": []
+  });
+});

vlmeval/VLMEvalKit_old/docs/zh-CN/_templates/autosummary/class.rst

@@ -0,0 +1,13 @@
+.. role:: hidden
+    :class: hidden-section
+.. currentmodule:: {{ module }}
+
+
+{{ name | underline}}
+
+.. autoclass:: {{ name }}
+    :members:
+
+..
+  autogenerated from _templates/autosummary/class.rst
+  note it does not have :inherited-members:

vlmeval/VLMEvalKit_old/docs/zh-CN/_templates/callable.rst

@@ -0,0 +1,14 @@
+.. role:: hidden
+    :class: hidden-section
+.. currentmodule:: {{ module }}
+
+
+{{ name | underline}}
+
+.. autoclass:: {{ name }}
+    :members:
+    :special-members: __call__
+
+..
+  autogenerated from _templates/callable.rst
+  note it does not have :inherited-members:

vlmeval/VLMEvalKit_old/docs/zh-CN/conf.py

@@ -0,0 +1,242 @@
+# flake8: noqa
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import ast
+import subprocess
+import sys
+
+import pytorch_sphinx_theme
+from sphinx.builders.html import StandaloneHTMLBuilder
+
+sys.path.insert(0, os.path.abspath('../../'))
+
+# -- Project information -----------------------------------------------------
+
+project = 'VLMEvalKit'
+copyright = '2023, VLMEvalKit'
+author = 'VLMEvalKit Authors'
+
+# The full version, including alpha/beta/rc tags
+version_file = '../../vlmeval/__init__.py'
+
+
+def get_version():
+    with open(version_file, 'r') as f:
+        file_content = f.read()
+    # Parse the file content into an abstract syntax tree (AST)
+    tree = ast.parse(file_content, filename=version_file)
+
+    # Iterate through the body of the AST, looking for an assignment to __version__
+    for node in tree.body:
+        if isinstance(node, ast.Assign):
+            for target in node.targets:
+                if isinstance(target, ast.Name) and target.id == '__version__':
+                    return node.value.s
+    raise ValueError('__version__ not found')
+
+
+release = get_version()
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.napoleon',
+    'sphinx.ext.viewcode',
+    'myst_parser',
+    'sphinx_copybutton',
+    'sphinx_tabs.tabs',
+    'notfound.extension',
+    'sphinxcontrib.jquery',
+    'sphinx_design',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+source_suffix = {
+    '.rst': 'restructuredtext',
+    '.md': 'markdown',
+}
+
+language = 'cn'
+
+# The master toctree document.
+root_doc = 'index'
+html_context = {
+    'github_version': 'latest',
+}
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'pytorch_sphinx_theme'
+html_theme_path = [pytorch_sphinx_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+# yapf: disable
+html_theme_options = {
+    'menu': [
+        {
+            'name': 'GitHub',
+            'url': 'https://github.com/open-compass/VLMEvalKit'
+        },
+    ],
+    # Specify the language of shared menu
+    'menu_lang': 'cn',
+    # Disable the default edit on GitHub
+    'default_edit_on_github': False,
+}
+# yapf: enable
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+html_css_files = [
+    'https://cdn.datatables.net/v/bs4/dt-1.12.1/datatables.min.css',
+    'css/readthedocs.css'
+]
+html_js_files = [
+    'https://cdn.datatables.net/v/bs4/dt-1.12.1/datatables.min.js',
+    'js/custom.js'
+]
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'vlmevalkitdoc'
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (root_doc, 'vlmevalkit.tex', 'VLMEvalKit Documentation', author,
+     'manual'),
+]
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [(root_doc, 'vlmevalkit', 'VLMEvalKit Documentation', [author],
+              1)]
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (root_doc, 'vlmevalkit', 'VLMEvalKit Documentation', author,
+     'VLMEvalKit Authors', 'AGI evaluation toolbox and benchmark.',
+     'Miscellaneous'),
+]
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+# set priority when building html
+StandaloneHTMLBuilder.supported_image_types = [
+    'image/svg+xml', 'image/gif', 'image/png', 'image/jpeg'
+]
+
+# -- Extension configuration -------------------------------------------------
+# Ignore >>> when copying code
+copybutton_prompt_text = r'>>> |\.\.\. '
+copybutton_prompt_is_regexp = True
+
+# Auto-generated header anchors
+myst_heading_anchors = 3
+# Enable "colon_fence" extension of myst.
+myst_enable_extensions = ['colon_fence', 'dollarmath']
+
+# Configuration for intersphinx
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3', None),
+    'numpy': ('https://numpy.org/doc/stable', None),
+    'torch': ('https://pytorch.org/docs/stable/', None),
+    'mmengine': ('https://mmengine.readthedocs.io/en/latest/', None),
+    'transformers':
+    ('https://huggingface.co/docs/transformers/main/en/', None),
+}
+napoleon_custom_sections = [
+    # Custom sections for data elements.
+    ('Meta fields', 'params_style'),
+    ('Data fields', 'params_style'),
+]
+
+# Disable docstring inheritance
+autodoc_inherit_docstrings = False
+# Mock some imports during generate API docs.
+autodoc_mock_imports = ['rich', 'attr', 'einops']
+# Disable displaying type annotations, these can be very verbose
+autodoc_typehints = 'none'
+
+# The not found page
+notfound_template = '404.html'
+
+
+def builder_inited_handler(app):
+    subprocess.run(['./cp_origin_docs.sh'])
+
+
+def setup(app):
+    app.connect('builder-inited', builder_inited_handler)

vlmeval/VLMEvalKit_old/docs/zh-CN/index.rst

@@ -0,0 +1,49 @@
+欢迎来到 VLMEvalKit 中文教程！
+==========================================
+
+VLMEvalKit 上手路线
+-------------------------------
+
+为了用户能够快速上手，我们推荐以下流程：
+
+- 对于想要使用 VLMEvalKit 的用户，我们推荐先阅读 开始你的第一步_ 部分来设置环境，并启动一个迷你实验熟悉流程。
+
+- 若您想进行更多模块的自定义，例如增加数据集和模型，我们提供了 进阶教程_ 。
+
+我们始终非常欢迎用户的 PRs 和 Issues 来完善 VLMEvalKit！
+
+.. _快速开始:
+.. toctree::
+   :maxdepth: 1
+   :caption: 快速开始
+
+   Quickstart.md
+
+
+.. .. _教程:
+.. .. toctree::
+..    :maxdepth: 1
+..    :caption: 教程
+
+..    user_guides/framework_overview.md
+
+.. _进阶教程:
+.. toctree::
+   :maxdepth: 1
+   :caption: 进阶教程
+
+   Development.md
+   ConfigSystem.md
+
+.. .. _其他说明:
+.. .. toctree::
+..    :maxdepth: 1
+..    :caption: 其他说明
+
+..    notes/contribution_guide.md
+
+索引与表格
+==================
+
+* :ref:`genindex`
+* :ref:`search`

vlmeval/VLMEvalKit_old/scripts/apires_scan.py

@@ -0,0 +1,55 @@
+import sys
+from vlmeval import *
+from vlmeval.dataset import SUPPORTED_DATASETS
+FAIL_MSG = 'Failed to obtain answer via API.'
+
+root = sys.argv[1]
+if root[-1] in '/\\':
+    root = root[:-1]
+
+model_name = root.split('/')[-1]
+
+for d in SUPPORTED_DATASETS:
+    fname = f'{model_name}_{d}.xlsx'
+    pth = osp.join(root, fname)
+    if osp.exists(pth):
+        data = load(pth)
+        # Detect Failure
+        assert 'prediction' in data
+        data['prediction'] = [str(x) for x in data['prediction']]
+        fail = [FAIL_MSG in x for x in data['prediction']]
+        if sum(fail):
+            nfail = sum(fail)
+            ntot = len(fail)
+            print(f'Model {model_name} x Dataset {d}: {nfail} out of {ntot} failed. {nfail / ntot * 100: .2f}%. ')
+
+        eval_files = ls(root, match=f'{model_name}_{d}_')
+        eval_files = [x for x in eval_files if listinstr([f'{d}_openai', f'{d}_gpt'], x) and x.endswith('.xlsx')]
+
+        if len(eval_files) == 0:
+            print(f'Model {model_name} x Dataset {d} openai missing')
+            continue
+        
+        assert len(eval_files) == 1
+        eval_file = eval_files[0]
+        data = load(eval_file)
+        
+        if 'MMVet' in d:
+            bad = [x for x in data['log'] if 'All 5 retries failed.' in str(x)]
+            if len(bad):
+                print(f'Model {model_name} x Dataset {d} Evaluation: {len(bad)} out of {len(data)} failed.')
+        elif 'MathVista' in d:
+            bad = [x for x in data['res'] if FAIL_MSG in str(x)]
+            if len(bad):
+                print(f'Model {model_name} x Dataset {d} Evaluation: {len(bad)} out of {len(data)} failed.')
+            
+        elif d == 'LLaVABench':
+            sub = data[data['gpt4_score'] == -1]
+            sub = sub[sub['gpt4_score'] == -1]
+            if len(sub):
+                print(f'Model {model_name} x Dataset {d} Evaluation: {len(sub)} out of {len(data)} failed.')
+        else:
+            bad = [x for x in data['log'] if FAIL_MSG in str(x)]
+            if len(bad):
+                print(f'Model {model_name} x Dataset {d} Evaluation: {len(bad)} out of {len(data)} failed.')
+                

vlmeval/VLMEvalKit_old/scripts/cover.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cp $DIR/../config.py $DIR/../vlmeval/
+cp $DIR/../misc/* $DIR/../vlmeval/vlm/misc/

vlmeval/VLMEvalKit_old/scripts/data_browser.py

@@ -0,0 +1,166 @@
+"""
+pip install gradio    # proxy_on first
+python vis_geochat_data.py
+# browse data in http://127.0.0.1:10064
+"""
+
+import os
+import io
+import json
+import copy
+import time
+import gradio as gr
+import base64
+from PIL import Image
+from io import BytesIO
+from argparse import Namespace
+# from llava import conversation as conversation_lib
+from typing import Sequence
+from vlmeval import *
+from vlmeval.dataset import SUPPORTED_DATASETS, build_dataset
+
+SYS = "You are a helpful assistant. Your job is to faithfully translate all provided text into Chinese faithfully. "
+
+# Translator = SiliconFlowAPI(model='Qwen/Qwen2.5-7B-Instruct', system_prompt=SYS)
+Translator = OpenAIWrapper(model='gpt-4o-mini', system_prompt=SYS)
+
+
+def image_to_mdstring(image):
+    return f"![image](data:image/jpeg;base64,{image})"
+
+
+def images_to_md(images):
+    return '\n\n'.join([image_to_mdstring(image) for image in images])
+
+
+def mmqa_display(question, target_size=768):
+    question = {k.lower() if len(k) > 1 else k: v for k, v in question.items()}
+    keys = list(question.keys())
+    keys = [k for k in keys if k not in ['index', 'image']]
+
+    idx = question.pop('index', 'XXX')
+    text = f'\n- INDEX: {idx}\n'
+
+    images = question.pop('image')
+    if images[0] == '[' and images[-1] == ']':
+        images = eval(images)
+    else:
+        images = [images]
+
+    qtext = question.pop('question', None)
+    if qtext is not None:
+        text += f'- QUESTION: {qtext}\n'
+
+    if 'A' in question:
+        text += f'- Choices: \n'
+        for k in string.ascii_uppercase:
+            if k in question:
+                text += f'\t-{k}: {question.pop(k)}\n'
+    answer = question.pop('answer', None)
+    
+    for k in question:
+        if not pd.isna(question[k]):
+            text += f'- {k.upper()}. {question[k]}\n'
+    
+    if answer is not None:
+        text += f'- ANSWER: {answer}\n'
+
+    image_md = images_to_md(images)
+
+    return text, image_md
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    # Essential Args, Setting the Names of Datasets and Models
+    parser.add_argument('--port', type=int, default=7860)
+    args = parser.parse_args()
+    return args
+
+
+def gradio_app_vis_dataset(port=7860):
+    data, loaded_obj = None, {}
+
+    def btn_submit_click(filename, ann_id):
+        if filename not in loaded_obj:
+            return filename_change(filename, ann_id)
+        nonlocal data
+        data_desc = gr.Markdown(f'Visualizing {filename}, {len(data)} samples in total. ')
+        if ann_id < 0 or ann_id >= len(data):
+            return filename, ann_id, data_desc, gr.Markdown('Invalid Index'), gr.Markdown(f'Index out of range [0, {len(data) - 1}]')
+        item = data.iloc[ann_id]
+        text, image_md = mmqa_display(item)
+        return filename, ann_id, data_desc, image_md, text
+
+    def btn_next_click(filename, ann_id):
+        return btn_submit_click(filename, ann_id + 1)
+
+    # def translate_click(anno_en):
+    #     return gr.Markdown(Translator.generate(anno_en))
+
+    def filename_change(filename, ann_id):
+        nonlocal data, loaded_obj
+
+        def legal_filename(filename):
+            LMURoot = LMUDataRoot()
+            if filename in SUPPORTED_DATASETS:
+                return build_dataset(filename).data
+            elif osp.exists(filename):
+                data = load(filename)
+                assert 'index' in data and 'image' in data
+                image_map = {i: image for i, image in zip(data['index'], data['image'])}
+                for k, v in image_map.items():
+                    if (not isinstance(v, str) or len(v) < 64) and v in image_map:
+                        image_map[k] = image_map[v]
+                data['image'] = [image_map[k] for k in data['index']]
+                return data
+            elif osp.exists(osp.join(LMURoot, filename)):
+                filename = osp.join(LMURoot, filename)
+                return legal_filename(filename)
+            else:
+                return None
+
+        data = legal_filename(filename)
+        if data is None:
+            return filename, 0, gr.Markdown(''), gr.Markdown("File not found"), gr.Markdown("File not found")
+        
+        loaded_obj[filename] = data
+        return btn_submit_click(filename, 0)
+
+    with gr.Blocks() as app:
+        
+        filename = gr.Textbox(
+            value='Dataset Name (supported by VLMEvalKit) or TSV FileName (Relative under `LMURoot` or Real Path)', 
+            label='Dataset', 
+            interactive=True,
+            visible=True)
+            
+        with gr.Row():
+            ann_id = gr.Number(0, label='Sample Index (Press Enter)', interactive=True, visible=True)
+            btn_next = gr.Button("Next")
+            # btn_translate = gr.Button('CN Translate')
+
+        with gr.Row():
+            data_desc = gr.Markdown('Dataset Description', label='Dataset Description')
+        
+        with gr.Row():
+            image_output = gr.Markdown('Image PlaceHolder', label='Image Visualization')
+            anno_en = gr.Markdown('Image Annotation', label='Image Annotation')
+            # anno_cn = gr.Markdown('Image Annotation (Chinese)', label='Image Annotation (Chinese)')
+
+        input_components = [filename, ann_id]
+        all_components = [filename, ann_id, data_desc, image_output, anno_en]
+
+        filename.submit(filename_change, input_components, all_components)
+        ann_id.submit(btn_submit_click, input_components, all_components)
+        btn_next.click(btn_next_click, input_components, all_components)
+        # btn_translate.click(translate_click, anno_en, anno_cn)
+
+    # app.launch()
+    app.launch(server_name='0.0.0.0', debug=True, show_error=True, server_port=port)
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    gradio_app_vis_dataset(port=args.port)
+

vlmeval/VLMEvalKit_old/scripts/mmb_eval_gradio.py

@@ -0,0 +1,101 @@
+from vlmeval.smp import *
+from vlmeval.tools import EVAL
+import gradio as gr
+
+HEADER = """
+# Welcome to MMBench👏👏
+We are delighted that you are willing to submit the evaluation results to the MMBench official website! The evaluation service currently can handle submissions of MMBench, MMBench-CN, and CCBench. We use `gpt-3.5-turbo-0125` to help answer matching. Evaluation Codes in VLMEvalKit: https://github.com/open-compass/VLMEvalKit. Please adopt / follow the implementation of VLMEvalKit to generate the submission files. 
+
+The evaluation script is available at https://github.com/open-compass/VLMEvalKit/tree/main/scripts/mmb_eval_gradio.py
+Please contact `[email protected]` for any inquirys about this script. 
+"""
+
+def upload_file(file):
+    file_path = file.name
+    return file_path
+
+def prepare_file(file_name):
+    file_md5 = md5(file_name)
+    root = LMUDataRoot()
+    root = osp.join(root, 'eval_server')
+    os.makedirs(root, exist_ok=True)
+    suffix = file_name.split('.')[-1]
+    if suffix not in ['xlsx', 'tsv', 'csv']:
+        return False, "Please submit a file that ends with `.xlsx`, `.tsv`, or `.csv`"
+    new_file_name = osp.join(root, f'{file_md5}.{suffix}')
+    shutil.move(file_name, new_file_name)
+    eval_file = new_file_name
+    try:
+        data = load(eval_file)
+    except:
+        return False, "Your excel file can not be successfully loaded by `pd.read_excel`, please double check and submit again. "
+    for k in data.keys():
+        data[k.lower() if k not in 'ABCD' else k] = data.pop(k)
+    if "index" not in data:
+        return False, "Your excel file should have a column named `index`, please double check and submit again" , {}
+    if "prediction" not in data:
+        return False, "Your excel file should have a column named `prediction`, please double check and submit again" , {}
+    for ch in 'ABCD':
+        if ch not in data:
+            return False, f"Your excel file should have a column named `{ch}`, please double check and submit again" , {}
+    dump(data, eval_file)
+    return True, eval_file
+
+def determine_dataset(eval_file):
+    data = load(eval_file)
+    def cn_ratio(data):
+        iscn = [cn_string(x) for x in data['question']]
+        return np.mean(iscn)
+    max_ind = np.max([int(x) for x in data['index'] if int(x) < 1e5])
+    if max_ind < 1000 and 'l2-category' not in data:
+        return 'CCBench' if cn_ratio(data) > 0.5 else "Unknown" 
+    elif max_ind < 3000 :
+        return 'MMBench_CN' if cn_ratio(data) > 0.5 else "MMBench"
+    else:
+        return 'MMBench_CN_V11' if cn_ratio(data) > 0.5 else "MMBench_V11"
+
+    
+def reformat_acc(acc):
+    splits = set(acc['split'])
+    keys = list(acc.keys())
+    keys.remove('split')
+    nacc = {'Category': []}
+    for sp in splits:
+        nacc[sp.upper()] = []
+    for k in keys:
+        nacc['Category'].append(k)
+        for sp in splits:
+            nacc[sp.upper()].append(acc[acc['split'] == sp].iloc[0][k] * 100)
+    return pd.DataFrame(nacc)
+
+def evaluate(file):
+    file_name = file.name
+    flag, eval_file = prepare_file(file_name)
+    if not flag:
+        return "Error: " + eval_file
+    dataset = determine_dataset(eval_file)
+    if dataset == 'Unknown':
+        return "Error: Cannot determine the dataset given your submitted file. " 
+
+    eval_id = eval_file.split('/')[-1].split('.')[0]
+    ret = f"Evaluation ID: {eval_id}\n"
+    timestamp = datetime.datetime.now().strftime('%Y.%m.%d  %H:%M:%S')
+    ret += f'Evaluation Timestamp: {timestamp}\n'
+    acc = EVAL(dataset, eval_file)
+    nacc = reformat_acc(acc).round(1)
+    return ret, nacc
+
+with gr.Blocks() as demo:
+    gr.Markdown(HEADER)
+    file_output = gr.File()
+    upload_button = gr.UploadButton("Click to upload you prediction files for a supported benchmark")
+    upload_button.upload(upload_file, upload_button, file_output)
+    
+    btn = gr.Button("🚀 Evaluate")
+    eval_log = gr.Textbox(label="Evaluation Log", placeholder="Your evaluation log will be displayed here")
+    df_empty = pd.DataFrame([], columns=['Evaluation Result'])
+    eval_result = gr.components.DataFrame(value=df_empty)
+    btn.click(evaluate, inputs=[file_output], outputs=[eval_log, eval_result])
+
+if __name__ == '__main__':
+    demo.launch(server_name='0.0.0.0', debug=True, show_error=True)

vlmeval/VLMEvalKit_old/scripts/run.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -x
+export GPU=$(nvidia-smi --list-gpus | wc -l)
+torchrun --nproc-per-node=$GPU run.py ${@:1}

vlmeval/VLMEvalKit_old/scripts/srun.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -x
+srun -n1 --ntasks-per-node=1 --partition $1 --gres=gpu:8 --quotatype=reserved --job-name vlmeval --cpus-per-task=64 torchrun --nproc-per-node=8 run.py ${@:2}

vlmeval/VLMEvalKit_old/scripts/visualize.ipynb

@@ -0,0 +1,266 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import copy as cp\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "import matplotlib.font_manager as fm\n",
+    "\n",
+    "def download_file(url, filename=None):\n",
+    "    from urllib.request import urlretrieve\n",
+    "    if filename is None:\n",
+    "        filename = url.split('/')[-1]\n",
+    "    urlretrieve(url, filename)\n",
+    "\n",
+    "font_URL = 'http://opencompass.openxlab.space/utils/Fonts/segoepr.ttf'\n",
+    "download_file(font_URL)\n",
+    "\n",
+    "font12 = fm.FontProperties(fname='segoepr.ttf', size=12)\n",
+    "font15 = fm.FontProperties(fname='segoepr.ttf', size=15, weight='bold')\n",
+    "font18 = fm.FontProperties(fname='segoepr.ttf', size=18, weight='bold')\n",
+    "\n",
+    "DATA_URL = 'http://opencompass.openxlab.space/utils/OpenVLM.json'\n",
+    "download_file(DATA_URL)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def pre_normalize(raw_data, labels):\n",
+    "    data_list = cp.deepcopy(raw_data)\n",
+    "    minimum, maximum, max_range, range_map = {}, {}, 0, {}\n",
+    "    for lb in labels:\n",
+    "        minimum[lb] = min([x[lb] for x in data_list])\n",
+    "        maximum[lb] = max([x[lb] for x in data_list])\n",
+    "        max_range = max(max_range, maximum[lb] - minimum[lb])\n",
+    "    max_range *= 1.25\n",
+    "    for lb in labels:\n",
+    "        mid = (minimum[lb] + maximum[lb]) / 2\n",
+    "        new_range = (mid - max_range / 2, mid + max_range / 2) if (mid + max_range / 2) < 100 else (100 - max_range, 100)\n",
+    "        range_map[lb] = new_range\n",
+    "        for item in data_list:\n",
+    "            assert new_range[0] <= item[lb] <= new_range[1]\n",
+    "            item[lb] = (item[lb] - new_range[0]) / max_range * 100\n",
+    "    return data_list, range_map\n",
+    "\n",
+    "# solve the problem that some benchmark score is too high and out of range\n",
+    "def log_normalize(raw_data, labels):\n",
+    "    data_list = cp.deepcopy(raw_data)\n",
+    "    minimum, maximum, max_range, range_map = {}, {}, 0, {}\n",
+    "    for lb in labels:\n",
+    "        minimum[lb] = min([np.log(x[lb]) for x in data_list])\n",
+    "        maximum[lb] = max([np.log(x[lb]) for x in data_list])\n",
+    "        max_range = max(max_range, maximum[lb] - minimum[lb])\n",
+    "    max_range *= 1.005\n",
+    "    for lb in labels:\n",
+    "        mid = (minimum[lb] + maximum[lb]) / 2\n",
+    "        new_range = (mid - max_range / 2, mid + max_range / 2) if (mid + max_range / 2) < 100 else (100 - max_range, 100)\n",
+    "        range_map[lb] = new_range\n",
+    "        for item in data_list:\n",
+    "            assert new_range[0] <= np.log(item[lb]) <= new_range[1]\n",
+    "            item[lb] = (np.log(item[lb]) - new_range[0]) / max_range * 100\n",
+    "    return data_list, range_map"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Draw MMBench Radar Graph\n",
+    "data = json.loads(open('OpenVLM.json').read())['results']\n",
+    "models = list(data)\n",
+    "print(models)\n",
+    "\n",
+    "# model2vis = [\n",
+    "#     'GPT-4v (detail: low)', 'GeminiProVision', 'Qwen-VL-Plus', \n",
+    "#     'InternLM-XComposer2-VL', 'LLaVA-v1.5-13B', 'CogVLM-17B-Chat',\n",
+    "#     'mPLUG-Owl2', 'Qwen-VL-Chat', 'IDEFICS-80B-Instruct'\n",
+    "# ]\n",
+    "\n",
+    "model2vis = [\n",
+    "    # 'GPT-4v (detail: low)', 'GeminiProVision', 'InternLM-XComposer2-VL', \n",
+    "    'GPT-4v (1106, detail-low)', 'Gemini-1.0-Pro', 'Gemini-1.5-Pro', #'Gemini-1.5-Flash', 'Qwen-VL-Plus', \n",
+    "    'InternLM-XComposer2', 'LLaVA-v1.5-13B', 'CogVLM-17B-Chat',\n",
+    "    'mPLUG-Owl2', 'Qwen-VL-Chat', 'IDEFICS-80B-Instruct'\n",
+    "]\n",
+    "\n",
+    "colors = [\n",
+    "    '#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', \n",
+    "    '#e377c2', '#7f7f7f', '#bcbd22'\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from collections import defaultdict\n",
+    "\n",
+    "split = 'MMBench_TEST_EN'\n",
+    "# data_sub = {k: v[split] for k, v in data.items()}\n",
+    "data_sub = {k: defaultdict(int, v)[split] for k, v in data.items()}\n",
+    "# solve the problem that some model lack the evaluation of MMBench_TEST_EN\n",
+    "\n",
+    "labels = list(data_sub[model2vis[0]])\n",
+    "labels.remove('Overall')\n",
+    "num_vars = len(labels)\n",
+    "\n",
+    "raw_data = [data_sub[m] for m in model2vis]\n",
+    "data_list, range_map = pre_normalize(raw_data, labels)\n",
+    "\n",
+    "alpha = 0.25\n",
+    "angles = np.linspace(0, 2 * np.pi, num_vars, endpoint=False).tolist()\n",
+    "angles_deg = np.linspace(0, 360, num_vars, endpoint=False).tolist()\n",
+    "fig, ax_base = plt.subplots(nrows=1, ncols=1, figsize=(10, 10), subplot_kw=dict(polar=True))\n",
+    "\n",
+    "for i in range(len(data_list)):\n",
+    "    item = data_list[i]\n",
+    "    model_name = model2vis[i]\n",
+    "    color = colors[i]\n",
+    "    tmp_angles = angles[:] + [angles[0]]\n",
+    "    tmp_values = [item[lb] for lb in labels] + [item[labels[0]]]\n",
+    "    ax_base.plot(tmp_angles, tmp_values, color=color, linewidth=1, linestyle='solid', label=model_name)\n",
+    "    ax_base.fill(tmp_angles, tmp_values, color=color, alpha=alpha)\n",
+    "    \n",
+    "angles += [angles[0]]\n",
+    "ax_base.set_ylim(0, 100)\n",
+    "ax_base.set_yticks([40, 60, 80, 100])\n",
+    "ax_base.set_yticklabels([''] * 4)\n",
+    "\n",
+    "ax_base.tick_params(pad=25)\n",
+    "ax_base.set_xticks(angles[:-1])\n",
+    "ax_base.set_xticklabels(labels, fontproperties=font18)\n",
+    "\n",
+    "leg = ax_base.legend(loc='center right', bbox_to_anchor=(1.6, 0.5), prop=font15, ncol=1, frameon=True, labelspacing=1.2)\n",
+    "for line in leg.get_lines():\n",
+    "    line.set_linewidth(2.5)\n",
+    "\n",
+    "cx, cy, sz = 0.44, 0.435, 0.34\n",
+    "axes = [fig.add_axes([cx - sz, cy - sz, cx + sz, cy + sz], projection='polar', label='axes%d' % i) for i in range(num_vars)]\n",
+    "    \n",
+    "for ax, angle, label in zip(axes, angles_deg, labels):\n",
+    "    ax.patch.set_visible(False)\n",
+    "    ax.grid(False)\n",
+    "    ax.xaxis.set_visible(False)\n",
+    "    cur_range = range_map[label]\n",
+    "    label_list = [cur_range[0] + (cur_range[1] - cur_range[0]) / 5 * i for i in range(2, 6)]\n",
+    "    label_list = [f'{x:.1f}' for x in label_list]\n",
+    "    ax.set_rgrids(range(40, 120, 20), angle=angle, labels=label_list, font_properties=font12)\n",
+    "    ax.spines['polar'].set_visible(False)\n",
+    "    ax.set_ylim(0, 100)\n",
+    "\n",
+    "title_text = f'{len(model2vis)} Representative VLMs on MMBench Test.'\n",
+    "plt.figtext(.7, .95, title_text, fontproperties=font18, ha='center')\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "labels = ['SEEDBench_IMG', 'CCBench', 'MMBench_TEST_EN', 'MMBench_TEST_CN', 'MME', 'MMVet', 'MMMU_VAL', 'MathVista', 'HallusionBench', 'LLaVABench']\n",
+    "num_vars = len(labels)\n",
+    "\n",
+    "raw_data = [{k: data[m][k]['Overall'] for k in labels} for m in model2vis]\n",
+    "data_list, range_map = pre_normalize(raw_data, labels)\n",
+    "\n",
+    "alpha = 0.25\n",
+    "angles = np.linspace(0, 2 * np.pi, num_vars, endpoint=False).tolist()\n",
+    "angles_deg = np.linspace(0, 360, num_vars, endpoint=False).tolist()\n",
+    "fig, ax_base = plt.subplots(nrows=1, ncols=1, figsize=(10, 10), subplot_kw=dict(polar=True))\n",
+    "\n",
+    "for i in range(len(data_list)):\n",
+    "    item = data_list[i]\n",
+    "    model_name = model2vis[i]\n",
+    "    color = colors[i]\n",
+    "    tmp_angles = angles[:] + [angles[0]]\n",
+    "    tmp_values = [item[lb] for lb in labels] + [item[labels[0]]]\n",
+    "    ax_base.plot(tmp_angles, tmp_values, color=color, linewidth=1, linestyle='solid', label=model_name)\n",
+    "    ax_base.fill(tmp_angles, tmp_values, color=color, alpha=alpha)\n",
+    "    \n",
+    "angles += [angles[0]]\n",
+    "ax_base.set_ylim(0, 100)\n",
+    "ax_base.set_yticks([40, 60, 80, 100])\n",
+    "ax_base.set_yticklabels([''] * 4)\n",
+    "\n",
+    "ax_base.tick_params(pad=15)\n",
+    "ax_base.set_xticks(angles[:-1])\n",
+    "ax_base.set_xticklabels(labels, fontproperties=font18)\n",
+    "\n",
+    "dataset_map = {\n",
+    "    'MMBench_TEST_EN': 'MMBench (Test)', \n",
+    "    'MMBench_TEST_CN': 'MMBenchCN (Test)', \n",
+    "    'MathVista': 'MathVista (TestMini)', \n",
+    "    'MMMU_VAL': 'MMMU (Val)'\n",
+    "}\n",
+    "for i, label in enumerate(ax_base.get_xticklabels()):\n",
+    "    x,y = label.get_position()\n",
+    "    text = label.get_text()\n",
+    "    text = dataset_map[text] if text in dataset_map else text\n",
+    "    lab = ax_base.text(x, y, text, transform=label.get_transform(),\n",
+    "                  ha=label.get_ha(), va=label.get_va(), font_properties=font15)\n",
+    "    lab.set_rotation(360 / num_vars * i + 270)\n",
+    "    labels.append(lab)\n",
+    "ax_base.set_xticklabels([])\n",
+    "\n",
+    "leg = ax_base.legend(loc='center right', bbox_to_anchor=(1.6, 0.5), prop=font15, ncol=1, frameon=True, labelspacing=1.2)\n",
+    "for line in leg.get_lines():\n",
+    "    line.set_linewidth(2.5)\n",
+    "\n",
+    "cx, cy, sz = 0.44, 0.435, 0.34\n",
+    "axes = [fig.add_axes([cx - sz, cy - sz, cx + sz, cy + sz], projection='polar', label='axes%d' % i) for i in range(num_vars)]\n",
+    "    \n",
+    "for ax, angle, label in zip(axes, angles_deg, labels):\n",
+    "    ax.patch.set_visible(False)\n",
+    "    ax.grid(False)\n",
+    "    ax.xaxis.set_visible(False)\n",
+    "    cur_range = range_map[label]\n",
+    "    label_list = [cur_range[0] + (cur_range[1] - cur_range[0]) / 5 * i for i in range(2, 6)]\n",
+    "    label_list = [f'{x:.1f}' for x in label_list]\n",
+    "    ax.set_rgrids(range(40, 120, 20), angle=angle, labels=label_list, font_properties=font12)\n",
+    "    ax.spines['polar'].set_visible(False)\n",
+    "    ax.set_ylim(0, 100)\n",
+    "\n",
+    "title_text = f'{len(model2vis)} Representative VLMs on {num_vars} Benchmarks in OpenCompass Multi-Modal Leaderboard.'\n",
+    "plt.figtext(.7, .95, title_text, fontproperties=font18, ha='center')\n",
+    "plt.show()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

vlmeval/VLMEvalKit_old/vlmeval/api/__init__.py

@@ -0,0 +1,26 @@
+from .gpt import OpenAIWrapper, GPT4V
+from .hf_chat_model import HFChatModel
+from .gemini import GeminiWrapper, GeminiProVision
+from .qwen_vl_api import QwenVLWrapper, QwenVLAPI, Qwen2VLAPI
+from .qwen_api import QwenAPI
+from .claude import Claude_Wrapper, Claude3V
+from .reka import Reka
+from .glm_vision import GLMVisionAPI
+from .cloudwalk import CWWrapper
+from .sensechat_vision import SenseChatVisionAPI
+from .siliconflow import SiliconFlowAPI, TeleMMAPI
+from .hunyuan import HunyuanVision
+from .bailingmm import bailingMMAPI
+from .bluelm_v_api import BlueLMWrapper, BlueLM_V_API
+from .jt_vl_chat import JTVLChatAPI
+from .taiyi import TaiyiAPI
+
+
+__all__ = [
+    'OpenAIWrapper', 'HFChatModel', 'GeminiWrapper', 'GPT4V',
+    'GeminiProVision', 'QwenVLWrapper', 'QwenVLAPI', 'QwenAPI',
+    'Claude3V', 'Claude_Wrapper', 'Reka', 'GLMVisionAPI',
+    'CWWrapper', 'SenseChatVisionAPI', 'HunyuanVision', 'Qwen2VLAPI',
+    'BlueLMWrapper', 'BlueLM_V_API', 'JTVLChatAPI', 'bailingMMAPI',
+    'TaiyiAPI', 'TeleMMAPI', 'SiliconFlowAPI'
+]

vlmeval/VLMEvalKit_old/vlmeval/api/bailingmm.py

@@ -0,0 +1,90 @@
+import base64
+from vlmeval.smp import *
+from vlmeval.api.base import BaseAPI
+from vlmeval.dataset import DATASET_TYPE
+from vlmeval.smp.vlm import encode_image_file_to_base64
+import time
+
+
+class bailingMMWrapper(BaseAPI):
+
+    is_api: bool = True
+
+    def __init__(self,
+                 model: str,
+                 retry: int = 5,
+                 wait: int = 5,
+                 key: str = None,
+                 verbose: bool = True,
+                 system_prompt: str = None,
+                 max_tokens: int = 1024,
+                 proxy: str = None,
+                 **kwargs):
+
+        self.model = model
+        self.fail_msg = 'Failed to obtain answer via bailingMM API.'
+        if key is None:
+            key = os.environ.get('BAILINGMM_API_KEY', None)
+        assert key is not None, ('Please set the API Key for bailingMM.')
+        self.key = key
+        self.headers = {"Content-Type": "application/json"}
+        super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
+
+    def image_to_base64(self, image_path):
+        with open(image_path, 'rb') as image_file:
+            encoded_string = str(base64.b64encode(image_file.read()), 'utf-8')
+            return encoded_string
+
+    def prepare_inputs(self, inputs):
+        msgs = cp.deepcopy(inputs)
+        content = []
+        for i, msg in enumerate(msgs):
+            if msg['type'] == 'text':
+                pass
+            else:
+                try:
+                    image_data = self.image_to_base64(msg['value'])
+                except Exception as e:
+                    if self.verbose:
+                        self.logger.error(e)
+                    image_data = ''
+                msg['value'] = image_data
+            content.append(msg)
+        return content
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        assert isinstance(inputs, str) or isinstance(inputs, list)
+        start = time.time()
+        inputs = [inputs] if isinstance(inputs, str) else inputs
+
+        messages = self.prepare_inputs(inputs)
+
+        service_url = "https://bailingchat.alipay.com/api/proxy/eval/antgmm/completions"
+
+        payload = {
+            "structInput": messages,
+            "sk": self.key,
+            "timeout": 180000
+        }
+        response = requests.post(service_url, headers=self.headers, json=payload)
+        if self.verbose:
+            self.logger.info('Time for requesting is:')
+            self.logger.info(time.time() - start)
+        try:
+            assert response.status_code == 200
+            output = json.loads(response.text)
+            answer = output['preds']['pred']
+            if self.verbose:
+                self.logger.info(f'inputs: {inputs}\nanswer: {answer}')
+            return 0, answer, 'Succeeded! '
+        except Exception as e:
+            if self.verbose:
+                self.logger.error(e)
+                self.logger.error(f'The input messages are {inputs}.')
+            return -1, self.fail_msg, ''
+
+
+class bailingMMAPI(bailingMMWrapper):
+
+    def generate(self, message, dataset=None):
+        return super(bailingMMAPI, self).generate(message, dataset=dataset)

vlmeval/VLMEvalKit_old/vlmeval/api/base.py

@@ -0,0 +1,289 @@
+import time
+import random as rd
+from abc import abstractmethod
+import os.path as osp
+import copy as cp
+from ..smp import get_logger, parse_file, concat_images_vlmeval, LMUDataRoot, md5, decode_base64_to_image_file
+
+
+class BaseAPI:
+
+    allowed_types = ['text', 'image']
+    INTERLEAVE = True
+    INSTALL_REQ = False
+
+    def __init__(self,
+                 retry=10,
+                 wait=3,
+                 system_prompt=None,
+                 verbose=True,
+                 fail_msg='Failed to obtain answer via API.',
+                 **kwargs):
+        """Base Class for all APIs.
+
+        Args:
+            retry (int, optional): The retry times for `generate_inner`. Defaults to 10.
+            wait (int, optional): The wait time after each failed retry of `generate_inner`. Defaults to 3.
+            system_prompt (str, optional): Defaults to None.
+            verbose (bool, optional): Defaults to True.
+            fail_msg (str, optional): The message to return when failed to obtain answer.
+                Defaults to 'Failed to obtain answer via API.'.
+            **kwargs: Other kwargs for `generate_inner`.
+        """
+
+        self.wait = wait
+        self.retry = retry
+        self.system_prompt = system_prompt
+        self.verbose = verbose
+        self.fail_msg = fail_msg
+        self.logger = get_logger('ChatAPI')
+
+        if len(kwargs):
+            self.logger.info(f'BaseAPI received the following kwargs: {kwargs}')
+            self.logger.info('Will try to use them as kwargs for `generate`. ')
+        self.default_kwargs = kwargs
+
+    @abstractmethod
+    def generate_inner(self, inputs, **kwargs):
+        """The inner function to generate the answer.
+
+        Returns:
+            tuple(int, str, str): ret_code, response, log
+        """
+        self.logger.warning('For APIBase, generate_inner is an abstract method. ')
+        assert 0, 'generate_inner not defined'
+        ret_code, answer, log = None, None, None
+        # if ret_code is 0, means succeed
+        return ret_code, answer, log
+
+    def working(self):
+        """If the API model is working, return True, else return False.
+
+        Returns:
+            bool: If the API model is working, return True, else return False.
+        """
+        self.old_timeout = None
+        if hasattr(self, 'timeout'):
+            self.old_timeout = self.timeout
+            self.timeout = 120
+
+        retry = 5
+        while retry > 0:
+            ret = self.generate('hello')
+            if ret is not None and ret != '' and self.fail_msg not in ret:
+                if self.old_timeout is not None:
+                    self.timeout = self.old_timeout
+                return True
+            retry -= 1
+
+        if self.old_timeout is not None:
+            self.timeout = self.old_timeout
+        return False
+
+    def check_content(self, msgs):
+        """Check the content type of the input. Four types are allowed: str, dict, liststr, listdict.
+
+        Args:
+            msgs: Raw input messages.
+
+        Returns:
+            str: The message type.
+        """
+        if isinstance(msgs, str):
+            return 'str'
+        if isinstance(msgs, dict):
+            return 'dict'
+        if isinstance(msgs, list):
+            types = [self.check_content(m) for m in msgs]
+            if all(t == 'str' for t in types):
+                return 'liststr'
+            if all(t == 'dict' for t in types):
+                return 'listdict'
+        return 'unknown'
+
+    def preproc_content(self, inputs):
+        """Convert the raw input messages to a list of dicts.
+
+        Args:
+            inputs: raw input messages.
+
+        Returns:
+            list(dict): The preprocessed input messages. Will return None if failed to preprocess the input.
+        """
+        if self.check_content(inputs) == 'str':
+            return [dict(type='text', value=inputs)]
+        elif self.check_content(inputs) == 'dict':
+            assert 'type' in inputs and 'value' in inputs
+            return [inputs]
+        elif self.check_content(inputs) == 'liststr':
+            res = []
+            for s in inputs:
+                mime, pth = parse_file(s)
+                if mime is None or mime == 'unknown':
+                    res.append(dict(type='text', value=s))
+                else:
+                    res.append(dict(type=mime.split('/')[0], value=pth))
+            return res
+        elif self.check_content(inputs) == 'listdict':
+            for item in inputs:
+                assert 'type' in item and 'value' in item
+                mime, s = parse_file(item['value'])
+                if mime is None:
+                    assert item['type'] == 'text', item['value']
+                else:
+                    assert mime.split('/')[0] == item['type']
+                    item['value'] = s
+            return inputs
+        else:
+            return None
+
+    # May exceed the context windows size, so try with different turn numbers.
+    def chat_inner(self, inputs, **kwargs):
+        _ = kwargs.pop('dataset', None)
+        while len(inputs):
+            try:
+                return self.generate_inner(inputs, **kwargs)
+            except Exception as e:
+                if self.verbose:
+                    self.logger.info(f'{type(e)}: {e}')
+                inputs = inputs[1:]
+                while len(inputs) and inputs[0]['role'] != 'user':
+                    inputs = inputs[1:]
+                continue
+        return -1, self.fail_msg + ': ' + 'Failed with all possible conversation turns.', None
+
+    def chat(self, messages, **kwargs1):
+        """The main function for multi-turn chatting. Will call `chat_inner` with the preprocessed input messages."""
+        assert hasattr(self, 'chat_inner'), 'The API model should has the `chat_inner` method. '
+        for msg in messages:
+            assert isinstance(msg, dict) and 'role' in msg and 'content' in msg, msg
+            assert self.check_content(msg['content']) in ['str', 'dict', 'liststr', 'listdict'], msg
+            msg['content'] = self.preproc_content(msg['content'])
+        # merge kwargs
+        kwargs = cp.deepcopy(self.default_kwargs)
+        kwargs.update(kwargs1)
+
+        answer = None
+        # a very small random delay [0s - 0.5s]
+        T = rd.random() * 0.5
+        time.sleep(T)
+
+        assert messages[-1]['role'] == 'user'
+
+        for i in range(self.retry):
+            try:
+                ret_code, answer, log = self.chat_inner(messages, **kwargs)
+                if ret_code == 0 and self.fail_msg not in answer and answer != '':
+                    if self.verbose:
+                        print(answer)
+                    return answer
+                elif self.verbose:
+                    if not isinstance(log, str):
+                        try:
+                            log = log.text
+                        except Exception as e:
+                            self.logger.warning(f'Failed to parse {log} as an http response: {str(e)}. ')
+                    self.logger.info(f'RetCode: {ret_code}\nAnswer: {answer}\nLog: {log}')
+            except Exception as err:
+                if self.verbose:
+                    self.logger.error(f'An error occured during try {i}: ')
+                    self.logger.error(f'{type(err)}: {err}')
+            # delay before each retry
+            T = rd.random() * self.wait * 2
+            time.sleep(T)
+
+        return self.fail_msg if answer in ['', None] else answer
+
+    def preprocess_message_with_role(self, message):
+        system_prompt = ''
+        new_message = []
+
+        for data in message:
+            assert isinstance(data, dict)
+            role = data.pop('role', 'user')
+            if role == 'system':
+                system_prompt += data['value'] + '\n'
+            else:
+                new_message.append(data)
+
+        if system_prompt != '':
+            if self.system_prompt is None:
+                self.system_prompt = system_prompt
+            else:
+                self.system_prompt += '\n' + system_prompt
+        return new_message
+
+    def generate(self, message, **kwargs1):
+        """The main function to generate the answer. Will call `generate_inner` with the preprocessed input messages.
+
+        Args:
+            message: raw input messages.
+
+        Returns:
+            str: The generated answer of the Failed Message if failed to obtain answer.
+        """
+        if self.check_content(message) == 'listdict':
+            message = self.preprocess_message_with_role(message)
+
+        assert self.check_content(message) in ['str', 'dict', 'liststr', 'listdict'], f'Invalid input type: {message}'
+        message = self.preproc_content(message)
+        assert message is not None and self.check_content(message) == 'listdict'
+        for item in message:
+            assert item['type'] in self.allowed_types, f'Invalid input type: {item["type"]}'
+
+        # merge kwargs
+        kwargs = cp.deepcopy(self.default_kwargs)
+        kwargs.update(kwargs1)
+
+        answer = None
+        # a very small random delay [0s - 0.5s]
+        T = rd.random() * 0.5
+        time.sleep(T)
+
+        for i in range(self.retry):
+            try:
+                ret_code, answer, log = self.generate_inner(message, **kwargs)
+                if ret_code == 0 and self.fail_msg not in answer and answer != '':
+                    if self.verbose:
+                        print(answer)
+                    return answer
+                elif self.verbose:
+                    if not isinstance(log, str):
+                        try:
+                            log = log.text
+                        except Exception as e:
+                            self.logger.warning(f'Failed to parse {log} as an http response: {str(e)}. ')
+                    self.logger.info(f'RetCode: {ret_code}\nAnswer: {answer}\nLog: {log}')
+            except Exception as err:
+                if self.verbose:
+                    self.logger.error(f'An error occured during try {i}: ')
+                    self.logger.error(f'{type(err)}: {err}')
+            # delay before each retry
+            T = rd.random() * self.wait * 2
+            time.sleep(T)
+
+        return self.fail_msg if answer in ['', None] else answer
+
+    def message_to_promptimg(self, message, dataset=None):
+        assert not self.INTERLEAVE
+        model_name = self.__class__.__name__
+        import warnings
+        warnings.warn(
+            f'Model {model_name} does not support interleaved input. '
+            'Will use the first image and aggregated texts as prompt. ')
+        num_images = len([x for x in message if x['type'] == 'image'])
+        if num_images == 0:
+            prompt = '\n'.join([x['value'] for x in message if x['type'] == 'text'])
+            image = None
+        elif num_images == 1:
+            prompt = '\n'.join([x['value'] for x in message if x['type'] == 'text'])
+            image = [x['value'] for x in message if x['type'] == 'image'][0]
+        else:
+            prompt = '\n'.join([x['value'] if x['type'] == 'text' else '<image>' for x in message])
+            if dataset == 'BLINK':
+                image = concat_images_vlmeval(
+                    [x['value'] for x in message if x['type'] == 'image'],
+                    target_size=512)
+            else:
+                image = [x['value'] for x in message if x['type'] == 'image'][0]
+        return prompt, image

vlmeval/VLMEvalKit_old/vlmeval/api/claude.py

@@ -0,0 +1,111 @@
+from vlmeval.smp import *
+from vlmeval.api.base import BaseAPI
+from time import sleep
+import base64
+import mimetypes
+from PIL import Image
+
+url = 'https://openxlab.org.cn/gw/alles-apin-hub/v1/claude/v1/text/chat'
+headers = {
+    'alles-apin-token': '',
+    'Content-Type': 'application/json'
+}
+
+
+class Claude_Wrapper(BaseAPI):
+
+    is_api: bool = True
+
+    def __init__(self,
+                 model: str = 'claude-3-opus-20240229',
+                 key: str = None,
+                 retry: int = 10,
+                 wait: int = 3,
+                 system_prompt: str = None,
+                 verbose: bool = True,
+                 temperature: float = 0,
+                 max_tokens: int = 1024,
+                 **kwargs):
+
+        self.model = model
+        self.headers = headers
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        if key is not None:
+            self.key = key
+        else:
+            self.key = os.environ.get('ALLES', '')
+        self.headers['alles-apin-token'] = self.key
+
+        super().__init__(retry=retry, wait=wait, verbose=verbose, system_prompt=system_prompt, **kwargs)
+
+    # inputs can be a lvl-2 nested list: [content1, content2, content3, ...]
+    # content can be a string or a list of image & text
+    def prepare_itlist(self, inputs):
+        assert np.all([isinstance(x, dict) for x in inputs])
+        has_images = np.sum([x['type'] == 'image' for x in inputs])
+        if has_images:
+            content_list = []
+            for msg in inputs:
+                if msg['type'] == 'text' and msg['value'] != '':
+                    content_list.append(dict(type='text', text=msg['value']))
+                elif msg['type'] == 'image':
+                    pth = msg['value']
+                    suffix = osp.splitext(pth)[-1].lower()
+                    media_type = mimetypes.types_map.get(suffix, None)
+                    assert media_type is not None
+
+                    content_list.append(dict(
+                        type='image',
+                        source={
+                            'type': 'base64',
+                            'media_type': media_type,
+                            'data': encode_image_file_to_base64(pth, target_size=4096)
+                        }))
+        else:
+            assert all([x['type'] == 'text' for x in inputs])
+            text = '\n'.join([x['value'] for x in inputs])
+            content_list = [dict(type='text', text=text)]
+        return content_list
+
+    def prepare_inputs(self, inputs):
+        input_msgs = []
+        assert isinstance(inputs, list) and isinstance(inputs[0], dict)
+        assert np.all(['type' in x for x in inputs]) or np.all(['role' in x for x in inputs]), inputs
+        if 'role' in inputs[0]:
+            assert inputs[-1]['role'] == 'user', inputs[-1]
+            for item in inputs:
+                input_msgs.append(dict(role=item['role'], content=self.prepare_itlist(item['content'])))
+        else:
+            input_msgs.append(dict(role='user', content=self.prepare_itlist(inputs)))
+        return input_msgs
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+
+        payload = json.dumps({
+            'model': self.model,
+            'max_tokens': self.max_tokens,
+            'messages': self.prepare_inputs(inputs),
+            'system': self.system_prompt,
+            **kwargs
+        })
+        response = requests.request('POST', url, headers=headers, data=payload)
+        ret_code = response.status_code
+        ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code
+        answer = self.fail_msg
+
+        try:
+            resp_struct = json.loads(response.text)
+            answer = resp_struct['data']['content'][0]['text'].strip()
+        except Exception as err:
+            if self.verbose:
+                self.logger.error(f'{type(err)}: {err}')
+                self.logger.error(response.text if hasattr(response, 'text') else response)
+
+        return ret_code, answer, response
+
+
+class Claude3V(Claude_Wrapper):
+
+    def generate(self, message, dataset=None):
+        return super(Claude_Wrapper, self).generate(message)

vlmeval/VLMEvalKit_old/vlmeval/api/cloudwalk.py

@@ -0,0 +1,107 @@
+from ..smp import *
+import os
+from .base import BaseAPI
+
+
+class CWWrapper(BaseAPI):
+
+    is_api: bool = True
+
+    def __init__(self,
+                 model: str = 'cw-congrong-v1.5',
+                 retry: int = 10,
+                 wait: int = 5,
+                 key: str = None,
+                 verbose: bool = True,
+                 system_prompt: str = None,
+                 temperature: float = 0,
+                 timeout: int = 600,
+                 api_base: str = 'http://cwapi-vlm01.cw_rb.azurebot.tk/v1/chat/completions',
+                 max_tokens: int = 1024,
+                 img_size: int = 512,
+                 img_detail: str = 'low',
+                 **kwargs):
+
+        self.model = model
+        self.cur_idx = 0
+        self.fail_msg = 'Failed to obtain answer via API. '
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+
+        base = os.environ.get('CW_API_BASE', None)
+        self.api_base = base if base is not None else api_base
+
+        env_key = os.environ.get('CW_API_KEY', None)
+        self.key = env_key if env_key is not None else key
+        assert self.key is not None, 'API key not provided. Please set CW_API_KEY environment variable or \
+            pass it to the constructor.'
+
+        assert img_size > 0 or img_size == -1
+        self.img_size = -1  # allways send full size image
+        assert img_detail in ['high', 'low']
+        self.img_detail = img_detail
+
+        self.vision = True
+        self.timeout = timeout
+
+        super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
+
+    # inputs can be a lvl-2 nested list: [content1, content2, content3, ...]
+    # content can be a string or a list of image & text
+    def prepare_inputs(self, inputs):
+        input_msgs = []
+        if self.system_prompt is not None:
+            input_msgs.append(dict(role='system', content=self.system_prompt))
+        has_images = np.sum([x['type'] == 'image' for x in inputs])
+        if has_images:
+            content_list = []
+            for msg in inputs:
+                if msg['type'] == 'text':
+                    content_list.append(dict(type='text', text=msg['value']))
+                elif msg['type'] == 'image':
+                    from PIL import Image
+                    img = Image.open(msg['value'])
+                    b64 = encode_image_to_base64(img, target_size=self.img_size)
+                    img_struct = dict(url=f"data:image/jpeg;base64,{b64}", detail=self.img_detail)
+                    content_list.append(dict(type='image_url', image_url=img_struct))
+            input_msgs.append(dict(role='user', content=content_list))
+        else:
+            assert all([x['type'] == 'text' for x in inputs])
+            text = '\n'.join([x['value'] for x in inputs])
+            input_msgs.append(dict(role='user', content=text))
+        return input_msgs
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        input_msgs = self.prepare_inputs(inputs)
+        temperature = kwargs.pop('temperature', self.temperature)
+        max_tokens = kwargs.pop('max_tokens', self.max_tokens)
+
+        if 0 < max_tokens <= 100:
+            self.logger.warning(
+                'Less than 100 tokens left, '
+                'may exceed the context window with some additional meta symbols. '
+            )
+        if max_tokens <= 0:
+            return 0, self.fail_msg + 'Input string longer than context window. ', 'Length Exceeded. '
+
+        headers = {'Content-Type': 'application/json', 'Authorization': f'{self.key}'}
+        payload = dict(
+            model=self.model,
+            messages=input_msgs,
+            max_tokens=max_tokens,
+            n=1,
+            temperature=temperature,
+            **kwargs)
+        response = requests.post(self.api_base, headers=headers, data=json.dumps(payload), timeout=self.timeout * 1.1)
+        ret_code = response.status_code
+        ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code
+        answer = self.fail_msg
+        try:
+            resp_struct = json.loads(response.text)
+            answer = resp_struct['choices'][0]['message']['content'].strip()
+        except Exception as err:
+            if self.verbose:
+                self.logger.error(f'{type(err)}: {err}')
+                self.logger.error(response.text if hasattr(response, 'text') else response)
+
+        return ret_code, answer, response

vlmeval/VLMEvalKit_old/vlmeval/api/gemini.py

@@ -0,0 +1,119 @@
+from vlmeval.smp import *
+from vlmeval.api.base import BaseAPI
+
+headers = 'Content-Type: application/json'
+
+
+class GeminiWrapper(BaseAPI):
+
+    is_api: bool = True
+
+    def __init__(self,
+                 model: str = 'gemini-1.0-pro',
+                 retry: int = 5,
+                 wait: int = 5,
+                 key: str = None,
+                 verbose: bool = True,
+                 temperature: float = 0.0,
+                 system_prompt: str = None,
+                 max_tokens: int = 1024,
+                 proxy: str = None,
+                 backend='genai',
+                 project_id='vlmeval',
+                 **kwargs):
+
+        assert model in ['gemini-1.0-pro', 'gemini-1.5-pro', 'gemini-1.5-flash']
+
+        self.model = model
+        self.fail_msg = 'Failed to obtain answer via API. '
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+        if key is None:
+            key = os.environ.get('GOOGLE_API_KEY', None)
+        # Try to load backend from environment variable
+        be = os.environ.get('GOOGLE_API_BACKEND', None)
+        if be is not None and be in ['genai', 'vertex']:
+            backend = be
+
+        assert backend in ['genai', 'vertex']
+        if backend == 'genai':
+            # We have not evaluated Gemini-1.5 w. GenAI backend
+            assert key is not None  # Vertex does not require API Key
+
+        self.backend = backend
+        self.project_id = project_id
+        self.api_key = key
+
+        if proxy is not None:
+            proxy_set(proxy)
+        super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
+
+    def build_msgs_genai(self, inputs):
+        messages = [] if self.system_prompt is None else [self.system_prompt]
+        for inp in inputs:
+            if inp['type'] == 'text':
+                messages.append(inp['value'])
+            elif inp['type'] == 'image':
+                messages.append(Image.open(inp['value']))
+        return messages
+
+    def build_msgs_vertex(self, inputs):
+        from vertexai.generative_models import Part, Image
+        messages = [] if self.system_prompt is None else [self.system_prompt]
+        for inp in inputs:
+            if inp['type'] == 'text':
+                messages.append(inp['value'])
+            elif inp['type'] == 'image':
+                messages.append(Part.from_image(Image.load_from_file(inp['value'])))
+        return messages
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        if self.backend == 'genai':
+            import google.generativeai as genai
+            assert isinstance(inputs, list)
+            pure_text = np.all([x['type'] == 'text' for x in inputs])
+            genai.configure(api_key=self.api_key)
+
+            if pure_text and self.model == 'gemini-1.0-pro':
+                model = genai.GenerativeModel('gemini-1.0-pro')
+            else:
+                assert self.model in ['gemini-1.5-pro', 'gemini-1.5-flash']
+                model = genai.GenerativeModel(self.model)
+
+            messages = self.build_msgs_genai(inputs)
+            gen_config = dict(max_output_tokens=self.max_tokens, temperature=self.temperature)
+            gen_config.update(kwargs)
+            try:
+                answer = model.generate_content(
+                    messages,
+                    generation_config=genai.types.GenerationConfig(**gen_config)).text
+                return 0, answer, 'Succeeded! '
+            except Exception as err:
+                if self.verbose:
+                    self.logger.error(f'{type(err)}: {err}')
+                    self.logger.error(f'The input messages are {inputs}.')
+
+                return -1, '', ''
+        elif self.backend == 'vertex':
+            import vertexai
+            from vertexai.generative_models import GenerativeModel
+            vertexai.init(project=self.project_id, location='us-central1')
+            model_name = 'gemini-1.0-pro-vision' if self.model == 'gemini-1.0-pro' else self.model
+            model = GenerativeModel(model_name=model_name)
+            messages = self.build_msgs_vertex(inputs)
+            try:
+                resp = model.generate_content(messages)
+                answer = resp.text
+                return 0, answer, 'Succeeded! '
+            except Exception as err:
+                if self.verbose:
+                    self.logger.error(f'{type(err)}: {err}')
+                    self.logger.error(f'The input messages are {inputs}.')
+
+                return -1, '', ''
+
+
+class GeminiProVision(GeminiWrapper):
+
+    def generate(self, message, dataset=None):
+        return super(GeminiProVision, self).generate(message)

vlmeval/VLMEvalKit_old/vlmeval/api/glm_vision.py

@@ -0,0 +1,95 @@
+import requests
+requests.packages.urllib3.disable_warnings()
+
+from vlmeval.smp import *
+from vlmeval.api.base import BaseAPI
+from vlmeval.dataset import DATASET_TYPE
+from vlmeval.smp.vlm import encode_image_file_to_base64
+
+
+class GLMVisionWrapper(BaseAPI):
+
+    is_api: bool = True
+
+    def __init__(self,
+                 model: str,
+                 retry: int = 5,
+                 wait: int = 5,
+                 key: str = None,
+                 verbose: bool = True,
+                 system_prompt: str = None,
+                 max_tokens: int = 4096,
+                 proxy: str = None,
+                 **kwargs):
+
+        self.model = model
+        self.fail_msg = 'Failed to obtain answer via API. '
+        self.default_params = {
+            'top_k': 1,
+            'best_of': 1,
+            'do_sample': False,
+            'stream': False,
+            'max_tokens': max_tokens,
+            "skip_moderation": True
+        }
+        if key is None:
+            key = os.environ.get('GLMV_API_KEY', None)
+        assert key is not None, (
+            'Please set the API Key (obtain it here: '
+            'https://open.bigmodel.cn/dev/howuse/introduction)'
+        )
+        self.key = key
+        super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
+
+    def build_msgs(self, msgs_raw, system_prompt=None, dataset=None):
+        msgs = cp.deepcopy(msgs_raw)
+        content = []
+        for i, msg in enumerate(msgs):
+            if msg['type'] == 'text':
+                content.append(dict(type='text', text=msg['value']))
+            elif msg['type'] == 'image':
+                content.append(dict(type='image_url', image_url=dict(url=encode_image_file_to_base64(msg['value']))))
+        if dataset in {'HallusionBench', 'POPE'}:
+            content.append(dict(type="text", text="Please answer yes or no."))
+        ret = [dict(role='user', content=content)]
+        return ret
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        assert isinstance(inputs, str) or isinstance(inputs, list)
+        inputs = [inputs] if isinstance(inputs, str) else inputs
+
+        messages = self.build_msgs(msgs_raw=inputs, dataset=kwargs.get('dataset', None))
+
+        url = 'https://api.chatglm.cn/v1/chat/completions'
+        headers = {
+            'Content-Type': 'application/json',
+            'Request-Id': 'remote-test',
+            'Authorization': f'Bearer {self.key}'
+        }
+        payload = {
+            'model': self.model,
+            'messages': messages,
+            **self.default_params
+        }
+        response = requests.post(url, headers=headers, data=json.dumps(payload), verify=False)
+        output = []
+        try:
+            assert response.status_code == 200
+            for line in response.iter_lines():
+                data = json.loads(line.decode('utf-8').lstrip('data: '))
+                output.append(data['choices'][0]['message']['content'])
+            answer = ''.join(output).replace('</s>', '')
+            if self.verbose:
+                self.logger.info(f'inputs: {inputs}\nanswer: {answer}')
+            return 0, answer, 'Succeeded! '
+        except Exception as err:
+            if self.verbose:
+                self.logger.error(f'{type(err)}: {err}')
+                self.logger.error(f'The input messages are {inputs}.')
+            return -1, self.fail_msg, ''
+
+
+class GLMVisionAPI(GLMVisionWrapper):
+
+    def generate(self, message, dataset=None):
+        return super(GLMVisionAPI, self).generate(message, dataset=dataset)

vlmeval/VLMEvalKit_old/vlmeval/api/hf_chat_model.py

@@ -0,0 +1,246 @@
+import os
+import sys
+import os.path as osp
+import torch
+from ..smp import *
+
+
+def get_gpu_num(model_name):
+    model_name = model_name.lower()
+    kws = {
+        8: ['65b', '70b'],
+        4: ['30b', '33b', '35b', '40b'],
+        2: ['13b', '14b', '20b'],
+        1: ['6b', '7b', 'moss'],
+    }
+    for k in [8, 4, 2, 1]:
+        for keyword in kws[k]:
+            if keyword in model_name:
+                return k
+    return 8
+
+
+validated_llms = [
+    'internlm/internlm-chat-7b', 'internlm/internlm-chat-7b-8k', 'internlm/internlm-chat-20b',
+    'Qwen/Qwen-7B-Chat', 'Qwen/Qwen-14B-Chat',
+    'THUDM/chatglm2-6b', 'THUDM/chatglm2-6b-32k', 'THUDM/chatglm3-6b', 'THUDM/chatglm3-6b-32k',
+    'baichuan-inc/Baichuan2-7B-Chat', 'baichuan-inc/Baichuan2-13B-Chat',
+    'lmsys/vicuna-7b-v1.5', 'lmsys/vicuna-13b-v1.5',
+    'meta-llama/Llama-2-7b-chat-hf'
+]
+Auto_model = ['chatglm']
+
+
+class HFChatModel:
+
+    def _get_context_length(self, model, model_path):
+        # By default, we use model.config.seq_length
+        model_path = model_path.lower()
+        if 'baichuan' in model_path:
+            context_window = model.config.model_max_length
+        elif 'internlm' in model_path or 'llama' in model_path:
+            context_window = model.config.max_position_embeddings
+        elif 'vicuna' in model_path:
+            context_window = model.generation_config.max_length
+        else:
+            # chatglm & qwen
+            context_window = model.config.seq_length
+        return context_window
+
+    def _get_context_length_robust(self, model, model_path):
+        try:
+            context_window = self._get_context_length(model, model_path)
+            return context_window
+        except Exception as err:
+            self.logger.critical(f'{type(err)}: {err}')
+            self.logger.critical(
+                'Failed to extract context_window information from config / generation_config. '
+                'Please read the above code and check if the logic works for you model path'
+            )
+            raise NotImplementedError
+
+    def __init__(self,
+                 model_path,
+                 system_prompt: str = None,
+                 **kwargs):
+
+        self.logger = get_logger('HFChatModel')
+        if 'vicuna' in model_path.lower():
+            try:
+                from fastchat.model import get_conversation_template
+            except Exception as err:
+                self.logger.critical('Please install fastchat first to use vicuna. ')
+                raise err
+
+        self.explicit_device = kwargs.pop('device', None)
+
+        if self.explicit_device is None:
+            # If CUDA_VISIBLE_DEVICES is not properly set
+            if 'CUDA_VISIBLE_DEVICES' not in os.environ or os.environ['CUDA_VISIBLE_DEVICES'] == '0,1,2,3,4,5,6,7':
+                num_gpu = get_gpu_num(model_path)
+                gpu_offset = kwargs.pop('gpu_offset', 0)
+                cuda_visible_devices = ','.join([str(i) for i in range(gpu_offset, gpu_offset + num_gpu)])
+                os.environ['CUDA_VISIBLE_DEVICES'] = cuda_visible_devices
+
+        from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModel
+        from transformers.generation import GenerationConfig
+
+        if model_path not in validated_llms:
+            self.logger.warning(f'{model_path} not in validated LLMs, may have inference troubles. ')
+
+        self.model_path = model_path
+        if listinstr(Auto_model, model_path):
+            LoadModel = AutoModel
+        else:
+            LoadModel = AutoModelForCausalLM
+
+        assert osp.exists(model_path) or len(model_path.split('/')) == 2
+
+        device = self.explicit_device if self.explicit_device else 'auto'
+
+        precision = {}
+        if 'internlm-chat-7b' in model_path:
+            precision = {'torch_dtype': torch.float16}
+        elif 'internlm-chat-20b' in model_path:
+            precision = {'torch_dtype': torch.bfloat16}
+
+        self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+        model = LoadModel.from_pretrained(model_path, trust_remote_code=True, device_map='cpu', **precision)
+        model = model.eval()
+
+        if device != 'cpu':
+            model = model.to(f'cuda:{device}' if isinstance(device, int) else 'cuda')
+        try:
+            model.generation_config = GenerationConfig.from_pretrained(
+                model_path, trust_remote_code=True, device_map=device)
+        except Exception as err:
+            self.logger.warning(f'{type(err)}: {err}')
+
+        torch.cuda.empty_cache()
+        self.model = model
+        self.context_length = self._get_context_length_robust(model=model, model_path=model_path)
+        self.answer_buffer = 192
+        self.system_prompt = system_prompt
+        for k, v in kwargs.items():
+            self.logger.info(f'Following args will be used for generation (If not set specifically), {k}: {v}. ')
+        self.kwargs = kwargs
+
+    def generate_str(self, input, **kwargs):
+        if 'baichuan' in self.model_path.lower():
+            messages = []
+            messages.append({'role': 'user', 'content': input})
+            resp = self.model.chat(self.tokenizer, messages, **kwargs)
+        elif 'vicuna' in self.model_path.lower():
+            from fastchat.model import get_conversation_template
+            conv = get_conversation_template('vicuna')
+            conv.append_message(conv.roles[0], input)
+            conv.append_message(conv.roles[1], None)
+            prompt = conv.get_prompt()
+            inputs = self.tokenizer([prompt], return_tensors='pt')
+            if torch.cuda.is_available():
+                for k in inputs:
+                    inputs[k] = inputs[k].cuda()
+
+            params = dict(do_sample=True, temperature=0.7, repetition_penalty=1.0, max_new_tokens=512)
+            params.update(self.kwargs)
+            params.update(kwargs)
+            outputs = self.model.generate(**inputs, **params)
+            resp = self.tokenizer.decode(
+                outputs[0][len(inputs['input_ids'][0]):],
+                skip_special_tokens=True,
+                spaces_between_special_tokens=False)
+
+        else:
+            params = self.kwargs
+            params.update(kwargs)
+            resp, _ = self.model.chat(self.tokenizer, input, history=[], **params)
+
+        return resp
+
+    def length_ok(self, inputs):
+        tot = len(self.tokenizer.encode(self.system_prompt)) if self.system_prompt is not None else 0
+        for s in inputs:
+            tot += len(self.tokenizer.encode(s))
+        return tot + self.answer_buffer < self.context_length
+
+    def generate_list(self, full_inputs, offset=0, **kwargs):
+        assert isinstance(full_inputs, list)
+
+        inputs = full_inputs[offset:]
+        if not self.length_ok(inputs):
+            return self.chat(full_inputs, offset + 1)
+
+        model_path = self.model_path.lower()
+
+        if sum([x in model_path for x in ['baichuan']]):
+            input_msgs = []
+            if self.system_prompt is not None:
+                input_msgs.append(dict(role='user', content=self.system_prompt))
+            if len(inputs):
+                assert isinstance(inputs, list) and isinstance(inputs[0], str)
+                roles = ['user', 'assistant'] if len(inputs) % 2 == 1 else ['assistant', 'user']
+                roles = roles * len(inputs)
+                for role, msg in zip(roles, inputs):
+                    input_msgs.append(dict(role=role, content=msg))
+            response = self.model.chat(self.tokenizer, input_msgs)
+        elif sum([x in model_path for x in ['vicuna']]):
+            from fastchat.model import get_conversation_template
+            conv = get_conversation_template('vicuna')
+            assert isinstance(inputs, list) and isinstance(inputs[0], str)
+            if len(inputs) % 2 == 1:
+                if self.system_prompt is not None:
+                    conv.append_message(conv.roles[0], self.system_prompt)
+                for i in range(len(inputs) // 2):
+                    conv.append_message(conv.roles[0], inputs[2 * i])
+                    conv.append_message(conv.roles[1], inputs[2 * i + 1])
+            else:
+                assert self.system_prompt is not None
+                conv.append_message(conv.roles[0], self.system_prompt)
+                conv.append_message(conv.roles[1], inputs[0])
+                for i in range(len(inputs) // 2 - 1):
+                    conv.append_message(conv.roles[0], inputs[2 * i + 1])
+                    conv.append_message(conv.roles[1], inputs[2 * i + 2])
+            conv.append_message(conv.roles[0], inputs[-1])
+            conv.append_message(conv.roles[1], None)
+            prompt = conv.get_prompt()
+            inputs = self.tokenizer([prompt], return_tensors='pt')
+            if torch.cuda.is_available():
+                for k in inputs:
+                    inputs[k] = inputs[k].cuda()
+
+            params = dict(do_sample=True, temperature=0.7, repetition_penalty=1.0, max_new_tokens=512)
+            params.update(self.kwargs)
+            params.update(kwargs)
+
+            outputs = self.model.generate(**inputs, **params)
+            response = self.tokenizer.decode(
+                outputs[0][len(inputs['input_ids'][0]):],
+                skip_special_tokens=True,
+                spaces_between_special_tokens=False)
+            response = response.lstrip('\n')
+        else:
+            # The default option, support internlm, chatglm, qwen
+            history, msg = [], None
+            if len(inputs) % 2 == 1:
+                if self.system_prompt is not None:
+                    history = [(self.system_prompt, '')]
+                for i in range(len(inputs) // 2):
+                    history.append((inputs[2 * i], inputs[2 * i + 1]))
+            else:
+                assert self.system_prompt is not None
+                history = [(self.system_prompt, inputs[0])]
+                for i in range(len(inputs) // 2 - 1):
+                    history.append((inputs[2 * i + 1], inputs[2 * i + 2]))
+            msg = inputs[-1]
+
+            params = self.kwargs
+            params.update(kwargs)
+            response, _ = self.model.chat(self.tokenizer, msg, history=history, **params)
+
+        return response, offset
+
+    def generate(self, inputs, **kwargs):
+        if isinstance(inputs, str):
+            return self.generate_str(inputs, **kwargs)
+        elif isinstance(inputs, list):
+            return self.generate_list(inputs, **kwargs)

vlmeval/VLMEvalKit_old/vlmeval/api/hunyuan.py

@@ -0,0 +1,147 @@
+from vlmeval.smp import *
+import os
+import sys
+from vlmeval.api.base import BaseAPI
+
+
+class HunyuanWrapper(BaseAPI):
+
+    is_api: bool = True
+    _apiVersion = '2023-09-01'
+    _service = 'hunyuan'
+
+    def __init__(self,
+                 model: str = 'hunyuan-vision',
+                 retry: int = 5,
+                 wait: int = 5,
+                 secret_key: str = None,
+                 secret_id: str = None,
+                 verbose: bool = True,
+                 system_prompt: str = None,
+                 temperature: float = 0,
+                 timeout: int = 60,
+                 api_base: str = 'hunyuan.tencentcloudapi.com',
+                 **kwargs):
+
+        self.model = model
+        self.cur_idx = 0
+        self.fail_msg = 'Failed to obtain answer via API. '
+        self.temperature = temperature
+
+        warnings.warn('You may need to set the env variable HUNYUAN_SECRET_ID & HUNYUAN_SECRET_KEY to use Hunyuan. ')
+
+        secret_key = os.environ.get('HUNYUAN_SECRET_KEY', secret_key)
+        assert secret_key is not None, 'Please set the environment variable HUNYUAN_SECRET_KEY. '
+        secret_id = os.environ.get('HUNYUAN_SECRET_ID', secret_id)
+        assert secret_id is not None, 'Please set the environment variable HUNYUAN_SECRET_ID. '
+
+        self.model = model
+        self.endpoint = api_base
+        self.secret_id = secret_id
+        self.secret_key = secret_key
+        self.timeout = timeout
+
+        try:
+            from tencentcloud.common import credential
+            from tencentcloud.common.profile.client_profile import ClientProfile
+            from tencentcloud.common.profile.http_profile import HttpProfile
+            from tencentcloud.hunyuan.v20230901 import hunyuan_client
+        except ImportError as err:
+            self.logger.critical('Please install tencentcloud-sdk-python to use Hunyuan API. ')
+            raise err
+
+        super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
+
+        cred = credential.Credential(self.secret_id, self.secret_key)
+        httpProfile = HttpProfile()
+        httpProfile.endpoint = self.endpoint
+        clientProfile = ClientProfile()
+        clientProfile.httpProfile = httpProfile
+        self.client = hunyuan_client.HunyuanClient(cred, 'ap-beijing', clientProfile)
+        self.logger.info(
+            f'Using Endpoint: {self.endpoint}; API Secret ID: {self.secret_id}; API Secret Key: {self.secret_key}'
+        )
+
+    # inputs can be a lvl-2 nested list: [content1, content2, content3, ...]
+    # content can be a string or a list of image & text
+    def prepare_itlist(self, inputs):
+        assert np.all([isinstance(x, dict) for x in inputs])
+        has_images = np.sum([x['type'] == 'image' for x in inputs])
+        if has_images:
+            content_list = []
+            for msg in inputs:
+                if msg['type'] == 'text':
+                    content_list.append(dict(Type='text', Text=msg['value']))
+                elif msg['type'] == 'image':
+                    from PIL import Image
+                    img = Image.open(msg['value'])
+                    b64 = encode_image_to_base64(img)
+                    img_struct = dict(Url=f'data:image/jpeg;base64,{b64}')
+                    content_list.append(dict(Type='image_url', ImageUrl=img_struct))
+        else:
+            assert all([x['type'] == 'text' for x in inputs])
+            text = '\n'.join([x['value'] for x in inputs])
+            content_list = [dict(Type='text', Text=text)]
+        return content_list
+
+    def prepare_inputs(self, inputs):
+        input_msgs = []
+        if self.system_prompt is not None:
+            input_msgs.append(dict(Role='system', Content=self.system_prompt))
+        assert isinstance(inputs, list) and isinstance(inputs[0], dict)
+        assert np.all(['type' in x for x in inputs]) or np.all(['role' in x for x in inputs]), inputs
+        if 'role' in inputs[0]:
+            assert inputs[-1]['role'] == 'user', inputs[-1]
+            for item in inputs:
+                input_msgs.append(dict(Role=item['role'], Contents=self.prepare_itlist(item['content'])))
+        else:
+            input_msgs.append(dict(Role='user', Contents=self.prepare_itlist(inputs)))
+        return input_msgs
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
+        from tencentcloud.hunyuan.v20230901 import models
+
+        input_msgs = self.prepare_inputs(inputs)
+        temperature = kwargs.pop('temperature', self.temperature)
+
+        payload = dict(
+            Model=self.model,
+            Messages=input_msgs,
+            Temperature=temperature,
+            **kwargs)
+
+        retry_counter = 0
+        while retry_counter < 3:
+            try:
+                req = models.ChatCompletionsRequest()
+                req.from_json_string(json.dumps(payload))
+                resp = self.client.ChatCompletions(req)
+                resp = json.loads(resp.to_json_string())
+                answer = resp['Choices'][0]['Message']['Content']
+                return 0, answer, resp
+            except TencentCloudSDKException as e:
+                self.logger.error(f'Got error code: {e.get_code()}')
+                if e.get_code() == 'ClientNetworkError':
+                    return -1, self.fail_msg + e.get_code(), None
+                elif e.get_code() in ['InternalError', 'ServerNetworkError']:
+                    if retry_counter == 3:
+                        return -1, self.fail_msg + e.get_code(), None
+                    retry_counter += 1
+                    continue
+                elif e.get_code() in ['LimitExceeded']:
+                    time.sleep(5)
+                    if retry_counter == 3:
+                        return -1, self.fail_msg + e.get_code(), None
+                    retry_counter += 1
+                    continue
+                else:
+                    return -1, self.fail_msg + str(e), None
+
+        return -1, self.fail_msg, None
+
+
+class HunyuanVision(HunyuanWrapper):
+
+    def generate(self, message, dataset=None):
+        return super(HunyuanVision, self).generate(message)

vlmeval/VLMEvalKit_old/vlmeval/api/jt_vl_chat.py

@@ -0,0 +1,239 @@
+import pandas as pd
+import requests
+import json
+import os
+import base64
+from vlmeval.smp import *
+from vlmeval.api.base import BaseAPI
+from vlmeval.dataset import DATASET_TYPE
+from vlmeval.dataset import img_root_map
+
+
+API_ENDPOINT = 'https://jiutian.10086.cn/kunlun/ingress/api/h3t-eeceff/92390745235a40a484d850be19e1f8b4/ai-5d7ae47ec93f4280953273c4001aafee/service-7544ea5ee3e841ad9d01e7af44acef7c/v1/chat/completions'  # noqa: E501
+APP_CODE = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI5ZGQwNmQ2ZjU4YTU0ZGY0OGEzNjRhMjQyNGMwODEyNSIsImlzcyI6ImFwaS1hdXRoLWtleSIsImV4cCI6NDg4MjkwNDA3OX0.k5t_T-955xWMndzBbx4WQQNAgm5DpMos9mHm7vkFipQ3yebCFMfyufpSxORSfEVpBaDS3Nly0dd8ygQYGnDgIQcC72vQ1xtkjCP49LNcqlceoET4rGc1zwRi76XLPSGFES4GcwvEmr7Ilth7XtqZNxcDF_Z7HyHyf1-zF0JIQETYSoxenqLU-gNteNfqRUnlyCgaKh03DscAbYvtoMUxEaFa2ZqyRSwekdHI_SPKCq9aC9G19yDPHTjeiwl1ubtyC5uMy5pERn_ClRsZS3Wyb-GmD5QQsFofrWvCiU_fVJuUiez39pYZvEP8awH0R9B7SkpQ4XOzj3fdytTPYy3g6g'  # noqa: E501
+
+
+class JTVLChatWrapper(BaseAPI):
+    is_api: bool = True
+    INTERLEAVE = False
+
+    def __init__(self,
+                 model: str = 'jt-vl-chat',
+                 retry: int = 5,
+                 wait: int = 5,
+                 api_base: str = API_ENDPOINT,
+                 key: str = APP_CODE,
+                 verbose: bool = True,
+                 system_prompt: str = None,
+                 temperature: float = 0.7,
+                 max_tokens: int = 256,
+                 proxy: str = None,
+                 **kwargs):
+        self.model = model
+
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self.api_base = api_base
+
+        if key is None:
+            key = os.environ.get('JTVLChat_API_KEY', None)
+        assert key is not None, (
+            'Please set the API Key (also called app_code, obtain it here: https://github.com/jiutiancv/JT-VL-Chat)'
+        )
+
+        self.key = key
+        super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
+
+    def dump_image(self, line, dataset):
+        """Dump the image(s) of the input line to the corresponding dataset folder.
+
+        Args:
+            line (line of pd.DataFrame): The raw input line.
+            dataset (str): The name of the dataset.
+
+        Returns:
+            str | list[str]: The paths of the dumped images.
+        """
+        ROOT = LMUDataRoot()
+        assert isinstance(dataset, str)
+
+        img_root = os.path.join(ROOT, 'images', img_root_map(dataset) if dataset in img_root_map(dataset) else dataset)
+        os.makedirs(img_root, exist_ok=True)
+        if 'image' in line:
+            if isinstance(line['image'], list):
+                tgt_path = []
+                assert 'image_path' in line
+                for img, im_name in zip(line['image'], line['image_path']):
+                    path = osp.join(img_root, im_name)
+                    if not read_ok(path):
+                        decode_base64_to_image_file(img, path)
+                    tgt_path.append(path)
+            else:
+                tgt_path = osp.join(img_root, f"{line['index']}.jpg")
+                if not read_ok(tgt_path):
+                    decode_base64_to_image_file(line['image'], tgt_path)
+                tgt_path = [tgt_path]
+        else:
+            assert 'image_path' in line
+            tgt_path = toliststr(line['image_path'])
+
+        return tgt_path
+
+    def use_custom_prompt(self, dataset):
+        assert dataset is not None
+        if listinstr(['MMMU_DEV_VAL','MMMU_TEST'], dataset):
+            return False
+        else:
+            return True
+
+    def build_multi_choice_prompt(self, line, dataset=None):
+        question = line['question']
+        hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None
+        if hint is not None:
+            question = hint + '\n' + question
+
+        options = {
+            cand: line[cand]
+            for cand in string.ascii_uppercase
+            if cand in line and not pd.isna(line[cand])
+        }
+        for key, item in options.items():
+            question += f'\n{key}. {item}'
+        prompt = question
+
+        if len(options):
+            prompt += '\n请直接回答选项字母。' if cn_string(
+                prompt) else "\nAnswer with the option's letter from the given choices directly."
+        else:
+            prompt += '\n请直接回答问题。' if cn_string(prompt) else '\nAnswer the question directly.'
+
+        return prompt
+
+    def build_prompt(self, line, dataset=None):
+        assert self.use_custom_prompt(dataset)
+        assert dataset is None or isinstance(dataset, str)
+
+        tgt_path = self.dump_image(line, dataset)
+
+        if dataset is not None and listinstr(['MME'], dataset):
+            question = line['question']
+            prompt = question + ' Answer the question using a single word or phrase.'
+        elif dataset is not None and listinstr(['HallusionBench'], dataset):
+            question = line['question']
+            prompt = question + ' Please answer yes or no. Answer the question using a single word or phrase.'
+        elif dataset is not None and DATASET_TYPE(dataset) == 'MCQ':
+            prompt = self.build_multi_choice_prompt(line, dataset)
+        elif dataset is not None and DATASET_TYPE(dataset) == 'VQA':
+            if listinstr(['MathVista', 'MathVision'], dataset):
+                prompt = line['question']
+            elif listinstr(['LLaVABench'], dataset):
+                question = line['question']
+                prompt = question + '\nAnswer this question in detail.'
+            elif listinstr(['MMVet'], dataset):
+                prompt = line['question']
+            else:
+                question = line['question']
+                prompt = question + '\nAnswer the question using a single word or phrase.'
+        else:
+            prompt = line['question']
+        message = [dict(type='text', value=prompt)]
+        message.extend([dict(type='image', value=s) for s in tgt_path])
+        return message
+
+    def message_to_promptimg(self, message, dataset=None):
+        assert not self.INTERLEAVE
+        model_name = self.__class__.__name__
+        import warnings
+        warnings.warn(
+            f'Model {model_name} does not support interleaved input. '
+            'Will use the first image and aggregated texts as prompt. ')
+        num_images = len([x for x in message if x['type'] == 'image'])
+        if num_images == 0:
+            prompt = '\n'.join([x['value'] for x in message if x['type'] == 'text'])
+            image = None
+        else:
+            prompt = '\n'.join([x['value'] for x in message if x['type'] == 'text'])
+            if dataset == 'BLINK':
+                image = concat_images_vlmeval(
+                    [x['value'] for x in message if x['type'] == 'image'],
+                    target_size=512)
+            else:
+                image = [x['value'] for x in message if x['type'] == 'image'][0]
+        return prompt, image
+
+    def get_send_data(self,prompt, image_path, temperature, max_tokens):
+        image = ''
+        with open(image_path, 'rb') as f:
+            image = str(base64.b64encode(f.read()), 'utf-8')
+        send_data = {
+            "messages": [
+                {
+                    "role": "user",
+                    "content": prompt
+                }
+            ],
+            "image_base64": image,
+            "max_tokens": max_tokens,
+            "temperature": temperature
+        }
+        return send_data
+
+    def get_send_data_no_image(self,prompt, temperature, max_tokens):
+        send_data = {
+            "messages": [
+                {
+                    "role": "user",
+                    "content": prompt
+                }
+            ],
+            "max_tokens": max_tokens,
+            "temperature": temperature
+        }
+        return send_data
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        assert isinstance(inputs, str) or isinstance(inputs, list)
+        inputs = [inputs] if isinstance(inputs, str) else inputs
+        dataset = kwargs.get('dataset', None)
+        prompt, image_path = self.message_to_promptimg(message=inputs, dataset=dataset)
+        # print("prompt:",prompt)
+        if image_path:
+            send_data = self.get_send_data(
+                prompt=prompt,
+                image_path=image_path,
+                temperature=self.temperature,
+                max_tokens=self.max_tokens)
+        else:
+            send_data = self.get_send_data_no_image(
+                prompt=prompt,
+                temperature=self.temperature,
+                max_tokens=self.max_tokens)
+
+        json_data = json.dumps(send_data)
+
+        header_dict = {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + self.key}
+
+        r = requests.post(self.api_base, headers=header_dict, data=json_data, timeout=3000)
+        try:
+            assert r.status_code == 200
+            r_json = r.json()
+            output = r_json['choices'][0]['message']['content']
+            if self.verbose:
+                self.logger.info(f'inputs: {inputs}\nanswer: {output}')
+
+            return 0,output,'Succeeded! '
+
+        except:
+            error_msg = f'Error! code {r.status_code} content: {r.content}'
+            error_con = r.content.decode('utf-8')
+            if self.verbose:
+                self.logger.error(error_msg)
+                self.logger.error(error_con)
+                self.logger.error(f'The input messages are {inputs}.')
+            return -1,error_msg,''
+
+
+class JTVLChatAPI(JTVLChatWrapper):
+
+    def generate(self, message, dataset=None):
+        return super(JTVLChatAPI, self).generate(message, dataset=dataset)

vlmeval/VLMEvalKit_old/vlmeval/api/qwen_api.py

@@ -0,0 +1,75 @@
+from http import HTTPStatus
+import os
+from vlmeval.api.base import BaseAPI
+from vlmeval.smp import *
+
+
+# Note: This is a pure language model API.
+class QwenAPI(BaseAPI):
+
+    is_api: bool = True
+
+    def __init__(self,
+                 model: str = 'qwen-max-1201',
+                 retry: int = 5,
+                 wait: int = 5,
+                 verbose: bool = True,
+                 seed: int = 2680,
+                 temperature: float = 0.0,
+                 system_prompt: str = None,
+                 key: str = None,
+                 max_tokens: int = 1024,
+                 proxy: str = None,
+                 **kwargs):
+
+        assert model in ['qwen-turbo', 'qwen-plus', 'qwen-max', 'qwen-max-1201', 'qwen-max-longcontext']
+        self.model = model
+        import dashscope
+        self.fail_msg = 'Failed to obtain answer via API. '
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+        self.seed = seed
+        if key is None:
+            key = os.environ.get('DASHSCOPE_API_KEY', None)
+        assert key is not None, (
+            'Please set the API Key (obtain it here: '
+            'https://help.aliyun.com/zh/dashscope/developer-reference/vl-plus-quick-start)'
+        )
+        dashscope.api_key = key
+        if proxy is not None:
+            proxy_set(proxy)
+        super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
+
+    @staticmethod
+    def build_msgs(msgs_raw, system_prompt=None):
+        msgs = cp.deepcopy(msgs_raw)
+        ret = []
+        if system_prompt is not None:
+            ret.append(dict(role='system', content=system_prompt))
+        for i, msg in enumerate(msgs):
+            role = 'user' if i % 2 == 0 else 'assistant'
+            ret.append(dict(role=role, content=msg))
+        return ret
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        from dashscope import MultiModalConversation
+        assert isinstance(inputs, str) or isinstance(inputs, list)
+        inputs = [inputs] if isinstance(inputs, str) else inputs
+        messages = self.build_msgs(msgs_raw=inputs, system_prompt=self.system_prompt)
+
+        import dashscope
+        response = dashscope.Generation.call(
+            model=self.model,
+            messages=messages,
+            seed=self.seed,
+            temperature=self.temperature,
+            max_tokens=self.max_tokens,
+            result_format='message',  # set the result to be "message" format.
+        )
+        if response.status_code != HTTPStatus.OK:
+            return -1, 'Error: Bad Response Statuse Code. ', f'The response status code is {response.status_code}. '
+
+        try:
+            return 0, response['output']['choices'][0]['message']['content'].strip(), 'Succeeded! '
+        except Exception as err:
+            return -1, f'Error: Failed to parse the response. {err}', response

vlmeval/VLMEvalKit_old/vlmeval/api/qwen_vl_api.py

@@ -0,0 +1,219 @@
+from __future__ import annotations
+
+import os
+import warnings
+
+from vlmeval.smp import *
+from vlmeval.api.base import BaseAPI
+from vlmeval.vlm.qwen2_vl.prompt import Qwen2VLPromptMixin
+
+
+def ensure_image_url(image: str) -> str:
+    prefixes = ['http://', 'https://', 'file://', 'data:image;']
+    if any(image.startswith(prefix) for prefix in prefixes):
+        return image
+    if os.path.exists(image):
+        return 'file://' + image
+    raise ValueError(f'Invalid image: {image}')
+
+
+class Qwen2VLAPI(Qwen2VLPromptMixin, BaseAPI):
+    is_api: bool = True
+
+    def __init__(
+        self,
+        model: str = 'qwen-vl-max-0809',
+        key: str | None = None,
+        min_pixels: int | None = None,
+        max_pixels: int | None = None,
+        max_length=1024,
+        top_p=0.001,
+        top_k=1,
+        temperature=0.01,
+        repetition_penalty=1.0,
+        presence_penalty=0.0,
+        seed=3407,
+        use_custom_prompt: bool = True,
+        **kwargs,
+    ):
+        import dashscope
+
+        self.model = model
+        self.min_pixels = min_pixels
+        self.max_pixels = max_pixels
+        self.generate_kwargs = dict(
+            max_length=max_length,
+            top_p=top_p,
+            top_k=top_k,
+            temperature=temperature,
+            repetition_penalty=repetition_penalty,
+            presence_penalty=presence_penalty,
+            seed=seed,
+        )
+
+        key = os.environ.get('DASHSCOPE_API_KEY', None) if key is None else key
+        assert key is not None, (
+            'Please set the API Key (obtain it here: '
+            'https://help.aliyun.com/zh/dashscope/developer-reference/vl-plus-quick-start)'
+        )
+        dashscope.api_key = key
+        super().__init__(use_custom_prompt=use_custom_prompt, **kwargs)
+
+    def _prepare_content(self, inputs: list[dict[str, str]], dataset: str | None = None) -> list[dict[str, str]]:
+        """
+        inputs list[dict[str, str]], each dict has keys: ['type', 'value']
+        """
+        content = []
+        for s in inputs:
+            if s['type'] == 'image':
+                item = {'type': 'image', 'image': ensure_image_url(s['value'])}
+                if dataset == 'OCRBench':
+                    item['min_pixels'] = 10 * 10 * 28 * 28
+                    warnings.warn(f"OCRBench dataset uses custom min_pixels={item['min_pixels']}")
+                    if self.max_pixels is not None:
+                        item['max_pixels'] = self.max_pixels
+                else:
+                    if self.min_pixels is not None:
+                        item['min_pixels'] = self.min_pixels
+                    if self.max_pixels is not None:
+                        item['max_pixels'] = self.max_pixels
+            elif s['type'] == 'text':
+                item = {'type': 'text', 'text': s['value']}
+            else:
+                raise ValueError(f"Invalid message type: {s['type']}, {s}")
+            content.append(item)
+        return content
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        import dashscope
+
+        messages = []
+        if self.system_prompt is not None:
+            messages.append({'role': 'system', 'content': self.system_prompt})
+        messages.append(
+            {'role': 'user', 'content': self._prepare_content(inputs, dataset=kwargs.get('dataset', None))}
+        )
+        if self.verbose:
+            print(f'\033[31m{messages}\033[0m')
+
+        # generate
+        generation_kwargs = self.generate_kwargs.copy()
+        kwargs.pop('dataset', None)
+        generation_kwargs.update(kwargs)
+        try:
+            response = dashscope.MultiModalConversation.call(
+                model=self.model,
+                messages=messages,
+                **generation_kwargs,
+            )
+            if self.verbose:
+                print(response)
+            answer = response.output.choices[0]['message']['content'][0]['text']
+            return 0, answer, 'Succeeded! '
+        except Exception as err:
+            if self.verbose:
+                self.logger.error(f'{type(err)}: {err}')
+                self.logger.error(f'The input messages are {inputs}.')
+            return -1, '', ''
+
+
+class QwenVLWrapper(BaseAPI):
+
+    is_api: bool = True
+
+    def __init__(self,
+                 model: str = 'qwen-vl-plus',
+                 retry: int = 5,
+                 wait: int = 5,
+                 key: str = None,
+                 verbose: bool = True,
+                 temperature: float = 0.0,
+                 system_prompt: str = None,
+                 max_tokens: int = 1024,
+                 proxy: str = None,
+                 **kwargs):
+
+        assert model in ['qwen-vl-plus', 'qwen-vl-max']
+        self.model = model
+        import dashscope
+        self.fail_msg = 'Failed to obtain answer via API. '
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+        if key is None:
+            key = os.environ.get('DASHSCOPE_API_KEY', None)
+        assert key is not None, (
+            'Please set the API Key (obtain it here: '
+            'https://help.aliyun.com/zh/dashscope/developer-reference/vl-plus-quick-start)'
+        )
+        dashscope.api_key = key
+        if proxy is not None:
+            proxy_set(proxy)
+        super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
+
+    # inputs can be a lvl-2 nested list: [content1, content2, content3, ...]
+    # content can be a string or a list of image & text
+    def prepare_itlist(self, inputs):
+        assert np.all([isinstance(x, dict) for x in inputs])
+        has_images = np.sum([x['type'] == 'image' for x in inputs])
+        if has_images:
+            content_list = []
+            for msg in inputs:
+                if msg['type'] == 'text':
+                    content_list.append(dict(text=msg['value']))
+                elif msg['type'] == 'image':
+                    content_list.append(dict(image='file://' + msg['value']))
+        else:
+            assert all([x['type'] == 'text' for x in inputs])
+            text = '\n'.join([x['value'] for x in inputs])
+            content_list = [dict(text=text)]
+        return content_list
+
+    def prepare_inputs(self, inputs):
+        input_msgs = []
+        if self.system_prompt is not None:
+            input_msgs.append(dict(role='system', content=self.system_prompt))
+        assert isinstance(inputs, list) and isinstance(inputs[0], dict)
+        assert np.all(['type' in x for x in inputs]) or np.all(['role' in x for x in inputs]), inputs
+        if 'role' in inputs[0]:
+            assert inputs[-1]['role'] == 'user', inputs[-1]
+            for item in inputs:
+                input_msgs.append(dict(role=item['role'], content=self.prepare_itlist(item['content'])))
+        else:
+            input_msgs.append(dict(role='user', content=self.prepare_itlist(inputs)))
+        return input_msgs
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        from dashscope import MultiModalConversation
+        assert isinstance(inputs, str) or isinstance(inputs, list)
+
+        if 'type' in inputs[0]:
+            pure_text = np.all([x['type'] == 'text' for x in inputs])
+        else:
+            pure_text = True
+            for inp in inputs:
+                if not np.all([x['type'] == 'text' for x in inp['content']]):
+                    pure_text = False
+                    break
+
+        assert not pure_text
+        messages = self.prepare_inputs(inputs)
+        gen_config = dict(max_output_tokens=self.max_tokens, temperature=self.temperature)
+        gen_config.update(kwargs)
+        try:
+            response = MultiModalConversation.call(model=self.model, messages=messages)
+            if self.verbose:
+                print(response)
+            answer = response.output.choices[0]['message']['content'][0]['text']
+            return 0, answer, 'Succeeded! '
+        except Exception as err:
+            if self.verbose:
+                self.logger.error(f'{type(err)}: {err}')
+                self.logger.error(f'The input messages are {inputs}.')
+
+            return -1, '', ''
+
+
+class QwenVLAPI(QwenVLWrapper):
+
+    def generate(self, message, dataset=None):
+        return super(QwenVLAPI, self).generate(message)

vlmeval/VLMEvalKit_old/vlmeval/api/reka.py

@@ -0,0 +1,60 @@
+from vlmeval.smp import *
+from vlmeval.api.base import BaseAPI
+from time import sleep
+import mimetypes
+
+
+class Reka_Wrapper(BaseAPI):
+
+    is_api: bool = True
+    INTERLEAVE: bool = False
+
+    def __init__(self,
+                 model: str = 'reka-flash-20240226',
+                 key: str = None,
+                 retry: int = 10,
+                 wait: int = 3,
+                 system_prompt: str = None,
+                 verbose: bool = True,
+                 temperature: float = 0,
+                 max_tokens: int = 1024,
+                 **kwargs):
+
+        try:
+            import reka
+        except ImportError:
+            raise ImportError('Please install reka by running "pip install reka-api"')
+
+        self.model = model
+        default_kwargs = dict(temperature=temperature, request_output_len=max_tokens)
+        default_kwargs.update(kwargs)
+        self.kwargs = default_kwargs
+        if key is not None:
+            self.key = key
+        else:
+            self.key = os.environ.get('REKA_API_KEY', '')
+        super().__init__(retry=retry, wait=wait, verbose=verbose, system_prompt=system_prompt, **kwargs)
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        import reka
+        reka.API_KEY = self.key
+        dataset = kwargs.pop('dataset', None)
+        prompt, image_path = self.message_to_promptimg(inputs, dataset=dataset)
+        image_b64 = encode_image_file_to_base64(image_path)
+
+        response = reka.chat(
+            model_name=self.model,
+            human=prompt,
+            media_url=f'data:image/jpeg;base64,{image_b64}',
+            **self.kwargs)
+
+        try:
+            return 0, response['text'], response
+        except Exception as err:
+            return -1, self.fail_msg + str(err), response
+
+
+class Reka(Reka_Wrapper):
+
+    def generate(self, message, dataset=None):
+        return super(Reka_Wrapper, self).generate(message)

vlmeval/VLMEvalKit_old/vlmeval/api/siliconflow.py

@@ -0,0 +1,185 @@
+from vlmeval.smp import *
+from vlmeval.api.base import BaseAPI
+from vlmeval.dataset import img_root_map
+from vlmeval.dataset import DATASET_TYPE
+
+API_BASE = 'https://api.siliconflow.cn/v1/chat/completions'
+
+
+class SiliconFlowAPI(BaseAPI):
+
+    is_api: bool = True
+
+    def __init__(self,
+                 model: str = 'deepseek-ai/DeepSeek-V2.5',
+                 retry: int = 5,
+                 wait: int = 5,
+                 key: str = None,
+                 api_base: str = API_BASE,
+                 verbose: bool = True,
+                 system_prompt: str = None,
+                 timeout: int = 60,
+                 **kwargs):
+
+        self.model = model
+        self.api_base = api_base
+
+        default_kwargs = {
+            'stream': False,
+            'temperature': 0,
+            'frequency_penalty': 0,
+            'n': 1,
+            'max_tokens': 1024,
+        }
+        for k, v in default_kwargs.items():
+            if k not in kwargs:
+                kwargs[k] = default_kwargs[k]
+        if key is not None:
+            self.key = key
+        else:
+            self.key = os.environ.get('SiliconFlow_API_KEY', '')
+        headers = {
+            "Authorization": 'Bearer {}',
+            "Content-Type": "application/json"
+        }
+        headers['Authorization'] = headers['Authorization'].format(self.key)
+        self.headers = headers
+        super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
+
+    @staticmethod
+    def build_msgs(msgs_raw):
+        messages = []
+        message = {'role': 'user', 'content': []}
+
+        def encode_image_to_base64_PNG(image_dir):
+            image = Image.open(image_dir)
+            from io import BytesIO
+            byte_stream = BytesIO()
+            image.save(byte_stream, format="PNG")
+            byte_data = byte_stream.getvalue()
+            base64_encoded_data = base64.b64encode(byte_data)
+            base64_string = base64_encoded_data.decode("utf-8")
+
+            return base64_string
+        image_b64 = None
+        for msg in msgs_raw:
+            if msg['type'] == 'image' and not image_b64:
+                image_b64 = encode_image_to_base64_PNG(msg['value'])
+                message['content'].append({
+                    'image_url': {'url': image_b64},
+                    'type': 'image_url'
+                })
+            elif msg['type'] == 'text':
+                message['content'].append({
+                    'text': msg['value'],
+                    'type': 'text'
+                })
+
+        messages.append(message)
+        return messages
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        default_kwargs = self.default_kwargs
+        default_kwargs.update(kwargs)
+
+        payload = dict(
+            model=self.model,
+            messages=self.build_msgs(msgs_raw=inputs),
+            **default_kwargs)
+
+        response = requests.post(self.api_base, headers=self.headers, data=json.dumps(payload))
+        ret_code = response.status_code
+        ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code
+
+        answer = self.fail_msg
+        try:
+            resp_struct = json.loads(response.text)
+            answer = resp_struct['choices'][0]['message']['content'].strip()
+        except:
+            pass
+        return ret_code, answer, response
+
+
+class TeleMMAPI(SiliconFlowAPI):
+
+    is_api: bool = True
+
+    def __init__(self,
+                 model: str = 'TeleAI/TeleMM',
+                 key: str = None,
+                 **kwargs):
+        super().__init__(model=model, key=key, **kwargs)
+
+    def dump_image(self, line, dataset):
+        """Dump the image(s) of the input line to the corresponding dataset folder.
+
+        Args:
+            line (line of pd.DataFrame): The raw input line.
+            dataset (str): The name of the dataset.
+
+        Returns:
+            str | list[str]: The paths of the dumped images.
+        """
+        ROOT = LMUDataRoot()
+        assert isinstance(dataset, str)
+        # img_root = osp.join(ROOT, 'images', img_root_map[dataset] if dataset in img_root_map else dataset)
+        img_root = osp.join(ROOT, 'images', img_root_map(dataset))
+        os.makedirs(img_root, exist_ok=True)
+        if 'image' in line:
+            if isinstance(line['image'], list):
+                tgt_path = []
+                assert 'image_path' in line
+                for img, im_name in zip(line['image'], line['image_path']):
+                    path = osp.join(img_root, im_name)
+                    if not read_ok(path):
+                        decode_base64_to_image_file(img, path)
+                    tgt_path.append(path)
+            else:
+                tgt_path = osp.join(img_root, f"{line['index']}.jpg")
+                if not read_ok(tgt_path):
+                    decode_base64_to_image_file(line['image'], tgt_path)
+                tgt_path = [tgt_path]
+        else:
+            assert 'image_path' in line
+            tgt_path = toliststr(line['image_path'])
+        return tgt_path
+
+    def use_custom_prompt(self, dataset):
+        assert dataset is not None
+        if listinstr(['MMDU', 'MME-RealWorld', 'MME-RealWorld-CN'], dataset):
+            # For Multi-Turn we don't have custom prompt
+            return False
+        if 'mmmu' in dataset.lower():
+            return True
+        return False
+
+    def build_mmmu(self, line):
+        question = line['question']
+        options = {
+            cand: line[cand]
+            for cand in string.ascii_uppercase
+            if cand in line and not pd.isna(line[cand])
+        }
+        options_prompt = 'Options:\n'
+        for key, item in options.items():
+            options_prompt += f'{key}. {item}\n'
+        hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None
+        prompt = ''
+        if hint is not None:
+            prompt += f'Hint: {hint}\n'
+        prompt += f'Question: {question}\n'
+        if len(options):
+            prompt += options_prompt
+            prompt += 'Please select the correct answer from the options above. \n'
+        return prompt
+
+    def build_prompt(self, line, dataset=None):
+        assert dataset is None or isinstance(dataset, str)
+        assert self.use_custom_prompt(dataset)
+        tgt_path = self.dump_image(line, dataset)
+        if 'mmmu' in dataset.lower():
+            prompt = self.build_mmmu(line)
+
+        ret = [dict(type='text', value=prompt)]
+        ret.extend([dict(type='image', value=s) for s in tgt_path])
+        return ret

vlmeval/VLMEvalKit_old/vlmeval/api/stepai.py

@@ -0,0 +1,87 @@
+from vlmeval.smp import *
+from vlmeval.api.base import BaseAPI
+
+url = 'https://api.stepfun.com/v1/chat/completions'
+headers = {
+    'Content-Type': 'application/json',
+    'Authorization': 'Bearer {}',
+}
+
+
+class StepAPI_INT(BaseAPI):
+
+    is_api: bool = True
+
+    def __init__(self,
+                 model: str = 'step-1v-8k',
+                 retry: int = 10,
+                 wait: int = 3,
+                 key: str = None,
+                 temperature: float = 0,
+                 max_tokens: int = 300,
+                 verbose: bool = True,
+                 system_prompt: str = None,
+                 **kwargs):
+        self.model = model
+        self.fail_msg = 'Fail to obtain answer via API.'
+        self.headers = headers
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self.system_prompt = system_prompt
+        if key is not None:
+            self.key = key
+        else:
+            self.key = os.environ.get('STEPAI_API_KEY', '')
+        headers['Authorization'] = headers['Authorization'].format(self.key)
+
+        super().__init__(retry=retry, wait=wait, verbose=verbose, system_prompt=system_prompt, **kwargs)
+
+    @staticmethod
+    def build_msgs(msgs_raw):
+        messages = []
+        message = {'role': 'user', 'content': []}
+
+        for msg in msgs_raw:
+            if msg['type'] == 'image':
+                image_b64 = encode_image_file_to_base64(msg['value'])
+                message['content'].append({
+                    'image_url': {'url': 'data:image/webp;base64,%s' % (image_b64)},
+                    'type': 'image_url'
+                })
+            elif msg['type'] == 'text':
+                message['content'].append({
+                    'text': msg['value'],
+                    'type': 'text'
+                })
+
+        messages.append(message)
+        return messages
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+        print(inputs, '\n')
+        payload = dict(
+            model=self.model,
+            max_tokens=self.max_tokens,
+            temperature=self.temperature,
+            messages=self.build_msgs(msgs_raw=inputs),
+            **kwargs)
+        response = requests.post(url, headers=headers, data=json.dumps(payload))
+        ret_code = response.status_code
+        ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code
+
+        answer = self.fail_msg
+        try:
+            resp_struct = json.loads(response.text)
+            answer = resp_struct['choices'][0]['message']['content'].strip()
+        except Exception as err:
+            if self.verbose:
+                self.logger.error(f'{type(err)}: {err}')
+                self.logger.error(response.text if hasattr(response, 'text') else response)
+
+        return ret_code, answer, response
+
+
+class Step1V_INT(StepAPI_INT):
+
+    def generate(self, message, dataset=None):
+        return super(StepAPI_INT, self).generate(message)

vlmeval/VLMEvalKit_old/vlmeval/api/taiyi.py

@@ -0,0 +1,192 @@
+from vlmeval.smp import *
+from vlmeval.api.base import BaseAPI
+from vlmeval.dataset import DATASET_TYPE, img_root_map
+
+
+class TaiyiWrapper(BaseAPI):
+
+    is_api: bool = True
+
+    def __init__(self,
+                 model: str = 'taiyi',
+                 retry: int = 5,
+                 wait: int = 5,
+                 key: str = None,
+                 verbose: bool = False,
+                 system_prompt: str = None,
+                 temperature: float = 0,
+                 timeout: int = 60,
+                 url: str = "https://taiyi.megvii.com/v1/chat/completions",
+                 max_tokens: int = 1024,
+                 **kwargs):
+
+        self.model = model
+        self.fail_msg = 'Failed to obtain answer via API. '
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+
+        if key is None:
+            key = os.environ.get('TAIYI_API_KEY', None)
+        assert key is not None, ('Please set the API Key ')
+        self.key = key
+
+        self.timeout = timeout
+        super().__init__(wait=wait, retry=retry, system_prompt=system_prompt, verbose=verbose, **kwargs)
+        assert url is not None, ('Please set the url ')
+        self.url = url
+        self.logger.info(f'Using url: {self.url}; API Key: {self.key}')
+
+    def use_custom_prompt(self, dataset):
+        if DATASET_TYPE(dataset) == 'Y/N' or DATASET_TYPE(dataset) == 'MCQ' or DATASET_TYPE(dataset) == 'VQA':
+            return True
+        return False
+
+    def prepare_inputs(self, inputs):
+        input_msgs = []
+        if self.system_prompt is not None:
+            input_msgs.append(dict(role='system', content=self.system_prompt))
+        has_images = np.sum([x['type'] == 'image' for x in inputs])
+        if has_images:
+            content_list = []
+            for msg in inputs:
+                if msg['type'] == 'text':
+                    content_list.append(dict(type='text', text=msg['value']))
+                elif msg['type'] == 'image':
+                    imgbytes = open(msg['value'],'rb').read()
+                    b64 = base64.b64encode(imgbytes).decode('ascii')
+                    img_struct = dict(url=f'data:image/jpeg;base64,{b64}')
+                    content_list.append(dict(type='image_url', image_url=img_struct))
+            input_msgs.append(dict(role='user', content=content_list))
+        else:
+            assert all([x['type'] == 'text' for x in inputs])
+            text = '\n'.join([x['value'] for x in inputs])
+            input_msgs.append(dict(role='user', content=text))
+        return input_msgs
+
+    def set_dump_image(self, dump_image_func):
+        self.dump_image_func = dump_image_func
+
+    def dump_image(self, line, dataset):
+        return self.dump_image_func(line)
+
+    def image_first(self, msgs):
+        nr_img = 0
+        for s in msgs:
+            if s['type'] == 'image':
+                nr_img += 1
+
+        if nr_img == 1:
+            new_msgs = []
+            img_msg = None
+            for s in msgs:
+                if s['type'] == 'text':
+                    new_msgs.append(s)
+                else:
+                    img_msg = s
+            new_msgs.insert(0, img_msg)
+        else:
+            new_msgs = msgs
+
+        return new_msgs
+
+    def build_multi_choice_prompt(self, line, dataset=None):
+        question = line['question']
+        hint = line['hint'] if ('hint' in line and not pd.isna(line['hint'])) else None
+        if hint is not None:
+            question = hint + '\n' + question
+
+        options = {
+            cand: line[cand]
+            for cand in string.ascii_uppercase
+            if cand in line and not pd.isna(line[cand])
+        }
+        for key, item in options.items():
+            question += f'\n{key}. {item}'
+        prompt = question
+
+        if len(options):
+            prompt += '\n请直接回答选项字母。' if cn_string(
+                prompt) else "\nAnswer with the option's letter from the given choices directly."
+        else:
+            prompt += '\n请直接回答问题。' if cn_string(prompt) else '\nAnswer the question directly.'
+
+        return prompt
+
+    def build_yorn_prompt(self, line, dataset=None):
+        if listinstr(['HallusionBench'], dataset):
+            pre_prompt = 'Read the following question carefully, think and solve it step by step.\n\n'
+        else:
+            pre_prompt = ''
+
+        prompt = pre_prompt + line['question'] + ' Please answer yes or no as the final answer.'
+
+        return prompt
+
+    def build_vqa_prompt(self, line, dataset=None):
+        if listinstr(['OCRBench'], dataset):
+            pre_prompt = 'Carefully identify the text in the image and answer the question.\n\n'
+        else:
+            pre_prompt = ''
+
+        if listinstr(['MMVet'], dataset):
+            post_prompt = '\nAnswer this question in detail.'
+        else:
+            post_prompt = ''
+
+        prompt = pre_prompt + line['question'] + post_prompt
+
+        return prompt
+
+    def build_prompt(self, line, dataset=None):
+        assert self.use_custom_prompt(dataset)
+        assert dataset is None or isinstance(dataset, str)
+        tgt_path = self.dump_image(line, dataset)
+
+        if DATASET_TYPE(dataset) == 'MCQ':
+            prompt = self.build_multi_choice_prompt(line, dataset)
+        elif DATASET_TYPE(dataset) == 'Y/N':
+            prompt = self.build_yorn_prompt(line, dataset)
+        elif DATASET_TYPE(dataset) == 'VQA':
+            prompt = self.build_vqa_prompt(line, dataset)
+        else:
+            raise RuntimeError(f'Invalid dataset type: {DATASET_TYPE(dataset)}')
+        message = []
+        message.extend([dict(type='image', value=s) for s in tgt_path])
+        message.extend([dict(type='text', value=prompt)])
+
+        # interleave dataset
+        if dataset.startswith('MMMU_'):
+            from .. import MMMUDataset
+            message = MMMUDataset.split_MMMU(message)
+            message = self.image_first(message)
+
+        return message
+
+    def generate_inner(self, inputs, **kwargs) -> str:
+
+        input_msgs = self.prepare_inputs(inputs)
+        temperature = kwargs.pop('temperature', self.temperature)
+
+        headers = {'Authorization': f'Bearer {self.key}'}
+        payload = dict(
+            model=self.model,
+            messages=input_msgs,
+            n=1,
+            temperature=temperature,
+            **kwargs)
+        response = requests.post(self.url, headers=headers, data=json.dumps(payload), timeout=self.timeout * 1.1)
+        ret_code = response.status_code
+        ret_code = 0 if (200 <= int(ret_code) < 300) else ret_code
+        answer = self.fail_msg
+        try:
+            resp_struct = json.loads(response.text)
+            answer = resp_struct['choices'][0]['message']['content'].strip()
+        except:
+            pass
+        return ret_code, answer, response
+
+
+class TaiyiAPI(TaiyiWrapper):
+
+    def generate(self, message, dataset=None):
+        return super(TaiyiAPI, self).generate(message)