Upload folder using huggingface_hub

README.md

@@ -11,7 +11,7 @@ pipeline_tag: image-text-to-text
 
 ## Introduction
 
-We are excited to announce the release of InternVL 2.0, the latest addition to the InternVL series of multimodal large language models. InternVL 2.0 features a variety of **instruction-tuned models**, ranging from 2 billion to 108 billion parameters. This repository contains the instruction-tuned InternVL2-40B model.
+We are excited to announce the release of InternVL 2.0, the latest addition to the InternVL series of multimodal large language models. InternVL 2.0 features a variety of **instruction-tuned models**, ranging from 1 billion to 108 billion parameters. This repository contains the instruction-tuned InternVL2-40B model.
 
 Compared to the state-of-the-art open-source multimodal large language models, InternVL 2.0 surpasses most open-source models. It demonstrates competitive performance on par with proprietary commercial models across various capabilities, including document and chart comprehension, infographics QA, scene text understanding and OCR tasks, scientific and mathematical problem solving, as well as cultural understanding and integrated multimodal capabilities.
 
@@ -60,8 +60,8 @@ InternVL 2.0 is a multimodal large language model series, featuring models of va
 |      Model Size      |   -    |   34B    |       34B        |     25.5B     |      40B      |
 |                      |        |          |                  |               |               |
 |       MVBench        |   -    |    -     |        -         |     67.5      |     72.5      |
-| Video-MME<br>wo subs |  59.9  |   59.0   |       52.0       |      TBD      |      TBD      |
-| Video-MME<br>w/ subs |  63.3  |   59.4   |       54.9       |      TBD      |      TBD      |
+| Video-MME<br>wo subs |  59.9  |   59.0   |       52.0       |     TODO      |     TODO      |
+| Video-MME<br>w/ subs |  63.3  |   59.4   |       54.9       |     TODO      |     TODO      |
 
 - We evaluate our models on MVBench by extracting 16 frames from each video, and each frame was resized to a 448x448 image.
 
@@ -71,6 +71,8 @@ Limitations: Although we have made efforts to ensure the safety of the model dur
 
 We provide an example code to run InternVL2-40B using `transformers`.
 
+We also welcome you to experience the InternVL2 series models in our [online demo](https://internvl.opengvlab.com/). Currently, due to the limited GPU resources with public IP addresses, we can only deploy models up to a maximum of 26B. We will expand soon and deploy larger models to the online demo.
+
 > Please use transformers==4.37.2 to ensure the model works normally.
 
 ```python
@@ -162,21 +164,15 @@ def load_image(image_file, input_size=448, max_num=6):
 
 
 path = 'OpenGVLab/InternVL2-40B'
-# If you have an 80G A100 GPU, you can put the entire model on a single GPU.
+# You need to set device_map='auto' to use multiple GPUs for inference.
+import os
+os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
 model = AutoModel.from_pretrained(
     path,
     torch_dtype=torch.bfloat16,
     low_cpu_mem_usage=True,
-    trust_remote_code=True).eval().cuda()
-# Otherwise, you need to set device_map='auto' to use multiple GPUs for inference.
-# import os
-# os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
-# model = AutoModel.from_pretrained(
-#     path,
-#     torch_dtype=torch.bfloat16,
-#     low_cpu_mem_usage=True,
-#     trust_remote_code=True,
-#     device_map='auto').eval()
+    trust_remote_code=True,
+    device_map='auto').eval()
 
 tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)
 # set the max number of tiles in `max_num`
@@ -340,9 +336,9 @@ from lmdeploy import pipeline, TurbomindEngineConfig, ChatTemplateConfig
 from lmdeploy.vl import load_image
 
 model = 'OpenGVLab/InternVL2-40B'
-system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。'
+system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。'
 image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg')
-chat_template_config = ChatTemplateConfig('yi-34b')
+chat_template_config = ChatTemplateConfig('internvl-zh-hermes2')
 chat_template_config.meta_instruction = system_prompt
 pipe = pipeline(model, chat_template_config=chat_template_config,
                 backend_config=TurbomindEngineConfig(session_len=8192))
@@ -356,14 +352,16 @@ If `ImportError` occurs while executing this case, please install the required d
 
 When dealing with multiple images, you can put them all in one list. Keep in mind that multiple images will lead to a higher number of input tokens, and as a result, the size of the context window typically needs to be increased.
 
+> Warning: Due to the scarcity of multi-image conversation data, the performance on multi-image tasks may be unstable, and it may require multiple attempts to achieve satisfactory results.
+
 ```python
 from lmdeploy import pipeline, TurbomindEngineConfig, ChatTemplateConfig
 from lmdeploy.vl import load_image
 from lmdeploy.vl.constants import IMAGE_TOKEN
 
 model = 'OpenGVLab/InternVL2-40B'
-system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。'
-chat_template_config = ChatTemplateConfig('yi-34b')
+system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。'
+chat_template_config = ChatTemplateConfig('internvl-zh-hermes2')
 chat_template_config.meta_instruction = system_prompt
 pipe = pipeline(model, chat_template_config=chat_template_config,
                 backend_config=TurbomindEngineConfig(session_len=8192))
@@ -388,8 +386,8 @@ from lmdeploy import pipeline, TurbomindEngineConfig, ChatTemplateConfig
 from lmdeploy.vl import load_image
 
 model = 'OpenGVLab/InternVL2-40B'
-system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。'
-chat_template_config = ChatTemplateConfig('yi-34b')
+system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。'
+chat_template_config = ChatTemplateConfig('internvl-zh-hermes2')
 chat_template_config.meta_instruction = system_prompt
 pipe = pipeline(model, chat_template_config=chat_template_config,
                 backend_config=TurbomindEngineConfig(session_len=8192))
@@ -412,8 +410,8 @@ from lmdeploy import pipeline, TurbomindEngineConfig, ChatTemplateConfig, Genera
 from lmdeploy.vl import load_image
 
 model = 'OpenGVLab/InternVL2-40B'
-system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。'
-chat_template_config = ChatTemplateConfig('yi-34b')
+system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。'
+chat_template_config = ChatTemplateConfig('internvl-zh-hermes2')
 chat_template_config.meta_instruction = system_prompt
 pipe = pipeline(model, chat_template_config=chat_template_config,
                 backend_config=TurbomindEngineConfig(session_len=8192))
@@ -428,12 +426,12 @@ print(sess.response.text)
 
 #### Service
 
-For lmdeploy v0.5.0, please configure the chat template config first. Create the following JSON file `chat_template.json`.
+To deploy InternVL2 as an API, please configure the chat template config first. Create the following JSON file `chat_template.json`.
 
 ```json
 {
     "model_name":"internvl-zh-hermes2",
-    "meta_instruction":"我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。",
+    "meta_instruction":"我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。",
     "stop_words":["<|im_start|>", "<|im_end|>"]
 }
 ```
@@ -441,16 +439,50 @@ For lmdeploy v0.5.0, please configure the chat template config first. Create the
 LMDeploy's `api_server` enables models to be easily packed into services with a single command. The provided RESTful APIs are compatible with OpenAI's interfaces. Below are an example of service startup:
 
 ```shell
-lmdeploy serve api_server OpenGVLab/InternVL2-40B --backend turbomind --chat-template chat_template.json
+lmdeploy serve api_server OpenGVLab/InternVL2-40B --model-name InternVL2-40B --backend turbomind --server-port 23333 --chat-template chat_template.json
 ```
 
-The default port of `api_server` is `23333`. After the server is launched, you can communicate with server on terminal through `api_client`:
+To use the OpenAI-style interface, you need to install OpenAI:
 
 ```shell
-lmdeploy serve api_client http://0.0.0.0:23333
+pip install openai
+```
+
+Then, use the code below to make the API call:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(api_key='YOUR_API_KEY', base_url='http://0.0.0.0:23333/v1')
+model_name = client.models.list().data[0].id
+response = client.chat.completions.create(
+    model="InternVL2-40B",
+    messages=[{
+        'role':
+        'user',
+        'content': [{
+            'type': 'text',
+            'text': 'describe this image',
+        }, {
+            'type': 'image_url',
+            'image_url': {
+                'url':
+                'https://modelscope.oss-cn-beijing.aliyuncs.com/resource/tiger.jpeg',
+            },
+        }],
+    }],
+    temperature=0.8,
+    top_p=0.8)
+print(response)
 ```
 
-You can overview and try out `api_server` APIs online by swagger UI at `http://0.0.0.0:23333`, or you can also read the API specification from [here](https://github.com/InternLM/lmdeploy/blob/main/docs/en/serving/restful_api.md).
+### vLLM
+
+TODO
+
+### Ollama
+
+TODO
 
 ## License
 
@@ -477,7 +509,7 @@ If you find this project useful in your research, please consider citing:
 
 ## 简介
 
-我们很高兴宣布 InternVL 2.0 的发布，这是 InternVL 系列多模态大语言模型的最新版本。InternVL 2.0 提供了多种**指令微调**的模型，参数从 20 亿到 1080 亿不等。此仓库包含经过指令微调的 InternVL2-40B 模型。
+我们很高兴宣布 InternVL 2.0 的发布，这是 InternVL 系列多模态大语言模型的最新版本。InternVL 2.0 提供了多种**指令微调**的模型，参数从 10 亿到 1080 亿不等。此仓库包含经过指令微调的 InternVL2-40B 模型。
 
 与最先进的开源多模态大语言模型相比，InternVL 2.0 超越了大多数开源模型。它在各种能力上表现出与闭源商业模型相媲美的竞争力，包括文档和图表理解、信息图表问答、场景文本理解和 OCR 任务、科学和数学问题解决，以及文化理解和综合多模态能力。
 
@@ -526,8 +558,8 @@ InternVL 2.0 是一个多模态大语言模型系列，包含各种规模的模
 |       模型大小       |   -    |   34B    |       34B        |     25.5B     |      40B      |
 |                      |        |          |                  |               |               |
 |       MVBench        |   -    |    -     |        -         |     67.5      |     72.5      |
-| Video-MME<br>wo subs |  59.9  |   59.0   |       52.0       |      TBD      |      TBD      |
-| Video-MME<br>w/ subs |  63.3  |   59.4   |       54.9       |      TBD      |      TBD      |
+| Video-MME<br>wo subs |  59.9  |   59.0   |       52.0       |     TODO      |     TODO      |
+| Video-MME<br>w/ subs |  63.3  |   59.4   |       54.9       |     TODO      |     TODO      |
 
 - 我们通过从每个视频中提取16帧来评估我们的模型在MVBench上的性能，每个视频帧被调整为448x448的图像。
 
@@ -537,6 +569,8 @@ InternVL 2.0 是一个多模态大语言模型系列，包含各种规模的模
 
 我们提供了一个示例代码，用于使用 `transformers` 运行 InternVL2-40B。
 
+我们也欢迎你在我们的[在线demo](https://internvl.opengvlab.com/)中体验InternVL2的系列模型。目前，由于具备公网IP地址的GPU资源有限，我们目前只能部署最大到26B的模型。我们会在不久之后进行扩容，把更大的模型部署到在线demo上，敬请期待。
+
 > 请使用 transformers==4.37.2 以确保模型正常运行。
 
 示例代码请[点击这里](#quick-start)。
@@ -560,9 +594,9 @@ from lmdeploy import pipeline, TurbomindEngineConfig, ChatTemplateConfig
 from lmdeploy.vl import load_image
 
 model = 'OpenGVLab/InternVL2-40B'
-system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。'
+system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。'
 image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg')
-chat_template_config = ChatTemplateConfig('yi-34b')
+chat_template_config = ChatTemplateConfig('internvl-zh-hermes2')
 chat_template_config.meta_instruction = system_prompt
 pipe = pipeline(model, chat_template_config=chat_template_config,
                 backend_config=TurbomindEngineConfig(session_len=8192))
@@ -582,8 +616,8 @@ from lmdeploy.vl import load_image
 from lmdeploy.vl.constants import IMAGE_TOKEN
 
 model = 'OpenGVLab/InternVL2-40B'
-system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。'
-chat_template_config = ChatTemplateConfig('yi-34b')
+system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。'
+chat_template_config = ChatTemplateConfig('internvl-zh-hermes2')
 chat_template_config.meta_instruction = system_prompt
 pipe = pipeline(model, chat_template_config=chat_template_config,
                 backend_config=TurbomindEngineConfig(session_len=8192))
@@ -607,8 +641,8 @@ from lmdeploy import pipeline, TurbomindEngineConfig, ChatTemplateConfig
 from lmdeploy.vl import load_image
 
 model = 'OpenGVLab/InternVL2-40B'
-system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。'
-chat_template_config = ChatTemplateConfig('yi-34b')
+system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。'
+chat_template_config = ChatTemplateConfig('internvl-zh-hermes2')
 chat_template_config.meta_instruction = system_prompt
 pipe = pipeline(model, chat_template_config=chat_template_config,
                 backend_config=TurbomindEngineConfig(session_len=8192))
@@ -631,8 +665,8 @@ from lmdeploy import pipeline, TurbomindEngineConfig, ChatTemplateConfig, Genera
 from lmdeploy.vl import load_image
 
 model = 'OpenGVLab/InternVL2-40B'
-system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。'
-chat_template_config = ChatTemplateConfig('yi-34b')
+system_prompt = '我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。'
+chat_template_config = ChatTemplateConfig('internvl-zh-hermes2')
 chat_template_config.meta_instruction = system_prompt
 pipe = pipeline(model, chat_template_config=chat_template_config,
                 backend_config=TurbomindEngineConfig(session_len=8192))
@@ -647,12 +681,12 @@ print(sess.response.text)
 
 #### API部署
 
-对于 lmdeploy v0.5.0，请先配置聊天模板配置文件。创建如下的 JSON 文件 `chat_template.json`。
+为了将InternVL2部署成API，请先配置聊天模板配置文件。创建如下的 JSON 文件 `chat_template.json`。
 
 ```json
 {
     "model_name":"internvl-zh-hermes2",
-    "meta_instruction":"我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。",
+    "meta_instruction":"我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。",
     "stop_words":["<|im_start|>", "<|im_end|>"]
 }
 ```
@@ -660,16 +694,50 @@ print(sess.response.text)
 LMDeploy 的 `api_server` 使模型能够通过一个命令轻松打包成服务。提供的 RESTful API 与 OpenAI 的接口兼容。以下是服务启动的示例：
 
 ```shell
-lmdeploy serve api_server OpenGVLab/InternVL2-40B --backend turbomind --chat-template chat_template.json
+lmdeploy serve api_server OpenGVLab/InternVL2-40B --model-name InternVL2-40B --backend turbomind --server-port 23333 --chat-template chat_template.json
 ```
 
-`api_server` 的默认端口是 `23333`。服务器启动后，你可以通过 `api_client` 在终端与服务器通信：
+为了使用OpenAI风格的API接口，您需要安装OpenAI:
 
 ```shell
-lmdeploy serve api_client http://0.0.0.0:23333
+pip install openai
 ```
 
-你可以通过 `http://0.0.0.0:23333` 的 swagger UI 在线查看和试用 `api_server` 的 API，也可以从 [这里](https://github.com/InternLM/lmdeploy/blob/main/docs/en/serving/restful_api.md) 阅读 API 规范。
+然后，使用下面的代码进行API调用:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(api_key='YOUR_API_KEY', base_url='http://0.0.0.0:23333/v1')
+model_name = client.models.list().data[0].id
+response = client.chat.completions.create(
+    model="InternVL2-40B",
+    messages=[{
+        'role':
+        'user',
+        'content': [{
+            'type': 'text',
+            'text': 'describe this image',
+        }, {
+            'type': 'image_url',
+            'image_url': {
+                'url':
+                'https://modelscope.oss-cn-beijing.aliyuncs.com/resource/tiger.jpeg',
+            },
+        }],
+    }],
+    temperature=0.8,
+    top_p=0.8)
+print(response)
+```
+
+### vLLM
+
+TODO
+
+### Ollama
+
+TODO
 
 ## 开源许可证
 

config.json

@@ -87,7 +87,7 @@
     "tie_word_embeddings": false,
     "tokenizer_class": null,
     "top_k": 50,
-    "top_p": null,
+    "top_p": 1.0,
     "torch_dtype": "bfloat16",
     "torchscript": false,
     "transformers_version": "4.37.2",

configuration_intern_vit.py

@@ -1,6 +1,6 @@
 # --------------------------------------------------------
 # InternVL
-# Copyright (c) 2023 OpenGVLab
+# Copyright (c) 2024 OpenGVLab
 # Licensed under The MIT License [see LICENSE for details]
 # --------------------------------------------------------
 import os

configuration_internvl_chat.py

@@ -1,6 +1,6 @@
 # --------------------------------------------------------
 # InternVL
-# Copyright (c) 2023 OpenGVLab
+# Copyright (c) 2024 OpenGVLab
 # Licensed under The MIT License [see LICENSE for details]
 # --------------------------------------------------------
 

conversation.py

@@ -330,13 +330,16 @@ def get_conv_template(name: str) -> Conversation:
     return conv_templates[name].copy()
 
 
-# Note that for inference, using the Hermes-2 and internlm2-chat templates is equivalent.
+# Both Hermes-2 and internlm2-chat are chatml-format conversation templates. The difference
+# is that during training, the preprocessing function for the Hermes-2 template doesn't add
+# <s> at the beginning of the tokenized sequence, while the internlm2-chat template does.
+# Therefore, they are completely equivalent during inference.
 register_conv_template(
     Conversation(
         name='Hermes-2',
         system_template='<|im_start|>system\n{system_message}',
         # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。',
+        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。',
         system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
         roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
         sep_style=SeparatorStyle.MPT,
@@ -357,7 +360,7 @@ register_conv_template(
         name='internlm2-chat',
         system_template='<|im_start|>system\n{system_message}',
         # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。',
+        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。',
         system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
         roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
         sep_style=SeparatorStyle.MPT,
@@ -376,7 +379,7 @@ register_conv_template(
         name='phi3-chat',
         system_template='<|system|>\n{system_message}',
         # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。人工智能实验室致力于原始技术创新，开源开放，共享共创，推动科技进步和产业发展。',
+        # system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室及多家合作单位联合开发的多模态大语言模型。',
         system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
         roles=('<|user|>\n', '<|assistant|>\n'),
         sep_style=SeparatorStyle.MPT,

modeling_intern_vit.py

@@ -1,6 +1,6 @@
 # --------------------------------------------------------
 # InternVL
-# Copyright (c) 2023 OpenGVLab
+# Copyright (c) 2024 OpenGVLab
 # Licensed under The MIT License [see LICENSE for details]
 # --------------------------------------------------------
 from typing import Optional, Tuple, Union