Upload tokenizer

README.md

@@ -5,14 +5,14 @@ license_link: https://huggingface.co/Qwen/Qwen2.5-72B-Instruct/blob/main/LICENSE
 pipeline_tag: image-text-to-text
 library_name: transformers
 base_model:
-  - OpenGVLab/InternVL3-2B-Instruct
+- OpenGVLab/InternVL3-2B-Instruct
 base_model_relation: finetune
 datasets:
-  - OpenGVLab/MMPR-v1.2
+- OpenGVLab/MMPR-v1.2
 language:
-  - multilingual
+- multilingual
 tags:
-  - internvl
+- internvl
 ---
 
 # InternVL3-2B Transformers 🤗 Implementation

special_tokens_map.json

@@ -13,16 +13,72 @@
     "<|vision_pad|>",
     "<|image_pad|>",
     "<|video_pad|>",
-    "<img>",
-    "</img>",
-    "<IMG_CONTEXT>",
-    "<quad>",
-    "</quad>",
-    "<ref>",
-    "</ref>",
-    "<box>",
-    "</box>"
+    {
+      "content": "<img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<IMG_CONTEXT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
   ],
+  "context_image_token": "<IMG_CONTEXT>",
+  "end_image_token": "</img>",
   "eos_token": {
     "content": "<|im_end|>",
     "lstrip": false,
@@ -36,5 +92,6 @@
     "normalized": false,
     "rstrip": false,
     "single_word": false
-  }
+  },
+  "start_image_token": "<img>"
 }

tokenizer_config.json

@@ -277,14 +277,20 @@
   ],
   "bos_token": null,
   "clean_up_tokenization_spaces": false,
+  "context_image_token": "<IMG_CONTEXT>",
+  "end_image_token": "</img>",
   "eos_token": "<|im_end|>",
   "errors": "replace",
-  "extra_special_tokens": {},
+  "extra_special_tokens": {
+    "context_image_token": "<IMG_CONTEXT>",
+    "end_image_token": "</img>",
+    "start_image_token": "<img>"
+  },
   "model_max_length": 8192,
   "pad_token": "<|endoftext|>",
-  "processor_class": "InternVLProcessor",
   "return_token_type_ids": false,
   "split_special_tokens": false,
+  "start_image_token": "<img>",
   "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null
 }