Upload nanoVLM using push_to_hub

config.json

@@ -2,33 +2,37 @@
     "vit_hidden_dim": 768,
     "vit_inter_dim": 3072,
     "vit_patch_size": 16,
-    "vit_img_size": 224,
+    "vit_img_size": 256,
     "vit_n_heads": 12,
     "vit_dropout": 0.0,
     "vit_n_blocks": 12,
     "vit_ln_eps": 1e-06,
     "vit_cls_flag": false,
-    "vit_model_type": "google/siglip-base-patch16-224",
-    "lm_hidden_dim": 576,
-    "lm_inter_dim": 1536,
+    "vit_model_type": "google/siglip2-base-patch16-256",
+    "lm_hidden_dim": 960,
+    "lm_inter_dim": 2560,
     "lm_rms_eps": 1e-05,
     "lm_re_base": 100000,
     "lm_max_position_embeddings": 8192,
-    "lm_vocab_size": 49152,
-    "lm_n_heads": 9,
-    "lm_n_kv_heads": 3,
+    "lm_base_vocab_size": 49152,
+    "extra_token_amount": 1,
+    "lm_vocab_size": 49153,
+    "lm_n_heads": 15,
+    "lm_n_kv_heads": 5,
     "lm_dropout": 0.0,
-    "lm_n_blocks": 30,
+    "lm_n_blocks": 32,
     "lm_attn_scaling": 1.0,
-    "IMAGE_TOKEN_LENGTH": 49,
-    "TOTAL_SEQUENCE_LENGTH": 128,
-    "lm_max_length": 79,
+    "lm_max_length": 512,
     "lm_use_tokens": false,
     "lm_tie_weights": true,
-    "lm_model_type": "HuggingFaceTB/SmolLM2-135M",
-    "lm_tokenizer": "HuggingFaceTB/cosmo2-tokenizer",
+    "lm_model_type": "HuggingFaceTB/SmolLM2-360M-Instruct",
+    "lm_tokenizer": "HuggingFaceTB/SmolLM2-360M-Instruct",
     "lm_eos_token_id": 0,
     "mp_pixel_shuffle_factor": 2,
+    "mp_image_token_length": 64,
+    "vlm_extra_tokens": {
+        "image_token": "<|image|>"
+    },
     "vlm_load_backbone_weights": true,
     "vlm_checkpoint_path": "checkpoints",
     "hf_repo_name": "nanoVLM"

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2fe6ed72fd068054bc3e56e3c930316582ecbc75e6f3510009fed7a46174989c
-size 888372464
+oid sha256:ffb3065b7015b9e03eed41c82c173e9cb5bb45eb492a9ad1bd8f1ae129944a01
+size 1802506128