Upload 11 files

config.json

@@ -0,0 +1,71 @@
+{
+  "_name_or_path": "jinaai/jina-clip-v1",
+  "add_projections": false,
+  "architectures": [
+    "JinaCLIPModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "jinaai/jina-clip-implementation--configuration_clip.JinaCLIPConfig",
+    "AutoModel": "jinaai/jina-clip-implementation--modeling_clip.JinaCLIPModel"
+  },
+  "initializer_factor": 1.0,
+  "logit_scale_init_value": 2.6592,
+  "model_type": "jina_clip",
+  "projection_dim": 768,
+  "text_config": {
+    "_name_or_path": "",
+    "embed_dim": 768,
+    "hf_model_config_kwargs": {
+      "use_flash_attn": false
+    },
+    "hf_model_name_or_path": "jinaai/jina-bert-flash-implementation",
+    "model_type": "jina_clip_text",
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "pooler_type": "mean_pooler",
+    "proj_bias": false,
+    "proj_type": null,
+    "transformers_version": "4.36.2",
+    "use_bfloat16": false
+  },
+  "torch_dtype": "float32",
+  "transformers_version": null,
+  "vision_config": {
+    "_name_or_path": "",
+    "embed_dim": 768,
+    "fused_layer_norm": false,
+    "head_width": 64,
+    "image_size": 224,
+    "intp_freq": false,
+    "layers": 12,
+    "ls_init_value": null,
+    "mlp_ratio": 2.6667,
+    "model_type": "jina_clip_vision",
+    "naive_swiglu": true,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_dropout": 0.1,
+    "patch_size": 16,
+    "post_norm": false,
+    "prefix": null,
+    "problem_type": null,
+    "proj_type": null,
+    "pruned_heads": {},
+    "pt_hw_seq_len": 14,
+    "qkv_bias": true,
+    "remove_invalid_values": false,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rope_embeddings": true,
+    "subln": true,
+    "tie_word_embeddings": true,
+    "transformers_version": "4.36.2",
+    "use_bfloat16": false,
+    "width": 768,
+    "x_attention": false
+  }
+}

config_sentence_transformers.json

@@ -0,0 +1,10 @@
+{
+  "__version__": {
+    "sentence_transformers": "3.1.0",
+    "transformers": "4.41.2",
+    "pytorch": "2.3.1+cu121"
+  },
+  "prompts": {},
+  "default_prompt_name": null,
+  "similarity_fn_name": "cosine"
+}

custom_st.py

@@ -0,0 +1,203 @@
+import base64
+import json
+import os
+from io import BytesIO
+from typing import Any, Dict, List, Literal, Optional, Union
+
+import requests
+import torch
+from PIL import Image
+from torch import nn
+from transformers import AutoConfig, AutoImageProcessor, AutoModel, AutoTokenizer
+
+
+class Transformer(nn.Module):
+    """Huggingface AutoModel to generate token embeddings.
+    Loads the correct class, e.g. BERT / RoBERTa etc.
+
+    Args:
+        model_name_or_path: Huggingface models name
+            (https://huggingface.co/models)
+        max_seq_length: Truncate any inputs longer than max_seq_length
+        model_args: Keyword arguments passed to the Huggingface
+            Transformers model
+        tokenizer_args: Keyword arguments passed to the Huggingface
+            Transformers tokenizer
+        config_args: Keyword arguments passed to the Huggingface
+            Transformers config
+        cache_dir: Cache dir for Huggingface Transformers to store/load
+            models
+        do_lower_case: If true, lowercases the input (independent if the
+            model is cased or not)
+        tokenizer_name_or_path: Name or path of the tokenizer. When
+            None, then model_name_or_path is used
+    """
+
+    def __init__(
+        self,
+        model_name_or_path: str,
+        max_seq_length: Optional[int] = None,
+        model_args: Optional[Dict[str, Any]] = None,
+        tokenizer_args: Optional[Dict[str, Any]] = None,
+        config_args: Optional[Dict[str, Any]] = None,
+        cache_dir: Optional[str] = None,
+        do_lower_case: bool = False,
+        tokenizer_name_or_path: str = None,
+        backend: Literal['torch', 'onnx', 'openvino'] = 'torch',
+        **_,
+    ) -> None:
+        super(Transformer, self).__init__()
+        if backend != 'torch':
+            raise ValueError(
+                f'Backend \'{backend}\' is not supported, please use \'torch\' instead'
+            )
+
+        self.config_keys = ["max_seq_length", "do_lower_case"]
+        self.do_lower_case = do_lower_case
+        if model_args is None:
+            model_args = {}
+        if tokenizer_args is None:
+            tokenizer_args = {}
+        if config_args is None:
+            config_args = {}
+
+        config = AutoConfig.from_pretrained(
+            model_name_or_path, **config_args, cache_dir=cache_dir
+        )
+        self.jina_clip = AutoModel.from_pretrained(
+            model_name_or_path, config=config, cache_dir=cache_dir, **model_args
+        )
+
+        if max_seq_length is not None and "model_max_length" not in tokenizer_args:
+            tokenizer_args["model_max_length"] = max_seq_length
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            (
+                tokenizer_name_or_path
+                if tokenizer_name_or_path is not None
+                else model_name_or_path
+            ),
+            cache_dir=cache_dir,
+            **tokenizer_args,
+        )
+        self.preprocessor = AutoImageProcessor.from_pretrained(
+            (
+                tokenizer_name_or_path
+                if tokenizer_name_or_path is not None
+                else model_name_or_path
+            ),
+            cache_dir=cache_dir,
+            **tokenizer_args,
+        )
+
+        # No max_seq_length set. Try to infer from model
+        if max_seq_length is None:
+            if (
+                hasattr(self.jina_clip, "config")
+                and hasattr(self.jina_clip.config, "max_position_embeddings")
+                and hasattr(self.tokenizer, "model_max_length")
+            ):
+                max_seq_length = min(
+                    self.jina_clip.config.max_position_embeddings,
+                    self.tokenizer.model_max_length,
+                )
+
+        self.max_seq_length = max_seq_length
+
+        if tokenizer_name_or_path is not None:
+            self.jina_clip.config.tokenizer_class = self.tokenizer.__class__.__name__
+
+    def forward(
+        self, features: Dict[str, torch.Tensor]
+    ) -> Dict[str, torch.Tensor]:
+        """Returns token_embeddings, cls_token"""
+        if "input_ids" in features:
+            embedding = self.jina_clip.get_text_features(
+                input_ids=features["input_ids"]
+            )
+        else:
+            embedding = self.jina_clip.get_image_features(
+                pixel_values=features["pixel_values"]
+            )
+        return {"sentence_embedding": embedding}
+
+    def get_word_embedding_dimension(self) -> int:
+        return self.config.text_config.embed_dim
+
+    def decode_data_image(data_image_str):
+        header, data = data_image_str.split(',', 1)
+        image_data = base64.b64decode(data)
+        return Image.open(BytesIO(image_data))
+
+    def tokenize(
+        self, batch: Union[List[str]], padding: Union[str, bool] = True
+    ) -> Dict[str, torch.Tensor]:
+        """Tokenizes a text and maps tokens to token-ids"""
+        images = []
+        texts = []
+        for sample in batch:
+            if isinstance(sample, str):
+                if sample.startswith('http'):
+                    response = requests.get(sample)
+                    images.append(Image.open(BytesIO(response.content)).convert('RGB'))
+                elif sample.startswith('data:image/'):
+                    images.append(self.decode_data_image(sample).convert('RGB'))
+                else:
+                    # TODO: Make sure that Image.open fails for non-image files
+                    try:
+                        images.append(Image.open(sample).convert('RGB'))
+                    except:
+                        texts.append(sample)
+            elif isinstance(sample, Image.Image):
+                images.append(sample.convert('RGB'))
+
+        if images and texts:
+            raise ValueError('Batch must contain either images or texts, not both')
+
+        if texts:
+            return self.tokenizer(
+                texts,
+                padding=padding,
+                truncation="longest_first",
+                return_tensors="pt",
+                max_length=self.max_seq_length,
+            )
+        elif images:
+            return self.preprocessor(images)
+        return {}
+
+    def save(self, output_path: str, safe_serialization: bool = True) -> None:
+        self.jina_clip.save_pretrained(
+            output_path, safe_serialization=safe_serialization
+        )
+        self.tokenizer.save_pretrained(output_path)
+        self.preprocessor.save_pretrained(output_path)
+
+    @staticmethod
+    def load(input_path: str) -> "Transformer":
+        # Old classes used other config names than 'sentence_bert_config.json'
+        for config_name in [
+            "sentence_bert_config.json",
+            "sentence_roberta_config.json",
+            "sentence_distilbert_config.json",
+            "sentence_camembert_config.json",
+            "sentence_albert_config.json",
+            "sentence_xlm-roberta_config.json",
+            "sentence_xlnet_config.json",
+        ]:
+            sbert_config_path = os.path.join(input_path, config_name)
+            if os.path.exists(sbert_config_path):
+                break
+
+        with open(sbert_config_path) as fIn:
+            config = json.load(fIn)
+        # Don't allow configs to set trust_remote_code
+        if "model_args" in config and "trust_remote_code" in config["model_args"]:
+            config["model_args"].pop("trust_remote_code")
+        if (
+            "tokenizer_args" in config
+            and "trust_remote_code" in config["tokenizer_args"]
+        ):
+            config["tokenizer_args"].pop("trust_remote_code")
+        if "config_args" in config and "trust_remote_code" in config["config_args"]:
+            config["config_args"].pop("trust_remote_code")
+        return Transformer(model_name_or_path=input_path, **config)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5140d6df851d217296a10b3961ece2850d22b35bff37af948f2d9db33ae4aec2
+size 890733860

modules.json

@@ -0,0 +1,14 @@
+[
+    {
+        "idx":0,
+        "name":"0",
+        "path":"",
+        "type":"custom_st.Transformer"
+    },
+    {
+        "idx":2,
+        "name":"2",
+        "path":"2_Normalize",
+        "type":"sentence_transformers.models.Normalize"
+    }
+]

preprocessor_config.json

@@ -0,0 +1,22 @@
+{
+  "auto_map": {
+    "AutoImageProcessor": "jinaai/jina-clip-implementation--processing_clip.JinaCLIPImageProcessor",
+    "AutoProcessor": "jinaai/jina-clip-implementation--processing_clip.JinaCLIPProcessor"
+  },
+  "fill_color": 0,
+  "image_processor_type": "JinaCLIPImageProcessor",
+  "interpolation": "bicubic",
+  "mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "processor_class": "JinaCLIPProcessor",
+  "resize_mode": "shortest",
+  "size": 224,
+  "std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ]
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc0a69757ff58c708bc773bd911d99a6d1e25be2b0eb4fb2e06640848dbb44a9
+size 890820542

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,57 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 8192,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}