Update README.md

README.md

@@ -3692,41 +3692,80 @@ The `GME` models support three types of input: **text**, **image**, and **image-
 |[`gme-Qwen2-VL-7B`](https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-7B-Instruct) | 8.29B | 32768 | 3584 | 67.48 | 69.73 | 67.44 |
 
 ## Usage 
+**Use with sentence_transformers**
+```python
+from sentence_transformers import SentenceTransformer
+
+
+t2i_prompt = 'Find an image that matches the given text.'
+texts = [
+    "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023.",
+    "Alibaba office.",
+]
+images = [
+    'https://upload.wikimedia.org/wikipedia/commons/e/e9/Tesla_Cybertruck_damaged_window.jpg',
+    'https://upload.wikimedia.org/wikipedia/commons/e/e0/TaobaoCity_Alibaba_Xixi_Park.jpg',
+]
+
+
+gme_st = SentenceTransformer("Alibaba-NLP/gme-Qwen2-VL-2B-Instruct")
+
+# Single-modal embedding
+e_text = gme_st.encode(texts, convert_to_tensor=True)
+e_image = gme_st.encode([dict(image=i) for i in images], convert_to_tensor=True)
+print('Single-modal', (e_text @ e_image.T).tolist())
+## Single-modal [[0.356201171875, 0.06536865234375], [0.041717529296875, 0.37890625]]
+
+# How to set embedding instruction
+e_query = gme_st.encode([dict(text=t, prompt=t2i_prompt) for t in texts], convert_to_tensor=True)
+# If no prompt, we always use the default instruction.
+e_corpus = gme_st.encode([dict(image=i) for i in images], convert_to_tensor=True)
+print('Single-modal with instruction', (e_query @ e_corpus.T).tolist())
+## Single-modal with instruction [[0.425537109375, 0.1158447265625], [0.049835205078125, 0.413818359375]]
+
+# Fused-modal embedding
+e_fused = gme_st.encode([dict(text=t, image=i) for t, i in zip(texts, images)], convert_to_tensor=True)
+print('Fused-modal', (e_fused @ e_fused.T).tolist())
+## Fused-modal [[0.99951171875, 0.0556640625], [0.0556640625, 0.99951171875]]
+```
+
+
 **Use with custom code**
 
 ```python
 # You can find the script gme_inference.py in https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct/blob/main/gme_inference.py
 from gme_inference import GmeQwen2VL
 
+t2i_prompt = 'Find an image that matches the given text.'
 texts = [
-    "What kind of car is this?",
-    "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023."
+    "The Tesla Cybertruck is a battery electric pickup truck built by Tesla, Inc. since 2023.",
+    "Alibaba office.",
 ]
 images = [
-    'https://en.wikipedia.org/wiki/File:Tesla_Cybertruck_damaged_window.jpg',
-    'https://en.wikipedia.org/wiki/File:2024_Tesla_Cybertruck_Foundation_Series,_front_left_(Greenwich).jpg',
+    'https://upload.wikimedia.org/wikipedia/commons/e/e9/Tesla_Cybertruck_damaged_window.jpg',
+    'https://upload.wikimedia.org/wikipedia/commons/e/e0/TaobaoCity_Alibaba_Xixi_Park.jpg',
 ]
 
+
 gme = GmeQwen2VL("Alibaba-NLP/gme-Qwen2-VL-2B-Instruct")
 
 # Single-modal embedding
 e_text = gme.get_text_embeddings(texts=texts)
 e_image = gme.get_image_embeddings(images=images)
-print((e_text * e_image).sum(-1))
-## tensor([0.2281, 0.6001], dtype=torch.float16)
+print('Single-modal', (e_text @ e_image.T).tolist())
+## [[0.359619140625, 0.0655517578125], [0.04180908203125, 0.374755859375]]
 
 # How to set embedding instruction
-e_query = gme.get_text_embeddings(texts=texts, instruction='Find an image that matches the given text.')
+e_query = gme.get_text_embeddings(texts=texts, instruction=t2i_prompt)
 # If is_query=False, we always use the default instruction.
 e_corpus = gme.get_image_embeddings(images=images, is_query=False)
-print((e_query * e_corpus).sum(-1))
-## tensor([0.2433, 0.7051], dtype=torch.float16)
+print('Single-modal with instruction', (e_query @ e_corpus.T).tolist())
+## [[0.429931640625, 0.11505126953125], [0.049835205078125, 0.409423828125]]
 
 # Fused-modal embedding
 e_fused = gme.get_fused_embeddings(texts=texts, images=images)
-print((e_fused[0] * e_fused[1]).sum())
-## tensor(0.6108, dtype=torch.float16)
-
+print('Fused-modal', (e_fused @ e_fused.T).tolist())
+## [[1.0, 0.05511474609375], [0.05511474609375, 1.0]]
 ```
 
 ## Evaluation