Update README.md

README.md

@@ -53,9 +53,12 @@ prompt_engine = MRPromptV3()
 
 sys_prompt = 'You are a helpful AI assistant built by MediaTek Research. The user you are helping speaks Traditional Chinese and comes from Taiwan.'
 
-def _inference(prompt, tokenizer, model, generation_config):
+def _inference(tokenizer, model, generation_config, prompt, pixel_values=None):
     inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-    output_tensors = model.generate(**inputs, generation_config=generation_config)
+    if pixel_values is None:
+        output_tensors = model.generate(**inputs, generation_config=generation_config)
+    else:
+        output_tensors = model.generate(**inputs, generation_config=generation_config, pixel_values=pixel_values.to(model.dtype))
     output_str = tokenizer.decode(output_tensors[0])
     return output_str
 ```
@@ -69,7 +72,7 @@ conversations = [
 ]
 
 prompt = prompt_engine.get_prompt(conversations)
-output_str = _inference(prompt, tokenizer, model, generation_config)
+output_str = _inference(tokenizer, model, generation_config, prompt)
 result = prompt_engine.parse_generated_str(output_str)
 print(result)
 # {'role': 'assistant', 'content': '深度學習是一種人工智慧技術，主要是透過模仿生物神經網路的結構和功能來實現。它利用大量數據進行訓練，以建立複雜的模型並使其能夠自主學習、預測或分類輸入資料。\n\n在深度學習中，通常使用多層的神經網路，每一層都包含許多相互連接的節點（稱為神經元）。這些神經元可以處理不同特徵的輸入資料，並將結果傳遞給下一層的神經元。隨著資料流向更高層次，這個過程逐漸捕捉到更抽象的概念或模式。\n\n深度學習已被廣泛應用於各種領域，如圖像識別、自然語言處理、語音識別以及遊戲等。它提供了比傳統機器學習方法更好的表現，因為它能夠從複雜且非線性的數據中提取出有用的資訊。'}
@@ -78,7 +81,24 @@ print(result)
 ## Feature: Visual Instruction Following
 
 ```python
+conversations = [
+    # {"role": "system", "content": sys_prompt},
+    {"role": "user", "content": [
+        {
+            "type": "image",
+            "image_path": "image.jpg",
+        },
+        {
+            "type": "text",
+            "text": "請問這是哪裡？"
+        },
+    ]},
+]
 
+prompt, pixel_values = prompt_engine.get_prompt(conversations)
+output_str = _inference(tokenizer, model, generation_config, prompt, pixel_values=pixel_values)
+result = prompt_engine.parse_generated_str(output_str)
+print(result)
 ```
 
 
@@ -122,7 +142,7 @@ conversations = [
 
 prompt = prompt_engine.get_prompt(conversations, functions=functions)
 
-output_str = _inference(prompt, tokenizer, model, generation_config)
+output_str = _inference(tokenizer, model, generation_config, prompt)
 result = prompt_engine.parse_generated_str(output_str)
 
 print(result)
@@ -152,7 +172,7 @@ conversations.append(
 
 prompt = prompt_engine.get_prompt(conversations, functions=functions)
 
-output_str2 = _inference(prompt, tokenizer, model, generation_config)
+output_str2 = _inference(tokenizer, model, generation_config, prompt)
 result2 = prompt_engine.parse_generated_str(output_str2)
 print(result2)
 # {'role': 'assistant', 'content': '台北目前的溫度是攝氏30度。'}