Update app.py

app.py

@@ -16,6 +16,7 @@ import cv2
 from transformers import (
     Qwen2VLForConditionalGeneration,
     Qwen2_5_VLForConditionalGeneration,
+    AutoModelForImageTextToText,
     AutoProcessor,
     TextIteratorStreamer,
 )
@@ -55,6 +56,15 @@ model_v = Qwen2_5_VLForConditionalGeneration.from_pretrained(
     torch_dtype=torch.float16
 ).to(device).eval()
 
+# Load aya-vision-8b
+MODEL_ID_A = "CohereForAI/aya-vision-8b"
+processor_a = AutoProcessor.from_pretrained(MODEL_ID_A, trust_remote_code=True)
+model_a = AutoModelForImageTextToText.from_pretrained(
+    MODEL_ID_A,
+    trust_remote_code=True,
+    torch_dtype=torch.float16
+).to(device).eval()
+
 def downsample_video(video_path):
     """
     Downsamples the video to evenly spaced frames.
@@ -95,6 +105,9 @@ def generate_image(model_name: str, text: str, image: Image.Image,
     elif model_name == "Nanonets-OCR-s":
         processor = processor_v
         model = model_v
+    elif model_name == "Aya-Vision":
+        processor = processor_a
+        model = model_a
     else:
         yield "Invalid model selected."
         return
@@ -126,6 +139,7 @@ def generate_image(model_name: str, text: str, image: Image.Image,
     buffer = ""
     for new_text in streamer:
         buffer += new_text
+        buffer = buffer.replace("<|im_end|>", "")
         time.sleep(0.01)
         yield buffer
 
@@ -148,6 +162,9 @@ def generate_video(model_name: str, text: str, video_path: str,
     elif model_name == "Nanonets-OCR-s":
         processor = processor_v
         model = model_v
+    elif model_name == "Aya-Vision":
+        processor = processor_a
+        model = model_a
     else:
         yield "Invalid model selected."
         return
@@ -190,6 +207,7 @@ def generate_video(model_name: str, text: str, video_path: str,
     buffer = ""
     for new_text in streamer:
         buffer += new_text
+        buffer = buffer.replace("<|im_end|>", "")
         time.sleep(0.01)
         yield buffer
 
@@ -200,6 +218,7 @@ image_examples = [
 ]
 
 video_examples = [
+    ["Explain the Ad in Detail", "videos/1.mp4"],
     ["Identify the main actions in the cartoon video", "videos/2.mp4"]
 ]
 
@@ -244,7 +263,7 @@ with gr.Blocks(css=css, theme="bethecloud/storj_theme") as demo:
         with gr.Column():
             output = gr.Textbox(label="Output", interactive=False, lines=2, scale=2)
             model_choice = gr.Radio(
-                choices=["Nanonets-OCR-s", "Qwen2-VL-OCR-2B-Instruct", "RolmOCR"],
+                choices=["Nanonets-OCR-s", "Qwen2-VL-OCR-2B-Instruct", "RolmOCR", "Aya-Vision"],
                 label="Select Model",
                 value="Nanonets-OCR-s"
             )
@@ -253,6 +272,7 @@ with gr.Blocks(css=css, theme="bethecloud/storj_theme") as demo:
             gr.Markdown("> [Qwen2-VL-OCR-2B-Instruct](https://huggingface.co/prithivMLmods/Qwen2-VL-OCR-2B-Instruct): qwen2-vl-ocr-2b-instruct model is a fine-tuned version of qwen2-vl-2b-instruct, tailored for tasks that involve [messy] optical character recognition (ocr), image-to-text conversion, and math problem solving with latex formatting.")
             gr.Markdown("> [Nanonets-OCR-s](https://huggingface.co/nanonets/Nanonets-OCR-s): nanonets-ocr-s is a powerful, state-of-the-art image-to-markdown ocr model that goes far beyond traditional text extraction. it transforms documents into structured markdown with intelligent content recognition and semantic tagging.")
             gr.Markdown("> [RolmOCR](https://huggingface.co/reducto/RolmOCR): rolmocr, high-quality, openly available approach to parsing pdfs and other complex documents oprical character recognition. it is designed to handle a wide range of document types, including scanned documents, handwritten text, and complex layouts.")
+            gr.Markdown("> [Aya-Vision](https://huggingface.co/CohereLabs/aya-vision-8b): cohere labs aya vision 8b is an open weights research release of an 8-billion parameter model with advanced capabilities optimized for a variety of vision-language use cases, including ocr, captioning, visual reasoning, summarization, question answering, code, and more.")
 
     image_submit.click(
         fn=generate_image,