Update app.py

app.py

@@ -8,8 +8,38 @@ from decouple import Config
 config = Config('.env')
 
 
-def query_vectara(question):
-    user_message = question
+def process_image_with_openai(image):
+    image_data = convert_image_to_required_format(image)
+    openai_api_key = config('OPENAI_API_KEY')  # Make sure to have this in your .env file
+
+    data_payload = {
+        "model": "gpt-4-vision-preview",
+        "messages": [
+            {
+                "role": "user",
+                "content": image_data
+            }
+        ],
+        "max_tokens": 300
+    }
+
+    response = requests.post(
+        "https://api.openai.com/v1/chat/completions",
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {openai_api_key}"
+        },
+        json=data_payload
+    )
+
+    if response.status_code == 200:
+        return response.json()['choices'][0]['message']['content']
+    else:
+        raise Exception(f"OpenAI Error: {response.status_code}")
+
+
+def query_vectara(text):
+    user_message = text
 
     # Read authentication parameters from the .env file
     CUSTOMER_ID = config('CUSTOMER_ID')
@@ -132,10 +162,27 @@ def convert_to_markdown(vectara_response_json):
         return f"{markdown_summary}**Sources:**\n{markdown_sources}"  
     else:  
         return "No data found in the response."  
-  
+# Main function to handle the Gradio interface logic
+def process_and_query(text, image):
+    try:
+        # If an image is provided, process it with OpenAI and use the response as the text query for Vectara
+        if image is not None:
+            text = process_image_with_openai(image)
+        
+        # Now, use the text (either provided by the user or obtained from OpenAI) to query Vectara
+        vectara_response_json = query_vectara(text)
+        markdown_output = convert_to_markdown(vectara_response_json)
+        return markdown_output
+    except Exception as e:
+        return str(e)
+
+# Define the Gradio interface
 iface = gr.Interface(
-    fn=lambda text: convert_to_markdown(query_vectara(text)),
-    inputs=[gr.Textbox(label="Input Text")],
+    fn=process_and_query,
+    inputs=[
+        gr.Textbox(label="Input Text", optional=True),
+        gr.Image(label="Upload Image", optional=True)
+    ],
     outputs=[gr.Markdown(label="Output Text")],
     title="👋🏻Welcome to ⚕🗣️😷MultiMed - Access Chat ⚕🗣️😷",
     description = '''