Upload 4 files

Vit_3895_best_model_epoch25.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39065c8493865b8f783eb481307ba87dd494f740f8f9f379fd4e83b3f615cce0
+size 849042907

Vit_3895_label_encoder_best.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9416a32d4f5550387283d0fccc2fdbaa37cc71d18fc7f7e476a8f6aa0ee1131
+size 37191

app.py

@@ -0,0 +1,121 @@
+import os
+os.environ["STREAMLIT_HOME"] = os.getcwd()
+os.environ["STREAMLIT_RUNTIME_METRICS_ENABLED"] = "false"
+os.makedirs(".streamlit", exist_ok=True)
+
+import streamlit as st
+import torch
+import joblib
+import numpy as np
+import random
+from PIL import Image
+from transformers import AutoTokenizer, AutoModel, ViTModel, ViTImageProcessor
+
+# CPU device only
+device = torch.device("cpu")
+
+# Define Swahili VQA Model
+class SwahiliVQAModel(torch.nn.Module):
+    def __init__(self, num_answers):
+        super().__init__()
+        self.vision_encoder = ViTModel.from_pretrained('google/vit-base-patch16-224-in21k')
+        self.text_encoder = AutoModel.from_pretrained("benjamin/roberta-base-wechsel-swahili")
+        self.fusion = torch.nn.Sequential(
+            torch.nn.Linear(768 + 768, 512),
+            torch.nn.ReLU(),
+            torch.nn.Dropout(0.3),
+            torch.nn.LayerNorm(512)
+        )
+        self.classifier = torch.nn.Linear(512, num_answers)
+
+    def forward(self, image, input_ids, attention_mask):
+        vision_outputs = self.vision_encoder(pixel_values=image)
+        image_feats = vision_outputs.last_hidden_state[:, 0, :]
+        text_outputs = self.text_encoder(input_ids=input_ids, attention_mask=attention_mask)
+        text_feats = text_outputs.last_hidden_state[:, 0, :]
+        combined = torch.cat([image_feats, text_feats], dim=1)
+        fused = self.fusion(combined)
+        return self.classifier(fused)
+
+# Load label encoder
+le = joblib.load("Vit_3895_label_encoder_best.pkl")
+
+# Load model weights normally — no override
+model = SwahiliVQAModel(num_answers=len(le.classes_)).to(device)
+state_dict = torch.load("Vit_3895_best_model_epoch25.pth", map_location=device)
+model.load_state_dict(state_dict)
+model.eval()
+
+# Load tokenizer and processor
+tokenizer = AutoTokenizer.from_pretrained("benjamin/roberta-base-wechsel-swahili")
+vit_processor = ViTImageProcessor.from_pretrained('google/vit-base-patch16-224-in21k')
+
+# Streamlit UI
+st.set_page_config(page_title="Swahili VQA", layout="wide")
+st.title("🦜 Swahili Visual Question Answering (VQA)")
+
+uploaded_image = st.file_uploader("📂 Pakia picha hapa:", type=["jpg", "jpeg", "png"])
+
+def generate_random_color():
+    return f"rgb({random.randint(150, 255)}, {random.randint(80, 200)}, {random.randint(80, 200)})"
+
+col1, col2 = st.columns([1, 2], gap="large")
+
+with col1:
+    if uploaded_image:
+        st.image(uploaded_image, caption="Picha Iliyopakiwa", use_container_width=True)
+        st.markdown("<div style='margin-bottom: 25px;'></div>", unsafe_allow_html=True)
+
+with col2:
+    st.markdown("<div style='padding-top: 15px;'>", unsafe_allow_html=True)
+    question = st.text_input("💬Andika swali lako hapa:", key="question_input")
+    submit_button = st.button("📩Tuma")
+    st.markdown("</div>", unsafe_allow_html=True)
+
+    if submit_button and uploaded_image and question:
+        with st.spinner("🔍 Inachakata jibu..."):
+            image = Image.open(uploaded_image).convert("RGB")
+            image_tensor = vit_processor(images=image, return_tensors="pt")["pixel_values"]
+
+            inputs = tokenizer(question, max_length=128, padding="max_length", truncation=True, return_tensors="pt")
+            input_ids = inputs["input_ids"]
+            attention_mask = inputs["attention_mask"]
+
+            with torch.no_grad():
+                logits = model(image_tensor, input_ids, attention_mask)
+                probs = torch.softmax(logits, dim=1)
+
+            top_probs, top_indices = torch.topk(probs, 5)
+            decoded_answers = le.inverse_transform(top_indices.cpu().numpy()[0])
+
+            results = [
+                {"answer": ans, "confidence": round(prob * 100, 2)}
+                for ans, prob in zip(decoded_answers, top_probs[0].tolist())
+            ]
+
+            results = sorted(results, key=lambda x: x["confidence"], reverse=True)
+
+            st.subheader("Majibu Yanayowezekana:")
+            max_confidence = max(result["confidence"] for result in results)
+
+            for i, pred in enumerate(results):
+                bar_width = (pred["confidence"] / max_confidence) * 70
+                color = generate_random_color()
+                st.markdown(
+                    f"""
+                    <div style="margin: 4px 0; padding: 2px 0; {'border-bottom: 1px solid rgba(150, 150, 150, 0.1);' if i < len(results)-1 else ''}">
+                        <div style="font-size: 14px; font-weight: bold; margin-bottom: 2px;">
+                            {pred['answer']}
+                        </div>
+                        <div style="display: flex; align-items: center; gap: 6px;">
+                            <div style="width: {bar_width}%; height: 8px; border-radius: 3px; background: {color};"></div>
+                            <div style="font-size: 13px; min-width: 45px;">
+                                {pred['confidence']}%
+                            </div>
+                        </div>
+                    </div>
+                    """,
+                    unsafe_allow_html=True
+                )
+    else:
+        st.info("📥 Pakia picha na andika swali kisha bonyeza Tuma ili kupata jibu.")

requirements.txt

@@ -1,3 +1,5 @@
-altair
-pandas
-streamlit
+torch==2.1.2
+transformers==4.36.2
+streamlit==1.30.0
+joblib==1.3.2
+Pillow>=10.0.0