Merge pull request #1 from edadaltocg/master

.gitignore

@@ -138,4 +138,5 @@ dmypy.json
 cython_debug/
 
 .DS_Store
-.vscode
+.vscode
+data/

README.md

@@ -16,7 +16,7 @@ Out-of-distribution (OOD) detection is an essential safety measure for machine l
 
 This demo is [online](https://huggingface.co/spaces/edadaltocg/ood-detection) at `https://huggingface.co/spaces/edadaltocg/ood-detection`
 
-## Running Gradio app locally:
+## Running Gradio app locally
 
 1. Install dependencies:
 
@@ -31,9 +31,3 @@ python app.py
 ```
 
 3. Open the app in your browser at `http://localhost:7860`.
-
-## Methods implemented
-
-- [ ] [Mahalanobis Distance](https://arxiv.org/abs/1807.03888)
-- [x] [Maximum Softmax Probability](https://arxiv.org/abs/1610.02136)
-- [x] [Energy Based Out-of-Distribution Detection](https://arxiv.org/abs/2010.03759)

app.py

@@ -3,22 +3,21 @@ Gradio demo of image classification with OOD detection.
 
 If the image example is probably OOD, the model will abstain from the prediction.
 """
-import os
-import pickle
 import json
+import logging
+import pickle
 from glob import glob
 
 import gradio as gr
-from gradio.components import Image, Label, JSON
 import numpy as np
-import torch
 import timm
+import torch
+import torch.nn.functional as F
+from gradio.components import JSON, Image, Label
 from timm.data import resolve_data_config
 from timm.data.transforms_factory import create_transform
 from torchvision.models.feature_extraction import create_feature_extractor, get_graph_node_names
 
-import logging
-
 _logger = logging.getLogger(__name__)
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -26,7 +25,7 @@ TOPK = 3
 
 # load model
 print("Loading model...")
-model = timm.create_model("resnet50", pretrained=True)
+model = timm.create_model("resnet50.tv2_in1k", pretrained=True)
 model.to(device)
 model.eval()
 
@@ -34,28 +33,25 @@ model.eval()
 idx2label = json.loads(open("ilsvrc2012.json").read())
 idx2label = {int(k): v for k, v in idx2label.items()}
 print(idx2label)
+print(idx2label.values())
 
 # transformation
 config = resolve_data_config({}, model=model)
 config["is_training"] = False
 transform = create_transform(**config)
 
-# print features names
-print(get_graph_node_names(model)[0])
-
-# load train scores
+# create feature extractor
 penultimate_features_key = "global_pool.flatten"
 logits_key = "fc"
 features_names = [penultimate_features_key, logits_key]
 
-# create feature extractor
 feature_extractor = create_feature_extractor(model, features_names)
 
-# OOD dtector thresholds
+centroids = torch.from_numpy(pickle.load(open("centroids_resnet50.tv2_in1k_igeood_logits.pkl", "rb"))).to(device)
+# OOD detector thresholds
 msp_threshold = 0.3796
 energy_threshold = 0.3781
-
-## unpickle detectors
+igeood_threshold = 2.4984
 
 
 def mahalanobis_penult(features):
@@ -72,9 +68,18 @@ def energy(logits):
     return torch.logsumexp(logits, dim=1).item()
 
 
+def igeoodlogits_vec(logits, temperature, centroids, epsilon=1e-12):
+    logits = torch.sqrt(F.softmax(logits / temperature, dim=1))
+    centroids = torch.sqrt(F.softmax(centroids / temperature, dim=1))
+    mult = logits @ centroids.T
+    stack = 2 * torch.acos(torch.clamp(mult, -1 + epsilon, 1 - epsilon))
+    return stack.mean(dim=1).item()
+
+
 def predict(image):
     # forward pass
     inputs = transform(image).unsqueeze(0)
+    inputs = inputs.to(device)
     with torch.no_grad():
         features = feature_extractor(inputs)
 
@@ -86,13 +91,16 @@ def predict(image):
 
     result = {idx2label[i.item()]: v.item() for i, v in zip(class_idxs.squeeze(), softmax.squeeze())}
     # OOD
-    msp_score = msp(features[logits_key])
-    energy_score = energy(features[logits_key])
+    msp_score = round(msp(features[logits_key]), 4)
+    energy_score = round(energy(features[logits_key]), 4)
+    igeood_scores = round(igeoodlogits_vec(features[logits_key], 1, centroids), 4)
     ood_scores = {
-        "msp": msp_score,
-        "msp_is_ood": msp_score < msp_threshold,
-        "energy": energy_score,
-        "energy_is_ood": energy_score < energy_threshold,
+        "MSP": msp_score,
+        "MSP, is the input OOD?": msp_score < msp_threshold,
+        "Energy": energy_score,
+        "Energy, is the input OOD?": energy_score < energy_threshold,
+        "Igeood": igeood_scores,
+        "Igeood, is the input OOD?": igeood_scores < igeood_threshold,
     }
     _logger.info(ood_scores)
     return result, ood_scores
@@ -100,9 +108,9 @@ def predict(image):
 
 def main():
     # image examples for demo shuffled
-    examples = glob("images/imagenet/*.jpg") + glob("images/ood/*.jpg")
+    examples = glob("images/imagenet/*") + glob("images/ood/*")
     np.random.seed(42)
-    np.random.shuffle(examples)
+    # np.random.shuffle(examples)
 
     # gradio interface
     interface = gr.Interface(
@@ -117,11 +125,24 @@ def main():
         allow_flagging="never",
         theme="default",
         title="OOD Detection 🧐",
-        description="Out-of-distribution (OOD) detection is an essential safety measure for machine learning models. This app demonstrates how these methods can be useful. They try to determine wether we can trust the predictions of a ResNet-50 model trained on ImageNet-1K. Enjoy the demo!",
-    )
-    interface.launch(
-        server_port=7860,
+        description=(
+            "Out-of-distribution (OOD) detection is an essential safety measure for machine learning models. "
+            "The objective of an OOD detector is to determine wether the input sample comes from the distribution known by the AI model. "
+            "For instance, an input that does not belong to any of the known classes or is from a different domain should be flagged by the detector.\n"
+            "In this demo we will display the decision of three OOD detectors on a ResNet-50 model trained to classify on the ImageNet-1K dataset (top-1 accuracy 80%)."
+            "This model can classify among 1000 classes from several categories, including `animals`, `vehicles`, `clothing`, `instruments`, `plants`, etc. "
+            "For the complete hierarchy of classes, please check the website https://observablehq.com/@mbostock/imagenet-hierarchy. "
+            "\n\n"
+            "## Instructions:\n"
+            "1. Upload an image of your choice or select one from the examples bar.\n"
+            "2. The model will predict the top 3 most likely classes for the image.\n"
+            "3. The OOD detectors will output their scores and decision on the image. The smaller the score, the least confident the detector is on the sample being in-distribution.\n"
+            "4. If the image is OOD, the model will abstain from the prediction and flag it to the practicioner.\n"
+            "\n\n\nEnjoy the demo!"
+        ),
+        cache_examples=True,
     )
+    interface.launch(server_port=7860)
     interface.close()
 
 

centroids_resnet50.tv2_in1k_igeood_logits.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8079b4fc02b6542210d147d98d08b6220372534a18ba7ef9e844b17ab0a1d7e
+size 4000163

imagenet_ood.py

@@ -0,0 +1,132 @@
+import logging
+import os
+from typing import Callable, Optional
+
+from torchvision.datasets import ImageFolder
+from torchvision.datasets.utils import check_integrity, download_and_extract_archive, verify_str_arg
+
+_logger = logging.getLogger(__name__)
+
+
+class ImageNetA(ImageFolder):
+    """ImageNetA dataset.
+
+    - Paper: [https://arxiv.org/abs/1907.07174](https://arxiv.org/abs/1907.07174).
+    """
+
+    base_folder = "imagenet-a"
+    url = "https://people.eecs.berkeley.edu/~hendrycks/imagenet-a.tar"
+    filename = "imagenet-a.tar"
+    tgz_md5 = "c3e55429088dc681f30d81f4726b6595"
+
+    def __init__(self, root: str, split=None, transform: Optional[Callable] = None, download: bool = False, **kwargs):
+        self.root = root
+
+        if download:
+            self.download()
+
+        if not self._check_integrity():
+            raise RuntimeError("Dataset not found or corrupted." + " You can use download=True to download it")
+
+        super().__init__(root=os.path.join(root, self.base_folder), transform=transform, **kwargs)
+
+    def _check_exists(self) -> bool:
+        return os.path.exists(os.path.join(self.root, self.base_folder))
+
+    def _check_integrity(self) -> bool:
+        return check_integrity(os.path.join(self.root, self.filename), self.tgz_md5)
+
+    def download(self) -> None:
+        if self._check_integrity() and self._check_exists():
+            _logger.debug("Files already downloaded and verified")
+            return
+        download_and_extract_archive(self.url, self.root, filename=self.filename, md5=self.tgz_md5)
+
+
+class ImageNetO(ImageNetA):
+    """ImageNetO datasets.
+
+    Contains unknown classes to ImageNet-1k.
+
+
+    - Paper: [https://arxiv.org/abs/1907.07174](https://arxiv.org/abs/1907.07174)
+    """
+
+    base_folder = "imagenet-o"
+    url = "https://people.eecs.berkeley.edu/~hendrycks/imagenet-o.tar"
+    filename = "imagenet-o.tar"
+    tgz_md5 = "86bd7a50c1c4074fb18fc5f219d6d50b"
+
+
+class ImageNetR(ImageNetA):
+    """ImageNet-R(endition) dataset.
+
+    Contains art, cartoons, deviantart, graffiti, embroidery, graphics, origami, paintings,
+    patterns, plastic objects,plush objects, sculptures, sketches, tattoos, toys,
+    and video game renditions of ImageNet-1k classes.
+
+    - Paper: [https://arxiv.org/abs/2006.16241](https://arxiv.org/abs/2006.16241)
+    """
+
+    base_folder = "imagenet-r"
+    url = "https://people.eecs.berkeley.edu/~hendrycks/imagenet-r.tar"
+    filename = "imagenet-r.tar"
+    tgz_md5 = "a61312130a589d0ca1a8fca1f2bd3337"
+
+
+class NINCOFull(ImageFolder):
+    """`NINCO` Dataset subset.
+
+    Args:
+        root (string): Root directory of dataset where directory
+            exists or will be saved to if download is set to True.
+        split (string, optional): The dataset split, not used.
+        transform (callable, optional): A function/transform that takes in an PIL image
+            and returns a transformed version. E.g, `transforms.RandomCrop`.
+        download (bool, optional): If true, downloads the dataset from the internet and
+            puts it in root directory. If dataset is already downloaded, it is not
+            downloaded again.
+        **kwargs: Additional arguments passed to :class:`~torchvision.datasets.ImageFolder`.
+    """
+
+    PAPER_URL = "https://arxiv.org/pdf/2306.00826.pdf"
+    base_folder = "ninco"
+    filename = "NINCO_all.tar.gz"
+    file_md5 = "b9ffae324363cd900a81ce3c367cd834"
+    url = "https://zenodo.org/record/8013288/files/NINCO_all.tar.gz"
+    # size: 15393
+
+    def __init__(
+        self, root: str, split=None, transform: Optional[Callable] = None, download: bool = False, **kwargs
+    ) -> None:
+        self.root = os.path.expanduser(root)
+        self.dataset_folder = os.path.join(self.root, self.base_folder)
+        self.archive = os.path.join(self.root, self.filename)
+
+        if download:
+            self.download()
+
+        if not self._check_integrity():
+            raise RuntimeError("Dataset not found or corrupted." + " You can use download=True to download it")
+
+        super().__init__(self.dataset_folder, transform=transform, **kwargs)
+
+    def _check_integrity(self) -> bool:
+        return check_integrity(self.archive, self.file_md5)
+
+    def _check_exists(self) -> bool:
+        return os.path.exists(self.dataset_folder)
+
+    def download(self) -> None:
+        if self._check_integrity() and self._check_exists():
+            return
+        download_and_extract_archive(
+            self.url, download_root=self.root, extract_root=self.dataset_folder, md5=self.file_md5
+        )
+
+
+if __name__ == "__main__":
+    ImageNetR(root="data", download=True)
+    ImageNetO(root="data", download=True)
+    ImageNetA(root="data", download=True)
+    NINCOFull(root="data", download=True)

requirements.txt

@@ -1,5 +1,87 @@
-torch
-torchvision
-transformers
-timm
-gradio
+aiofiles==23.2.1
+aiohttp==3.8.3
+aiosignal==1.3.1
+altair==5.1.0
+anyio==3.6.2
+async-timeout==4.0.2
+attrs==23.1.0
+bcrypt==4.0.1
+black==22.10.0
+certifi==2022.9.24
+cffi==1.15.1
+charset-normalizer==2.1.1
+click==8.1.3
+contourpy==1.0.6
+cryptography==38.0.4
+cycler==0.11.0
+fastapi==0.88.0
+ffmpy==0.3.0
+filelock==3.8.1
+fonttools==4.38.0
+frozenlist==1.3.3
+fsspec==2022.11.0
+gradio==3.41.2
+gradio_client==0.5.0
+h11==0.12.0
+httpcore==0.15.0
+httpx==0.23.1
+huggingface-hub==0.16.4
+idna==3.4
+importlib-resources==6.0.1
+Jinja2==3.1.2
+jsonschema==4.19.0
+jsonschema-specifications==2023.7.1
+kiwisolver==1.4.4
+linkify-it-py==1.0.3
+markdown-it-py==2.1.0
+MarkupSafe==2.1.1
+matplotlib==3.6.2
+mdit-py-plugins==0.3.2
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.0.3
+mypy-extensions==0.4.3
+networkx==3.1
+numpy==1.23.5
+orjson==3.8.3
+packaging==21.3
+pandas==1.5.2
+paramiko==2.12.0
+pathspec==0.10.2
+Pillow==9.3.0
+platformdirs==2.5.4
+pycparser==2.21
+pycryptodome==3.16.0
+pydantic==1.10.2
+pydub==0.25.1
+PyNaCl==1.5.0
+pyparsing==3.0.9
+python-dateutil==2.8.2
+python-multipart==0.0.5
+pytz==2022.6
+PyYAML==6.0
+referencing==0.30.2
+regex==2022.10.31
+requests==2.28.1
+rfc3986==1.5.0
+rpds-py==0.10.0
+safetensors==0.3.3
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.22.0
+sympy==1.12
+timm==0.9.5
+tokenizers==0.13.2
+tomli==2.0.1
+toolz==0.12.0
+torch==2.0.1
+torchvision==0.15.2
+tqdm==4.64.1
+transformers==4.32.1
+typing_extensions==4.4.0
+uc-micro-py==1.0.1
+urllib3==1.26.13
+uvicorn==0.20.0
+websockets==10.4
+yarl==1.8.2