first commit

README.md

@@ -1,13 +1,46 @@
 ---
 title: PanopticQuality
-emoji: 🏢
-colorFrom: yellow
-colorTo: red
+tags:
+- evaluate
+- metric
+description: >-
+  PanopticQuality score
 sdk: gradio
-sdk_version: 4.27.0
+sdk_version: 3.19.1
 app_file: app.py
 pinned: false
-license: apache-2.0
+emoji: 🕵️
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# SEA-AI/PanopticQuality
+
+This hugging face metric uses `seametrics.segmentation.PanopticQuality` under the hood to compute a panoptic quality score. It is a wrapper class for the torchmetrics class [`torchmetrics.detection.PanopticQuality`](https://lightning.ai/docs/torchmetrics/stable/detection/panoptic_quality.html). 
+
+## Getting Started
+
+To get started with PanopticQuality, make sure you have the necessary dependencies installed. This metric relies on the `evaluate`, `seametrics` and `seametrics[segmentation]`libraries for metric calculation and integration with FiftyOne datasets.
+
+### Installation
+
+First, ensure you have Python 3.8 or later installed. Then, install det-metrics using pip:
+
+```sh
+pip install evaluate git+https://github.com/SEA-AI/seametrics@develop
+```
+
+### Basic Usage
+
+
+## Metric Settings
+
+## Output Values
+
+## Further References
+
+- **seametrics Library**: Explore the [seametrics GitHub repository](https://github.com/SEA-AI/seametrics/tree/main) for more details on the underlying library.
+- **Torchmetrics**: https://lightning.ai/docs/torchmetrics/stable/detection/panoptic_quality.html
+- **Understanding Metrics**: The Panoptic Segmentation task, as well as Panoptic Quality as the evaluation metric, were introduced [in this paper](https://arxiv.org/pdf/1801.00868.pdf).
+
+## Contribution
+
+Your contributions are welcome! If you'd like to improve SEA-AI/PanopticQuality or add new features, please feel free to fork the repository, make your changes, and submit a pull request.

app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("SEA-AI/PanopticQuality")
+launch_gradio_widget(module)

gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

pq.py

@@ -0,0 +1,180 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""TODO: Add a description here."""
+
+from typing import Set
+from deprecated import deprecated
+
+import evaluate
+import datasets
+import numpy as np
+
+from seametrics.segmentation import PanopticQuality
+
+_CITATION = """\
+@inproceedings{DBLP:conf/cvpr/KirillovHGRD19,
+  author       = {Alexander Kirillov and
+                  Kaiming He and
+                  Ross B. Girshick and
+                  Carsten Rother and
+                  Piotr Doll{\'{a}}r},
+  title        = {Panoptic Segmentation},
+  booktitle    = {{IEEE} Conference on Computer Vision and Pattern Recognition, {CVPR}
+                  2019, Long Beach, CA, USA, June 16-20, 2019},
+  pages        = {9404--9413},
+  publisher    = {Computer Vision Foundation / {IEEE}},
+  year         = {2019},
+  url          = {http://openaccess.thecvf.com/content\_CVPR\_2019/html/Kirillov\_Panoptic\_Segmentation\_CVPR\_2019\_paper.html
+}
+"""
+
+_DESCRIPTION = """\
+This evaluation metric calculates Panoptic Quality (PQ) for panoptic segmentation masks.
+"""
+
+
+_KWARGS_DESCRIPTION = """
+Calculates PQ-score given predicted and ground truth panoptic segmentation masks.
+Args:
+    predictions: a 4-d array of shape (batch_size, img_height, img_width, 2).
+        The last dimension should hold the category index at position 0, and
+        the instance ID at position 1.
+    references:  a 4-d array of shape (batch_size, img_height, img_width, 2).
+        The last dimension should hold the category index at position 0, and
+        the instance ID at position 1.
+Returns:
+    A single float number in range [0, 1] that represents the PQ score.
+    1 is perfect panoptic segmentation, 0 is worst possible panoptic segmentation.
+Examples:
+    >>> import evaluate
+    >>> from seametrics.fo_utils.utils import fo_to_payload
+    >>> MODEL_FIELD = ["maskformer-27k-100ep"]
+    >>> payload = fo_to_payload("SAILING_PANOPTIC_DATASET_QA",
+    >>>                         gt_field="ground_truth_det",
+    >>>                         models=MODEL_FIELD,
+    >>>                         sequence_list=["Trip_55_Seq_2", "Trip_197_Seq_1", "Trip_197_Seq_68"],
+    >>>                         excluded_classes=[""])
+    >>> module = evaluate.load("SEA-AI/PanopticQuality")
+    >>> module.add_payload(payload, model_name=MODEL_FIELD[0])
+    >>> module.compute()
+    100%|██████████| 3/3 [00:03<00:00,  1.30s/it]
+    Added data ...
+    Start computing ...
+    Finished!
+    tensor(0.2082, dtype=torch.float64)
+"""
+
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class PQMetric(evaluate.Metric):
+    def __init__(
+        self,
+        label2id: dict = None,
+        stuff: list = None,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        DEFAULT_LABEL2ID = {'WATER': 0,
+                            'SKY': 1,
+                            'LAND': 2,
+                            'MOTORBOAT': 3,
+                            'FAR_AWAY_OBJECT': 4,
+                            'SAILING_BOAT_WITH_CLOSED_SAILS': 5,
+                            'SHIP': 6,
+                            'WATERCRAFT': 7,
+                            'SPHERICAL_BUOY': 8,
+                            'CONSTRUCTION': 9,
+                            'FLOTSAM': 10,
+                            'SAILING_BOAT_WITH_OPEN_SAILS': 11,
+                            'CONTAINER': 12,
+                            'PILLAR_BUOY': 13}
+        DEFAULT_STUFF = ["WATER", "SKY", "LAND", "CONSTRUCTION", "ICE", "OWN_BOAT"]
+        self.label2id = label2id if label2id is not None else DEFAULT_LABEL2ID
+        self.stuff =  stuff if stuff is not None else DEFAULT_STUFF
+        self.pq_metric = PanopticQuality(
+            things=set([self.label2id[label] for label in self.label2id.keys() if label not in self.stuff]), 
+            stuffs=set([self.label2id[label] for label in self.label2id.keys() if label in self.stuff])
+        )
+
+    def _info(self):
+        return evaluate.MetricInfo(
+            # This is the description that will appear on the modules page.
+            module_type="metric",
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            # This defines the format of each prediction and reference
+            features=datasets.Features(
+                {
+                    "predictions": datasets.Sequence(
+                                        datasets.Sequence(
+                                            datasets.Sequence(
+                                                datasets.Sequence(datasets.Value("float"))
+                                            )
+                                         ),
+                                    ),
+                    "references": datasets.Sequence( # batch
+                                        datasets.Sequence( # img height
+                                            datasets.Sequence( # img width
+                                                datasets.Sequence(datasets.Value("float")) # 2
+                                            )
+                                         ),
+                                    ),
+                }
+            ),
+            # Additional links to the codebase or references
+            codebase_urls=[
+                "https://lightning.ai/docs/torchmetrics/stable/detection/panoptic_quality.html"
+            ],
+        )
+
+    def add(self, *, prediction, reference, **kwargs):
+        """Adds a batch of predictions and references to the metric"""
+        # in case the inputs are lists, convert them to numpy arrays
+
+        self.pq_metric.update(prediction, reference)
+
+        # does not impact the metric, but is required for the interface x_x
+        super(evaluate.Metric, self).add(
+            prediction=self._postprocess(prediction),
+            references=self._postprocess(reference),
+            **kwargs
+        )
+
+    def _compute(self, *, predictions, references, **kwargs):
+        """Called within the evaluate.Metric.compute() method"""
+        return self.pq_metric.compute()
+
+    def add_payload(self, payload: dict, model_name: str = None):
+        """Converts the payload to the format expected by the metric"""
+        # import only if needed since fiftyone is not a direct dependency
+        from seametrics.segmentation.utils import payload_to_seg_metric
+
+        predictions, references, label2id = payload_to_seg_metric(payload, model_name, self.label2id)
+        self.label2id = label2id
+        self.add(prediction=predictions, reference=references)
+
+    def _postprocess(self, np_array):
+        """Converts the numpy arrays to lists for type checking"""
+        return self._np_to_lists(np_array)
+
+    def _np_to_lists(self, d):
+        """datasets does not support numpy arrays for type checking"""
+        if isinstance(d, np.ndarray):
+            if d.ndim == 1:
+                return d.tolist()
+            else:
+                return [self._np_to_lists(sub_arr) for sub_arr in d]
+        else:
+            return d

requirements.txt

@@ -0,0 +1,3 @@
+git+https://github.com/huggingface/evaluate@main
+git+https://github.com/SEA-AI/seametrics@develop
+fiftyone