proper use of "add" and "compute" + "add_from_payload"

det-metrics.py

@@ -13,15 +13,15 @@
 # limitations under the License.
 """TODO: Add a description here."""
 
-from typing import List, Tuple, Dict, Literal
+from typing import List, Tuple, Literal
 
 import evaluate
 import datasets
 import numpy as np
 
 from seametrics.detection import PrecisionRecallF1Support
-from seametrics.fo_utils.utils import _fo_dets_to_metrics_dict
-from seametrics.fo_utils.utils import _add_batch
+from seametrics.detection.utils import payload_to_det_metric
+from seametrics.payload import Payload
 
 _CITATION = """\
 @InProceedings{coco:2020,
@@ -53,13 +53,16 @@ It is based on a modified version of the commonly used COCO-evaluation metrics.
 _KWARGS_DESCRIPTION = """
 Calculates object detection metrics given predicted and ground truth bounding boxes for a single image.
 Args:
-    predictions: list of predictions to score. Each prediction should
-        be a list containing the four co-ordinates that specify the bounding box.
-        Co-ordinate format is as defined when instantiating the metric
-        (parameter: bbox_type, defaults to xywh).
-    references: list of reference for each prediction. Each prediction should
-        be a list containing the four co-ordinates that specify the bounding box.
-        Bounding box format should be the same as for the predictions.
+    predictions: list of predictions for each image. Each prediction should
+        be a dict containing the following
+        - 'boxes': list of bounding boxes, xywh in absolute pixel values
+        - 'labels': list of labels for each bounding box
+        - 'scores': list of scores for each bounding box
+    references: list of ground truth annotations for each image. Each reference should
+        be a dict containing the following
+        - 'boxes': list of bounding boxes, xywh in absolute pixel values
+        - 'labels': list of labels for each bounding box
+        - 'area': list of areas for each bounding box
 Returns:
     dict containing dicts for each specified area range with following items:
         'range': specified area with [max_px_area, max_px_area]
@@ -84,11 +87,11 @@ Returns:
         'nImgs': number of images considered in evaluation
 Examples:
     >>> import evaluate
-    >>> from seametrics.fo_to_payload.utils import fo_to_payload
-    >>> payload = fo_to_payload(..., models=model_list)
-    >>> for model in payload["models"]:
-        >>> module = evaluate.load("./detection_metric.py", iou_thresholds=0.9)
-        >>> module.add_batch(payload)
+    >>> from seametrics.payload import PayloadProcessor
+    >>> payload = PayloadProcessor(...).payload  
+    >>> for model in payload.models:
+        >>> module = evaluate.load("SEA-AI/det-metrics", ...)
+        >>> module.add_from_payload(payload)
         >>> result = module.compute()
         >>> print(result)
             {'all': {
@@ -113,27 +116,29 @@ Examples:
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class DetectionMetric(evaluate.Metric):
     def __init__(
-            self, 
-            area_ranges_tuples: List[Tuple[str, List[int]]] = [("all", [0, 1e5 ** 2])], 
-            iou_threshold: float = 1e-10, 
-            class_agnostic: bool = True,
-            bbox_format: str = "xywh",
-            iou_type: Literal["bbox", "segm"] = "bbox",
-            **kwargs
-        ):
+        self,
+        area_ranges_tuples: List[Tuple[str, List[int]]] = [("all", [0, 1e5**2])],
+        iou_threshold: List[float] = [1e-10],
+        class_agnostic: bool = True,
+        bbox_format: str = "xywh",
+        iou_type: Literal["bbox", "segm"] = "bbox",
+        **kwargs
+    ):
         super().__init__(**kwargs)
         area_ranges = [v for _, v in area_ranges_tuples]
         area_ranges_labels = [k for k, _ in area_ranges_tuples]
+        iou_threshold = (
+            [iou_threshold] if not isinstance(iou_threshold, list) else iou_threshold
+        )
 
-        metric_params = dict(
-            iou_thresholds=[iou_threshold],
+        self.coco_metric = PrecisionRecallF1Support(
+            iou_thresholds=iou_threshold,
             area_ranges=area_ranges,
             area_ranges_labels=area_ranges_labels,
             class_agnostic=class_agnostic,
             iou_type=iou_type,
-            box_format=bbox_format
+            box_format=bbox_format,
         )
-        self.coco_metric = PrecisionRecallF1Support(**metric_params)
 
     def _info(self):
         return evaluate.MetricInfo(
@@ -145,46 +150,63 @@ class DetectionMetric(evaluate.Metric):
             # This defines the format of each prediction and reference
             features=datasets.Features(
                 {
-                'predictions': datasets.Sequence(feature=datasets.Sequence(datasets.Value("float"))),
-                'references': datasets.Sequence(feature=datasets.Sequence(datasets.Value("float"))),
+                    "predictions": [
+                        datasets.Features(
+                            {
+                                "boxes": datasets.Sequence(
+                                    datasets.Sequence(datasets.Value("float"))
+                                ),
+                                "labels": datasets.Sequence(datasets.Value("int64")),
+                                "scores": datasets.Sequence(datasets.Value("float")),
+                            }
+                        )
+                    ],
+                    "references": [
+                        datasets.Features(
+                            {
+                                "boxes": datasets.Sequence(
+                                    datasets.Sequence(datasets.Value("float"))
+                                ),
+                                "labels": datasets.Sequence(datasets.Value("int64")),
+                                "area": datasets.Sequence(datasets.Value("float")),
+                            }
+                        )
+                    ],
                 }
             ),
             # Additional links to the codebase or references
-            codebase_urls=["https://github.com/SEA-AI/metrics/tree/main",
-                           "https://github.com/cocodataset/cocoapi/tree/master"]
+            codebase_urls=[
+                "https://github.com/SEA-AI/seametrics/tree/main",
+                "https://lightning.ai/docs/torchmetrics/stable/detection/mean_average_precision.html",
+            ],
         )
-    
-    def add_batch(
-            self, 
-            data: dict,
-            model: str = None
-        ):
-        """Add predictions and ground truths of a single image to update the metric.
-
-        Args:
-            data (dict): containing standard payload of data that should be evaluated
-                format should be as returned by function `fo_to_payload()` in seametrics library
-            model (str): should be one out of values given in data["models"]
-                if not defined, defaults to data["models"][0], as only one model can be evaluated a time.
-        """
-        # populate two empty lists in format suitable for hugging face metric
-        # nothing is computed based on them but prevents huggingface error
-
-
-        self, predictions,references = _add_batch(self, data, model)    
-
-        # prevents hugging face error, doesn't do a lot       
-        super(evaluate.Metric, self).add_batch(
-            predictions=predictions,
-            references=references
+
+    def add(self, *, prediction, reference, **kwargs):
+        """Adds a batch of predictions and references to the metric"""
+        self.coco_metric.update(prediction, reference)
+
+        # does not impact the metric, but is required for the interface x_x
+        super(evaluate.Metric, self).add(
+            prediction=[self._np_to_lists(p) for p in prediction],
+            references=[self._np_to_lists(r) for r in reference],
+            **kwargs
         )
 
+    def _np_to_lists(self, d):
+        """datasets does not support numpy arrays for type checking"""
+        for k, v in d.items():
+            if isinstance(v, dict):
+                self._np_to_lists(v)
+            elif isinstance(v, np.ndarray):
+                d[k] = v.tolist()
+        return d
 
-    def _compute(
-            self, 
-            predictions, 
-            references
-        ):
+    def _compute(self, *, predictions, references, **kwargs):
         """Returns the scores"""
-        result = self.coco_metric.compute()["metrics"]
-        return result
+        return self.coco_metric.compute()["metrics"]
+
+    def add_from_payload(self, payload: Payload, model_name: str = None):
+        """Converts the payload to the format expected by the metric"""
+        predictions, references = payload_to_det_metric(payload, model_name)
+        self.add(prediction=predictions, reference=references)
+        return self