diff --git a/CHANGELOG.md b/CHANGELOG.md index 742b90749b8..dfcc70232ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed cached network in `FeatureShare` not being moved to the correct device ([#2348](https://github.com/Lightning-AI/torchmetrics/pull/2348)) +- Fix naming of statistics in `MeanAveragePrecision` with custom max det thresholds ([#2367](https://github.com/Lightning-AI/torchmetrics/pull/2367)) + + - Fixed custom aggregation in retrieval metrics ([#2364](https://github.com/Lightning-AI/torchmetrics/pull/2364)) diff --git a/src/torchmetrics/detection/mean_ap.py b/src/torchmetrics/detection/mean_ap.py index e087b812d46..fb59c07b8f6 100644 --- a/src/torchmetrics/detection/mean_ap.py +++ b/src/torchmetrics/detection/mean_ap.py @@ -128,9 +128,12 @@ class MeanAveragePrecision(Metric): - map_small: (:class:`~torch.Tensor`), mean average precision for small objects - map_medium:(:class:`~torch.Tensor`), mean average precision for medium objects - map_large: (:class:`~torch.Tensor`), mean average precision for large objects - - mar_1: (:class:`~torch.Tensor`), mean average recall for 1 detection per image - - mar_10: (:class:`~torch.Tensor`), mean average recall for 10 detections per image - - mar_100: (:class:`~torch.Tensor`), mean average recall for 100 detections per image + - mar_{mdt[0]}: (:class:`~torch.Tensor`), mean average recall for `max_detection_thresholds[0]` (default 1) + detection per image + - mar_{mdt[1]}: (:class:`~torch.Tensor`), mean average recall for `max_detection_thresholds[1]` (default 10) + detection per image + - mar_{mdt[1]}: (:class:`~torch.Tensor`), mean average recall for `max_detection_thresholds[2]` (default 100) + detection per image - mar_small: (:class:`~torch.Tensor`), mean average recall for small objects - mar_medium: (:class:`~torch.Tensor`), mean average recall for medium objects - mar_large: (:class:`~torch.Tensor`), mean average recall for large objects @@ -140,8 +143,8 @@ class MeanAveragePrecision(Metric): IoU=0.75 - map_per_class: (:class:`~torch.Tensor`) (-1 if class metrics are disabled), mean average precision per observed class - - mar_100_per_class: (:class:`~torch.Tensor`) (-1 if class metrics are disabled), mean average recall for 100 - detections per image per observed class + - mar_{mdt[2]}_per_class: (:class:`~torch.Tensor`) (-1 if class metrics are disabled), mean average recall for + `max_detection_thresholds[2]` (default 100) detections per image per observed class - classes (:class:`~torch.Tensor`), list of all observed classes For an example on how to use this metric check the `torchmetrics mAP example`_. @@ -184,8 +187,7 @@ class MeanAveragePrecision(Metric): with step ``0.01``. Else provide a list of floats. max_detection_thresholds: Thresholds on max detections per image. If set to `None` will use thresholds ``[1, 10, 100]``. - Else, please provide a list of ints. If the `pycocotools` backend is used then the list needs to have - length 3. If this is a problem, shift to `faster_coco_eval` which supports more detection thresholds. + Else, please provide a list of ints of length 3, which is the only supported length by both backends. class_metrics: Option to enable per-class metrics for mAP and mAR_100. Has a performance impact that scales linearly with the number of classes in the dataset. @@ -410,10 +412,10 @@ def __init__( f"Expected argument `max_detection_thresholds` to either be `None` or a list of ints" f" but got {max_detection_thresholds}" ) - if max_detection_thresholds is not None and backend == "pycocotools" and len(max_detection_thresholds) != 3: + if max_detection_thresholds is not None and len(max_detection_thresholds) != 3: raise ValueError( - "When using `pycocotools` backend the number of max detection thresholds should be 3 else" - f" it will not work correctly with the backend. Got value {len(max_detection_thresholds)}." + "When providing a list of max detection thresholds it should have length 3." + " Got value {len(max_detection_thresholds)}" ) max_det_thr, _ = torch.sort(torch.tensor(max_detection_thresholds or [1, 10, 100], dtype=torch.int)) self.max_detection_thresholds = max_det_thr.tolist() @@ -556,7 +558,7 @@ def compute(self) -> dict: coco_eval.params.maxDets = self.max_detection_thresholds map_per_class_list = [] - mar_100_per_class_list = [] + mar_per_class_list = [] for class_id in self._get_classes(): coco_eval.params.catIds = [class_id] with contextlib.redirect_stdout(io.StringIO()): @@ -566,18 +568,18 @@ def compute(self) -> dict: class_stats = coco_eval.stats map_per_class_list.append(torch.tensor([class_stats[0]])) - mar_100_per_class_list.append(torch.tensor([class_stats[8]])) + mar_per_class_list.append(torch.tensor([class_stats[8]])) map_per_class_values = torch.tensor(map_per_class_list, dtype=torch.float32) - mar_100_per_class_values = torch.tensor(mar_100_per_class_list, dtype=torch.float32) + mar_per_class_values = torch.tensor(mar_per_class_list, dtype=torch.float32) else: map_per_class_values = torch.tensor([-1], dtype=torch.float32) - mar_100_per_class_values = torch.tensor([-1], dtype=torch.float32) + mar_per_class_values = torch.tensor([-1], dtype=torch.float32) prefix = "" if len(self.iou_type) == 1 else f"{i_type}_" result_dict.update( { f"{prefix}map_per_class": map_per_class_values, - f"{prefix}mar_100_per_class": mar_100_per_class_values, + f"{prefix}mar_{self.max_detection_thresholds[-1]}_per_class": mar_per_class_values, }, ) result_dict.update({"classes": torch.tensor(self._get_classes(), dtype=torch.int32)}) @@ -616,9 +618,9 @@ def _get_coco_datasets(self, average: Literal["macro", "micro"]) -> Tuple[object return coco_preds, coco_target - @staticmethod - def _coco_stats_to_tensor_dict(stats: List[float], prefix: str) -> Dict[str, Tensor]: + def _coco_stats_to_tensor_dict(self, stats: List[float], prefix: str) -> Dict[str, Tensor]: """Converts the output of COCOeval.stats to a dict of tensors.""" + mdt = self.max_detection_thresholds return { f"{prefix}map": torch.tensor([stats[0]], dtype=torch.float32), f"{prefix}map_50": torch.tensor([stats[1]], dtype=torch.float32), @@ -626,9 +628,9 @@ def _coco_stats_to_tensor_dict(stats: List[float], prefix: str) -> Dict[str, Ten f"{prefix}map_small": torch.tensor([stats[3]], dtype=torch.float32), f"{prefix}map_medium": torch.tensor([stats[4]], dtype=torch.float32), f"{prefix}map_large": torch.tensor([stats[5]], dtype=torch.float32), - f"{prefix}mar_1": torch.tensor([stats[6]], dtype=torch.float32), - f"{prefix}mar_10": torch.tensor([stats[7]], dtype=torch.float32), - f"{prefix}mar_100": torch.tensor([stats[8]], dtype=torch.float32), + f"{prefix}mar_{mdt[0]}": torch.tensor([stats[6]], dtype=torch.float32), + f"{prefix}mar_{mdt[1]}": torch.tensor([stats[7]], dtype=torch.float32), + f"{prefix}mar_{mdt[2]}": torch.tensor([stats[8]], dtype=torch.float32), f"{prefix}mar_small": torch.tensor([stats[9]], dtype=torch.float32), f"{prefix}mar_medium": torch.tensor([stats[10]], dtype=torch.float32), f"{prefix}mar_large": torch.tensor([stats[11]], dtype=torch.float32), diff --git a/tests/unittests/detection/test_map.py b/tests/unittests/detection/test_map.py index 1dc587dc87c..51cb264fffb 100644 --- a/tests/unittests/detection/test_map.py +++ b/tests/unittests/detection/test_map.py @@ -861,6 +861,10 @@ def test_many_detection_thresholds(self, backend): else: assert round(res["map"].item(), 5) == 0.6 + assert "mar_1" in res + assert "mar_10" in res + assert "mar_1000" in res + @pytest.mark.parametrize("max_detection_thresholds", [[1, 10], [1, 10, 50, 100]]) def test_with_more_and_less_detection_thresholds(self, max_detection_thresholds, backend): """Test how metric is working when list of max detection thresholds is not 3. @@ -869,25 +873,7 @@ def test_with_more_and_less_detection_thresholds(self, max_detection_thresholds, https://github.com/ppwwyyxx/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py#L461 """ - preds = [ - { - "boxes": torch.tensor([[258.0, 41.0, 606.0, 285.0]]), - "scores": torch.tensor([0.536]), - "labels": torch.tensor([0]), - } - ] - target = [ - { - "boxes": torch.tensor([[214.0, 41.0, 562.0, 285.0]]), - "labels": torch.tensor([0]), - } - ] - - if backend == "pycocotools": - with pytest.raises( - ValueError, match="When using `pycocotools` backend the number of max detection thresholds should.*" - ): - metric = MeanAveragePrecision(max_detection_thresholds=max_detection_thresholds, backend=backend) - else: - metric = MeanAveragePrecision(max_detection_thresholds=max_detection_thresholds, backend=backend) - metric(preds, target) + with pytest.raises( + ValueError, match="When providing a list of max detection thresholds it should have length 3.*" + ): + MeanAveragePrecision(max_detection_thresholds=max_detection_thresholds, backend=backend)