Feature: Plotting of metrics

.github/workflows/ci-tests-full.yml

@@ -108,6 +108,7 @@ jobs:
 
     - name: Install all dependencies
       run: |
+        pip install cython  # needed for installing `pycocotools` in latest config
         python ./requirements/adjust-versions.py requirements.txt
         for fpath in `ls requirements/*.txt`; do
             python ./requirements/adjust-versions.py $fpath

CHANGELOG.md

@@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
--
+- Added support for plotting of metrics through `.plot()` method ([#1328](https://github.com/Lightning-AI/metrics/pull/1328))
 
 
 ### Changed

docs/source/conf.py

@@ -101,8 +101,12 @@ def _transform_changelog(path_in: str, path_out: str) -> None:
     "sphinx_paramlinks",
     "sphinx.ext.githubpages",
     "pt_lightning_sphinx_theme.extensions.lightning",
+    "matplotlib.sphinxext.plot_directive",
 ]
 
+# Set that source code from plotting is always included
+plot_include_source = True
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 

examples/plotting.py

@@ -0,0 +1,95 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+
+import matplotlib.pyplot as plt
+import torch
+
+
+def accuracy_example():
+    from torchmetrics.classification import MulticlassAccuracy
+
+    p = lambda: torch.randn(20, 5)
+    t = lambda: torch.randint(5, (20,))
+
+    # plot single value
+    metric = MulticlassAccuracy(num_classes=5)
+    metric.update(p(), t())
+    fig, ax = metric.plot()
+
+    # plot a value per class
+    metric = MulticlassAccuracy(num_classes=5, average=None)
+    metric.update(p(), t())
+    fig, ax = metric.plot()
+
+    # plot two values as a series
+    metric = MulticlassAccuracy(num_classes=5)
+    val1 = metric(p(), t())
+    val2 = metric(p(), t())
+    fig, ax = metric.plot([val1, val2])
+
+    # plot a series of values per class
+    metric = MulticlassAccuracy(num_classes=5, average=None)
+    val1 = metric(p(), t())
+    val2 = metric(p(), t())
+    fig, ax = metric.plot([val1, val2])
+    return fig, ax
+
+
+def mean_squared_error_example():
+    from torchmetrics.regression import MeanSquaredError
+
+    p = lambda: torch.randn(20)
+    t = lambda: torch.randn(20)
+
+    # single val
+    metric = MeanSquaredError()
+    metric.update(p(), t())
+    fig, ax = metric.plot()
+
+    # multiple values
+    metric = MeanSquaredError()
+    vals = [metric(p(), t()) for _ in range(10)]
+    fig, ax = metric.plot(vals)
+    return fig, ax
+
+
+def confusion_matrix_example():
+    from torchmetrics.classification import MulticlassConfusionMatrix
+
+    p = lambda: torch.randn(20, 5)
+    t = lambda: torch.randint(5, (20,))
+
+    # plot single value
+    metric = MulticlassConfusionMatrix(num_classes=5)
+    metric.update(p(), t())
+    fig, ax = metric.plot()
+    return fig, ax
+
+
+if __name__ == "__main__":
+
+    metrics_func = {
+        "accuracy": accuracy_example,
+        "mean_squared_error": mean_squared_error_example,
+        "confusion_matrix": confusion_matrix_example,
+    }
+
+    parser = argparse.ArgumentParser(description="Example script for plotting metrics.")
+    parser.add_argument("metric", type=str, nargs="?", choices=list(metrics_func.keys()), default="accuracy")
+    args = parser.parse_args()
+
+    fig, ax = metrics_func[args.metric]()
+
+    plt.show()

requirements/devel.txt

@@ -10,6 +10,7 @@
 # -r detection.txt  # version collision with min versio of PyTorch
 -r audio.txt
 -r multimodal.txt
+-r visual.txt
 
 # add extra testing
 -r image_test.txt

requirements/docs.txt

@@ -13,3 +13,4 @@ sphinx-copybutton>=0.3
 
 # integrations
 -r integrate.txt
+-r visual.txt

requirements/visual.txt

@@ -0,0 +1 @@
+matplotlib>=3.2.0

src/torchmetrics/classification/accuracy.py

@@ -11,14 +11,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any, Optional
+from typing import Any, Optional, Sequence, Union
 
 import torch
 from torch import Tensor
 from typing_extensions import Literal
 
 from torchmetrics.functional.classification.accuracy import _accuracy_reduce
 from torchmetrics.metric import Metric
+from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE, plot_single_or_multi_val
+
+if not _MATPLOTLIB_AVAILABLE:
+    __doctest_skip__ = ["BinaryAccuracy.plot", "MulticlassAccuracy.plot"]
 
 from torchmetrics.classification.stat_scores import (  # isort:skip
     BinaryStatScores,
@@ -95,12 +100,61 @@ class BinaryAccuracy(BinaryStatScores):
     is_differentiable = False
     higher_is_better = True
     full_state_update: bool = False
+    plot_options: dict = {"lower_bound": 0.0, "upper_bound": 1.0}
 
     def compute(self) -> Tensor:
         """Computes accuracy based on inputs passed in to ``update`` previously."""
         tp, fp, tn, fn = self._final_state()
         return _accuracy_reduce(tp, fp, tn, fn, average="binary", multidim_average=self.multidim_average)
 
+    def plot(
+        self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None
+    ) -> _PLOT_OUT_TYPE:
+        """Plot a single or multiple values from the metric.
+
+        Args:
+            val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
+                If no value is provided, will automatically call `metric.compute` and plot that result.
+            ax: An matplotlib axis object. If provided will add plot to that axis
+
+        Returns:
+            fig: Figure object
+            ax: Axes object
+
+        Raises:
+            ModuleNotFoundError:
+                If `matplotlib` is not installed
+
+        Examples:
+
+        .. plot::
+            :scale: 75
+
+            >>> # Example plotting a single value
+            >>> import torch
+            >>> from torchmetrics.classification import BinaryAccuracy
+            >>> metric = BinaryAccuracy()
+            >>> metric.update(torch.rand(10), torch.randint(2,(10,)))
+            >>> fig_, ax_ = metric.plot()
+
+        .. plot::
+            :scale: 75
+
+            >>> # Example plotting multiple values
+            >>> import torch
+            >>> from torchmetrics.classification import BinaryAccuracy
+            >>> metric = BinaryAccuracy()
+            >>> values = [ ]
+            >>> for _ in range(10):
+            ...     values.append(metric(torch.rand(10), torch.randint(2,(10,))))
+            >>> fig_, ax_ = metric.plot(values)
+        """
+        val = val or self.compute()
+        fig, ax = plot_single_or_multi_val(
+            val, ax=ax, higher_is_better=self.higher_is_better, **self.plot_options, name=self.__class__.__name__
+        )
+        return fig, ax
+
 
 class MulticlassAccuracy(MulticlassStatScores):
     r"""Computes `Accuracy`_ for multiclass tasks:
@@ -200,12 +254,61 @@ class MulticlassAccuracy(MulticlassStatScores):
     is_differentiable = False
     higher_is_better = True
     full_state_update: bool = False
+    plot_options = {"lower_bound": 0.0, "upper_bound": 1.0, "legend_name": "Class"}
 
     def compute(self) -> Tensor:
         """Computes accuracy based on inputs passed in to ``update`` previously."""
         tp, fp, tn, fn = self._final_state()
         return _accuracy_reduce(tp, fp, tn, fn, average=self.average, multidim_average=self.multidim_average)
 
+    def plot(
+        self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None
+    ) -> _PLOT_OUT_TYPE:
+        """Plot a single or multiple values from the metric.
+
+        Args:
+            val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
+                If no value is provided, will automatically call `metric.compute` and plot that result.
+            ax: An matplotlib axis object. If provided will add plot to that axis
+
+        Returns:
+            fig: Figure object
+            ax: Axes object
+
+        Raises:
+            ModuleNotFoundError:
+                If `matplotlib` is not installed
+
+        Examples:
+
+        .. plot::
+            :scale: 75
+
+            >>> # Example plotting a single value per class
+            >>> import torch
+            >>> from torchmetrics.classification import MulticlassAccuracy
+            >>> metric = MulticlassAccuracy(num_classes=3, average=None)
+            >>> metric.update(torch.randint(3, (20,)), torch.randint(3, (20,)))
+            >>> fig_, ax_ = metric.plot()
+
+        .. plot::
+            :scale: 75
+
+            >>> # Example plotting a multiple values per class
+            >>> import torch
+            >>> from torchmetrics.classification import MulticlassAccuracy
+            >>> metric = MulticlassAccuracy(num_classes=3, average=None)
+            >>> values = []
+            >>> for _ in range(20):
+            ...     values.append(metric(torch.randint(3, (20,)), torch.randint(3, (20,))))
+            >>> fig_, ax_ = metric.plot(values)
+        """
+        val = val or self.compute()
+        fig, ax = plot_single_or_multi_val(
+            val, ax=ax, higher_is_better=self.higher_is_better, **self.plot_options, name=self.__class__.__name__
+        )
+        return fig, ax
+
 
 class MultilabelAccuracy(MultilabelStatScores):
     r"""Computes `Accuracy`_ for multilabel tasks:
@@ -303,6 +406,7 @@ class MultilabelAccuracy(MultilabelStatScores):
     is_differentiable = False
     higher_is_better = True
     full_state_update: bool = False
+    plot_options: dict = {"lower_bound": 0.0, "upper_bound": 1.0, "legend_name": "Label"}
 
     def compute(self) -> Tensor:
         """Computes accuracy based on inputs passed in to ``update`` previously."""

src/torchmetrics/classification/confusion_matrix.py

@@ -35,6 +35,11 @@
     _multilabel_confusion_matrix_update,
 )
 from torchmetrics.metric import Metric
+from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
+from torchmetrics.utilities.plot import _PLOT_OUT_TYPE, plot_confusion_matrix
+
+if not _MATPLOTLIB_AVAILABLE:
+    __doctest_skip__ = ["MulticlassConfusionMatrix.plot"]
 
 
 class BinaryConfusionMatrix(Metric):
@@ -217,6 +222,36 @@ def compute(self) -> Tensor:
         """
         return _multiclass_confusion_matrix_compute(self.confmat, self.normalize)
 
+    def plot(self, val: Optional[Tensor] = None) -> _PLOT_OUT_TYPE:
+        """Plot a single or multiple values from the metric.
+
+        Args:
+            val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
+                If no value is provided, will automatically call `metric.compute` and plot that result.
+
+        Returns:
+            fig: Figure object
+            ax: Axes object
+
+        Raises:
+            ModuleNotFoundError:
+                If `matplotlib` is not installed
+
+        .. plot::
+            :scale: 75
+
+            >>> import torch
+            >>> from torchmetrics.classification import MulticlassConfusionMatrix
+            >>> metric = MulticlassConfusionMatrix(num_classes=5)
+            >>> metric.update(torch.randint(5, (20,)), torch.randint(5, (20,)))
+            >>> fig_, ax_ = metric.plot()
+        """
+        val = val or self.compute()
+        if not isinstance(val, Tensor):
+            raise TypeError(f"Expected val to be a single tensor but got {val}")
+        fig, ax = plot_confusion_matrix(val)
+        return fig, ax
+
 
 class MultilabelConfusionMatrix(Metric):
     r"""Computes the `confusion matrix`_ for multilabel tasks.

src/torchmetrics/functional/multimodal/clip_score.py

@@ -53,7 +53,7 @@ def _clip_score_update(
     device = images[0].device
     processed_input = processor(
         text=text, images=[i.cpu() for i in images], return_tensors="pt", padding=True
-    )  # type:ignore
+    )  # type: ignore
 
     img_features = model.get_image_features(processed_input["pixel_values"].to(device))
     img_features = img_features / img_features.norm(p=2, dim=-1, keepdim=True)

src/torchmetrics/metric.py

@@ -555,6 +555,10 @@ def compute(self) -> Any:
         """Override this method to compute the final metric value from state variables synchronized across the
         distributed backend."""
 
+    def plot(self, *_: Any, **__: Any) -> None:
+        """Override this method plot the metric value."""
+        raise NotImplementedError
+
     def reset(self) -> None:
         """This method automatically resets the metric state variables to their default value."""
         self._update_count = 0