Feature: Plotting of metrics

CHANGELOG.md

@@ -31,6 +31,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `normalize` argument to `Inception`, `FID`, `KID` metrics ([#1246](https://github.com/Lightning-AI/metrics/pull/1246))
 
 
+- Added support for plotting of metrics through `.plot()` method ([#1328](https://github.com/Lightning-AI/metrics/pull/1328))
+
+
 ### Changed
 
 - Changed `MeanAveragePrecision` to vectorize `_find_best_gt_match` operation ([#1259](https://github.com/Lightning-AI/metrics/pull/1259))
@@ -52,7 +55,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
-- 
+-
 
 
 ## [0.10.3] - 2022-11-16

docs/source/conf.py

@@ -101,8 +101,12 @@ def _transform_changelog(path_in: str, path_out: str) -> None:
     "sphinx_paramlinks",
     "sphinx.ext.githubpages",
     "pt_lightning_sphinx_theme.extensions.lightning",
+    "matplotlib.sphinxext.plot_directive",
 ]
 
+# Set that source code from plotting is always included
+plot_include_source = True
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 

examples/plotting.py

@@ -0,0 +1,96 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+
+import matplotlib.pyplot as plt
+import torch
+
+
+def accuracy_example():
+    from torchmetrics.classification import MulticlassAccuracy
+
+    p = lambda: torch.randn(20, 5)
+    t = lambda: torch.randint(5, (20,))
+
+    # plot single value
+    metric = MulticlassAccuracy(num_classes=5)
+    metric.update(p(), t())
+    fig, ax = metric.plot()
+
+    # plot a value per class
+    metric = MulticlassAccuracy(num_classes=5, average=None)
+    metric.update(p(), t())
+    fig, ax = metric.plot()
+
+    # plot two values as an series
+    metric = MulticlassAccuracy(num_classes=5)
+    val1 = metric(p(), t())
+    val2 = metric(p(), t())
+    fig, ax = metric.plot([val1, val2])
+
+    # plot a series of values per class
+    metric = MulticlassAccuracy(num_classes=5, average=None)
+    val1 = metric(p(), t())
+    val2 = metric(p(), t())
+    fig, ax = metric.plot([val1, val2])
+    return fig, ax
+
+
+def mean_squared_error_example():
+    from torchmetrics.regression import MeanSquaredError
+
+    p = lambda: torch.randn(20)
+    t = lambda: torch.randn(20)
+
+    # single val
+    metric = MeanSquaredError()
+    metric.update(p(), t())
+    fig, ax = metric.plot()
+
+    # multiple values
+    metric = MeanSquaredError()
+    vals = [metric(p(), t()) for _ in range(10)]
+    fig, ax = metric.plot(vals)
+    return fig, ax
+
+
+def confusion_matrix_example():
+    from torchmetrics.classification import MulticlassConfusionMatrix
+
+    p = lambda: torch.randn(20, 5)
+    t = lambda: torch.randint(5, (20,))
+
+    # plot single value
+    metric = MulticlassConfusionMatrix(num_classes=5)
+    metric.update(p(), t())
+    fig, ax = metric.plot()
+    return fig, ax
+
+
+if __name__ == "__main__":
+    list_of_choices = ["accuracy", "mean_squared_error", "confusion_matrix"]
+    parser = argparse.ArgumentParser(description="Example script for plotting metrics.")
+    parser.add_argument("metric", choices=list_of_choices)
+    args = parser.parse_args()
+
+    if args.metric == "accuracy":
+        fig, ax = accuracy_example()
+
+    if args.metric == "mean_squared_error":
+        fig, ax = mean_squared_error_example()
+
+    if args.metric == "confusion_matrix":
+        fig, ax = confusion_matrix_example()
+
+    plt.show()

requirements/integrate.txt

@@ -1 +1,2 @@
 pytorch-lightning>=1.5
+matplotlib>=3.0.0

src/torchmetrics/classification/accuracy.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any, Optional
+from typing import Any, Optional, Sequence, Union
 
 import torch
 from torch import Tensor, tensor
@@ -28,8 +28,13 @@
 )
 from torchmetrics.metric import Metric
 from torchmetrics.utilities.enums import AverageMethod, DataType
+from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
+from torchmetrics.utilities.plot import _PLOT_OUT_TYPE, plot_single_or_multi_val
 from torchmetrics.utilities.prints import rank_zero_warn
 
+if not _MATPLOTLIB_AVAILABLE:
+    __doctest_skip__ = ["BinaryAccuracy.plot", "MulticlassAccuracy.plot"]
+
 from torchmetrics.classification.stat_scores import (  # isort:skip
     StatScores,
     BinaryStatScores,
@@ -107,12 +112,58 @@ class BinaryAccuracy(BinaryStatScores):
     is_differentiable = False
     higher_is_better = True
     full_state_update: bool = False
+    plot_options: dict = {"lower_bound": 0.0, "upper_bound": 1.0}
 
     def compute(self) -> Tensor:
         """Computes accuracy based on inputs passed in to ``update`` previously."""
         tp, fp, tn, fn = self._final_state()
         return _accuracy_reduce(tp, fp, tn, fn, average="binary", multidim_average=self.multidim_average)
 
+    def plot(self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None) -> _PLOT_OUT_TYPE:
+        """Plot a single or multiple values from the metric.
+
+        Args:
+            val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
+                If no value is provided, will automatically call `metric.compute` and plot that result.
+
+        Returns:
+            fig: Figure object
+            ax: Axes object
+
+        Raises:
+            ModuleNotFoundError:
+                If `matplotlib` is not installed
+
+        Examples:
+
+        .. plot::
+            :scale: 75
+
+            >>> # Example plotting a single value
+            >>> import torch
+            >>> from torchmetrics.classification import BinaryAccuracy
+            >>> metric = BinaryAccuracy()
+            >>> metric.update(torch.rand(10), torch.randint(2,(10,)))
+            >>> metric.plot()
+
+        .. plot::
+            :scale: 75
+
+            >>> # Example plotting multiple values
+            >>> import torch
+            >>> from torchmetrics.classification import BinaryAccuracy
+            >>> metric = BinaryAccuracy()
+            >>> values = [ ]
+            >>> for _ in range(10):
+            ...     values.append(metric(torch.rand(10), torch.randint(2,(10,))))
+            >>> metric.plot(values)
+        """
+        val = val or self.compute()
+        fig, ax = plot_single_or_multi_val(
+            val, higher_is_better=self.higher_is_better, **self.plot_options, name=self.__class__.__name__
+        )
+        return fig, ax
+
 
 class MulticlassAccuracy(MulticlassStatScores):
     r"""Computes `Accuracy`_ for multiclass tasks:
@@ -213,12 +264,58 @@ class MulticlassAccuracy(MulticlassStatScores):
     is_differentiable = False
     higher_is_better = True
     full_state_update: bool = False
+    plot_options = {"lower_bound": 0.0, "upper_bound": 1.0, "legend_name": "Class"}
 
     def compute(self) -> Tensor:
         """Computes accuracy based on inputs passed in to ``update`` previously."""
         tp, fp, tn, fn = self._final_state()
         return _accuracy_reduce(tp, fp, tn, fn, average=self.average, multidim_average=self.multidim_average)
 
+    def plot(self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None) -> _PLOT_OUT_TYPE:
+        """Plot a single or multiple values from the metric.
+
+        Args:
+            val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
+                If no value is provided, will automatically call `metric.compute` and plot that result.
+
+        Returns:
+            fig: Figure object
+            ax: Axes object
+
+        Raises:
+            ModuleNotFoundError:
+                If `matplotlib` is not installed
+
+        Examples:
+
+        .. plot::
+            :scale: 75
+
+            >>> # Example plotting a single value per class
+            >>> import torch
+            >>> from torchmetrics.classification import MulticlassAccuracy
+            >>> metric = MulticlassAccuracy(num_classes=3, average=None)
+            >>> metric.update(torch.randint(3, (20,)), torch.randint(3, (20,)))
+            >>> metric.plot()
+
+        .. plot::
+            :scale: 75
+
+            >>> # Example plotting a multiple values per class
+            >>> import torch
+            >>> from torchmetrics.classification import MulticlassAccuracy
+            >>> metric = MulticlassAccuracy(num_classes=3, average=None)
+            >>> values = []
+            >>> for _ in range(20):
+            ...     values.append(metric(torch.randint(3, (20,)), torch.randint(3, (20,))))
+            >>> metric.plot(values)
+        """
+        val = val or self.compute()
+        fig, ax = plot_single_or_multi_val(
+            val, higher_is_better=self.higher_is_better, **self.plot_options, name=self.__class__.__name__
+        )
+        return fig, ax
+
 
 class MultilabelAccuracy(MultilabelStatScores):
     r"""Computes `Accuracy`_ for multilabel tasks:
@@ -317,6 +414,7 @@ class MultilabelAccuracy(MultilabelStatScores):
     is_differentiable = False
     higher_is_better = True
     full_state_update: bool = False
+    plot_options: dict = {"lower_bound": 0.0, "upper_bound": 1.0, "legend_name": "Label"}
 
     def compute(self) -> Tensor:
         """Computes accuracy based on inputs passed in to ``update`` previously."""

src/torchmetrics/classification/confusion_matrix.py

@@ -37,8 +37,13 @@
     _multilabel_confusion_matrix_update,
 )
 from torchmetrics.metric import Metric
+from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE
+from torchmetrics.utilities.plot import _PLOT_OUT_TYPE, plot_confusion_matrix
 from torchmetrics.utilities.prints import rank_zero_warn
 
+if not _MATPLOTLIB_AVAILABLE:
+    __doctest_skip__ = ["MulticlassConfusionMatrix.plot"]
+
 
 class BinaryConfusionMatrix(Metric):
     r"""Computes the `confusion matrix`_ for binary tasks.
@@ -220,6 +225,36 @@ def compute(self) -> Tensor:
         """
         return _multiclass_confusion_matrix_compute(self.confmat, self.normalize)
 
+    def plot(self, val: Optional[Tensor] = None) -> _PLOT_OUT_TYPE:
+        """Plot a single or multiple values from the metric.
+
+        Args:
+            val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
+                If no value is provided, will automatically call `metric.compute` and plot that result.
+
+        Returns:
+            fig: Figure object
+            ax: Axes object
+
+        Raises:
+            ModuleNotFoundError:
+                If `matplotlib` is not installed
+
+        .. plot::
+            :scale: 75
+
+            >>> import torch
+            >>> from torchmetrics.classification import MulticlassConfusionMatrix
+            >>> metric = MulticlassConfusionMatrix(num_classes=5)
+            >>> metric.update(torch.randint(5, (20,)), torch.randint(5, (20,)))
+            >>> metric.plot()
+        """
+        val = val or self.compute()
+        if not isinstance(val, Tensor):
+            raise TypeError(f"Expected val to be a single tensor but got {val}")
+        fig, ax = plot_confusion_matrix(val)
+        return fig, ax
+
 
 class MultilabelConfusionMatrix(Metric):
     r"""Computes the `confusion matrix`_ for multilabel tasks.

src/torchmetrics/metric.py

@@ -558,6 +558,10 @@ def compute(self) -> Any:
         """Override this method to compute the final metric value from state variables synchronized across the
         distributed backend."""
 
+    def plot(self, *_: Any, **__: Any) -> None:
+        """Override this method plot the metric value."""
+        raise NotImplementedError
+
     def reset(self) -> None:
         """This method automatically resets the metric state variables to their default value."""
         self._update_count = 0