RFC: a more robust plot norm/colorbar API

conftest.py

@@ -114,15 +114,6 @@ def pytest_configure(config):
     ):
         config.addinivalue_line("filterwarnings", value)
 
-    if MPL_VERSION < Version("3.0.0"):
-        config.addinivalue_line(
-            "filterwarnings",
-            (
-                "ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' "
-                "is deprecated since Python 3.3,and in 3.9 it will stop working:DeprecationWarning"
-            ),
-        )
-
     if MPL_VERSION < Version("3.5.2") and PILLOW_VERSION >= Version("9.1"):
         # see https://github.com/matplotlib/matplotlib/pull/22766
         config.addinivalue_line(

doc/source/visualizing/plots.rst

@@ -732,6 +732,9 @@ the axes unit labels.
 The same result could have been accomplished by explicitly setting the ``width``
 to ``(.01, 'Mpc')``.
 
+
+.. _set-image-units:
+
 Set image units
 ~~~~~~~~~~~~~~~
 
@@ -913,7 +916,7 @@ customization.
 Colormaps
 ~~~~~~~~~
 
-Each of these functions accept two arguments.  In all cases the first argument
+Each of these functions accepts at least two arguments.  In all cases the first argument
 is a field name.  This makes it possible to use different custom colormaps for
 different fields tracked by the plot object.
 
@@ -930,54 +933,137 @@ Use any of the colormaps listed in the :ref:`colormaps` section.
    slc.set_cmap(("gas", "density"), "RdBu_r")
    slc.save()
 
-The :meth:`~yt.visualization.plot_window.AxisAlignedSlicePlot.set_log` function
-accepts a field name and a boolean.  If the boolean is ``True``, the colormap
-for the field will be log scaled.  If it is ``False`` the colormap will be
-linear.
+Colorbar norms
+::::::::::::::
+
+Slice plots and similar plot classes default to log norms when all values are
+strictly positive, and symlog otherwise. yt supports two different interfaces to
+move away from the defaults. See **constrained norms** and **arbitrary norm**
+hereafter.
+
+.. note:: defaults can be configured on a per-field basis, see :ref:`per-field-plotconfig`
+
+**Constrained norms**
+
+The norm properties can be constrained via two methods
+
+- :meth:`~yt.visualization.plot_container.PlotContainer.set_zlim` controls the extrema
+  of the value range ``zmin`` and ``zmax``.
+- :meth:`~yt.visualization.plot_container.ImagePlotContainer.set_log` allows switching to
+  linear or symlog norms. With symlog, the linear threshold can be set
+  explicitly. Otherwise, yt will dynamically determine a reasonable value.
+
+Use the :meth:`~yt.visualization.plot_container.PlotContainer.set_zlim`
+method to set a custom colormap range.
+
+.. python-script::
+
+   import yt
+
+   ds = yt.load("IsolatedGalaxy/galaxy0030/galaxy0030")
+   slc = yt.SlicePlot(ds, "z", ("gas", "density"), width=(10, "kpc"))
+   slc.set_zlim(("gas", "density"), zmin=(1e-30, "g/cm**3"), zmax=(1e-25, "g/cm**3"))
+   slc.save()
+
+Units can be left out, in which case they implicitly match the current display
+units of the colorbar (controlled with the ``set_unit`` method, see
+:ref:`_set-image-units`).
+
+It is not required to specify both ``zmin`` and ``zmax``. Left unset, they will
+default to extremal values in the current view. This default beheviour can be
+enforced or restored by passing ``zmin="min"`` (reps. ``zmax="max"``)
+explicitly.
+
+
+:meth:`~yt.visualization.plot_container.ImagePlotContainer.set_log` takes a boolean argument
+to select log (``True``) or linear (``False``) scalings.
 
 .. python-script::
 
    import yt
 
    ds = yt.load("IsolatedGalaxy/galaxy0030/galaxy0030")
    slc = yt.SlicePlot(ds, "z", ("gas", "density"), width=(10, "kpc"))
-   slc.set_log(("gas", "density"), False)
+   slc.set_log(("gas", "density"), False) # switch to linear scaling
    slc.save()
 
-Specifically, a field containing both positive and negative values can be plotted
-with symlog scale, by setting the boolean to be ``True`` and either providing an extra
-parameter ``linthresh`` or setting ``symlog_auto = True``. In the region around zero
-(when the log scale approaches to infinity), the linear scale will be applied to the
-region ``(-linthresh, linthresh)`` and stretched relative to the logarithmic range.
-In some cases, if yt detects zeros present in the dataset and the user has selected
-``log`` scaling, yt automatically switches to ``symlog`` scaling and automatically
-chooses a ``linthresh`` value to avoid errors.  This is the same behavior you can
-achieve by setting the keyword ``symlog_auto`` to ``True``. In these cases, yt will
-choose the smallest non-zero value in a dataset to be the ``linthresh`` value.
-As an example,
+One can switch to `symlog
+<https://matplotlib.org/stable/api/_as_gen/matplotlib.colors.SymLogNorm.html?highlight=symlog#matplotlib.colors.SymLogNorm>`_
+by providing a "linear threshold" (``linthresh``) value.
+With ``linthresh="auto"`` yt will switch to symlog norm and guess an appropriate value
+automatically. Specifically the minimum absolute value in the image is used
+unless it's zero, in which case yt uses 1/1000 of the maximum value.
+
+.. python-script::
+
+   import yt
+
+   ds = yt.load_sample("IsolatedGalaxy")
+   slc = yt.SlicePlot(ds, "z", ("gas", "density"), width=(10, "kpc"))
+   slc.set_log(("gas", "density"), linthresh="auto")
+   slc.save()
+
+
+In some cases, you might find that the automatically selected linear threshold is not
+really suited to your dataset, for instance
+
+.. python-script::
+
+   import yt
+
+   ds = yt.load_sample("FIRE_M12i_ref11")
+   p = yt.ProjectionPlot(ds, "x", ("gas", "density"), width=(30, "Mpc"))
+   p.set_log(("gas", "density"), linthresh="auto")
+   p.save()
+
+An explicit value can be passed instead
 
 .. python-script::
 
    import yt
 
    ds = yt.load_sample("FIRE_M12i_ref11")
-   p = yt.ProjectionPlot(ds, "x", ("gas", "density"))
-   p.set_log(("gas", "density"), True, symlog_auto=True)
+   p = yt.ProjectionPlot(ds, "x", ("gas", "density"), width=(30, "Mpc"))
+   p.set_log(("gas", "density"), linthresh=(1e-22, "g/cm**2"))
    p.save()
 
-Symlog is very versatile, and will work with positive or negative dataset ranges.
-Here is an example using symlog scaling to plot a positive field with a linear range of
-``(0, linthresh)``.
+Similar to the ``zmin`` and ``zmax`` arguments of the ``set_zlim`` method, units
+can be left out in ``linthresh``.
+
+
+**Arbitrary norms**
+
+Alternatively, arbitrary matplotlib norms can be passed via the
+:meth:`~yt.visualization.plot_container.PlotContainer.set_norm` method. In that
+case, any numeric value is treated as having implicit units, matching the
+current display units. This alternative interface is more flexible, but
+considered experimental as of yt 4.1. Don't forget that with great power comes
+great responsibility.
+
 
 .. python-script::
 
    import yt
+   from matplotlib.colors import TwoSlopeNorm
 
    ds = yt.load("IsolatedGalaxy/galaxy0030/galaxy0030")
    slc = yt.SlicePlot(ds, "z", ("gas", "velocity_x"), width=(30, "kpc"))
-   slc.set_log(("gas", "velocity_x"), True, linthresh=1.0e1)
+   slc.set_norm(("gas", "velocity_x"), TwoSlopeNorm(vcenter=0))
+
+   # using a diverging colormap to emphasize that vcenter corresponds to the
+   # middle value in the color range
+   slc.set_cmap(("gas", "velocity_x"), "RdBu")
    slc.save()
 
+.. note:: When calling
+  :meth:`~yt.visualization.plot_container.PlotContainer.set_norm`, any constraints
+  previously set with
+  :meth:`~yt.visualization.plot_container.PlotContainer.set_log` or
+  :meth:`~yt.visualization.plot_container.PlotContainer.set_zlim` will be dropped.
+  Conversely, calling ``set_log`` or ``set_zlim`` will have the
+  effect of dropping any norm previously set via ``set_norm``.
+
+
 The :meth:`~yt.visualization.plot_container.ImagePlotContainer.set_background_color`
 function accepts a field name and a color (optional). If color is given, the function
 will set the plot's background color to that. If not, it will set it to the bottom
@@ -994,33 +1080,6 @@ value of the color map.
    slc.set_background_color(("gas", "density"), color="black")
    slc.save("black_background")
 
-If you would like to change the background for a plot and also hide the axes,
-you will need to make use of the ``draw_frame`` keyword argument for the ``hide_axes`` function. If you do not use this keyword argument, the call to
-``set_background_color`` will have no effect. Here is an example illustrating how to use the ``draw_frame`` keyword argument for ``hide_axes``:
-
-.. python-script::
-
-   import yt
-
-   ds = yt.load("IsolatedGalaxy/galaxy0030/galaxy0030")
-   field = ("deposit", "all_density")
-   slc = yt.ProjectionPlot(ds, "z", field, width=(1.5, "Mpc"))
-   slc.set_background_color(field)
-   slc.hide_axes(draw_frame=True)
-   slc.hide_colorbar()
-   slc.save("just_image")
-
-Lastly, the :meth:`~yt.visualization.plot_window.AxisAlignedSlicePlot.set_zlim`
-function makes it possible to set a custom colormap range.
-
-.. python-script::
-
-   import yt
-
-   ds = yt.load("IsolatedGalaxy/galaxy0030/galaxy0030")
-   slc = yt.SlicePlot(ds, "z", ("gas", "density"), width=(10, "kpc"))
-   slc.set_zlim(("gas", "density"), 1e-30, 1e-25)
-   slc.save()
 
 Annotations
 ~~~~~~~~~~~

nose_unit.cfg

@@ -6,5 +6,5 @@ nologcapture=1
 verbosity=2
 where=yt
 with-timer=1
-ignore-files=(test_load_errors.py|test_load_sample.py|test_commons.py|test_ambiguous_fields.py|test_field_access_pytest.py|test_save.py|test_line_annotation_unit.py|test_eps_writer.py|test_registration.py|test_invalid_origin.py|test_outputs_pytest\.py|test_normal_plot_api\.py|test_load_archive\.py|test_stream_particles\.py|test_file_sanitizer\.py|test_version\.py|\test_on_demand_imports\.py|test_add_field\.py)
+ignore-files=(test_load_errors.py|test_load_sample.py|test_commons.py|test_ambiguous_fields.py|test_field_access_pytest.py|test_save.py|test_line_annotation_unit.py|test_eps_writer.py|test_registration.py|test_invalid_origin.py|test_outputs_pytest\.py|test_normal_plot_api\.py|test_load_archive\.py|test_stream_particles\.py|test_file_sanitizer\.py|test_version\.py|\test_on_demand_imports\.py|test_set_zlim\.py|test_add_field\.py)
 exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF

setup.cfg

@@ -38,7 +38,7 @@ project_urls =
 packages = find:
 install_requires =
     cmyt>=0.2.2
-    matplotlib!=3.4.2,>=2.2.3  # keep in sync with tests/windows_conda_requirements.txt
+    matplotlib!=3.4.2,>=3.1  # keep in sync with tests/windows_conda_requirements.txt
     more-itertools>=8.4
     numpy>=1.14.5
     packaging>=20.9
@@ -49,6 +49,7 @@ install_requires =
     unyt>=2.8.0
     importlib-metadata>=1.4;python_version < '3.8'
     tomli>=1.2.3;python_version < '3.11'
+    typing-extensions>=4.2.0;python_version < '3.8'
 python_requires = >=3.7
 include_package_data = True
 scripts = scripts/iyt
@@ -102,7 +103,7 @@ mapserver =
     bottle
 minimal =
     cmyt==0.2.2
-    matplotlib==2.2.3
+    matplotlib==3.1
     more-itertools==8.4
     numpy==1.14.5
     pillow==6.2.0

tests/report_failed_answers.py

@@ -13,6 +13,7 @@
 import os
 import re
 import shutil
+import sys
 import tempfile
 import xml.etree.ElementTree as ET
 
@@ -425,6 +426,9 @@ def handle_error(error, testcase, missing_errors, missing_answers, failed_answer
                 + "\n"
             )
         response = upload_answers(failed_answers)
+        if response is None:
+            log.error("Failed to upload answers for failed tests !")
+            sys.exit(1)
         if response.ok:
             msg += (
                 FLAG_EMOJI
@@ -438,6 +442,9 @@ def handle_error(error, testcase, missing_errors, missing_answers, failed_answer
 
     if args.upload_missing_answers and missing_answers:
         response = upload_answers(missing_answers)
+        if response is None:
+            log.error("Failed to upload missing answers !")
+            sys.exit(1)
         if response.ok:
             msg = (
                 FLAG_EMOJI

tests/tests.yaml

@@ -95,7 +95,7 @@ answer_tests:
     - yt/frontends/owls/tests/test_outputs.py:test_snapshot_033
     - yt/frontends/owls/tests/test_outputs.py:test_OWLS_particlefilter
 
-  local_pw_044:  # PR 3640
+  local_pw_046:  # PR 3849
     - yt/visualization/tests/test_plotwindow.py:test_attributes
     - yt/visualization/tests/test_particle_plot.py:test_particle_projection_answers
     - yt/visualization/tests/test_particle_plot.py:test_particle_projection_filter
@@ -183,6 +183,16 @@ answer_tests:
   local_nc4_cm1_002: # PR  2176, 2998
     - yt/frontends/nc4_cm1/tests/test_outputs.py:test_cm1_mesh_fields
 
+  local_norm_api_008: # PR 3849
+    - yt/visualization/tests/test_norm_api_lineplot.py:test_lineplot_set_axis_properties
+    - yt/visualization/tests/test_norm_api_profileplot.py:test_profileplot_set_axis_properties
+    - yt/visualization/tests/test_norm_api_custom_norm.py:test_sliceplot_custom_norm
+    - yt/visualization/tests/test_norm_api_set_background_color.py:test_sliceplot_set_background_color
+    - yt/visualization/tests/test_norm_api_phaseplot_set_colorbar_implicit.py:test_phaseplot_set_colorbar_properties_implicit
+    - yt/visualization/tests/test_norm_api_phaseplot_set_colorbar_explicit.py:test_phaseplot_set_colorbar_properties_explicit
+    - yt/visualization/tests/test_norm_api_particleplot.py:test_particleprojectionplot_set_colorbar_properties
+    - yt/visualization/tests/test_norm_api_inf_zlim.py:test_inf_and_finite_values_zlim
+
   local_cf_radial_002: # PR  1990
     - yt/frontends/cf_radial/tests/test_outputs.py:test_cfradial_grid_field_values
 
@@ -206,6 +216,7 @@ other_tests:
      - "--ignore-files=test_normal_plot_api\\.py"
      - "--ignore-file=test_file_sanitizer\\.py"
      - "--ignore-files=test_version\\.py"
+     - "--ignore-files=test_set_zlim\\.py"
      - "--ignore-file=test_add_field\\.py"
      - "--exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF"
      - "--exclude-test=yt.frontends.adaptahop.tests.test_outputs"

tests/windows_conda_requirements.txt

@@ -2,5 +2,5 @@ numpy>=1.19.4
 cython>=0.29.21,<3.0
 cartopy>=0.20.1
 h5py~=3.1.0
-matplotlib!=3.4.2,>=2.2.3  # keep in sync with setup.cfg
+matplotlib!=3.4.2,>=3.1  # keep in sync with setup.cfg
 scipy~=1.5.0

yt/_maintenance/backports.py

@@ -80,3 +80,18 @@ def __get__(self, instance, owner=None):
 
 else:
     pass
+
+
+builtin_zip = zip
+if sys.version_info >= (3, 10):
+    zip = builtin_zip
+else:
+    # this function is deprecated in more_itertools
+    # because it is superseded by the standard library
+    from more_itertools import zip_equal
+
+    def zip(*args, strict=False):
+        if strict:
+            return zip_equal(*args)
+        else:
+            return builtin_zip(*args)

yt/_typing.py

@@ -1,5 +1,6 @@
 from typing import List, Optional, Tuple, Union
 
+import unyt as un
 from numpy import ndarray
 
 FieldDescT = Tuple[str, Tuple[str, List[str], Optional[str]]]
@@ -10,3 +11,10 @@
     Tuple[ndarray, ndarray, ndarray],  # xyz
     Union[float, ndarray],  # hsml
 ]
+
+
+# types that can be converted to un.Unit
+Unit = Union[un.Unit, str]
+
+# types that can be converted to un.unyt_quantity
+Quantity = Union[un.unyt_quantity, Tuple[float, Unit]]

yt/funcs.py

@@ -1260,6 +1260,12 @@ def dictWithFactory(factory: Callable[[Any], Any]) -> Type:
         A class to create new dictionaries handling missing keys.
     """
 
+    issue_deprecation_warning(
+        "yt.funcs.dictWithFactory will be removed in a future version of yt, please do not rely on it. "
+        "If you need it, copy paste this function from yt's source code",
+        since="4.1",
+    )
+
     class DictWithFactory(dict):
         def __init__(self, *args, **kwargs):
             self.factory = factory

yt/utilities/answer_testing/framework.py

@@ -278,6 +278,7 @@ def get(self, ds_name, default=None):
             return default
         # Read data using shelve
         answer_name = f"{ds_name}"
+        os.makedirs(os.path.dirname(self.reference_name), exist_ok=True)
         ds = shelve.open(self.reference_name, protocol=-1)
         try:
             result = ds[answer_name]