[ENH] Add custom thumbnails for failing examples (#1313)

Co-authored-by: Lucy Liu <jliu176@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Eric Larson <larson.eric.d@gmail.com>

doc/conf.py

@@ -393,6 +393,7 @@ def setup(app):
     "within_subsection_order": "FileNameSortKey",
     "expected_failing_examples": [
         "../examples/no_output/plot_raise.py",
+        "../examples/no_output/plot_raise_thumbnail.py",
         "../examples/no_output/plot_syntaxerror.py",
     ],
     "min_reported_time": min_reported_time,

doc/configuration.rst

@@ -99,6 +99,7 @@ Some options can also be set or overridden on a file-by-file basis:
 - ``# sphinx_gallery_line_numbers`` (:ref:`adding_line_numbers`)
 - ``# sphinx_gallery_thumbnail_number`` (:ref:`choosing_thumbnail`)
 - ``# sphinx_gallery_thumbnail_path`` (:ref:`providing_thumbnail`)
+- ``# sphinx_gallery_failing_thumbnail`` (:ref:`failing_thumbnail`)
 - ``# sphinx_gallery_dummy_images`` (:ref:`dummy_images`)
 - ``# sphinx_gallery_capture_repr`` (:ref:`capture_repr`)
 
@@ -1318,6 +1319,29 @@ Note that ``sphinx_gallery_thumbnail_number`` overrules
 :ref:`sphx_glr_auto_examples_plot_4b_provide_thumbnail.py` for an example of
 this functionality.
 
+.. _failing_thumbnail:
+
+Controlling thumbnail behaviour in failing examples
+===================================================
+
+By default, expected failing examples will have their thumbnail image as a
+stamp with the word "BROKEN". This behaviour is controlled by
+``sphinx_gallery_failing_thumbnail``, which is by default ``True``. In cases
+where control over the thumbnail image is desired, this should be set to
+``False``. This will return thumbnail behaviour to 'normal', whereby
+thumbnail will be either the first figure created (or the
+:ref:`default thumbnail <custom_default_thumb>` if no figure is created)
+or :ref:`provided thumbnail <providing_thumbnail>`::
+
+
+    # sphinx_gallery_failing_thumbnail = False
+
+Compare the thumbnails of
+:ref:`sphx_glr_auto_examples_no_output_plot_raise_thumbnail.py` (where the
+option is ``False``) and :ref:`sphx_glr_auto_examples_no_output_plot_raise.py`
+(where the option is the default ``True``) for an example of this
+functionality.
+
 
 .. _binder_links:
 

examples/no_output/plot_raise_thumbnail.py

@@ -0,0 +1,29 @@
+"""
+Example that fails to execute (with normal thumbnail behaviour)
+===============================================================
+
+By default, examples with code blocks that raise an error will have the broken
+image stamp as their gallery thumbnail. However, this may not be desired, e.g.
+if only part of the example is expected to fail and it should not look like the
+entire example fails.
+
+In these cases, the `sphinx_gallery_failing_thumbnail` variable can be set to
+``False``, which will change the thumbnail selection to the default behaviour
+as for non-failing examples.
+"""
+
+# Code source: Thomas S. Binns
+# License: BSD 3 clause
+# sphinx_gallery_line_numbers = True
+
+# sphinx_gallery_failing_thumbnail = False
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+plt.pcolormesh(np.random.randn(100, 100))
+
+# %%
+# This block will raise an AssertionError
+
+assert False

sphinx_gallery/gen_rst.py

@@ -395,7 +395,9 @@ def save_thumbnail(image_path_template, src_file, script_vars, file_conf, galler
     base_image_name = os.path.splitext(os.path.basename(src_file))[0]
     thumb_file = os.path.join(thumb_dir, f"sphx_glr_{base_image_name}_thumb.{ext}")
 
-    if "formatted_exception" in script_vars:
+    if "formatted_exception" in script_vars and file_conf.get(
+        "failing_thumbnail", True
+    ):
         img = os.path.join(glr_path_static(), "broken_example.png")
     elif os.path.exists(thumbnail_image_path):
         img = thumbnail_image_path

sphinx_gallery/tests/test_full.py

@@ -41,8 +41,8 @@
 #
 # total number of plot_*.py files in
 # (examples + examples_rst_index + examples_with_rst + examples_README_header)
-N_EXAMPLES = 15 + 3 + 2 + 1
-N_FAILING = 2
+N_EXAMPLES = 17 + 3 + 2 + 1
+N_FAILING = 4
 N_GOOD = N_EXAMPLES - N_FAILING  # galleries that run w/o error
 # passthroughs and non-executed examples in
 # (examples + examples_rst_index + examples_with_rst + examples_README_header)
@@ -193,7 +193,7 @@ def test_junit(sphinx_app, tmp_path):
     want = dict(
         errors="0",
         failures="0",
-        skipped="2",
+        skipped="4",
         tests=f"{N_EXAMPLES}",
         name="sphinx-gallery",
     )
@@ -241,18 +241,42 @@ def test_junit(sphinx_app, tmp_path):
     with open(junit_file, "rb") as fid:
         suite = lxml.etree.fromstring(fid.read())
     # this time we only ran the stale files
-    want.update(failures="2", skipped="1", tests="3")
+    want.update(failures="2", skipped="3", tests="5")
     got = dict(suite.attrib)
     del got["time"]
-    assert len(suite) == 3
-    assert suite[0].attrib["classname"] == "plot_numpy_matplotlib"
-    assert suite[0][0].tag == "failure", suite[0].attrib["classname"]
-    assert suite[0][0].attrib["message"].startswith("RuntimeError: Forcing")
-    assert suite[1].attrib["classname"] == "plot_scraper_broken"
-    assert suite[1][0].tag == "skipped", suite[1].attrib["classname"]
-    assert suite[2].attrib["classname"] == "plot_future_imports_broken"
-    assert suite[2][0].tag == "failure", suite[2].attrib["classname"]
-    assert suite[2][0].attrib["message"] == "Passed even though it was marked to fail"
+    skips_and_fails = [
+        {
+            "classname": "plot_failing_example",
+            "tag": "skipped",
+            "message": None,
+        },
+        {
+            "classname": "plot_failing_example_thumbnail",
+            "tag": "skipped",
+            "message": None,
+        },
+        {
+            "classname": "plot_numpy_matplotlib",
+            "tag": "failure",
+            "message": "RuntimeError: Forcing",
+        },
+        {
+            "classname": "plot_scraper_broken",
+            "tag": "skipped",
+            "message": None,
+        },
+        {
+            "classname": "plot_future_imports_broken",
+            "tag": "failure",
+            "message": "Passed even though it was marked to fail",
+        },
+    ]
+    assert len(suite) == len(skips_and_fails)
+    for this_suite, this_example in zip(suite, skips_and_fails):
+        assert this_suite.attrib["classname"] == this_example["classname"]
+        assert this_suite[0].tag == this_example["tag"], this_suite.attrib["classname"]
+        if this_example["message"] is not None:
+            assert this_suite[0].attrib["message"].startswith(this_example["message"])
     assert got == want
 
 
@@ -330,6 +354,47 @@ def test_negative_thumbnail_config(sphinx_app, tmpdir):
     assert corr > 0.99
 
 
+def test_thumbnail_expected_failing_examples(sphinx_app, tmpdir):
+    """Test thumbnail behaviour for expected failing examples."""
+    import numpy as np
+
+    # Get the "BROKEN" stamp for the default failing example thumbnail
+    stamp_fname = op.join(
+        sphinx_app.srcdir, "_static_nonstandard", "broken_example.png"
+    )
+    stamp_fname_scaled = str(tmpdir.join("new.png"))
+    scale_image(
+        stamp_fname,
+        stamp_fname_scaled,
+        *sphinx_app.config.sphinx_gallery_conf["thumbnail_size"],
+    )
+    Image = _get_image()
+    broken_stamp = np.asarray(Image.open(stamp_fname_scaled))
+    assert broken_stamp.shape[2] in (3, 4)  # optipng can strip the alpha channel
+
+    # Get thumbnail from example with failing example thumbnail behaviour
+    # (i.e. thumbnail should be "BROKEN" stamp)
+    thumb_fname = op.join(
+        sphinx_app.outdir, "_images", "sphx_glr_plot_failing_example_thumb.png"
+    )
+    thumbnail = np.asarray(Image.open(thumb_fname))
+    assert broken_stamp.shape[:2] == thumbnail.shape[:2]
+    corr = np.corrcoef(broken_stamp[..., :3].ravel(), thumbnail[..., :3].ravel())[0, 1]
+    assert corr > 0.99  # i.e. thumbnail and "BROKEN" stamp are identical
+
+    # Get thumbnail from example with default thumbnail behaviour
+    # (i.e. thumbnail should be the plot from the example, not the "BROKEN" stamp)
+    thumb_fname = op.join(
+        sphinx_app.outdir,
+        "_images",
+        "sphx_glr_plot_failing_example_thumbnail_thumb.png",
+    )
+    thumbnail = np.asarray(Image.open(thumb_fname))
+    assert broken_stamp.shape[:2] == thumbnail.shape[:2]
+    corr = np.corrcoef(broken_stamp[..., :3].ravel(), thumbnail[..., :3].ravel())[0, 1]
+    assert corr < 0.7  # i.e. thumbnail and "BROKEN" stamp are not identical
+
+
 def test_command_line_args_img(sphinx_app):
     generated_examples_dir = op.join(sphinx_app.outdir, "auto_examples")
     thumb_fname = "../_images/sphx_glr_plot_command_line_args_thumb.png"
@@ -797,6 +862,8 @@ def test_rebuild(tmpdir_factory, sphinx_app):
         "sg_api_usage",
         "plot_future_imports_broken",
         "plot_scraper_broken",
+        "plot_failing_example",
+        "plot_failing_example_thumbnail",
     )
     _assert_mtimes(generated_rst_0, generated_rst_1, ignore=ignore)
 
@@ -890,6 +957,8 @@ def _rerun(
     #
     # - auto_examples/future/plot_future_imports_broken
     # - auto_examples/future/sg_execution_times
+    # - auto_examples/plot_failing_example
+    # - auto_examples/plot_failing_example_thumbnail
     # - auto_examples/plot_scraper_broken
     # - auto_examples/sg_execution_times
     # - auto_examples_rst_index/sg_execution_times
@@ -904,9 +973,9 @@ def _rerun(
     # - auto_examples/index
     # - auto_examples/plot_numpy_matplotlib
     if how == "modify":
-        n_ch = "([3-9]|10|11)"
+        n_ch = "([3-9]|1[0-3])"  # 3-13
     else:
-        n_ch = "[1-9]"
+        n_ch = "([1-9]|1[01])"  # 1-11
     lines = "\n".join([f"\n{how} != {n_ch}:"] + lines)
     want = f".*updating environment:.*[0|1] added, {n_ch} changed, 0 removed.*"
     assert re.match(want, status, flags) is not None, lines
@@ -968,6 +1037,8 @@ def _rerun(
         # this one will not change even though it was retried
         "plot_future_imports_broken",
         "plot_scraper_broken",
+        "plot_failing_example",
+        "plot_failing_example_thumbnail",
     )
     # not reliable on Windows and one Ubuntu run
     bad = sys.platform.startswith("win") or os.getenv("BAD_MTIME", "0") == "1"
@@ -1381,10 +1452,10 @@ def test_recommend_n_examples(sphinx_app):
 
     assert '<p class="rubric">Related examples</p>' in html
     assert count == n_examples
-    # Check the same 3 related examples are shown
+    # Check the same 3 related examples are shown (can change when new examples added)
     assert "sphx-glr-auto-examples-plot-repr-py" in html
-    assert "sphx-glr-auto-examples-plot-webp-py" in html
-    assert "sphx-glr-auto-examples-plot-numpy-matplotlib-py" in html
+    assert "sphx-glr-auto-examples-plot-matplotlib-backend-py" in html
+    assert "sphx-glr-auto-examples-plot-second-future-imports-py" in html
 
 
 def test_sidebar_components_download_links(sphinx_app):

sphinx_gallery/tests/test_gen_rst.py

@@ -1239,6 +1239,5 @@ def test_newlines(log_collector_wrap):
     assert tee.newlines == tee.output.newlines
 
 
-# TODO: test that broken thumbnail does appear when needed
 # TODO: test that examples are executed after a no-plot and produce
 #       the correct image in the thumbnail

sphinx_gallery/tests/tinybuild/doc/conf.py

@@ -78,6 +78,8 @@
     "expected_failing_examples": [
         "../examples/future/plot_future_imports_broken.py",
         "../examples/plot_scraper_broken.py",
+        "../examples/plot_failing_example.py",
+        "../examples/plot_failing_example_thumbnail.py",
     ],
     "show_memory": False,
     "compress_images": ("images", "thumbnails"),

sphinx_gallery/tests/tinybuild/examples/plot_failing_example.py

@@ -0,0 +1,15 @@
+"""
+Failing example test
+====================
+Test failing thumbnail appears for files in expected_failing_examples.
+"""
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+plt.pcolormesh(np.random.randn(100, 100))
+
+# %%
+# will raise AssertionError
+
+assert False

sphinx_gallery/tests/tinybuild/examples/plot_failing_example_thumbnail.py

@@ -0,0 +1,18 @@
+"""
+Failing example test with normal thumbnail behaviour
+====================================================
+Test files in expected_failing_examples run, but normal thumbnail behaviour is
+retained when sphinx_gallery_failing_thumbnail = False.
+"""
+
+# sphinx_gallery_failing_thumbnail = False
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+plt.pcolormesh(np.random.randn(100, 100))
+
+# %%
+# will raise AssertionError
+
+assert False