Merge pull request #3849 from jobh/fix-initialization-imbalance

Fix pytest plugin issue

hypothesis-python/RELEASE.rst

@@ -0,0 +1,5 @@
+RELEASE_TYPE: patch
+
+Fix a spurious warning seen when running pytest's test
+suite, caused by never realizing we got out of
+initialization due to imbalanced hook calls.

hypothesis-python/src/_hypothesis_globals.py

@@ -16,9 +16,9 @@
 import os
 
 in_initialization = 1
-"""If nonzero, indicates that hypothesis is still initializing (importing or loading
+"""If >0, indicates that hypothesis is still initializing (importing or loading
 the test environment). `import hypothesis` will cause this number to be decremented,
-and the pytest plugin increments at load time, then decrements it just before the test
+and the pytest plugin increments at load time, then decrements it just before each test
 session starts. However, this leads to a hole in coverage if another pytest plugin
 imports hypothesis before our plugin is loaded. HYPOTHESIS_EXTEND_INITIALIZATION may
 be set to pre-increment the value on behalf of _hypothesis_pytestplugin, plugging the

hypothesis-python/src/_hypothesis_pytestplugin.py

@@ -101,7 +101,6 @@ def __call__(self, msg):
     # need balanced increment/decrement in configure/sessionstart to support nested
     # pytest (e.g. runpytest_inprocess), so this early increment in effect replaces
     # the first one in pytest_configure.
-    _configured = False
     if not os.environ.get("HYPOTHESIS_EXTEND_INITIALIZATION"):
         _hypothesis_globals.in_initialization += 1
         if "hypothesis" in sys.modules:
@@ -163,12 +162,6 @@ def pytest_report_header(config):
         return f"hypothesis profile {settings._current_profile!r}{settings_str}"
 
     def pytest_configure(config):
-        global _configured
-        # skip first increment because we pre-incremented at import time
-        if _configured:
-            _hypothesis_globals.in_initialization += 1
-        _configured = True
-
         config.addinivalue_line("markers", "hypothesis: Tests which use hypothesis.")
         if not _any_hypothesis_option(config):
             return
@@ -430,6 +423,7 @@ def pytest_collection_modifyitems(items):
                 item.add_marker("hypothesis")
 
     def pytest_sessionstart(session):
+        # Note: may be called multiple times, so we can go negative
         _hypothesis_globals.in_initialization -= 1
 
     # Monkeypatch some internals to prevent applying @pytest.fixture() to a

hypothesis-python/src/hypothesis/configuration.py

@@ -9,6 +9,7 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import os
+import sys
 import warnings
 from pathlib import Path
 
@@ -45,7 +46,7 @@ def storage_directory(*names, intent_to_write=True):
 
 
 def check_sideeffect_during_initialization(
-    what: str, *fmt_args: object, extra: str = ""
+    what: str, *fmt_args: object, is_restart: bool = False
 ) -> None:
     """Called from locations that should not be executed during initialization, for example
     touching disk or materializing lazy/deferred strategies from plugins. If initialization
@@ -60,13 +61,29 @@ def check_sideeffect_during_initialization(
     # notice_initialization_restarted() to be called if in_initialization changes away from zero.
     if _first_postinit_what is not None:
         return
-    elif _hypothesis_globals.in_initialization:
+    elif _hypothesis_globals.in_initialization > 0:
+        msg = what.format(*fmt_args)
+        if is_restart:
+            when = "between importing hypothesis and loading the hypothesis plugin"
+        elif "_hypothesis_pytestplugin" in sys.modules or os.getenv(
+            "HYPOTHESIS_EXTEND_INITIALIZATION"
+        ):
+            when = "during pytest plugin or conftest initialization"
+        else:  # pragma: no cover
+            # This can be triggered by Hypothesis plugins, but is really annoying
+            # to test automatically - drop st.text().example() in hypothesis.run()
+            # to manually confirm that we get the warning.
+            when = "at import time"
         # Note: -Werror is insufficient under pytest, as doesn't take effect until
         # test session start.
-        msg = what.format(*fmt_args)
+        text = (
+            f"Slow code in plugin: avoid {msg} {when}!  Set PYTHONWARNINGS=error "
+            "to get a traceback and show which plugin is responsible."
+        )
+        if is_restart:
+            text += " Additionally, set HYPOTHESIS_EXTEND_INITIALIZATION=1 to pinpoint the exact location."
         warnings.warn(
-            f"Slow code in plugin: avoid {msg} at import time!  Set PYTHONWARNINGS=error "
-            "to get a traceback and show which plugin is responsible." + extra,
+            text,
             HypothesisSideeffectWarning,
             stacklevel=3,
         )
@@ -87,5 +104,5 @@ def notice_initialization_restarted(*, warn: bool = True) -> None:
             check_sideeffect_during_initialization(
                 what,
                 *fmt_args,
-                extra=" Additionally, set HYPOTHESIS_EXTEND_INITIALIZATION=1 to pinpoint the exact location.",
+                is_restart=True,
             )

hypothesis-python/tests/conjecture/test_engine.py

@@ -811,7 +811,7 @@ def f(data):
             data.mark_interesting()
 
     monkeypatch.setattr(time, "perf_counter", fast_time)
-    runner = ConjectureRunner(f, settings=settings(database=None))
+    runner = ConjectureRunner(f, settings=settings(database=None, max_examples=100_000))
     runner.run()
     assert runner.exit_reason == ExitReason.very_slow_shrinking
     assert runner.statistics["stopped-because"] == "shrinking was very slow"

hypothesis-python/tests/cover/test_sideeffect_warnings.py

@@ -54,6 +54,7 @@ def test_sideeffect_delayed_warning(monkeypatch, _extend_initialization):
     monkeypatch.setattr(_hypothesis_globals, IN_INITIALIZATION_ATTR, 0)
     fs.check_sideeffect_during_initialization(what)
     fs.check_sideeffect_during_initialization("ignored since not first")
-    with pytest.warns(HypothesisSideeffectWarning, match=what):
+    # The warning should identify itself as happening after import but before plugin load
+    with pytest.warns(HypothesisSideeffectWarning, match=what + ".*between import"):
         monkeypatch.setattr(_hypothesis_globals, IN_INITIALIZATION_ATTR, 1)
         fs.notice_initialization_restarted()

hypothesis-python/tests/nocover/test_sampled_from.py

@@ -24,7 +24,6 @@ def test_filter_large_lists(n):
 
     @counts_calls
     def cond(x):
-        assert cond.calls < filter_limit
         return x % 2 != 0
 
     s = st.sampled_from(range(n)).filter(cond)

hypothesis-python/tests/pytest/test_sideeffect_warnings.py

@@ -50,6 +50,8 @@ def test_conftest_sideeffect_pinpoint_error(testdir, monkeypatch):
     script = testdir.makepyfile(TEST_SCRIPT)
     result = testdir.runpytest_subprocess(script)
     assert "HypothesisSideeffectWarning" in "\n".join(result.errlines)
+    # Plugin is always loaded before conftest, so "during pytest plugin initialization"
+    assert "during pytest" in "\n".join(result.errlines)
     assert SIDEEFFECT_STATEMENT in "\n".join(result.errlines)
 
 
@@ -62,4 +64,6 @@ def test_plugin_sideeffect_pinpoint_error(testdir, monkeypatch):
     script = testdir.makepyfile(TEST_SCRIPT)
     result = testdir.runpytest_subprocess(script, "-p", "sideeffect_plugin")
     assert "HypothesisSideeffectWarning" in "\n".join(result.errlines)
+    # Plugin order unknown, but certainly not at import time
+    assert "at import time" not in "\n".join(result.errlines)
     assert SIDEEFFECT_STATEMENT in "\n".join(result.errlines)