From b688f39236499b89dd8b6889f4f7bb37f755abad Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Sat, 23 Mar 2024 21:37:58 -0700 Subject: [PATCH 1/7] Parallel coverage tests --- build.sh | 4 ++-- hypothesis-python/setup.py | 2 +- hypothesis-python/tox.ini | 16 +++++----------- pytest.ini | 2 ++ requirements/coverage.in | 3 ++- requirements/coverage.txt | 10 +++++++--- requirements/fuzzing.txt | 14 +++++++++----- requirements/tools.txt | 18 ++++++++++++------ tooling/src/hypothesistooling/__main__.py | 6 +++--- 9 files changed, 43 insertions(+), 32 deletions(-) diff --git a/build.sh b/build.sh index e8600a3bcc..2817f0dfc4 100755 --- a/build.sh +++ b/build.sh @@ -25,8 +25,8 @@ if [ -n "${GITHUB_ACTIONS-}" ] || [ -n "${CODESPACES-}" ] ; then else # Otherwise, we install it from scratch # NOTE: tooling keeps this version in sync with ci_version in tooling - "$SCRIPTS/ensure-python.sh" 3.10.13 - PYTHON=$(pythonloc 3.10.13)/bin/python + "$SCRIPTS/ensure-python.sh" 3.10.14 + PYTHON=$(pythonloc 3.10.14)/bin/python fi TOOL_REQUIREMENTS="$ROOT/requirements/tools.txt" diff --git a/hypothesis-python/setup.py b/hypothesis-python/setup.py index faa4be13ef..e95cab1ad4 100644 --- a/hypothesis-python/setup.py +++ b/hypothesis-python/setup.py @@ -60,7 +60,7 @@ def local_file(name): "pytest": ["pytest>=4.6"], "dpcontracts": ["dpcontracts>=0.4"], "redis": ["redis>=3.0.0"], - "crosshair": ["hypothesis-crosshair>=0.0.2", "crosshair-tool>=0.0.51"], + "crosshair": ["hypothesis-crosshair>=0.0.2", "crosshair-tool>=0.0.53"], # zoneinfo is an odd one: every dependency is conditional, because they're # only necessary on old versions of Python or Windows systems or emscripten. "zoneinfo": [ diff --git a/hypothesis-python/tox.ini b/hypothesis-python/tox.ini index c0d5e38e41..6552fac2ea 100644 --- a/hypothesis-python/tox.ini +++ b/hypothesis-python/tox.ini @@ -168,18 +168,14 @@ setenv= commands_pre = rm -f branch-check pip install .[zoneinfo] - python -m coverage --version - python -m coverage debug sys - # Explicitly erase any old .coverage file so the report never sees it. - python -m coverage erase # Produce a coverage report even if the test suite fails. # (The tox task will still count as failed.) ignore_errors = true commands = - python -bb -X dev -m coverage run --rcfile=.coveragerc --source=hypothesis -m pytest -n0 --ff {posargs} \ + python -bb -X dev -m pytest -n auto --ff {posargs} \ + --cov=hypothesis.internal.conjecture --cov-config=.coveragerc \ tests/cover tests/conjecture tests/datetime tests/numpy tests/pandas tests/lark \ tests/redis tests/dpcontracts tests/codemods tests/typing_extensions tests/patching tests/test_annotated_types.py - python -m coverage report python scripts/validate_branch_check.py @@ -189,12 +185,10 @@ deps = setenv= PYTHONWARNDEFAULTENCODING=1 HYPOTHESIS_INTERNAL_COVERAGE=true -commands_pre = - python -m coverage erase -ignore_errors = true commands = - python -bb -X dev -m coverage run --rcfile=.coveragerc --source=hypothesis.internal.conjecture -m pytest -n0 --strict-markers tests/conjecture - python -m coverage report + python -bb -X dev \ + -m pytest -n auto tests/conjecture/ \ + --cov=hypothesis.internal.conjecture --cov-config=.coveragerc [testenv:examples3] diff --git a/pytest.ini b/pytest.ini index 0ad5627aaf..c0b5bbc2af 100644 --- a/pytest.ini +++ b/pytest.ini @@ -22,3 +22,5 @@ filterwarnings = default:`np\.bool` is a deprecated alias for the builtin `bool`:DeprecationWarning default:`np\.complex` is a deprecated alias for the builtin `complex`:DeprecationWarning default:`np\.object` is a deprecated alias for the builtin `object`:DeprecationWarning + # pytest-cov can't see into subprocesses; we'll see <100% covered if this is an issue + ignore:Module hypothesis.* was previously imported, but not measured diff --git a/requirements/coverage.in b/requirements/coverage.in index b4d6826c63..698196b4db 100644 --- a/requirements/coverage.in +++ b/requirements/coverage.in @@ -1,7 +1,6 @@ annotated-types black click -coverage dpcontracts fakeredis lark @@ -13,3 +12,5 @@ python-dateutil pytz typing-extensions -r test.in +# Need the unreleased compatibility fix for pytest-xdist rsyncdirs deprecation +git+https://github.com/pytest-dev/pytest-cov.git@9757222e2e044361e70125ebdd96e5eb87395983 diff --git a/requirements/coverage.txt b/requirements/coverage.txt index b1bba2f26f..b5d3c83431 100644 --- a/requirements/coverage.txt +++ b/requirements/coverage.txt @@ -16,8 +16,8 @@ click==8.1.7 # via # -r requirements/coverage.in # black -coverage==7.4.4 - # via -r requirements/coverage.in +coverage[toml]==7.4.4 + # via pytest-cov dpcontracts==0.6.0 # via -r requirements/coverage.in exceptiongroup==1.2.0 ; python_version < "3.11" @@ -59,12 +59,15 @@ pluggy==1.4.0 # via pytest ptyprocess==0.7.0 # via pexpect -pyarrow==15.0.1 +pyarrow==15.0.2 # via -r requirements/coverage.in pytest==8.1.1 # via # -r requirements/test.in + # pytest-cov # pytest-xdist +pytest-cov @ git+https://github.com/pytest-dev/pytest-cov.git@9757222e2e044361e70125ebdd96e5eb87395983 + # via -r requirements/coverage.in pytest-xdist==3.5.0 # via -r requirements/test.in python-dateutil==2.9.0.post0 @@ -88,6 +91,7 @@ sortedcontainers==2.4.0 tomli==2.0.1 # via # black + # coverage # pytest typing-extensions==4.10.0 # via diff --git a/requirements/fuzzing.txt b/requirements/fuzzing.txt index 90e11e79e0..ca66d479ae 100644 --- a/requirements/fuzzing.txt +++ b/requirements/fuzzing.txt @@ -29,10 +29,10 @@ click==8.1.7 # black # flask # hypothesis -coverage==7.4.4 +coverage[toml]==7.4.4 # via - # -r requirements/coverage.in # hypofuzz + # pytest-cov dash==2.16.1 # via hypofuzz dash-core-components==2.0.0 @@ -56,11 +56,11 @@ flask==3.0.2 # via dash hypofuzz==24.2.3 # via -r requirements/fuzzing.in -hypothesis[cli]==6.99.6 +hypothesis[cli]==6.99.12 # via hypofuzz idna==3.6 # via requests -importlib-metadata==7.0.2 +importlib-metadata==7.1.0 # via dash iniconfig==2.0.0 # via pytest @@ -116,7 +116,7 @@ psutil==5.9.8 # via hypofuzz ptyprocess==0.7.0 # via pexpect -pyarrow==15.0.1 +pyarrow==15.0.2 # via -r requirements/coverage.in pygments==2.17.2 # via rich @@ -124,7 +124,10 @@ pytest==8.1.1 # via # -r requirements/test.in # hypofuzz + # pytest-cov # pytest-xdist +pytest-cov @ git+https://github.com/pytest-dev/pytest-cov.git@9757222e2e044361e70125ebdd96e5eb87395983 + # via -r requirements/coverage.in pytest-xdist==3.5.0 # via -r requirements/test.in python-dateutil==2.9.0.post0 @@ -161,6 +164,7 @@ tenacity==8.2.3 tomli==2.0.1 # via # black + # coverage # pytest typing-extensions==4.10.0 # via diff --git a/requirements/tools.txt b/requirements/tools.txt index b4def4eaf2..c5f30a4e4e 100644 --- a/requirements/tools.txt +++ b/requirements/tools.txt @@ -8,7 +8,7 @@ alabaster==0.7.16 # via sphinx anyio==4.3.0 # via watchfiles -asgiref==3.7.2 +asgiref==3.8.1 # via django asttokens==2.4.1 # via stack-data @@ -87,7 +87,7 @@ idna==3.6 # requests imagesize==1.4.1 # via sphinx -importlib-metadata==7.0.2 +importlib-metadata==7.1.0 # via # keyring # twine @@ -97,6 +97,10 @@ ipython==8.22.2 # via -r requirements/tools.in jaraco-classes==3.3.1 # via keyring +jaraco-context==4.3.0 + # via keyring +jaraco-functools==4.0.0 + # via keyring jedi==0.19.1 # via ipython jeepney==0.8.0 @@ -109,7 +113,7 @@ jinja2==3.1.3 # sphinx jsonpointer==2.4 # via sphinx-jsonschema -keyring==24.3.1 +keyring==25.0.0 # via twine lark==1.1.9 # via -r requirements/tools.in @@ -128,7 +132,9 @@ matplotlib-inline==0.1.6 mdurl==0.1.2 # via markdown-it-py more-itertools==10.2.0 - # via jaraco-classes + # via + # jaraco-classes + # jaraco-functools mypy==1.9.0 # via -r requirements/tools.in mypy-extensions==1.0.0 @@ -194,7 +200,7 @@ pyproject-hooks==1.0.0 # via # build # pip-tools -pyright==1.1.354 +pyright==1.1.355 # via -r requirements/tools.in pytest==8.1.1 # via -r requirements/tools.in @@ -298,7 +304,7 @@ tomli==2.0.1 # pyproject-hooks # pytest # tox -tox==4.14.1 +tox==4.14.2 # via -r requirements/tools.in traitlets==5.14.2 # via diff --git a/tooling/src/hypothesistooling/__main__.py b/tooling/src/hypothesistooling/__main__.py index 6782b02a09..2944a1e5e6 100644 --- a/tooling/src/hypothesistooling/__main__.py +++ b/tooling/src/hypothesistooling/__main__.py @@ -427,9 +427,9 @@ def run_tox(task, version, *args): # When a version is added or removed, manually update the env lists in tox.ini and # workflows/main.yml, and the `Programming Language ::` specifiers in setup.py PYTHONS = { - "3.8": "3.8.18", - "3.9": "3.9.18", - "3.10": "3.10.13", + "3.8": "3.8.19", + "3.9": "3.9.19", + "3.10": "3.10.14", "3.11": "3.11.8", "3.12": "3.12.2", "3.13": "3.13.0a5", From ed9a34f4173dee7ea776c65082537159edc5d8cd Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Sat, 23 Mar 2024 21:37:58 -0700 Subject: [PATCH 2/7] Split windows tests --- .github/workflows/main.yml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7cf0561f95..848afa1672 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -123,22 +123,24 @@ jobs: runs-on: windows-latest strategy: matrix: - include: - - python-version: "3.9" - - python-version: "3.10" - - python-version: "3.11" - - python-version: "3.11" - python-architecture: "x86" + python: + - version: "3.9" + - version: "3.11" + - version: "3.11" + architecture: "x86" + whichtests: + - nocover + - cover+rest fail-fast: false steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} ${{ matrix.python-architecture }} + - name: Set up Python ${{ matrix.python.version }} ${{ matrix.python.architecture }} uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.python-architecture }} + python-version: ${{ matrix.python.version }} + architecture: ${{ matrix.python.architecture }} - name: Restore cache uses: actions/cache@v3 with: @@ -146,12 +148,12 @@ jobs: ~\appdata\local\pip\cache vendor\bundle .tox - key: deps-${{ runner.os }}-${{ matrix.python-architecture }}-${{ hashFiles('requirements/*.txt') }}-${{ matrix.python-version }} + key: deps-${{ runner.os }}-${{ matrix.python.architecture }}-${{ hashFiles('requirements/*.txt') }}-${{ matrix.python.version }} restore-keys: | - deps-${{ runner.os }}-${{ matrix.python-architecture }}-${{ hashFiles('requirements/*.txt') }} - deps-${{ runner.os }}-${{ matrix.python-architecture }} + deps-${{ runner.os }}-${{ matrix.python.architecture }}-${{ hashFiles('requirements/*.txt') }} + deps-${{ runner.os }}-${{ matrix.python.architecture }} - name: Use old pandas on win32 - if: matrix.python-architecture + if: matrix.python.architecture # See https://github.com/pandas-dev/pandas/issues/54979 run: | (Get-Content .\requirements\coverage.txt) -replace 'pandas==[0-9.]+', 'pandas==2.0.3' | Out-File .\requirements\coverage.txt @@ -162,7 +164,7 @@ jobs: pip install -r requirements/coverage.txt pip install hypothesis-python/[all] - name: Run tests - run: python -m pytest --numprocesses auto hypothesis-python/tests/ --ignore=hypothesis-python/tests/quality/ --ignore=hypothesis-python/tests/ghostwriter/ --ignore=hypothesis-python/tests/patching/ + run: python -m pytest --numprocesses auto ${{ matrix.whichtests == 'nocover' && 'hypothesis-python/tests/nocover' || 'hypothesis-python/tests/ --ignore=hypothesis-python/tests/nocover/ --ignore=hypothesis-python/tests/quality/ --ignore=hypothesis-python/tests/ghostwriter/ --ignore=hypothesis-python/tests/patching/' }} test-osx: runs-on: macos-latest From 284c99bcc3462d4c1eda1716eb00a88ade75ed2f Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Sat, 23 Mar 2024 21:37:58 -0700 Subject: [PATCH 3/7] Concurrent custom branch check --- .github/workflows/main.yml | 2 +- .gitignore | 2 +- hypothesis-python/scripts/validate_branch_check.py | 6 ++++-- hypothesis-python/src/hypothesis/internal/coverage.py | 2 +- hypothesis-python/tox.ini | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 848afa1672..c1716becc7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -117,7 +117,7 @@ jobs: path: | hypothesis-python/.coverage* !hypothesis-python/.coveragerc - hypothesis-python/branch-check + hypothesis-python/branch-check* test-win: runs-on: windows-latest diff --git a/.gitignore b/.gitignore index 5a404bcaaa..84fa305ae1 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ # generic build components .runtimes -/hypothesis-python/branch-check +/hypothesis-python/branch-check* /pythonpython3.* /pythonpypy3.* .pyodide-xbuildenv diff --git a/hypothesis-python/scripts/validate_branch_check.py b/hypothesis-python/scripts/validate_branch_check.py index 12daecfef7..a9fb7064f2 100644 --- a/hypothesis-python/scripts/validate_branch_check.py +++ b/hypothesis-python/scripts/validate_branch_check.py @@ -11,10 +11,12 @@ import json import sys from collections import defaultdict +from pathlib import Path if __name__ == "__main__": - with open("branch-check", encoding="utf-8") as i: - data = [json.loads(l) for l in i] + data = [] + for p in Path.cwd().glob("branch-check*"): + data.extend(json.loads(l) for l in p.read_text("utf-8").splitlines()) checks = defaultdict(set) diff --git a/hypothesis-python/src/hypothesis/internal/coverage.py b/hypothesis-python/src/hypothesis/internal/coverage.py index 40f609b75a..c71ce28642 100644 --- a/hypothesis-python/src/hypothesis/internal/coverage.py +++ b/hypothesis-python/src/hypothesis/internal/coverage.py @@ -61,7 +61,7 @@ def record_branch(name, value): if key in written: return written.add(key) - with open("branch-check", mode="a", encoding="utf-8") as log: + with open(f"branch-check-{os.getpid()}", mode="a", encoding="utf-8") as log: log.write(json.dumps({"name": name, "value": value}) + "\n") description_stack = [] diff --git a/hypothesis-python/tox.ini b/hypothesis-python/tox.ini index 6552fac2ea..131f99698a 100644 --- a/hypothesis-python/tox.ini +++ b/hypothesis-python/tox.ini @@ -166,7 +166,7 @@ setenv= PYTHONWARNDEFAULTENCODING=1 HYPOTHESIS_INTERNAL_COVERAGE=true commands_pre = - rm -f branch-check + rm -f branch-check* pip install .[zoneinfo] # Produce a coverage report even if the test suite fails. # (The tox task will still count as failed.) From 9a8422c3752ddc3c059904160867399bacdcd8cb Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Sat, 23 Mar 2024 21:37:58 -0700 Subject: [PATCH 4/7] Observe backend in how_generated --- hypothesis-python/RELEASE.rst | 4 ++++ hypothesis-python/src/hypothesis/core.py | 19 +++++++++++-------- .../src/hypothesis/internal/observability.py | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..9cb6875cbe --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,4 @@ +RELEASE_TYPE: patch + +This patch includes the :obj:`~hypothesis.settings.backend` setting in the +``how_generated`` field of our :doc:`observability output `. diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index 402382c6aa..ccd5c43b6e 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -786,7 +786,6 @@ def __init__(self, stuff, test, settings, random, wrapped_test): self.explain_traces = defaultdict(set) self._start_timestamp = time.time() self._string_repr = "" - self._jsonable_arguments = {} self._timing_features = {} @property @@ -913,7 +912,7 @@ def run(data): ), ) self._string_repr = printer.getvalue() - self._jsonable_arguments = { + data._observability_arguments = { **dict(enumerate(map(to_jsonable, args))), **{k: to_jsonable(v) for k, v in kwargs.items()}, } @@ -1085,19 +1084,23 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None: # Conditional here so we can save some time constructing the payload; in # other cases (without coverage) it's cheap enough to do that regardless. if TESTCASE_CALLBACKS: - if self.failed_normally or self.failed_due_to_deadline: - phase = "shrink" - elif runner := getattr(self, "_runner", None): + if runner := getattr(self, "_runner", None): phase = runner._current_phase + elif self.failed_normally or self.failed_due_to_deadline: + phase = "shrink" else: # pragma: no cover # in case of messing with internals phase = "unknown" + backend_desc = f", using backend={self.settings.backend!r}" * ( + self.settings.backend != "hypothesis" + and not getattr(runner, "_switch_to_hypothesis_provider", False) + ) tc = make_testcase( start_timestamp=self._start_timestamp, test_name_or_nodeid=self.test_identifier, data=data, - how_generated=f"generated during {phase} phase", + how_generated=f"during {phase} phase{backend_desc}", string_repr=self._string_repr, - arguments={**self._jsonable_arguments, **data._observability_args}, + arguments=data._observability_args, timing=self._timing_features, coverage=tractable_coverage_report(trace) or None, phase=phase, @@ -1217,7 +1220,7 @@ def run_engine(self): "status": "passed" if sys.exc_info()[0] else "failed", "status_reason": str(origin or "unexpected/flaky pass"), "representation": self._string_repr, - "arguments": self._jsonable_arguments, + "arguments": ran_example._observability_args, "how_generated": "minimal failing example", "features": { **{ diff --git a/hypothesis-python/src/hypothesis/internal/observability.py b/hypothesis-python/src/hypothesis/internal/observability.py index aff19d805c..a532d054cd 100644 --- a/hypothesis-python/src/hypothesis/internal/observability.py +++ b/hypothesis-python/src/hypothesis/internal/observability.py @@ -36,7 +36,7 @@ def make_testcase( start_timestamp: float, test_name_or_nodeid: str, data: ConjectureData, - how_generated: str = "unknown", + how_generated: str, string_repr: str = "", arguments: Optional[dict] = None, timing: Dict[str, float], From 2fda65f12b72e17e3b44685d39497a724628f84c Mon Sep 17 00:00:00 2001 From: Jonathan Plasse <13716151+JonathanPlasse@users.noreply.github.com> Date: Sat, 23 Mar 2024 21:37:58 -0700 Subject: [PATCH 5/7] Test CrossHair backend in the CI --- .github/workflows/main.yml | 3 +++ hypothesis-python/tests/common/setup.py | 4 ++-- hypothesis-python/tox.ini | 15 +++++++++++++++ tooling/src/hypothesistooling/__main__.py | 7 +++++-- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c1716becc7..b126372768 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -73,6 +73,9 @@ jobs: - check-pandas13 - check-pandas12 - check-pandas11 + # - check-crosshair-cover + # - check-crosshair-nocover + # - check-crosshair-niche - check-py38-oldestnumpy fail-fast: false steps: diff --git a/hypothesis-python/tests/common/setup.py b/hypothesis-python/tests/common/setup.py index 18a635cdc1..80e6812986 100644 --- a/hypothesis-python/tests/common/setup.py +++ b/hypothesis-python/tests/common/setup.py @@ -61,7 +61,7 @@ def run(): settings.register_profile("debug", settings(verbosity=Verbosity.debug)) - settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) - for backend in set(AVAILABLE_PROVIDERS) - {"hypothesis"}: settings.register_profile(backend, backend=backend) # e.g. "crosshair" + + settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) diff --git a/hypothesis-python/tox.ini b/hypothesis-python/tox.ini index 131f99698a..2b8c5bf356 100644 --- a/hypothesis-python/tox.ini +++ b/hypothesis-python/tox.ini @@ -103,6 +103,21 @@ commands = python -bb -X dev -m pytest tests/pandas -n auto # Adding a new pandas? See comment above! +[testenv:crosshair-{cover,nocover,niche,custom}] +deps = + -r../requirements/test.txt + -e .[crosshair] +allowlist_externals = + bash +setenv= + HYPOTHESIS_PROFILE=crosshair +commands = + # invoke with `./build.sh check-crosshair-cover -- -x -Wignore` + cover: python -bb -X dev -m pytest -n auto tests/cover/ tests/pytest/ {posargs} + nocover: python -bb -X dev -m pytest -n auto tests/nocover/ {posargs} + niche: bash scripts/other-tests.sh + custom: python -bb -X dev -m pytest {posargs} + [testenv:django32] commands = pip install .[pytz] diff --git a/tooling/src/hypothesistooling/__main__.py b/tooling/src/hypothesistooling/__main__.py index 2944a1e5e6..08cd84bab6 100644 --- a/tooling/src/hypothesistooling/__main__.py +++ b/tooling/src/hypothesistooling/__main__.py @@ -481,9 +481,9 @@ def tox(*args): run_tox(*args) -def standard_tox_task(name, *args, py=ci_version): +def standard_tox_task(name, py=ci_version): TASKS["check-" + name] = python_tests( - lambda: run_tox(name, PYTHONS.get(py, py), *args) + lambda *args: run_tox(name, PYTHONS.get(py, py), *args) ) @@ -498,6 +498,9 @@ def standard_tox_task(name, *args, py=ci_version): for n in [11, 12, 13, 14, 15, 20]: standard_tox_task(f"pandas{n}") +for kind in ("cover", "nocover", "niche"): + standard_tox_task(f"crosshair-{kind}") + standard_tox_task("py38-oldestnumpy", py="3.8") standard_tox_task("coverage") From 067fc761d1caaba1fbc7c673c6efab17452a0ddb Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Sun, 24 Mar 2024 01:14:46 -0700 Subject: [PATCH 6/7] Observe clearer status_reason --- hypothesis-python/src/hypothesis/extra/array_api.py | 4 ++-- .../src/hypothesis/internal/conjecture/data.py | 6 +++--- .../src/hypothesis/internal/conjecture/engine.py | 9 +++++++++ hypothesis-python/src/hypothesis/stateful.py | 2 +- .../src/hypothesis/strategies/_internal/datetime.py | 2 +- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/hypothesis-python/src/hypothesis/extra/array_api.py b/hypothesis-python/src/hypothesis/extra/array_api.py index ce0993ab3b..8c82f63114 100644 --- a/hypothesis-python/src/hypothesis/extra/array_api.py +++ b/hypothesis-python/src/hypothesis/extra/array_api.py @@ -424,12 +424,12 @@ def do_draw(self, data): while elements.more(): i = data.draw_integer(0, self.array_size - 1) if i in assigned: - elements.reject() + elements.reject("chose an array index we've already used") continue val = data.draw(self.elements_strategy) if self.unique: if val in seen: - elements.reject() + elements.reject("chose an element we've already used") continue else: seen.add(val) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/data.py b/hypothesis-python/src/hypothesis/internal/conjecture/data.py index 3f15a974ef..701105bd5e 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/data.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/data.py @@ -2273,13 +2273,13 @@ def _pop_ir_tree_node(self, ir_type: IRTypeName, kwargs: IRKWargsType) -> IRNode # (in fact, it is possible that giving up early here results in more time # for useful shrinks to run). if node.ir_type != ir_type: - self.mark_invalid() + self.mark_invalid(f"(internal) want a {ir_type} but have a {node.ir_type}") # if a node has different kwargs (and so is misaligned), but has a value # that is allowed by the expected kwargs, then we can coerce this node # into an aligned one by using its value. It's unclear how useful this is. if not ir_value_permitted(node.value, node.ir_type, kwargs): - self.mark_invalid() + self.mark_invalid(f"(internal) got a {ir_type} but outside the valid range") return node @@ -2348,7 +2348,7 @@ def draw( strategy.validate() if strategy.is_empty: - self.mark_invalid("strategy is empty") + self.mark_invalid(f"empty strategy {self!r}") if self.depth >= MAX_DEPTH: self.mark_invalid("max depth exceeded") diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index 6791b28e04..ddf8c0e090 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -809,6 +809,15 @@ def generate_new_examples(self): self.test_function(data) + if ( + data.status == Status.OVERRUN + and max_length < BUFFER_SIZE + and "invalid because" not in data.events + ): + data.events["invalid because"] = ( + "reduced max size for early examples (avoids flaky health checks)" + ) + self.generate_mutations_from(data) # Although the optimisations are logically a distinct phase, we diff --git a/hypothesis-python/src/hypothesis/stateful.py b/hypothesis-python/src/hypothesis/stateful.py index d5af39b5c0..8c8272df7b 100644 --- a/hypothesis-python/src/hypothesis/stateful.py +++ b/hypothesis-python/src/hypothesis/stateful.py @@ -478,7 +478,7 @@ def do_draw(self, data): machine = data.draw(self_strategy) bundle = machine.bundle(self.name) if not bundle: - data.mark_invalid() + data.mark_invalid(f"Cannot draw from empty bundle {self.name!r}") # Shrink towards the right rather than the left. This makes it easier # to delete data generated earlier, as when the error is towards the # end there can be a lot of hard to remove padding. diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py b/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py index f2c33fa8c5..427d8c5ed2 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/datetime.py @@ -155,7 +155,7 @@ def do_draw(self, data): # If we happened to end up with a disallowed imaginary time, reject it. if (not self.allow_imaginary) and datetime_does_not_exist(result): - data.mark_invalid("nonexistent datetime") + data.mark_invalid(f"{result} does not exist (usually a DST transition)") return result def draw_naive_datetime_and_combine(self, data, tz): From 17296f06946b659ea310282f94da7ff3368acda2 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Sun, 24 Mar 2024 02:00:32 -0700 Subject: [PATCH 7/7] Another workaround for black --- hypothesis-python/src/hypothesis/extra/_patching.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hypothesis-python/src/hypothesis/extra/_patching.py b/hypothesis-python/src/hypothesis/extra/_patching.py index 8f53076d72..d3678e3f9f 100644 --- a/hypothesis-python/src/hypothesis/extra/_patching.py +++ b/hypothesis-python/src/hypothesis/extra/_patching.py @@ -121,7 +121,7 @@ def __call_node_to_example_dec(self, node, via): cst.Module([]).code_for_node(via), mode=black.FileMode(line_length=self.line_length), ) - except ImportError: + except (ImportError, AttributeError): return None # See https://github.com/psf/black/pull/4224 via = cst.parse_expression(pretty.strip()) return cst.Decorator(via)