Skip to content

Commit

Permalink
chore(pytest): fail fast on typos in notimpl, notyet, and never marks
Browse files Browse the repository at this point in the history
A typo in one of our custom marks would just lead to a test
failure (usually) because a test that we expected to xfail wasn't
xfailed.

This is annoying because a) it's surprising and b) you have to run
possibly the whole test suite to hit it (and then you have to figure out
why).

I added a collection hook to flag any typos in the arguments to our
custom marks, and it spits them all out, along with path and test name,
then aborts the test run altogether.
  • Loading branch information
gforsyth authored and cpcloud committed Jun 5, 2023
1 parent 80d2fb7 commit e6fb703
Showing 1 changed file with 19 additions and 1 deletion.
20 changes: 19 additions & 1 deletion ibis/backends/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@
'ibis/backends/tests/test_window.py',
]

ALL_BACKENDS = set(_get_backend_names())


@pytest.fixture(scope='session')
def script_directory() -> Path:
Expand Down Expand Up @@ -279,7 +281,6 @@ def pytest_ignore_collect(path, config):


def pytest_collection_modifyitems(session, config, items):
# add the backend marker to any tests are inside "ibis/backends"
all_backends = _get_backend_names()
additional_markers = []

Expand All @@ -288,7 +289,20 @@ def pytest_collection_modifyitems(session, config, items):
except ImportError:
pyspark = None

unrecognized_backends = set()
for item in items:
# Yell loudly if unrecognized backend in notimpl, notyet or never
for name in ("notimpl", "notyet", "never"):
for mark in item.iter_markers(name=name):
if backend := set(util.promote_list(mark.args[0])).difference(
ALL_BACKENDS
):
unrecognized_backends.add(
f"""Unrecognize backend(s) {backend} passed to {name} marker in
{item.path}::{item.originalname}"""
)

# add the backend marker to any tests are inside "ibis/backends"
parts = item.path.parts
backend = _get_backend_from_parts(parts)
if backend is not None and backend in all_backends:
Expand All @@ -305,6 +319,7 @@ def pytest_collection_modifyitems(session, config, items):
if not any(item.iter_markers(name="benchmark")):
item.add_marker(pytest.mark.core)

# skip or xfail pyspark tests that run afoul of our non-ancient stack
for _ in item.iter_markers(name="pyspark"):
if not isinstance(item, pytest.DoctestItem):
additional_markers.append(
Expand All @@ -329,6 +344,9 @@ def pytest_collection_modifyitems(session, config, items):
)
)

if unrecognized_backends:
raise pytest.PytestCollectionWarning("\n" + "\n".join(unrecognized_backends))

for item, markers in additional_markers:
for marker in markers:
item.add_marker(marker)
Expand Down

0 comments on commit e6fb703

Please sign in to comment.