From e62d9b30c8da1b2a0614664b215ae78eb3a0eee3 Mon Sep 17 00:00:00 2001 From: Paul Ollis Date: Sun, 5 Nov 2023 14:02:05 +0000 Subject: [PATCH 1/5] Add new test. --- testing/acceptance_test.py | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 79fe2cd0..b81964c3 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -122,6 +122,45 @@ def test_fail2(): assert result.ret == 2 result.stdout.fnmatch_lines(["*Interrupted: stopping*1*", "*1 failed*"]) + def test_exitfail_waits_for_workers_to_finish( + self, pytester: pytest.Pytester + ) -> None: + """The DSession waits for workers before exiting early on failure. + + When -x/--exitfail is set, the DSession wait for the workers to finish + before raising an Interrupt exception. This prevents reports from the + faiing test and other tests from being discarded. + """ + p1 = pytester.makepyfile( + """ + import time + + def test_fail1(): + time.sleep(0.1) + assert 0 + def test_fail2(): + time.sleep(0.2) + def test_fail3(): + time.sleep(0.3) + assert 0 + def test_fail4(): + time.sleep(0.3) + def test_fail5(): + time.sleep(0.3) + def test_fail6(): + time.sleep(0.3) + """ + ) + result = pytester.runpytest(p1, "-x", "-rA", "-v", "-n2") + assert result.ret == 2 + result.stdout.fnmatch_lines(["*Interrupted: stopping*1*"]) + m = re.search(r"== (\d+) failed, (\d+) passed in ", str(result.stdout)) + assert m + n_failed, n_passed = (int(s) for s in m.groups()) + assert 1 <= n_failed <= 2 + assert 1 <= n_passed <= 3 + assert (n_passed + n_failed) < 6 + def test_basetemp_in_subprocesses(self, pytester: pytest.Pytester) -> None: p1 = pytester.makepyfile( """ From 16201f357463063ecdb56ec78b50c59e33cc69cc Mon Sep 17 00:00:00 2001 From: Paul Ollis Date: Sun, 5 Nov 2023 12:05:55 +0000 Subject: [PATCH 2/5] Let workers exit before aborting test run. --- src/xdist/dsession.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index 39d1afa6..3e1e1f3a 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -118,11 +118,14 @@ def pytest_runtestloop(self): assert self.sched is not None self.shouldstop = False + pending_exception = None while not self.session_finished: self.loop_once() - if self.shouldstop: + if self.shouldstop and not self.shuttingdown: self.triggershutdown() - raise Interrupted(str(self.shouldstop)) + pending_exception = Interrupted(str(self.shouldstop)) + if pending_exception: + raise pending_exception return True def loop_once(self): @@ -351,7 +354,11 @@ def _failed_worker_collectreport(self, node, rep): def _handlefailures(self, rep): if rep.failed: self.countfailures += 1 - if self.maxfail and self.countfailures >= self.maxfail: + if ( + self.maxfail + and self.countfailures >= self.maxfail + and not self.shouldstop + ): self.shouldstop = f"stopping after {self.countfailures} failures" def triggershutdown(self): From 08115abbd34a17561572918e5f80150ab5197d72 Mon Sep 17 00:00:00 2001 From: Paul Ollis Date: Sun, 5 Nov 2023 16:51:01 +0000 Subject: [PATCH 3/5] Tweak tests and code. --- src/xdist/dsession.py | 5 +++-- testing/acceptance_test.py | 17 ++--------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index 3e1e1f3a..b70975ef 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -121,8 +121,9 @@ def pytest_runtestloop(self): pending_exception = None while not self.session_finished: self.loop_once() - if self.shouldstop and not self.shuttingdown: - self.triggershutdown() + if self.shouldstop: + if not self.shuttingdown: + self.triggershutdown() pending_exception = Interrupted(str(self.shouldstop)) if pending_exception: raise pending_exception diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index b81964c3..60edd9cc 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -109,19 +109,6 @@ def test_skip(): ) assert result.ret == 1 - def test_n1_fail_minus_x(self, pytester: pytest.Pytester) -> None: - p1 = pytester.makepyfile( - """ - def test_fail1(): - assert 0 - def test_fail2(): - assert 0 - """ - ) - result = pytester.runpytest(p1, "-x", "-v", "-n1") - assert result.ret == 2 - result.stdout.fnmatch_lines(["*Interrupted: stopping*1*", "*1 failed*"]) - def test_exitfail_waits_for_workers_to_finish( self, pytester: pytest.Pytester ) -> None: @@ -153,7 +140,7 @@ def test_fail6(): ) result = pytester.runpytest(p1, "-x", "-rA", "-v", "-n2") assert result.ret == 2 - result.stdout.fnmatch_lines(["*Interrupted: stopping*1*"]) + result.stdout.re_match_lines([".*Interrupted: stopping.*[12].*"]) m = re.search(r"== (\d+) failed, (\d+) passed in ", str(result.stdout)) assert m n_failed, n_passed = (int(s) for s in m.groups()) @@ -1189,7 +1176,7 @@ def test_aaa1(crasher): """ ) result = pytester.runpytest_subprocess("--maxfail=1", "-n1") - result.stdout.fnmatch_lines(["* 1 error in *"]) + result.stdout.re_match_lines([".* [12] errors? in .*"]) assert "INTERNALERROR" not in result.stderr.str() From 02506c564e44c434fab387d8cfbfa2aae74922cc Mon Sep 17 00:00:00 2001 From: Paul Ollis Date: Wed, 8 Nov 2023 15:39:14 +0000 Subject: [PATCH 4/5] Move self.shuttingdown into triggershutdown. As suggested by Bruno Oliveira. --- src/xdist/dsession.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index b70975ef..9dcfc2a6 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -122,8 +122,7 @@ def pytest_runtestloop(self): while not self.session_finished: self.loop_once() if self.shouldstop: - if not self.shuttingdown: - self.triggershutdown() + self.triggershutdown() pending_exception = Interrupted(str(self.shouldstop)) if pending_exception: raise pending_exception @@ -363,10 +362,11 @@ def _handlefailures(self, rep): self.shouldstop = f"stopping after {self.countfailures} failures" def triggershutdown(self): - self.log("triggering shutdown") - self.shuttingdown = True - for node in self.sched.nodes: - node.shutdown() + if not self.shuttingdown: + self.log("triggering shutdown") + self.shuttingdown = True + for node in self.sched.nodes: + node.shutdown() def handle_crashitem(self, nodeid, worker): # XXX get more reporting info by recording pytest_runtest_logstart? From a1bd56e9872b7e5d48c227e5f30210b633b56b96 Mon Sep 17 00:00:00 2001 From: Paul Ollis Date: Wed, 8 Nov 2023 20:24:05 +0000 Subject: [PATCH 5/5] Add changelong fragment. --- changelog/963.improvement.rst | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/963.improvement.rst diff --git a/changelog/963.improvement.rst b/changelog/963.improvement.rst new file mode 100644 index 00000000..6cf96e24 --- /dev/null +++ b/changelog/963.improvement.rst @@ -0,0 +1,5 @@ +Wait for workers to finish reporting when test run stops early. + +This makes sure that the results of in-progress tests are displayed. +Previously these reports were being discarded, losing information about the +test run.