Skip to content

Commit

Permalink
[3.12] gh-108834: Sync libregrtest with the main branch (#108966)
Browse files Browse the repository at this point in the history
* gh-108834: regrtest reruns failed tests in subprocesses (#108839)

When using --rerun option, regrtest now re-runs failed tests
in verbose mode in fresh worker processes to have more
deterministic behavior. So it can write its final report even
if a test killed a worker progress.

Add --fail-rerun option to regrtest: exit with non-zero exit code
if a test failed pass passed when re-run in verbose mode (in a
fresh process). That's now more useful since tests can pass
when re-run in a fresh worker progress, whereas they failed
when run after other tests when tests are run sequentially.

Rename --verbose2 option (-w) to --rerun. Keep --verbose2 as a
deprecated alias.

Changes:

* Fix and enhance statistics in regrtest summary. Add "(filtered)"
  when --match and/or --ignore options are used.
* Add RunTests class.
* Add TestResult.get_rerun_match_tests() method
* Rewrite code to serialize/deserialize worker arguments as JSON
  using a new WorkerJob class.
* Fix stats when a test is run with --forever --rerun.
* If failed test names cannot be parsed, log a warning and don't
  filter tests.
* test_regrtest.test_rerun_success() now uses a marker file, since
  the test is re-run in a separated process.
* Add tests on normalize_test_name() function.
* Add test_success() and test_skip() tests to test_regrtest.

(cherry picked from commit 31c2945)

* gh-108834: regrtest --fail-rerun exits with code 5 (#108896)

When the --fail-rerun option is used and a test fails and then pass,
regrtest now uses exit code 5 ("rerun) instead of 2 ("bad test").

(cherry picked from commit 1170d5a)

* gh-108416: Mark slow but not CPU bound test methods with requires_resource('walltime') (GH-108480)

(cherry picked from commit 1e0d627)

* Manually sync Lib/test/libregrtest/ from main

---------

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
  • Loading branch information
vstinner and serhiy-storchaka authored Sep 8, 2023
1 parent eeaae92 commit 0855b2c
Show file tree
Hide file tree
Showing 29 changed files with 892 additions and 499 deletions.
3 changes: 3 additions & 0 deletions Lib/test/_test_multiprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -675,6 +675,7 @@ def test_close(self):

close_queue(q)

@support.requires_resource('walltime')
def test_many_processes(self):
if self.TYPE == 'threads':
self.skipTest('test not appropriate for {}'.format(self.TYPE))
Expand Down Expand Up @@ -4953,6 +4954,7 @@ def test_wait_slow(self):
def test_wait_socket_slow(self):
self.test_wait_socket(True)

@support.requires_resource('walltime')
def test_wait_timeout(self):
from multiprocessing.connection import wait

Expand Down Expand Up @@ -4981,6 +4983,7 @@ def signal_and_sleep(cls, sem, period):
sem.release()
time.sleep(period)

@support.requires_resource('walltime')
def test_wait_integer(self):
from multiprocessing.connection import wait

Expand Down
7 changes: 4 additions & 3 deletions Lib/test/bisect_cmd.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,10 @@ def parse_args():

def main():
args = parse_args()
if '-w' in args.test_args or '--verbose2' in args.test_args:
print("WARNING: -w/--verbose2 option should not be used to bisect!")
print()
for opt in ('-w', '--rerun', '--verbose2'):
if opt in args.test_args:
print(f"WARNING: {opt} option should not be used to bisect!")
print()

if args.input:
with open(args.input) as fp:
Expand Down
15 changes: 11 additions & 4 deletions Lib/test/libregrtest/cmdline.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@
cpu - Used for certain CPU-heavy tests.
walltime - Long running but not CPU-bound tests.
subprocess Run all tests for the subprocess module.
urlfetch - It is okay to download files required on testing.
Expand All @@ -129,7 +131,7 @@


ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network',
'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui')
'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime')

# Other resources excluded from --use=all:
#
Expand All @@ -156,7 +158,7 @@ def __init__(self, **kwargs) -> None:
self.coverdir = 'coverage'
self.runleaks = False
self.huntrleaks = False
self.verbose2 = False
self.rerun = False
self.verbose3 = False
self.print_slow = False
self.random_seed = None
Expand Down Expand Up @@ -213,8 +215,10 @@ def _create_parser():
group = parser.add_argument_group('Verbosity')
group.add_argument('-v', '--verbose', action='count',
help='run tests in verbose mode with output to stdout')
group.add_argument('-w', '--verbose2', action='store_true',
group.add_argument('-w', '--rerun', action='store_true',
help='re-run failed tests in verbose mode')
group.add_argument('--verbose2', action='store_true', dest='rerun',
help='deprecated alias to --rerun')
group.add_argument('-W', '--verbose3', action='store_true',
help='display test output on failure')
group.add_argument('-q', '--quiet', action='store_true',
Expand Down Expand Up @@ -309,6 +313,9 @@ def _create_parser():
group.add_argument('--fail-env-changed', action='store_true',
help='if a test file alters the environment, mark '
'the test as failed')
group.add_argument('--fail-rerun', action='store_true',
help='if a test failed and then passed when re-run, '
'mark the tests as failed')

group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME',
help='writes JUnit-style XML results to the specified '
Expand Down Expand Up @@ -380,7 +387,7 @@ def _parse_args(args, **kwargs):
ns.python = shlex.split(ns.python)
if ns.failfast and not (ns.verbose or ns.verbose3):
parser.error("-G/--failfast needs either -v or -W")
if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3):
if ns.pgo and (ns.verbose or ns.rerun or ns.verbose3):
parser.error("--pgo/-v don't go together!")
if ns.pgo_extended:
ns.pgo = True # pgo_extended implies pgo
Expand Down
Loading

0 comments on commit 0855b2c

Please sign in to comment.