From 1c8b523c86f9c2b9ed71104334a1bad8c41079cf Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 9 Dec 2024 17:39:12 +0100 Subject: [PATCH] mtest: tap: accept out-of-order or partly-numbered tests Resolves: #13802 Signed-off-by: Paolo Bonzini --- mesonbuild/mtest.py | 23 ++++++++++++++---- unittests/taptests.py | 56 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index 556451c5e6ab..d0added78f8c 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -326,6 +326,8 @@ class Version(T.NamedTuple): plan: T.Optional[Plan] = None lineno = 0 num_tests = 0 + last_test = 0 + highest_test = 0 yaml_lineno: T.Optional[int] = None yaml_indent = '' state = _MAIN @@ -396,10 +398,11 @@ def parse_line(self, line: T.Optional[str]) -> T.Iterator[TYPE_TAPResult]: yield self.Error('unexpected test after late plan') self.found_late_test = True self.num_tests += 1 - num = self.num_tests if m.group(2) is None else int(m.group(2)) - if num != self.num_tests: - yield self.Error('out of order test numbers') - yield from self.parse_test(m.group(1) == 'ok', num, + self.last_test = self.last_test + 1 if m.group(2) is None else int(m.group(2)) + self.highest_test = max(self.highest_test, self.last_test) + if self.plan and self.last_test > self.plan.num_tests: + yield self.Error('test number exceeds maximum specified in test plan') + yield from self.parse_test(m.group(1) == 'ok', self.last_test, m.group(3), m.group(4), m.group(5)) self.state = self._AFTER_TEST return @@ -449,11 +452,21 @@ def parse_line(self, line: T.Optional[str]) -> T.Iterator[TYPE_TAPResult]: if self.state == self._YAML: yield self.Error(f'YAML block not terminated (started on line {self.yaml_lineno})') - if not self.bailed_out and self.plan and self.num_tests != self.plan.num_tests: + if self.bailed_out: + return + + if self.plan and self.num_tests != self.plan.num_tests: if self.num_tests < self.plan.num_tests: yield self.Error(f'Too few tests run (expected {self.plan.num_tests}, got {self.num_tests})') else: yield self.Error(f'Too many tests run (expected {self.plan.num_tests}, got {self.num_tests})') + return + + if self.highest_test != self.num_tests: + if self.highest_test < self.num_tests: + yield self.Error(f'Duplicate test numbers (expected {self.num_tests}, got test numbered {self.highest_test}') + else: + yield self.Error(f'Missing test numbers (expected {self.num_tests}, got test numbered {self.highest_test}') class TestLogger: def flush(self) -> None: diff --git a/unittests/taptests.py b/unittests/taptests.py index 26d96eafdec4..e91194cb564c 100644 --- a/unittests/taptests.py +++ b/unittests/taptests.py @@ -163,10 +163,57 @@ def test_one_test_late_plan(self): self.assert_plan(events, num_tests=1, late=True) self.assert_last(events) + def test_low_max_early_plan(self): + events = self.parse_tap('1..2\nok 1\nok 1') + self.assert_plan(events, num_tests=2, late=False) + self.assert_test(events, number=1, name='', result=TestResult.OK) + self.assert_test(events, number=1, name='', result=TestResult.OK) + self.assert_error(events) # incorrect high test number + self.assert_last(events) + + def test_low_max_late_plan(self): + events = self.parse_tap('ok 1\nok 1\n1..2') + self.assert_test(events, number=1, name='', result=TestResult.OK) + self.assert_test(events, number=1, name='', result=TestResult.OK) + self.assert_plan(events, num_tests=2, late=True) + self.assert_error(events) # incorrect high test number + self.assert_last(events) + + def test_high_max_early_plan(self): + events = self.parse_tap('1..2\nok 2\nok 3') + self.assert_plan(events, num_tests=2, late=False) + self.assert_test(events, number=2, name='', result=TestResult.OK) + self.assert_error(events) # high id + self.assert_test(events, number=3, name='', result=TestResult.OK) + self.assert_error(events) # incorrect high test number + self.assert_last(events) + + def test_high_max_late_plan(self): + events = self.parse_tap('ok 2\nok 3\n1..2') + self.assert_test(events, number=2, name='', result=TestResult.OK) + self.assert_test(events, number=3, name='', result=TestResult.OK) + self.assert_plan(events, num_tests=2, late=True) + self.assert_error(events) + self.assert_last(events) + def test_out_of_order(self): + events = self.parse_tap('1..2\nok 2\nok 1') + self.assert_plan(events, num_tests=2, late=False) + self.assert_test(events, number=2, name='', result=TestResult.OK) + self.assert_test(events, number=1, name='', result=TestResult.OK) + self.assert_last(events) + + def test_out_of_order_no_plan(self): events = self.parse_tap('ok 2') + self.assert_test(events, number=2, name='', result=TestResult.OK) self.assert_error(events) + + def test_out_of_order_missing_numbers(self): + events = self.parse_tap('1..3\nok 2\nok\nok 1') + self.assert_plan(events, num_tests=3, late=False) self.assert_test(events, number=2, name='', result=TestResult.OK) + self.assert_test(events, number=3, name='', result=TestResult.OK) + self.assert_test(events, number=1, name='', result=TestResult.OK) self.assert_last(events) def test_middle_plan(self): @@ -184,7 +231,7 @@ def test_too_many_plans(self): self.assert_test(events, number=1, name='', result=TestResult.OK) self.assert_last(events) - def test_too_many(self): + def test_too_many_late_plan(self): events = self.parse_tap('ok 1\nnot ok 2\n1..1') self.assert_test(events, number=1, name='', result=TestResult.OK) self.assert_test(events, number=2, name='', result=TestResult.FAIL) @@ -192,14 +239,16 @@ def test_too_many(self): self.assert_error(events) self.assert_last(events) + def test_too_many_early_plan(self): events = self.parse_tap('1..1\nok 1\nnot ok 2') self.assert_plan(events, num_tests=1, late=False) self.assert_test(events, number=1, name='', result=TestResult.OK) + self.assert_error(events) # test number too high self.assert_test(events, number=2, name='', result=TestResult.FAIL) - self.assert_error(events) + self.assert_error(events) # too many tests run self.assert_last(events) - def test_too_few(self): + def test_too_few_late_plan(self): events = self.parse_tap('ok 1\nnot ok 2\n1..3') self.assert_test(events, number=1, name='', result=TestResult.OK) self.assert_test(events, number=2, name='', result=TestResult.FAIL) @@ -207,6 +256,7 @@ def test_too_few(self): self.assert_error(events) self.assert_last(events) + def test_too_few_early_plan(self): events = self.parse_tap('1..3\nok 1\nnot ok 2') self.assert_plan(events, num_tests=3, late=False) self.assert_test(events, number=1, name='', result=TestResult.OK)