mtest: tap: accept out-of-order or partly-numbered tests

Resolves: #13802
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

mesonbuild/mtest.py

@@ -348,6 +348,8 @@ class Version(T.NamedTuple):
     plan: T.Optional[Plan] = None
     lineno = 0
     num_tests = 0
+    last_test = 0
+    highest_test = 0
     yaml_lineno: T.Optional[int] = None
     yaml_indent = ''
     state = _MAIN
@@ -418,10 +420,11 @@ def parse_line(self, line: T.Optional[str]) -> T.Iterator[TYPE_TAPResult]:
                     yield self.Error('unexpected test after late plan')
                     self.found_late_test = True
                 self.num_tests += 1
-                num = self.num_tests if m.group(2) is None else int(m.group(2))
-                if num != self.num_tests:
-                    yield self.Error('out of order test numbers')
-                yield from self.parse_test(m.group(1) == 'ok', num,
+                self.last_test = self.last_test + 1 if m.group(2) is None else int(m.group(2))
+                self.highest_test = max(self.highest_test, self.last_test)
+                if self.plan and self.last_test > self.plan.num_tests:
+                    yield self.Error('test number exceeds maximum specified in test plan')
+                yield from self.parse_test(m.group(1) == 'ok', self.last_test,
                                            m.group(3), m.group(4), m.group(5))
                 self.state = self._AFTER_TEST
                 return
@@ -471,11 +474,21 @@ def parse_line(self, line: T.Optional[str]) -> T.Iterator[TYPE_TAPResult]:
             if self.state == self._YAML:
                 yield self.Error(f'YAML block not terminated (started on line {self.yaml_lineno})')
 
-            if not self.bailed_out and self.plan and self.num_tests != self.plan.num_tests:
+            if self.bailed_out:
+                return
+
+            if self.plan and self.num_tests != self.plan.num_tests:
                 if self.num_tests < self.plan.num_tests:
                     yield self.Error(f'Too few tests run (expected {self.plan.num_tests}, got {self.num_tests})')
                 else:
                     yield self.Error(f'Too many tests run (expected {self.plan.num_tests}, got {self.num_tests})')
+                return
+
+            if self.highest_test != self.num_tests:
+                if self.highest_test < self.num_tests:
+                    yield self.Error(f'Duplicate test numbers (expected {self.num_tests}, got test numbered {self.highest_test}')
+                else:
+                    yield self.Error(f'Missing test numbers (expected {self.num_tests}, got test numbered {self.highest_test}')
 
 class TestLogger:
     def flush(self) -> None:

unittests/taptests.py

@@ -163,10 +163,57 @@ def test_one_test_late_plan(self):
         self.assert_plan(events, num_tests=1, late=True)
         self.assert_last(events)
 
+    def test_low_max_early_plan(self):
+        events = self.parse_tap('1..2\nok 1\nok 1')
+        self.assert_plan(events, num_tests=2, late=False)
+        self.assert_test(events, number=1, name='', result=TestResult.OK)
+        self.assert_test(events, number=1, name='', result=TestResult.OK)
+        self.assert_error(events) # incorrect high test number
+        self.assert_last(events)
+
+    def test_low_max_late_plan(self):
+        events = self.parse_tap('ok 1\nok 1\n1..2')
+        self.assert_test(events, number=1, name='', result=TestResult.OK)
+        self.assert_test(events, number=1, name='', result=TestResult.OK)
+        self.assert_plan(events, num_tests=2, late=True)
+        self.assert_error(events) # incorrect high test number
+        self.assert_last(events)
+
+    def test_high_max_early_plan(self):
+        events = self.parse_tap('1..2\nok 2\nok 3')
+        self.assert_plan(events, num_tests=2, late=False)
+        self.assert_test(events, number=2, name='', result=TestResult.OK)
+        self.assert_error(events) # high id
+        self.assert_test(events, number=3, name='', result=TestResult.OK)
+        self.assert_error(events) # incorrect high test number
+        self.assert_last(events)
+
+    def test_high_max_late_plan(self):
+        events = self.parse_tap('ok 2\nok 3\n1..2')
+        self.assert_test(events, number=2, name='', result=TestResult.OK)
+        self.assert_test(events, number=3, name='', result=TestResult.OK)
+        self.assert_plan(events, num_tests=2, late=True)
+        self.assert_error(events)
+        self.assert_last(events)
+
     def test_out_of_order(self):
+        events = self.parse_tap('1..2\nok 2\nok 1')
+        self.assert_plan(events, num_tests=2, late=False)
+        self.assert_test(events, number=2, name='', result=TestResult.OK)
+        self.assert_test(events, number=1, name='', result=TestResult.OK)
+        self.assert_last(events)
+
+    def test_out_of_order_no_plan(self):
         events = self.parse_tap('ok 2')
+        self.assert_test(events, number=2, name='', result=TestResult.OK)
         self.assert_error(events)
+
+    def test_out_of_order_missing_numbers(self):
+        events = self.parse_tap('1..3\nok 2\nok\nok 1')
+        self.assert_plan(events, num_tests=3, late=False)
         self.assert_test(events, number=2, name='', result=TestResult.OK)
+        self.assert_test(events, number=3, name='', result=TestResult.OK)
+        self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_last(events)
 
     def test_middle_plan(self):
@@ -184,29 +231,32 @@ def test_too_many_plans(self):
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_last(events)
 
-    def test_too_many(self):
+    def test_too_many_late_plan(self):
         events = self.parse_tap('ok 1\nnot ok 2\n1..1')
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_test(events, number=2, name='', result=TestResult.FAIL)
         self.assert_plan(events, num_tests=1, late=True)
         self.assert_error(events)
         self.assert_last(events)
 
+    def test_too_many_early_plan(self):
         events = self.parse_tap('1..1\nok 1\nnot ok 2')
         self.assert_plan(events, num_tests=1, late=False)
         self.assert_test(events, number=1, name='', result=TestResult.OK)
+        self.assert_error(events) # test number too high
         self.assert_test(events, number=2, name='', result=TestResult.FAIL)
-        self.assert_error(events)
+        self.assert_error(events) # too many tests run
         self.assert_last(events)
 
-    def test_too_few(self):
+    def test_too_few_late_plan(self):
         events = self.parse_tap('ok 1\nnot ok 2\n1..3')
         self.assert_test(events, number=1, name='', result=TestResult.OK)
         self.assert_test(events, number=2, name='', result=TestResult.FAIL)
         self.assert_plan(events, num_tests=3, late=True)
         self.assert_error(events)
         self.assert_last(events)
 
+    def test_too_few_early_plan(self):
         events = self.parse_tap('1..3\nok 1\nnot ok 2')
         self.assert_plan(events, num_tests=3, late=False)
         self.assert_test(events, number=1, name='', result=TestResult.OK)