Skip to content

Commit

Permalink
twister: stats: fix suite statistics
Browse files Browse the repository at this point in the history 
suite stats were not correct, a mixup between skipped and filtered
suites was leading to inconsistent numbers. This is now fixed.

(cherry picked from commit a43a67e)

Original-Signed-off-by: Anas Nashif <anas.nashif@intel.com>
GitOrigin-RevId: a43a67e
Cr-Build-Id: 8731390868330359073
Cr-Build-Url: https://cr-buildbucket.appspot.com/build/8731390868330359073
Copybot-Job-Name: zephyr-main-copybot-downstream
Change-Id: I38bddeb1b68c8d16cf3f9d14753c19195e62057b
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/zephyr/+/6017554
Reviewed-by: Al Semjonovs <asemjonovs@google.com>
Tested-by: ChromeOS Prod (Robot) <chromeos-ci-prod@chromeos-bot.iam.gserviceaccount.com>
Tested-by: Al Semjonovs <asemjonovs@google.com>
Commit-Queue: Al Semjonovs <asemjonovs@google.com>
  • Loading branch information
@nashif
nashif authored and Chromeos LUCI committed Nov 13, 2024
1 parent 858bd7c commit 2033cb9
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 21 deletions.
37 changes: 25 additions & 12 deletions scripts/pylib/twister/twisterlib/runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ def __init__(self, total=0):
# updated by report_out() in pipeline
self._error = Value('i', 0)
self._failed = Value('i', 0)
self._skipped = Value('i', 0)

# initialized to number of test instances
self._total = Value('i', total)
Expand Down Expand Up @@ -145,30 +146,27 @@ def _find_number_length(n):

def summary(self):
selected_cases = self.cases - self.filtered_cases
completed_configs = self.done - self.filtered_static
completed_configs = self.done - self.filtered_static - self.filtered_runtime

# Find alignment length for aesthetic printing
suites_n_length = self._find_number_length(self.total if self.total > self.done else self.done)
processed_suites_n_length = self._find_number_length(self.done)
completed_suites_n_length = self._find_number_length(completed_configs)
skipped_suites_n_length = self._find_number_length(self.filtered_configs)
filtered_suites_n_length = self._find_number_length(self.filtered_configs)
total_cases_n_length = self._find_number_length(self.cases)
selected_cases_n_length = self._find_number_length(selected_cases)

print("--------------------------------------------------")
print(f"{'Total test suites: ':<23}{self.total:>{suites_n_length}}") # actually test instances
print(f"{'Processed test suites: ':<23}{self.done:>{suites_n_length}}")
print(f"├─ {'Filtered test suites (static): ':<37}{self.filtered_static:>{processed_suites_n_length}}")
print(f"└─ {'Selected test suites: ':<37}{completed_configs:>{processed_suites_n_length}}")
print(f" ├─ {'Skipped test suites: ':<37}{self.filtered_runtime:>{completed_suites_n_length}}")
print(f"└─{'Filtered test suites: ':<21}{self.filtered_configs}")
print(f" ├─ {'Filtered test suites (static): ':<37}{self.filtered_static:>{filtered_suites_n_length}}")
print(f" └─ {'Filtered test suites (at runtime): ':<37}{self.filtered_runtime:>{filtered_suites_n_length}}")
print(f"└─ {'Selected test suites: ':<37}{completed_configs:>{completed_suites_n_length}}")
print(f" ├─ {'Skipped test suites: ':<37}{self.skipped:>{completed_suites_n_length}}")
print(f" ├─ {'Passed test suites: ':<37}{self.passed:>{completed_suites_n_length}}")
print(f" ├─ {'Built only test suites: ':<37}{self.notrun:>{completed_suites_n_length}}")
print(f" ├─ {'Failed test suites: ':<37}{self.failed:>{completed_suites_n_length}}")
print(f" └─ {'Errors in test suites: ':<37}{self.error:>{completed_suites_n_length}}")
print(f"")
print(f"{'Filtered test suites: ':<21}{self.filtered_configs}")
print(f"├─ {'Filtered test suites (static): ':<37}{self.filtered_static:>{skipped_suites_n_length}}")
print(f"└─ {'Filtered test suites (at runtime): ':<37}{self.filtered_runtime:>{skipped_suites_n_length}}")
print("---------------------- ----------------------")
print(f"{'Total test cases: ':<18}{self.cases}")
print(f"├─ {'Filtered test cases: ':<21}{self.filtered_cases:>{total_cases_n_length}}")
Expand Down Expand Up @@ -341,6 +339,20 @@ def started_cases_increment(self, value=1):
with self._started_cases.get_lock():
self._started_cases.value += value

@property
def skipped(self):
with self._skipped.get_lock():
return self._skipped.value

@skipped.setter
def skipped(self, value):
with self._skipped.get_lock():
self._skipped.value = value

def skipped_increment(self, value=1):
with self._skipped.get_lock():
self._skipped.value += value

@property
def error(self):
with self._error.get_lock():
Expand Down Expand Up @@ -892,6 +904,7 @@ def process(self, pipeline, done, message, lock, results):
logger.debug("filtering %s" % self.instance.name)
self.instance.status = TwisterStatus.FILTER
self.instance.reason = "runtime filter"
results.filtered_runtime_increment()
self.instance.add_missing_case_status(TwisterStatus.FILTER)
next_op = 'report'
else:
Expand Down Expand Up @@ -951,7 +964,7 @@ def process(self, pipeline, done, message, lock, results):
# Count skipped cases during build, for example
# due to ram/rom overflow.
if self.instance.status == TwisterStatus.SKIP:
results.filtered_runtime_increment()
results.skipped_increment()
self.instance.add_missing_case_status(TwisterStatus.SKIP, self.instance.reason)

if ret.get('returncode', 1) > 0:
Expand Down Expand Up @@ -1381,7 +1394,7 @@ def report_out(self, results):
if not self.options.verbose:
self.log_info_file(self.options.inline_logs)
elif instance.status == TwisterStatus.SKIP:
results.filtered_configs_increment()
results.skipped_increment()
elif instance.status == TwisterStatus.FILTER:
results.filtered_configs_increment()
elif instance.status == TwisterStatus.PASS:
Expand Down
16 changes: 7 additions & 9 deletions scripts/tests/twister/test_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,17 +209,15 @@ def test_executioncounter(capfd):
'--------------------------------------------------\n'
'Total test suites: 12\n'
'Processed test suites: 9\n'
'├─ Filtered test suites (static): 2\n'
'└─ Completed test suites: 7\n'
' ├─ Filtered test suites (at runtime): 1\n'
'└─Filtered test suites: 3\n'
' ├─ Filtered test suites (static): 2\n'
' └─ Filtered test suites (at runtime): 1\n'
'└─ Selected test suites: 6\n'
' ├─ Skipped test suites: 0\n'
' ├─ Passed test suites: 6\n'
' ├─ Built only test suites: 0\n'
' ├─ Failed test suites: 1\n'
' └─ Errors in test suites: 2\n'
'\n'
'Filtered test suites: 3\n'
'├─ Filtered test suites (static): 2\n'
'└─ Filtered test suites (at runtime): 1\n'
'---------------------- ----------------------\n'
'Total test cases: 25\n'
'├─ Filtered test cases: 0\n'
Expand Down Expand Up @@ -914,7 +912,7 @@ def mock_getsize(filename, *args, **kwargs):
{'op': 'report', 'test': mock.ANY},
TwisterStatus.FILTER,
'runtime filter',
0,
1,
(TwisterStatus.FILTER,)
),
(
Expand Down Expand Up @@ -1091,7 +1089,7 @@ def mock_getsize(filename, *args, **kwargs):
{'op': 'gather_metrics', 'test': mock.ANY},
mock.ANY,
mock.ANY,
1,
0,
(TwisterStatus.SKIP, mock.ANY)
),
(
Expand Down

0 comments on commit 2033cb9

Please sign in to comment.