Skip to content

Commit

Permalink
Add test for consistent random ordering
Browse files Browse the repository at this point in the history
  • Loading branch information
jbytheway authored and horenmar committed Apr 14, 2020
1 parent f696ab8 commit da9e3ee
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 0 deletions.
2 changes: 2 additions & 0 deletions projects/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,8 @@ set_tests_properties(TestsInFile::InvalidTestNames-1 PROPERTIES PASS_REGULAR_EXP
add_test(NAME TestsInFile::InvalidTestNames-2 COMMAND $<TARGET_FILE:SelfTest> "-f ${CATCH_DIR}/projects/SelfTest/Misc/invalid-test-names.input")
set_tests_properties(TestsInFile::InvalidTestNames-2 PROPERTIES PASS_REGULAR_EXPRESSION "No tests ran")

add_test(NAME RandomTestOrdering COMMAND ${PYTHON_EXECUTABLE}
${CATCH_DIR}/projects/TestScripts/testRandomOrder.py $<TARGET_FILE:SelfTest>)

if (CATCH_USE_VALGRIND)
add_test(NAME ValgrindRunTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest>)
Expand Down
53 changes: 53 additions & 0 deletions projects/TestScripts/testRandomOrder.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
#!/usr/bin/env python3

import subprocess
import sys

def list_tests(self_test_exe, tags, rng_seed):
cmd = [self_test_exe, '--list-test-names-only', '--order', 'rand',
'--rng-seed', str(rng_seed)]
tags_arg = ','.join('[{}]'.format(t) for t in tags)
if tags_arg:
cmd.append(tags_arg + '~[.]')
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if stderr:
raise RuntimeError("Unexpected error output:\n" + process.stderr)
result = stdout.split(b'\n')
result = [s for s in result if s]
if len(result) < 2:
raise RuntimeError("Unexpectedly few tests listed (got {})".format(
len(result)))
return result

def check_is_sublist_of(shorter, longer):
assert len(shorter) < len(longer)
assert len(set(longer)) == len(longer)

indexes_in_longer = {s: i for i, s in enumerate(longer)}
for s1, s2 in zip(shorter, shorter[1:]):
assert indexes_in_longer[s1] < indexes_in_longer[s2], (
'{} comes before {} in longer list.\n'
'Longer: {}\nShorter: {}'.format(s2, s1, longer, shorter))

def main():
self_test_exe, = sys.argv[1:]

list_one_tag = list_tests(self_test_exe, ['generators'], 1)
list_two_tags = list_tests(self_test_exe, ['generators', 'matchers'], 1)
list_all = list_tests(self_test_exe, [], 1)

# First, verify that restricting to a subset yields the same order
check_is_sublist_of(list_two_tags, list_all)
check_is_sublist_of(list_one_tag, list_two_tags)

# Second, verify that different seeds yield different orders
num_seeds = 100
seeds = range(1, num_seeds + 1) # Avoid zero since that's special
lists = {tuple(list_tests(self_test_exe, [], seed)) for seed in seeds}
assert len(lists) == num_seeds, (
'Got {} distict lists from {} seeds'.format(len(lists), num_seeds))

if __name__ == '__main__':
sys.exit(main())

0 comments on commit da9e3ee

Please sign in to comment.