Skip to content

Commit

Permalink
Enhancement: Allow to order tests by duration
Browse files Browse the repository at this point in the history
  • Loading branch information
@localheinz @sebastianbergmann
localheinz authored and sebastianbergmann committed Sep 8, 2018
1 parent 5a9fc54 commit 755c1a8
Show file tree
Hide file tree
Showing 3 changed files with 181 additions and 4 deletions.
2 changes: 2 additions & 0 deletions phpunit.xsd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,12 +176,14 @@
<xs:restriction base="xs:string">
<xs:enumeration value="default"/>
<xs:enumeration value="defects"/>
<xs:enumeration value="duration"/>
<xs:enumeration value="depends"/>
<xs:enumeration value="depends,defects"/>
<xs:enumeration value="random"/>
<xs:enumeration value="reverse"/>
<xs:enumeration value="depends,random"/>
<xs:enumeration value="depends,reverse"/>
<xs:enumeration value="depends,duration"/>
</xs:restriction>
</xs:simpleType>
<xs:complexType name="fileFilterType">
Expand Down
33 changes: 30 additions & 3 deletions src/Runner/TestSuiteSorter.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ final class TestSuiteSorter
*/
public const ORDER_DEFECTS_FIRST = 3;

/**
* @var int
*/
public const ORDER_DURATION = 4;

/**
* List of sorting weights for all test result codes. A higher number gives higher priority.
*/
Expand Down Expand Up @@ -73,11 +78,12 @@ public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDepend
self::ORDER_DEFAULT,
self::ORDER_REVERSED,
self::ORDER_RANDOMIZED,
self::ORDER_DURATION,
];

if (!\in_array($order, $allowedOrders, true)) {
throw new Exception(
'$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED'
'$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED, or TestSuiteSorter::ORDER_DURATION'
);
}

Expand Down Expand Up @@ -115,6 +121,8 @@ private function sort(TestSuite $suite, int $order, bool $resolveDependencies, i
$suite->setTests($this->reverse($suite->tests()));
} elseif ($order === self::ORDER_RANDOMIZED) {
$suite->setTests($this->randomize($suite->tests()));
} elseif ($order === self::ORDER_DURATION && $this->cache !== null) {
$suite->setTests($this->sortByDuration($suite->tests()));
}

if ($orderDefects === self::ORDER_DEFECTS_FIRST && $this->cache !== null) {
Expand Down Expand Up @@ -175,6 +183,18 @@ function ($left, $right) {
return $tests;
}

private function sortByDuration(array $tests): array
{
\usort(
$tests,
function ($left, $right) {
return $this->cmpDuration($left, $right);
}
);

return $tests;
}

/**
* Comparator callback function to sort tests for "reach failure as fast as possible":
* 1. sort tests by defect weight defined in self::DEFECT_SORT_WEIGHT
Expand All @@ -192,14 +212,21 @@ private function cmpDefectPriorityAndTime(Test $a, Test $b): int
}

if ($priorityA || $priorityB) {
// Sort test duration ascending
return $this->cache->getTime($a->getName()) <=> $this->cache->getTime($b->getName());
return $this->cmpDuration($a, $b);
}

// do not change execution order
return 0;
}

/**
* Compares test duration for sorting tests by duration ascending.
*/
private function cmpDuration(Test $a, Test $b): int
{
return $this->cache->getTime($a->getName()) <=> $this->cache->getTime($b->getName());
}

/**
* Reorder Tests within a TestCase in such a way as to resolve as many dependencies as possible.
* The algorithm will leave the tests in original running order when it can.
Expand Down
150 changes: 149 additions & 1 deletion tests/unit/Runner/TestSuiteSorterTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public function testThrowsExceptionWhenUsingInvalidOrderOption(): void
$sorter = new TestSuiteSorter();

$this->expectException(Exception::class);
$this->expectExceptionMessage('$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED');
$this->expectExceptionMessage('$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED, or TestSuiteSorter::ORDER_DURATION');
$sorter->reorderTestsInSuite($suite, -1, false, TestSuiteSorter::ORDER_DEFAULT);
}

Expand Down Expand Up @@ -99,6 +99,154 @@ public function testCanSetRandomizationWithASeedAndResolveDependencies(): void
$this->assertSame(['testTwo', 'testFive', 'testOne', 'testThree', 'testFour'], $this->getTestExecutionOrder($suite));
}

/**
* @dataProvider orderDurationWithoutCacheProvider
*/
public function testOrderDurationWithoutCache(bool $resolveDependencies, array $expected): void
{
$suite = new TestSuite;

$suite->addTestSuite(\MultiDependencyTest::class);

$sorter = new TestSuiteSorter();

$sorter->reorderTestsInSuite(
$suite,
TestSuiteSorter::ORDER_DURATION,
$resolveDependencies,
TestSuiteSorter::ORDER_DEFAULT
);

$this->assertSame($expected, $this->getTestExecutionOrder($suite));
}

public function orderDurationWithoutCacheProvider(): array
{
return [
'dependency-ignore' => [
self::IGNORE_DEPENDENCIES,
[
'testOne',
'testTwo',
'testThree',
'testFour',
'testFive',
],
],
'dependency-resolve' => [
self::RESOLVE_DEPENDENCIES,
[
'testOne',
'testTwo',
'testThree',
'testFour',
'testFive',
],
],
];
}

/**
* @dataProvider orderDurationWithCacheProvider
*/
public function testOrderDurationWithCache(bool $resolveDependencies, array $testTimes, array $expected): void
{
$suite = new TestSuite;

$suite->addTestSuite(\MultiDependencyTest::class);

$cache = new TestResultCache();

foreach ($testTimes as $testName => $time) {
$cache->setTime($testName, $time);
}

$sorter = new TestSuiteSorter($cache);

$sorter->reorderTestsInSuite(
$suite,
TestSuiteSorter::ORDER_DURATION,
$resolveDependencies,
TestSuiteSorter::ORDER_DEFAULT
);

$this->assertSame($expected, $this->getTestExecutionOrder($suite));
}

public function orderDurationWithCacheProvider(): array
{
return [
'duration-same-dependency-ignore' => [
self::IGNORE_DEPENDENCIES,
[
'testOne' => 1,
'testTwo' => 1,
'testThree' => 1,
'testFour' => 1,
'testFive' => 1,
],
[
'testOne',
'testTwo',
'testThree',
'testFour',
'testFive',
],
],
'duration-same-dependency-resolve' => [
self::RESOLVE_DEPENDENCIES,
[
'testOne' => 1,
'testTwo' => 1,
'testThree' => 1,
'testFour' => 1,
'testFive' => 1,
],
[
'testOne',
'testTwo',
'testThree',
'testFour',
'testFive',
],
],
'duration-different-dependency-ignore' => [
self::IGNORE_DEPENDENCIES,
[
'testOne' => 5,
'testTwo' => 3,
'testThree' => 4,
'testFour' => 1,
'testFive' => 2,
],
[
'testFour',
'testFive',
'testTwo',
'testThree',
'testOne',
],
],
'duration-different-dependency-resolve' => [
self::RESOLVE_DEPENDENCIES,
[
'testOne' => 5,
'testTwo' => 3,
'testThree' => 4,
'testFour' => 1,
'testFive' => 2,
],
[
'testFive',
'testTwo',
'testOne',
'testThree',
'testFour',
],
],
];
}

/**
* @dataProvider defectsSorterOptionsProvider
*/
Expand Down

0 comments on commit 755c1a8

Please sign in to comment.