Skip to content

Commit

Permalink
Breakfast: rerun defects first
Browse files Browse the repository at this point in the history
  • Loading branch information
epdenouden authored and sebastianbergmann committed Jun 24, 2018
1 parent ac9e3e7 commit 76241c5
Show file tree
Hide file tree
Showing 40 changed files with 1,570 additions and 95 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@
/tests/TextUI/*.out
/tests/TextUI/*.php
/vendor

/.phpunit.result.cache
1 change: 1 addition & 0 deletions phpunit.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="phpunit.xsd"
bootstrap="tests/bootstrap.php"
cacheResult="true"
verbose="true">
<testsuites>
<testsuite name="small">
Expand Down
10 changes: 9 additions & 1 deletion phpunit.xsd
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,13 @@
<xs:simpleType name="executionOrderType">
<xs:restriction base="xs:string">
<xs:enumeration value="default"/>
<xs:enumeration value="reverse"/>
<xs:enumeration value="defects"/>
<xs:enumeration value="depends"/>
<xs:enumeration value="depends,defects"/>
<xs:enumeration value="random"/>
<xs:enumeration value="reverse"/>
<xs:enumeration value="depends,random"/>
<xs:enumeration value="depends,reverse"/>
</xs:restriction>
</xs:simpleType>
<xs:complexType name="fileFilterType">
Expand Down Expand Up @@ -216,6 +221,8 @@
<xs:attribute name="backupGlobals" type="xs:boolean" default="false"/>
<xs:attribute name="backupStaticAttributes" type="xs:boolean" default="false"/>
<xs:attribute name="bootstrap" type="xs:anyURI"/>
<xs:attribute name="cacheResult" type="xs:boolean"/>
<xs:attribute name="cacheResultFile" type="xs:anyURI"/>
<xs:attribute name="cacheTokens" type="xs:boolean"/>
<xs:attribute name="colors" type="xs:boolean" default="false"/>
<xs:attribute name="columns" type="columnsType" default="80"/>
Expand All @@ -228,6 +235,7 @@
<xs:attribute name="printerClass" type="xs:string" default="PHPUnit\TextUI\ResultPrinter"/>
<xs:attribute name="printerFile" type="xs:anyURI"/>
<xs:attribute name="processIsolation" type="xs:boolean" default="false"/>
<xs:attribute name="stopOnDefect" type="xs:boolean" default="false"/>
<xs:attribute name="stopOnError" type="xs:boolean" default="false"/>
<xs:attribute name="stopOnFailure" type="xs:boolean" default="false"/>
<xs:attribute name="stopOnWarning" type="xs:boolean" default="false"/>
Expand Down
21 changes: 17 additions & 4 deletions src/Framework/TestResult.php
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,11 @@ class TestResult implements Countable
*/
protected $stopOnSkipped = false;

/**
* @var bool
*/
protected $stopOnDefect = false;

/**
* @var bool
*/
Expand Down Expand Up @@ -229,7 +234,7 @@ public function addError(Test $test, Throwable $t, float $time): void
$test->markAsRisky();
}

if ($this->stopOnRisky) {
if ($this->stopOnRisky || $this->stopOnDefect) {
$this->stop();
}
} elseif ($t instanceof IncompleteTest) {
Expand Down Expand Up @@ -274,7 +279,7 @@ public function addError(Test $test, Throwable $t, float $time): void
*/
public function addWarning(Test $test, Warning $e, float $time): void
{
if ($this->stopOnWarning) {
if ($this->stopOnWarning || $this->stopOnDefect) {
$this->stop();
}

Expand All @@ -301,7 +306,7 @@ public function addFailure(Test $test, AssertionFailedError $e, float $time): vo
$test->markAsRisky();
}

if ($this->stopOnRisky) {
if ($this->stopOnRisky || $this->stopOnDefect) {
$this->stop();
}
} elseif ($e instanceof IncompleteTest) {
Expand All @@ -322,7 +327,7 @@ public function addFailure(Test $test, AssertionFailedError $e, float $time): vo
$this->failures[] = new TestFailure($test, $e);
$notifyMethod = 'addFailure';

if ($this->stopOnFailure) {
if ($this->stopOnFailure || $this->stopOnDefect) {
$this->stop();
}
}
Expand Down Expand Up @@ -1011,6 +1016,14 @@ public function stopOnSkipped(bool $flag): void
$this->stopOnSkipped = $flag;
}

/**
* Enables or disables the stopping for defects: error, failure, warning
*/
public function stopOnDefect(bool $flag): void
{
$this->stopOnDefect = $flag;
}

/**
* Returns the time spent running the tests.
*/
Expand Down
101 changes: 101 additions & 0 deletions src/Runner/ResultCacheExtension.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
<?php
/*
* This file is part of PHPUnit.
*
* (c) Sebastian Bergmann <sebastian@phpunit.de>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace PHPUnit\Runner;

final class ResultCacheExtension implements AfterSuccessfulTestHook, AfterSkippedTestHook, AfterRiskyTestHook, AfterIncompleteTestHook, AfterTestErrorHook, AfterTestWarningHook, AfterTestFailureHook, AfterLastTestHook
{
/**
* @var TestResultCacheInterface
*/
private $cache;

public function __construct(TestResultCache $cache)
{
$this->cache = $cache;
}

public function flush(): void
{
$this->cache->persist();
}

public function executeAfterSuccessfulTest(string $test, float $time): void
{
$testName = $this->getTestName($test);
$this->cache->setTime($testName, \round($time, 3));
}

public function executeAfterIncompleteTest(string $test, string $message, float $time): void
{
$testName = $this->getTestName($test);
$this->cache->setTime($testName, \round($time, 3));
$this->cache->setState($testName, BaseTestRunner::STATUS_INCOMPLETE);
}

public function executeAfterRiskyTest(string $test, string $message, float $time): void
{
$testName = $this->getTestName($test);
$this->cache->setTime($testName, \round($time, 3));
$this->cache->setState($testName, BaseTestRunner::STATUS_RISKY);
}

public function executeAfterSkippedTest(string $test, string $message, float $time): void
{
$testName = $this->getTestName($test);
$this->cache->setTime($testName, \round($time, 3));
$this->cache->setState($testName, BaseTestRunner::STATUS_SKIPPED);
}

public function executeAfterTestError(string $test, string $message, float $time): void
{
$testName = $this->getTestName($test);
$this->cache->setTime($testName, \round($time, 3));
$this->cache->setState($testName, BaseTestRunner::STATUS_ERROR);
}

public function executeAfterTestFailure(string $test, string $message, float $time): void
{
$testName = $this->getTestName($test);
$this->cache->setTime($testName, \round($time, 3));
$this->cache->setState($testName, BaseTestRunner::STATUS_FAILURE);
}

public function executeAfterTestWarning(string $test, string $message, float $time): void
{
$testName = $this->getTestName($test);
$this->cache->setTime($testName, \round($time, 3));
$this->cache->setState($testName, BaseTestRunner::STATUS_WARNING);
}

public function executeAfterLastTest(): void
{
$this->flush();
}

/**
* @param string $test A long description format of the current test
*
* @return string The test name without TestSuiteClassName:: and @dataprovider details
*/
private function getTestName(string $test): string
{
$matches = [];

if (\preg_match('/^(?:\S+::)?(?<name>\S+)(?:(?<data> with data set (?:#\d+|"[^"]+"))\s\()?/', $test, $matches)) {
$test = $matches['name'];

if (isset($matches['data'])) {
$test .= $matches['data'];
}
}

return $test;
}
}
104 changes: 99 additions & 5 deletions src/Runner/TestSuiteSorter.php
Original file line number Diff line number Diff line change
Expand Up @@ -31,27 +31,69 @@ final class TestSuiteSorter
*/
public const ORDER_REVERSED = 2;

/**
* @var int
*/
public const ORDER_DEFECTS_FIRST = 3;

/**
* List of sorting weights for all test result codes. A higher number gives higher priority.
*/
private const DEFECT_SORT_WEIGHT = [
BaseTestRunner::STATUS_ERROR => 6,
BaseTestRunner::STATUS_FAILURE => 5,
BaseTestRunner::STATUS_WARNING => 4,
BaseTestRunner::STATUS_INCOMPLETE => 3,
BaseTestRunner::STATUS_RISKY => 2,
BaseTestRunner::STATUS_SKIPPED => 1,
BaseTestRunner::STATUS_UNKNOWN => 0
];

/**
* @var array<string, int> Associative array of (string => DEFECT_SORT_WEIGHT) elements
*/
private $defectSortOrder = [];

/**
* @var TestResultCacheInterface
*/
private $cache;

public function __construct(?TestResultCacheInterface $cache = null)
{
$this->cache = $cache ?? new NullTestResultCache;
}

/**
* @throws Exception
*/
public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies): void
public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies, int $orderDefects): void
{
if ($order !== self::ORDER_DEFAULT && $order !== self::ORDER_REVERSED && $order !== self::ORDER_RANDOMIZED) {
throw new Exception(
'$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED'
);
}

if ($suite instanceof TestSuite && !empty($suite->tests())) {
if ($orderDefects !== self::ORDER_DEFAULT && $orderDefects !== self::ORDER_DEFECTS_FIRST) {
throw new Exception(
'$orderDefects must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_DEFECTS_FIRST'
);
}

if ($suite instanceof TestSuite) {
foreach ($suite as $_suite) {
$this->reorderTestsInSuite($_suite, $order, $resolveDependencies);
$this->reorderTestsInSuite($_suite, $order, $resolveDependencies, $orderDefects);
}

$this->sort($suite, $order, $resolveDependencies);
if ($orderDefects === self::ORDER_DEFECTS_FIRST) {
$this->addSuiteToDefectSortOrder($suite);
}
$this->sort($suite, $order, $resolveDependencies, $orderDefects);
}
}

private function sort(TestSuite $suite, int $order, bool $resolveDependencies): void
private function sort(TestSuite $suite, int $order, bool $resolveDependencies, int $orderDefects): void
{
if (empty($suite->tests())) {
return;
Expand All @@ -63,11 +105,29 @@ private function sort(TestSuite $suite, int $order, bool $resolveDependencies):
$suite->setTests($this->randomize($suite->tests()));
}

if ($orderDefects === self::ORDER_DEFECTS_FIRST && $this->cache !== null) {
$suite->setTests($this->sortDefectsFirst($suite->tests()));
}

if ($resolveDependencies && !($suite instanceof DataProviderTestSuite) && $this->suiteOnlyContainsTests($suite)) {
$suite->setTests($this->resolveDependencies($suite->tests()));
}
}

private function addSuiteToDefectSortOrder(TestSuite $suite): void
{
$max = 0;

foreach ($suite->tests() as $test) {
if (!isset($this->defectSortOrder[$test->getName()])) {
$this->defectSortOrder[$test->getName()] = self::DEFECT_SORT_WEIGHT[$this->cache->getState($test->getName())];
$max = \max($max, $this->defectSortOrder[$test->getName()]);
}
}

$this->defectSortOrder[$suite->getName()] = $max;
}

private function suiteOnlyContainsTests(TestSuite $suite): bool
{
return \array_reduce($suite->tests(), function ($carry, $test) {
Expand All @@ -87,6 +147,40 @@ private function randomize(array $tests): array
return $tests;
}

private function sortDefectsFirst(array $tests): array
{
\usort($tests, function ($left, $right) {
return $this->cmpDefectPriorityAndTime($left, $right);
});

return $tests;
}

/**
* Comparator callback function to sort tests for "reach failure as fast as possible":
* 1. sort tests by defect weight defined in self::DEFECT_SORT_WEIGHT
* 2. when tests are equally defective, sort the fastest to the front
* 3. do not reorder successful tests
*/
private function cmpDefectPriorityAndTime(Test $a, Test $b): int
{
$priorityA = $this->defectSortOrder[$a->getName()] ?? 0;
$priorityB = $this->defectSortOrder[$b->getName()] ?? 0;

if ($priorityB <=> $priorityA) {
// Sort defect weight descending
return $priorityB <=> $priorityA;
}

if ($priorityA || $priorityB) {
// Sort test duration ascending
return $this->cache->getTime($a->getName()) <=> $this->cache->getTime($b->getName());
}

// do not change execution order
return 0;
}

/**
* Reorder Tests within a TestCase in such a way as to resolve as many dependencies as possible.
* The algorithm will leave the tests in original running order when it can.
Expand Down
Loading

0 comments on commit 76241c5

Please sign in to comment.