Skip to content

Commit

Permalink
slang-test: retry failed test at the end. (#5255)
Browse files Browse the repository at this point in the history
  • Loading branch information
@csyonghe
csyonghe authored Oct 11, 2024
1 parent 80c1851 commit 466fb5b
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 29 deletions.
52 changes: 51 additions & 1 deletion tools/slang-test/slang-test-main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ struct TestOptions
bool isSynthesized = false;
};

struct FileTestInfoImpl : public FileTestInfo
{
String testName;
String filePath;
String outputStem;
TestOptions options;
};

struct TestDetails
{
TestDetails() {}
Expand Down Expand Up @@ -4042,13 +4050,29 @@ static SlangResult _runTestsOnFile(
if (_canIgnore(context, testDetails))
{
testResult = TestResult::Ignored;
context->getTestReporter()->addResult(testResult);
}
else
{
testResult = runTest(context, filePath, outputStem, testName, testDetails.options);
if (testResult == TestResult::Fail
&& !context->getTestReporter()->m_expectedFailureList.contains(testName))
{
RefPtr<FileTestInfoImpl> fileTestInfo = new FileTestInfoImpl();
fileTestInfo->filePath = filePath;
fileTestInfo->testName = testName;
fileTestInfo->outputStem = outputStem;
fileTestInfo->options = testDetails.options;

std::lock_guard lock(context->mutexFailedFileTests);
context->failedFileTests.add(fileTestInfo);
}
else
{
context->getTestReporter()->addResult(testResult);
}
}

context->getTestReporter()->addResult(testResult);

// Could determine if to continue or not here... based on result
}
Expand Down Expand Up @@ -4615,6 +4639,32 @@ SlangResult innerMain(int argc, char** argv)
TestReporter::set(nullptr);
}

// If we have a couple failed tests, they maybe intermittent failures due to parallel
// excution or driver instability. We can try running them again.
static constexpr int kFailedTestLimitForRetry = 16;
if (context.failedFileTests.getCount() <= kFailedTestLimitForRetry)
{
printf("Retrying %d failed tests...\n", (int)context.failedFileTests.getCount());
for (auto& test : context.failedFileTests)
{
FileTestInfoImpl* fileTestInfo = static_cast<FileTestInfoImpl*>(test.Ptr());
TestReporter::SuiteScope suiteScope(&reporter, "tests");
TestReporter::TestScope scope(&reporter, fileTestInfo->testName);
auto newResult = runTest(&context, fileTestInfo->filePath, fileTestInfo->outputStem, fileTestInfo->testName, fileTestInfo->options);
reporter.addResult(newResult);
}
}
else
{
// If there are too many failed tests, don't bother retrying.
for (auto& test : context.failedFileTests)
{
FileTestInfoImpl* fileTestInfo = static_cast<FileTestInfoImpl*>(test.Ptr());
TestReporter::TestScope scope(&reporter, fileTestInfo->testName);
reporter.addResult(TestResult::Fail);
}
}

reporter.outputSummary();
return reporter.didAllSucceed() ? SLANG_OK : SLANG_FAIL;
}
Expand Down
62 changes: 35 additions & 27 deletions tools/slang-test/test-context.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,62 +85,66 @@ struct TestRequirements
Slang::RenderApiFlags usedRenderApiFlags = 0; ///< Used render api flags (some might be implied)
};

struct FileTestInfo : public Slang::RefObject
{
};

class TestContext
{
public:
public:

typedef Slang::TestToolUtil::InnerMainFunc InnerMainFunc;

/// Get the slang session
SlangSession* getSession() const { return m_session; }
/// Get the slang session
SlangSession* getSession() const { return m_session; }

SlangResult init(const char* exePath);

/// Get the inner main function (from shared library)
/// Get the inner main function (from shared library)
InnerMainFunc getInnerMainFunc(const Slang::String& dirPath, const Slang::String& name);
/// Set the function for the shared library
/// Set the function for the shared library
void setInnerMainFunc(const Slang::String& name, InnerMainFunc func);

void setTestRequirements(TestRequirements* req);

TestRequirements* getTestRequirements() const;

/// If true tests aren't being run just the information on testing is being accumulated
/// If true tests aren't being run just the information on testing is being accumulated
bool isCollectingRequirements() const { return getTestRequirements() != nullptr; }
/// If set, then tests are executed
/// If set, then tests are executed
bool isExecuting() const { return getTestRequirements() == nullptr; }

/// True if a render API filter is enabled
/// True if a render API filter is enabled
bool isRenderApiFilterEnabled() const { return options.enabledApis != Slang::RenderApiFlag::AllOf && options.enabledApis != 0; }

/// True if a test with the requiredFlags can in principal run (it may not be possible if the API is not available though)
/// True if a test with the requiredFlags can in principal run (it may not be possible if the API is not available though)
bool canRunTestWithRenderApiFlags(Slang::RenderApiFlags requiredFlags);

/// True if can run unit tests
/// True if can run unit tests
bool canRunUnitTests() const { return options.apiOnly == false; }

/// Given a spawn type, return the final spawn type.
/// In particular we want 'Default' spawn type to vary by the environment (for example running on test server on CI)
/// Given a spawn type, return the final spawn type.
/// In particular we want 'Default' spawn type to vary by the environment (for example running on test server on CI)
SpawnType getFinalSpawnType(SpawnType spawnType);

SpawnType getFinalSpawnType();

/// Get compiler set
/// Get compiler set
Slang::DownstreamCompilerSet* getCompilerSet();
Slang::IDownstreamCompiler* getDefaultCompiler(SlangSourceLanguage sourceLanguage);

Slang::JSONRPCConnection* getOrCreateJSONRPCConnection();
void destroyRPCConnection();

/// Ctor
/// Ctor
TestContext();
/// Dtor
/// Dtor
~TestContext();

Options options;
TestCategorySet categorySet;

/// If set then tests are not run, but their requirements are set
/// If set then tests are not run, but their requirements are set

PassThroughFlags availableBackendFlags = 0;
Slang::RenderApiFlags availableRenderApiFlags = 0;
Expand All @@ -152,17 +156,17 @@ class TestContext
Slang::String dllDirectoryPath;
Slang::String exePath;

/// Timeout time for communication over connection.
/// NOTE! If the timeout is hit, the connection will be destroyed, and then recreated.
/// For tests that compile the stdlib, if that takes this time, the stdlib will be
/// repeatedly compiled and each time fail.
/// NOTE! This timeout may be altered in the ctor for a specific target, the initializatoin
/// value is just the default.
///
/// TODO(JS): We could split the stdlib compilation from other actions, and have timeout specific for
/// that. To do this we could have a 'compileStdLib' RPC method.
///
/// Current default is 60 seconds.
/// Timeout time for communication over connection.
/// NOTE! If the timeout is hit, the connection will be destroyed, and then recreated.
/// For tests that compile the stdlib, if that takes this time, the stdlib will be
/// repeatedly compiled and each time fail.
/// NOTE! This timeout may be altered in the ctor for a specific target, the initializatoin
/// value is just the default.
///
/// TODO(JS): We could split the stdlib compilation from other actions, and have timeout specific for
/// that. To do this we could have a 'compileStdLib' RPC method.
///
/// Current default is 60 seconds.
Slang::Int connectionTimeOutInMs = 60 * 1000;

void setThreadIndex(int index);
Expand All @@ -175,6 +179,10 @@ class TestContext
std::mutex mutex;
Slang::RefPtr<Slang::JSONRPCConnection> m_languageServerConnection;


std::mutex mutexFailedFileTests;
Slang::List<Slang::RefPtr<FileTestInfo>> failedFileTests;

Slang::IFileCheck* getFileCheck() { return m_fileCheck; };

protected:
Expand Down
2 changes: 1 addition & 1 deletion tools/slang-test/test-reporter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ enum class TestOutputMode

class TestReporter : public ITestReporter
{
public:
public:

struct TestInfo
{
Expand Down

0 comments on commit 466fb5b

Please sign in to comment.