diff --git a/tools/slang-test/slang-test-main.cpp b/tools/slang-test/slang-test-main.cpp index d387c815a6..b493b40b0b 100644 --- a/tools/slang-test/slang-test-main.cpp +++ b/tools/slang-test/slang-test-main.cpp @@ -103,6 +103,14 @@ struct TestOptions bool isSynthesized = false; }; +struct FileTestInfoImpl : public FileTestInfo +{ + String testName; + String filePath; + String outputStem; + TestOptions options; +}; + struct TestDetails { TestDetails() {} @@ -4042,13 +4050,29 @@ static SlangResult _runTestsOnFile( if (_canIgnore(context, testDetails)) { testResult = TestResult::Ignored; + context->getTestReporter()->addResult(testResult); } else { testResult = runTest(context, filePath, outputStem, testName, testDetails.options); + if (testResult == TestResult::Fail + && !context->getTestReporter()->m_expectedFailureList.contains(testName)) + { + RefPtr fileTestInfo = new FileTestInfoImpl(); + fileTestInfo->filePath = filePath; + fileTestInfo->testName = testName; + fileTestInfo->outputStem = outputStem; + fileTestInfo->options = testDetails.options; + + std::lock_guard lock(context->mutexFailedFileTests); + context->failedFileTests.add(fileTestInfo); + } + else + { + context->getTestReporter()->addResult(testResult); + } } - context->getTestReporter()->addResult(testResult); // Could determine if to continue or not here... based on result } @@ -4615,6 +4639,32 @@ SlangResult innerMain(int argc, char** argv) TestReporter::set(nullptr); } + // If we have a couple failed tests, they maybe intermittent failures due to parallel + // excution or driver instability. We can try running them again. + static constexpr int kFailedTestLimitForRetry = 16; + if (context.failedFileTests.getCount() <= kFailedTestLimitForRetry) + { + printf("Retrying %d failed tests...\n", (int)context.failedFileTests.getCount()); + for (auto& test : context.failedFileTests) + { + FileTestInfoImpl* fileTestInfo = static_cast(test.Ptr()); + TestReporter::SuiteScope suiteScope(&reporter, "tests"); + TestReporter::TestScope scope(&reporter, fileTestInfo->testName); + auto newResult = runTest(&context, fileTestInfo->filePath, fileTestInfo->outputStem, fileTestInfo->testName, fileTestInfo->options); + reporter.addResult(newResult); + } + } + else + { + // If there are too many failed tests, don't bother retrying. + for (auto& test : context.failedFileTests) + { + FileTestInfoImpl* fileTestInfo = static_cast(test.Ptr()); + TestReporter::TestScope scope(&reporter, fileTestInfo->testName); + reporter.addResult(TestResult::Fail); + } + } + reporter.outputSummary(); return reporter.didAllSucceed() ? SLANG_OK : SLANG_FAIL; } diff --git a/tools/slang-test/test-context.h b/tools/slang-test/test-context.h index c1bd82c8c1..a472248eb0 100644 --- a/tools/slang-test/test-context.h +++ b/tools/slang-test/test-context.h @@ -85,62 +85,66 @@ struct TestRequirements Slang::RenderApiFlags usedRenderApiFlags = 0; ///< Used render api flags (some might be implied) }; +struct FileTestInfo : public Slang::RefObject +{ +}; + class TestContext { - public: +public: typedef Slang::TestToolUtil::InnerMainFunc InnerMainFunc; - /// Get the slang session - SlangSession* getSession() const { return m_session; } + /// Get the slang session + SlangSession* getSession() const { return m_session; } SlangResult init(const char* exePath); - /// Get the inner main function (from shared library) + /// Get the inner main function (from shared library) InnerMainFunc getInnerMainFunc(const Slang::String& dirPath, const Slang::String& name); - /// Set the function for the shared library + /// Set the function for the shared library void setInnerMainFunc(const Slang::String& name, InnerMainFunc func); void setTestRequirements(TestRequirements* req); TestRequirements* getTestRequirements() const; - /// If true tests aren't being run just the information on testing is being accumulated + /// If true tests aren't being run just the information on testing is being accumulated bool isCollectingRequirements() const { return getTestRequirements() != nullptr; } - /// If set, then tests are executed + /// If set, then tests are executed bool isExecuting() const { return getTestRequirements() == nullptr; } - /// True if a render API filter is enabled + /// True if a render API filter is enabled bool isRenderApiFilterEnabled() const { return options.enabledApis != Slang::RenderApiFlag::AllOf && options.enabledApis != 0; } - /// True if a test with the requiredFlags can in principal run (it may not be possible if the API is not available though) + /// True if a test with the requiredFlags can in principal run (it may not be possible if the API is not available though) bool canRunTestWithRenderApiFlags(Slang::RenderApiFlags requiredFlags); - /// True if can run unit tests + /// True if can run unit tests bool canRunUnitTests() const { return options.apiOnly == false; } - /// Given a spawn type, return the final spawn type. - /// In particular we want 'Default' spawn type to vary by the environment (for example running on test server on CI) + /// Given a spawn type, return the final spawn type. + /// In particular we want 'Default' spawn type to vary by the environment (for example running on test server on CI) SpawnType getFinalSpawnType(SpawnType spawnType); SpawnType getFinalSpawnType(); - /// Get compiler set + /// Get compiler set Slang::DownstreamCompilerSet* getCompilerSet(); Slang::IDownstreamCompiler* getDefaultCompiler(SlangSourceLanguage sourceLanguage); Slang::JSONRPCConnection* getOrCreateJSONRPCConnection(); void destroyRPCConnection(); - /// Ctor + /// Ctor TestContext(); - /// Dtor + /// Dtor ~TestContext(); Options options; TestCategorySet categorySet; - /// If set then tests are not run, but their requirements are set + /// If set then tests are not run, but their requirements are set PassThroughFlags availableBackendFlags = 0; Slang::RenderApiFlags availableRenderApiFlags = 0; @@ -152,17 +156,17 @@ class TestContext Slang::String dllDirectoryPath; Slang::String exePath; - /// Timeout time for communication over connection. - /// NOTE! If the timeout is hit, the connection will be destroyed, and then recreated. - /// For tests that compile the stdlib, if that takes this time, the stdlib will be - /// repeatedly compiled and each time fail. - /// NOTE! This timeout may be altered in the ctor for a specific target, the initializatoin - /// value is just the default. - /// - /// TODO(JS): We could split the stdlib compilation from other actions, and have timeout specific for - /// that. To do this we could have a 'compileStdLib' RPC method. - /// - /// Current default is 60 seconds. + /// Timeout time for communication over connection. + /// NOTE! If the timeout is hit, the connection will be destroyed, and then recreated. + /// For tests that compile the stdlib, if that takes this time, the stdlib will be + /// repeatedly compiled and each time fail. + /// NOTE! This timeout may be altered in the ctor for a specific target, the initializatoin + /// value is just the default. + /// + /// TODO(JS): We could split the stdlib compilation from other actions, and have timeout specific for + /// that. To do this we could have a 'compileStdLib' RPC method. + /// + /// Current default is 60 seconds. Slang::Int connectionTimeOutInMs = 60 * 1000; void setThreadIndex(int index); @@ -175,6 +179,10 @@ class TestContext std::mutex mutex; Slang::RefPtr m_languageServerConnection; + + std::mutex mutexFailedFileTests; + Slang::List> failedFileTests; + Slang::IFileCheck* getFileCheck() { return m_fileCheck; }; protected: diff --git a/tools/slang-test/test-reporter.h b/tools/slang-test/test-reporter.h index 1d2857463d..775732745b 100644 --- a/tools/slang-test/test-reporter.h +++ b/tools/slang-test/test-reporter.h @@ -23,7 +23,7 @@ enum class TestOutputMode class TestReporter : public ITestReporter { - public: +public: struct TestInfo {