[Merge-on-Red] - Implement Test Process Watcher

src/coreclr/CMakeLists.txt

@@ -113,6 +113,11 @@ else()
     endif()
 endif()
 
+#----------------------------------------------------
+# Build the test watchdog alongside the CLR
+#----------------------------------------------------
+add_subdirectory("${CLR_SRC_NATIVE_DIR}/watchdog" test-watchdog)
+
 # Add this subdir. We install the headers for the jit.
 add_subdirectory(pal/prebuilt/inc)
 
@@ -264,3 +269,4 @@ endif(CLR_CMAKE_HOST_WIN32)
 if(CLR_CROSS_COMPONENTS_BUILD)
   include(crosscomponents.cmake)
 endif(CLR_CROSS_COMPONENTS_BUILD)
+

src/native/watchdog/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_executable(watchdog ${CMAKE_CURRENT_LIST_DIR}/watchdog.cpp)
+

src/native/watchdog/watchdog.cpp

@@ -0,0 +1,134 @@
+#include <cstdio>
+#include <cstdlib>
+#include <errno.h>
+#include <signal.h>
+
+#ifdef _WIN32
+#include <windows.h>
+#include <string>
+#else
+#include <chrono>
+#include <sys/wait.h>
+#include <thread>
+#include <unistd.h>
+#include <vector>
+#endif
+
+int run_timed_process(const long, const int, const char *[]);
+
+int main(const int argc, const char *argv[])
+{
+    if (argc < 3)
+    {
+        printf("There are missing arguments. Got %d instead of 3+ :(\n", argc);
+        return EXIT_FAILURE;
+    }
+
+    const long timeout_sec = strtol(argv[1], nullptr, 10);
+    int exit_code = run_timed_process(timeout_sec * 1000L, argc-2, &argv[2]);
+
+    printf("App Exit Code: %d\n", exit_code);
+    return exit_code;
+}
+
+int run_timed_process(const long timeout_ms, const int proc_argc, const char *proc_argv[])
+{
+#ifdef _WIN32
+    std::string cmdline(proc_argv[0]);
+
+    for (int i = 1; i < proc_argc; i++)
+    {
+        cmdline.append(" ");
+        cmdline.append(proc_argv[i]);
+    }
+
+    STARTUPINFOA startup_info;
+    PROCESS_INFORMATION proc_info;
+    unsigned long exit_code;
+
+    ZeroMemory(&startup_info, sizeof(startup_info));
+    startup_info.cb = sizeof(startup_info);
+    ZeroMemory(&proc_info, sizeof(proc_info));
+
+    if (!CreateProcessA(NULL, &cmdline[0], NULL, NULL, FALSE, 0, NULL, NULL,
+                       &startup_info, &proc_info))
+    {
+        int error_code = GetLastError();
+        printf("Process creation failed... Code %d.\n", error_code);
+        return error_code;
+    }
+
+    WaitForSingleObject(proc_info.hProcess, timeout);
+    GetExitCodeProcess(proc_info.hProcess, &exit_code);
+
+    CloseHandle(proc_info.hProcess);
+    CloseHandle(proc_info.hThread);
+    return exit_code;
+
+#else
+    // TODO: Describe what the 'ms_factor' is, and why it's being used here.
+    const int ms_factor = 40;
+    const int check_interval = 1000 / ms_factor;
+
+    int check_count = 0;
+    std::vector<const char*> args;
+
+    pid_t child_pid;
+    int child_status;
+    int wait_code;
+
+    for (int i = 0; i < proc_argc; i++)
+    {
+        args.push_back(proc_argv[i]);
+    }
+    args.push_back(NULL);
+
+    // This is just for development. Will remove it when it's ready to be submitted :)
+    for (int j = 0; j < proc_argc; j++)
+    {
+        printf("[%d]: %s\n", j, args[j]);
+    }
+
+    child_pid = fork();
+
+    if (child_pid < 0)
+    {
+        // Fork failed. No memory remaining available :(
+        printf("Fork failed... Returning ENOMEM.\n");
+        return ENOMEM;
+    }
+    else if (child_pid == 0)
+    {
+        // Instructions for child process!
+        printf("Running child process...\n");
+        execv(args[0], const_cast<char* const*>(args.data()));
+    }
+    else
+    {
+        do
+        {
+            // Instructions for the parent process!
+            wait_code = waitpid(child_pid, &child_status, WNOHANG);
+
+            if (wait_code == -1)
+                return EINVAL;
+
+            std::this_thread::sleep_for(std::chrono::milliseconds(check_interval));
+
+            if (wait_code)
+            {
+                if (WIFEXITED(child_status))
+                    return WEXITSTATUS(child_status);
+            }
+            check_count += ms_factor;
+
+        } while (check_count < ((timeout_ms / check_interval) * ms_factor));
+    }
+
+    printf("Child process took too long. Timed out... Exiting...\n");
+    kill(child_pid, SIGKILL);
+
+#endif
+    return ETIMEDOUT;
+}
+

src/tests/Common/CLRTest.Execute.Bash.targets

@@ -172,14 +172,27 @@ fi
         <Description>Set CORE_ROOT to the specified value before running the test.</Description>
       </BashCLRTestExecutionScriptArgument>
 
+      <BashCLRTestExecutionScriptArgument Include="watcher">
+        <HasParam>true</HasParam>
+        <ParamText>=*</ParamText> <!-- Bash specific -->
+        <ParamName>watcherFullPath</ParamName>
+        <Command><![CDATA[        export _WatcherFullPath="${i#*=}"
+        if [ ! -f "$_WatcherFullPath" ]
+        then
+            echo "The Watcher FullPath %5C%22${_WatcherFullPath}%5C%22 does not exist"
+            usage
+        fi]]></Command>
+        <Description>Path to the test watcher that will look over the test.</Description>
+      </BashCLRTestExecutionScriptArgument>
+
       <BashCLRTestExecutionScriptArgument Include="e;env">
         <HasParam>true</HasParam>
         <ParamText>=*</ParamText> <!-- Bash specific -->
         <ParamName>dotenvFullPath</ParamName>
         <Command><![CDATA[        export __DotEnv="${i#*=}"
         if [ ! -f "$__DotEnv" ]
         then
-            echo "The Debugger FullPath %5C%22${__DotEnv}%5C%22 does not exist"
+            echo "The dotenv file FullPath %5C%22${__DotEnv}%5C%22 does not exist"
             usage
         fi
         export __DotEnvArg=-e ${__DotEnv}]]></Command>
@@ -242,10 +255,11 @@ then
     exit 1
   fi
 
-  # Copy CORECLR native binaries to $LinkBin,
+  # Copy CORECLR native binaries and the test watcher (if provided) to $LinkBin,
   # so that we can run the test based on that directory
   cp $CORE_ROOT/*.so $LinkBin/
   cp $CORE_ROOT/corerun $LinkBin/
+  cp $_WatcherFullPath/watchdog $LinkBin/
 
   # Copy some files that may be arguments
   for f in *.txt;
@@ -255,6 +269,7 @@ then
 
   ExePath=$LinkBin/$(InputAssemblyName)
   export CORE_ROOT=$PWD/$LinkBin
+  export _WatcherFullPath=$PWD/$LinkBin/watchdog
 fi
 ]]>
           </BashLinkerTestLaunchCmds>
@@ -275,6 +290,7 @@ fi
       </PropertyGroup>
       <PropertyGroup>
       <CLRTestRunFile Condition="'$(CLRTestIsHosted)'=='true'">"$CORE_ROOT/corerun" $(CoreRunArgs)  ${__DotEnvArg}</CLRTestRunFile>
+      <WatcherRunFile>"$_WatcherFullPath" 300</WatcherRunFile>
 
       <!-- Note that this overwrites CLRTestBashPreCommands rather than adding to it. -->
       <CLRTestBashPreCommands Condition="'$(CLRTestKind)' == 'BuildAndRun' and '$(TargetArchitecture)' == 'wasm'"><![CDATA[
@@ -311,7 +327,7 @@ if [ ! -z "$CLRCustomTestLauncher" ];
 then
     LAUNCHER="$CLRCustomTestLauncher $PWD/"
 else
-    LAUNCHER="$_DebuggerFullPath $_DebuggerArgsSeparator $(CLRTestRunFile)"
+    LAUNCHER="$_DebuggerFullPath $_DebuggerArgsSeparator $(WatcherRunFile) $(CLRTestRunFile)"
 fi
 
 $(BashIlrtTestLaunchCmds)
@@ -476,7 +492,7 @@ usage()
 for i in "$@"
     do
         case $i in
-        -?|-h|--help)
+          -?|-h|--help|/?|/h|/help)
         usage
         %3B%3B
 @(BashCLRTestExecutionScriptArgument -> '        -%(Identity)%(ParamText)|/%(Identity)%(ParamText))

src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs

@@ -341,11 +341,11 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe
         builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Running test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
         builder.AppendLine($"{_outputRecorderIdentifier}.ResetTestOutput();");
         builder.AppendLine(testExecutionExpression);
-        builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput());");
+        builder.AppendLine($"{_summaryLocalIdentifier}.ReportPassedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw);");
         builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Passed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
         builder.AppendLine("}");
         builder.AppendLine("catch (System.Exception ex) {");
-        builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput());");
+        builder.AppendLine($"{_summaryLocalIdentifier}.ReportFailedTest({test.TestNameExpression}, \"{test.ContainingType}\", @\"{test.Method}\", stopwatch.Elapsed - testStart, ex, {_outputRecorderIdentifier}.GetTestOutput(), tempLogSw);");
         builder.AppendLine($"System.Console.WriteLine(\"{{0:HH:mm:ss.fff}} Failed test: {{1}}\", System.DateTime.Now, {test.TestNameExpression});");
         builder.AppendLine("}");
 
@@ -359,6 +359,6 @@ public string WrapTestExecutionWithReporting(string testExecutionExpression, ITe
 
     public string GenerateSkippedTestReporting(ITestInfo skippedTest)
     {
-        return $"{_summaryLocalIdentifier}.ReportSkippedTest({skippedTest.TestNameExpression}, \"{skippedTest.ContainingType}\", @\"{skippedTest.Method}\", System.TimeSpan.Zero, string.Empty);";
+        return $"{_summaryLocalIdentifier}.ReportSkippedTest({skippedTest.TestNameExpression}, \"{skippedTest.ContainingType}\", @\"{skippedTest.Method}\", System.TimeSpan.Zero, string.Empty, tempLogSw);";
     }
 }

src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs

@@ -155,7 +155,9 @@ private static string GenerateFullTestRunner(ImmutableArray<ITestInfo> testInfos
         builder.AppendLine("System.Collections.Generic.HashSet<string> testExclusionList = XUnitWrapperLibrary.TestFilter.LoadTestExclusionList();");
 
         builder.AppendLine("XUnitWrapperLibrary.TestFilter filter = new (args, testExclusionList);");
-        builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new();");
+        builder.AppendLine($@"XUnitWrapperLibrary.TestSummary summary = new();");
+        // builder.AppendLine($@"XUnitWrapperLibrary.TestSummary summary = new(""{assemblyName}"");");
+        // builder.AppendLine("summary.OpenTempLog();");
         builder.AppendLine("System.Diagnostics.Stopwatch stopwatch = System.Diagnostics.Stopwatch.StartNew();");
         builder.AppendLine("XUnitWrapperLibrary.TestOutputRecorder outputRecorder = new(System.Console.Out);");
         builder.AppendLine("System.Console.SetOut(outputRecorder);");
@@ -177,27 +179,35 @@ private static string GenerateFullTestRunner(ImmutableArray<ITestInfo> testInfos
                 if (testsLeftInCurrentTestExecutor == 0)
                 {
                     if (currentTestExecutor != 0)
+                    {
+                        testExecutorBuilder.AppendLine("}");
                         testExecutorBuilder.AppendLine("}");
+                    }
+
                     currentTestExecutor++;
                     testExecutorBuilder.AppendLine($"void TestExecutor{currentTestExecutor}(){{");
+                    testExecutorBuilder.AppendLine($@"using (System.IO.StreamWriter tempLogSw = System.IO.File.AppendText(""{assemblyName}_templog.xml"")) {{");
                     builder.AppendLine($"TestExecutor{currentTestExecutor}();");
                     testsLeftInCurrentTestExecutor = 50; // Break test executors into groups of 50, which empircally seems to work well
                 }
+
                 testExecutorBuilder.AppendLine(test.GenerateTestExecution(reporter));
                 totalTestsEmitted++;
                 testsLeftInCurrentTestExecutor--;
             }
+
+            testExecutorBuilder.AppendLine("}");
             testExecutorBuilder.AppendLine("}");
         }
 
+        // builder.AppendLine($@"summary.CloseTempLog();");
         builder.AppendLine($@"string testResults = summary.GetTestResultOutput(""{assemblyName}"");");
         builder.AppendLine($@"string workitemUploadRoot = System.Environment.GetEnvironmentVariable(""HELIX_WORKITEM_UPLOAD_ROOT"");");
         builder.AppendLine($@"if (workitemUploadRoot != null) System.IO.File.WriteAllText(System.IO.Path.Combine(workitemUploadRoot, ""{assemblyName}.testResults.xml.txt""), testResults);");
         builder.AppendLine($@"System.IO.File.WriteAllText(""{assemblyName}.testResults.xml"", testResults);");
         builder.AppendLine("return 100;");
 
         builder.Append(testExecutorBuilder);
-
         builder.AppendLine("public static class TestCount { public const int Count = " + totalTestsEmitted.ToString() + "; }");
         return builder.ToString();
     }