From 728fd85bc7ad04f5a0ea2ad0d4d8afe371ff9b64 Mon Sep 17 00:00:00 2001 From: Ivan Diaz Sanchez Date: Fri, 10 Mar 2023 12:06:53 -0800 Subject: [PATCH] [Merge-on-Red] - Implement Test Process Watcher (#78742) Initial implementation of the test watcher that looks out for hangs and freezes during test runs. --- src/coreclr/CMakeLists.txt | 6 + src/native/watchdog/CMakeLists.txt | 4 + src/native/watchdog/watchdog.cpp | 136 ++++++++++++++++++ src/tests/Common/CLRTest.Execute.Bash.targets | 23 ++- .../Common/CLRTest.Execute.Batch.targets | 18 ++- src/tests/Common/helixpublishwitharcade.proj | 8 +- 6 files changed, 186 insertions(+), 9 deletions(-) create mode 100644 src/native/watchdog/CMakeLists.txt create mode 100644 src/native/watchdog/watchdog.cpp diff --git a/src/coreclr/CMakeLists.txt b/src/coreclr/CMakeLists.txt index e7cd64d24922a..67b773bf78787 100644 --- a/src/coreclr/CMakeLists.txt +++ b/src/coreclr/CMakeLists.txt @@ -119,6 +119,11 @@ else() endif() endif() +#---------------------------------------------------- +# Build the test watchdog alongside the CLR +#---------------------------------------------------- +add_subdirectory("${CLR_SRC_NATIVE_DIR}/watchdog" test-watchdog) + # Add this subdir. We install the headers for the jit. add_subdirectory(pal/prebuilt/inc) @@ -275,3 +280,4 @@ endif(NOT CLR_CMAKE_HOST_MACCATALYST AND NOT CLR_CMAKE_HOST_IOS AND NOT CLR_CMAK if(CLR_CROSS_COMPONENTS_BUILD) include(crosscomponents.cmake) endif(CLR_CROSS_COMPONENTS_BUILD) + diff --git a/src/native/watchdog/CMakeLists.txt b/src/native/watchdog/CMakeLists.txt new file mode 100644 index 0000000000000..723e105d7373f --- /dev/null +++ b/src/native/watchdog/CMakeLists.txt @@ -0,0 +1,4 @@ +add_executable_clr(watchdog ${CMAKE_CURRENT_LIST_DIR}/watchdog.cpp) +install_clr(TARGETS watchdog DESTINATIONS . COMPONENT hosts) +install_clr(TARGETS watchdog DESTINATIONS . COMPONENT nativeaot) + diff --git a/src/native/watchdog/watchdog.cpp b/src/native/watchdog/watchdog.cpp new file mode 100644 index 0000000000000..1dc6f74fb06b6 --- /dev/null +++ b/src/native/watchdog/watchdog.cpp @@ -0,0 +1,136 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include +#include +#include +#include + +#ifdef TARGET_WINDOWS + +#include +#include + +#else // !TARGET_WINDOWS + +#include +#include +#include +#include +#include + +#endif // TARGET_WINDOWS + +int run_timed_process(const long, const int, const char *[]); + +#ifdef TARGET_X86 +int __cdecl main(const int argc, const char *argv[]) +#else +int main(const int argc, const char *argv[]) +#endif +{ + if (argc < 3) + { + printf("There are missing arguments. Got %d instead of 3+ :(\n", argc); + return EXIT_FAILURE; + } + + const long timeout_sec = strtol(argv[1], nullptr, 10); + int exit_code = run_timed_process(timeout_sec * 1000L, argc-2, &argv[2]); + + printf("App Exit Code: %d\n", exit_code); + return exit_code; +} + +int run_timed_process(const long timeout_ms, const int proc_argc, const char *proc_argv[]) +{ +#ifdef TARGET_WINDOWS + std::string cmdline(proc_argv[0]); + + for (int i = 1; i < proc_argc; i++) + { + cmdline.append(" "); + cmdline.append(proc_argv[i]); + } + + STARTUPINFOA startup_info; + PROCESS_INFORMATION proc_info; + unsigned long exit_code; + + ZeroMemory(&startup_info, sizeof(startup_info)); + startup_info.cb = sizeof(startup_info); + ZeroMemory(&proc_info, sizeof(proc_info)); + + if (!CreateProcessA(NULL, &cmdline[0], NULL, NULL, FALSE, 0, NULL, NULL, + &startup_info, &proc_info)) + { + int error_code = GetLastError(); + printf("Process creation failed... Code %d.\n", error_code); + return error_code; + } + + WaitForSingleObject(proc_info.hProcess, timeout_ms); + GetExitCodeProcess(proc_info.hProcess, &exit_code); + + CloseHandle(proc_info.hProcess); + CloseHandle(proc_info.hThread); + return exit_code; + +#else // !TARGET_WINDOWS + + const int check_interval_ms = 25; + int check_count = 0; + std::vector args; + + pid_t child_pid; + int child_status; + int wait_code; + + for (int i = 0; i < proc_argc; i++) + { + args.push_back(proc_argv[i]); + } + args.push_back(NULL); + + child_pid = fork(); + + if (child_pid < 0) + { + // Fork failed. No memory remaining available :( + printf("Fork failed... Returning ENOMEM.\n"); + return ENOMEM; + } + else if (child_pid == 0) + { + // Instructions for child process! + execv(args[0], const_cast(args.data())); + } + else + { + do + { + // Instructions for the parent process! + wait_code = waitpid(child_pid, &child_status, WNOHANG); + + if (wait_code == -1) + return EINVAL; + + std::this_thread::sleep_for(std::chrono::milliseconds(check_interval_ms)); + + if (wait_code) + { + if (WIFEXITED(child_status)) + return WEXITSTATUS(child_status); + } + check_count++; + + } while (check_count < (timeout_ms / check_interval_ms)); + } + + printf("Child process took too long. Timed out... Exiting...\n"); + kill(child_pid, SIGKILL); + +#endif // TARGET_WINDOWS + return ETIMEDOUT; +} + diff --git a/src/tests/Common/CLRTest.Execute.Bash.targets b/src/tests/Common/CLRTest.Execute.Bash.targets index 27557752e3137..5cc68ec20eff0 100644 --- a/src/tests/Common/CLRTest.Execute.Bash.targets +++ b/src/tests/Common/CLRTest.Execute.Bash.targets @@ -187,12 +187,18 @@ fi A dotenv file to pass to corerun to set environment variables for the test run. + + + false + + Run the tests using the test watcher. + @@ -250,10 +256,11 @@ then exit 1 fi - # Copy CORECLR native binaries to $LinkBin, + # Copy CORECLR native binaries and the test watcher to $LinkBin, # so that we can run the test based on that directory cp $CORE_ROOT/*.so $LinkBin/ cp $CORE_ROOT/corerun $LinkBin/ + cp $CORE_ROOT/watchdog $LinkBin/ # Copy some files that may be arguments for f in *.txt; @@ -283,6 +290,7 @@ fi "$CORE_ROOT/corerun" $(CoreRunArgs) ${__DotEnvArg} + "$CORE_ROOT/watchdog" 300 ' -%(Identity)%(ParamText)|/%(Identity)%(ParamText)) @@ -534,6 +548,7 @@ ReleaseLock() } cd "$%28dirname "${BASH_SOURCE[0]}")" LockFile="lock" +_RunWithWatcher=0 # The __TestEnv variable may be used to specify a script to source before the test. diff --git a/src/tests/Common/CLRTest.Execute.Batch.targets b/src/tests/Common/CLRTest.Execute.Batch.targets index db74a66be45b5..2e0d05e8c635b 100644 --- a/src/tests/Common/CLRTest.Execute.Batch.targets +++ b/src/tests/Common/CLRTest.Execute.Batch.targets @@ -216,6 +216,14 @@ Exit /b 0 ]]> Set CORE_ROOT to the specified value before running the test. + + + false + + Run the tests using the test watcher. + @@ -260,17 +268,18 @@ IF defined DoLink ( Exit /b 1 ) - REM Copy CORECLR native binaries to %LinkBin%, so that we can run the test based on that directory + REM Copy CORECLR native binaries and the test watcher to %LinkBin%, so that we can run the test based on that directory copy %CORE_ROOT%\clrjit.dll %LinkBin% > nul 2> nul copy %CORE_ROOT%\coreclr.dll %LinkBin% > nul 2> nul copy %CORE_ROOT%\mscorrc.dll %LinkBin% > nul 2> nul copy %CORE_ROOT%\CoreRun.exe %LinkBin% > nul 2> nul + copy %CORE_ROOT%\watchdog.exe %LinkBin% > nul 2> nul REM Copy some files that may be arguments copy *.txt %LinkBin% > nul 2> nul set ExePath=%LinkBin%\$(InputAssemblyName) - set CORE_ROOT=%scriptPath%LinkBin% + set CORE_ROOT=%scriptPath%\%LinkBin% ) ]]> @@ -289,6 +298,8 @@ if defined DoLink ( "%CORE_ROOT%\corerun.exe" $(CoreRunArgs) %__DotEnvArg% + "%CORE_ROOT%\watchdog.exe" 300 + - - + + @@ -722,7 +722,9 @@ - + + +