Apply Job Object in delayed child process (dotnet#654)

Co-authored-by: Theodore Tsirpanis <teo@tsirpanis.gr>
caaavik-msft · Nov 5, 2023 · 26b2f9f · 26b2f9f
1 parent 1d0bf34
commit 26b2f9f
Show file tree

Hide file tree

Showing 7 changed files with 88 additions and 3 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -25,7 +25,7 @@
 
     <!-- Lots of our dependencies are not strong-named -->
     <NoWarn>$(NoWarn);8002</NoWarn>
-    <LangVersion>8.0</LangVersion>    
+    <LangVersion>latest</LangVersion>    
   </PropertyGroup>
 
 </Project>
diff --git a/Microsoft.Crank.sln b/Microsoft.Crank.sln
@@ -50,6 +50,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Crank.Jobs.H2Load
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Crank.PullRequestBot", "src\Microsoft.Crank.PullRequestBot\Microsoft.Crank.PullRequestBot.csproj", "{A2B9140B-46E5-451D-9246-ECA019487F5C}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Crank.JobObjectWrapper", "src\Microsoft.Crank.JobOjectWrapper\Microsoft.Crank.JobObjectWrapper.csproj", "{D02CC5A5-A6EA-42CF-9EA5-E3D1CE0FFBFE}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -116,6 +118,10 @@ Global
 		{A2B9140B-46E5-451D-9246-ECA019487F5C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{A2B9140B-46E5-451D-9246-ECA019487F5C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{A2B9140B-46E5-451D-9246-ECA019487F5C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D02CC5A5-A6EA-42CF-9EA5-E3D1CE0FFBFE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D02CC5A5-A6EA-42CF-9EA5-E3D1CE0FFBFE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D02CC5A5-A6EA-42CF-9EA5-E3D1CE0FFBFE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D02CC5A5-A6EA-42CF-9EA5-E3D1CE0FFBFE}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -136,6 +142,7 @@ Global
 		{7BAB625A-3338-4BF1-BD0C-D5ECCAC254B5} = {995FCFF9-E5F6-4BDD-8E5B-FBDEA21145F9}
 		{C45AB9D7-5346-4610-8D44-C6F6B31BEC2D} = {995FCFF9-E5F6-4BDD-8E5B-FBDEA21145F9}
 		{A2B9140B-46E5-451D-9246-ECA019487F5C} = {995FCFF9-E5F6-4BDD-8E5B-FBDEA21145F9}
+		{D02CC5A5-A6EA-42CF-9EA5-E3D1CE0FFBFE} = {995FCFF9-E5F6-4BDD-8E5B-FBDEA21145F9}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {C48AD7EE-82B1-4307-A869-3FC14AC9B21F}

diff --git a/src/Microsoft.Crank.Agent/Microsoft.Crank.Agent.csproj b/src/Microsoft.Crank.Agent/Microsoft.Crank.Agent.csproj
@@ -39,6 +39,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\Microsoft.Crank.JobOjectWrapper\Microsoft.Crank.JobObjectWrapper.csproj" />
     <ProjectReference Include="..\Microsoft.Crank.Models\Microsoft.Crank.Models.csproj" />
   </ItemGroup>
 

diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs
@@ -4612,6 +4612,7 @@ private static async Task<Process> StartProcess(string hostname, string benchmar
                     {
                         Log.Info($"Tracking child process id: {childProcessId}");
                         job.ChildProcessId = childProcessId;
+                        stopwatch.Restart();
                     }
                 }
             };
@@ -4636,6 +4637,33 @@ private static async Task<Process> StartProcess(string hostname, string benchmar
                 process.WaitForExit();
             };
 
+            var useWindowsLimiter = OperatingSystem == OperatingSystem.Windows && (job.MemoryLimitInBytes > 0 || job.CpuLimitRatio > 0 || !String.IsNullOrWhiteSpace(job.CpuSet));
+
+            // If useWindowsLimiter is true we need to use a proxy app that will get a JobObject attached such that the actual application can account for the limits
+            // when it starts. Otherwise the dotnet app would start while the JobOject is not yet defined.
+
+            if (useWindowsLimiter)
+            {
+                // This won't be necessary once https://github.com/dotnet/runtime/issues/94127 is implemented.
+                // At that point the process can be started suspended while the JobObject is assigned to it.
+
+                Console.WriteLine("Invoking JobObject Wrapper");
+
+                var filename = process.StartInfo.FileName;
+                var arguments = process.StartInfo.Arguments;
+
+                // The executable should be in the same folder as the agent since it references the Console project
+                process.StartInfo.FileName = "Microsoft.Crank.JobObjectWrapper.exe";
+                process.StartInfo.Arguments = filename + " " + arguments;
+
+                // .NET doesn't respect a cpu affinity if a ratio is not set too. https://github.com/dotnet/runtime/issues/94364
+                if (!String.IsNullOrWhiteSpace(job.CpuSet) && job.CpuLimitRatio == 0)
+                {
+                    process.StartInfo.EnvironmentVariables.Add("DOTNET_PROCESSOR_COUNT", CalculateCpuList(job.CpuSet).Count.ToString(CultureInfo.InvariantCulture));
+                    process.StartInfo.EnvironmentVariables.Add("DOTNET_Thread_AssignCpuGroups", "0");
+                }
+            }
+
             stopwatch.Start();
             process.Start();
 
@@ -4650,7 +4678,7 @@ private static async Task<Process> StartProcess(string hostname, string benchmar
                 RunAndTrace();
             }
 
-            if ((job.MemoryLimitInBytes > 0 || job.CpuLimitRatio > 0 || !String.IsNullOrWhiteSpace(job.CpuSet)) && RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            if (useWindowsLimiter)
             {
                 var limiter = new WindowsLimiter(job, (uint)process.Id);
 

diff --git a/src/Microsoft.Crank.Agent/WindowsLimiter.cs b/src/Microsoft.Crank.Agent/WindowsLimiter.cs
@@ -91,7 +91,7 @@ public void LimitProcess()
                 var cpuSets = _cpuSet.Select(i => ssi[i].CpuSet.Id).ToArray();
 
                 var result = SetProcessDefaultCpuSets(_safeProcess, cpuSets, (uint)cpuSets.Length);
-                Log.Info($"Limiting CpuSet ids: {String.Join(',', cpuSets)}, {result} ({_job.Service}:{_job.Id})");
+                Log.Info($"Limiting CpuSet ids: {String.Join(',', cpuSets)}, for ({_job.Service}:{_job.Id}): {(result ? "SUCCESS" : "FAILED")}");
 
                 foreach (var csi in ssi)
                 {

diff --git a/src/Microsoft.Crank.JobOjectWrapper/Microsoft.Crank.JobObjectWrapper.csproj b/src/Microsoft.Crank.JobOjectWrapper/Microsoft.Crank.JobObjectWrapper.csproj
@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net6.0;net7.0</TargetFrameworks>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>
diff --git a/src/Microsoft.Crank.JobOjectWrapper/Program.cs b/src/Microsoft.Crank.JobOjectWrapper/Program.cs
@@ -0,0 +1,39 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Diagnostics;
+
+if (args.Length == 0)
+{
+    Console.WriteLine("Arguments are missing.");
+    Environment.Exit(-1);
+}
+
+Console.WriteLine("Job Object Wrapper");
+Console.WriteLine("Waiting for Job Object to be setup...");
+
+await Task.Delay(1000);
+
+var process = new Process()
+{
+    StartInfo = {
+        FileName = args[0],
+        Arguments = string.Join(" ", args[1..]),
+        UseShellExecute = false,
+        RedirectStandardOutput = true,
+        RedirectStandardError = true,
+    }
+};
+
+Console.WriteLine("Starting process...");
+Console.WriteLine($"Filename: {process.StartInfo.FileName}");
+Console.WriteLine($"Args: {process.StartInfo.Arguments}");
+
+process.Start();
+Console.WriteLine($"##ChildProcessId:{process.Id}");
+process.WaitForExit();
+
+Console.WriteLine(process.StandardOutput.ReadToEnd());
+
+await Task.Delay(1000);