Merge pull request #1796 from tgstation/v640 [TGSDeploy]

v6.4.0

.github/workflows/ci-pipeline.yml

@@ -1420,7 +1420,12 @@ jobs:
     - name: Build ReleaseNotes
       run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj
 
-    - name: Run ReleaseNotes Create CI Completion Check
+    - name: Run ReleaseNotes Create CI Completion Check (PR HEAD)
+      if: github.event_name != 'push' && github.event_name != 'schedule'
+      run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ci-completion-check ${{ github.event.pull_request.head.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }}
+
+    - name: Run ReleaseNotes Create CI Completion Check (Branch)
+      if: github.event_name == 'push' || github.event_name == 'schedule'
       run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ci-completion-check ${{ github.sha }} ${{ secrets.TGS_CI_GITHUB_APP_TOKEN_BASE64 }}
 
   deployment-gate:

.github/workflows/stable-merge.yml

@@ -17,7 +17,7 @@ jobs:
         fetch-depth: 0
 
     - name: Merge master into dev
-      uses: robotology/gh-action-nightly-merge@22f5e45d028f22837d617fa07512925457eec184 #v1.3.3
+      uses: robotology/gh-action-nightly-merge@14b4a4cf358f7479aa708bee05cf8a794d6a2516 #v1.5.0
       with:
         stable_branch: 'master'
         development_branch: 'dev'

README.md

@@ -157,6 +157,7 @@ docker run \
 	--network="host" \ # Not recommended, eases networking setup if your sql server is on the same machine
 	--name="tgs" \ # Name for the container
 	--cap-add=sys_nice \ # Recommended, allows TGS to lower the niceness of child processes if it sees fit
+	--cap-add=sys_resource \ # Recommended, allows TGS to not be killed by the OOM killer before its child processes
 	--init \ #Highly recommended, reaps potential zombie processes
 	-p 5000:5000 \ # Port bridge for accessing TGS, you can change this if you need
 	-p 0.0.0.0:<public game port>:<public game port> \ # Port bridge for accessing DreamDaemon

build/Version.props

@@ -3,7 +3,7 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="WebpanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>6.3.2</TgsCoreVersion>
+    <TgsCoreVersion>6.4.0</TgsCoreVersion>
     <TgsConfigVersion>5.1.0</TgsConfigVersion>
     <TgsApiVersion>10.2.0</TgsApiVersion>
     <TgsCommonLibraryVersion>7.0.0</TgsCommonLibraryVersion>

src/Tgstation.Server.Api/Tgstation.Server.Api.csproj

@@ -27,7 +27,7 @@
     <!-- Usage: HTTP constants reference -->
     <PackageReference Include="Microsoft.AspNetCore.Http.Extensions" Version="2.2.0" />
     <!-- Usage: Decoding the 'nbf' property of JWTs -->
-    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="7.3.1" />
+    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="7.4.0" />
     <!-- Usage: Primary JSON library -->
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <!-- Usage: Data model annotating -->

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -855,11 +855,12 @@ async ValueTask<bool> RunDreamMaker(IEngineExecutableLock engineLock, Models.Com
 			var environment = await engineLock.LoadEnv(logger, true, cancellationToken);
 			var arguments = engineLock.FormatCompilerArguments($"{job.DmeName}.{DmeExtension}");
 
-			await using var dm = processExecutor.LaunchProcess(
+			await using var dm = await processExecutor.LaunchProcess(
 				engineLock.CompilerExePath,
 				ioManager.ResolvePath(
 					job.DirectoryName!.Value.ToString()),
 				arguments,
+				cancellationToken,
 				environment,
 				readStandardHandles: true,
 				noShellExecute: true);

src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs

@@ -241,10 +241,11 @@ await HandleExtremelyLongPathOperation(
 				async shortenedPath =>
 				{
 					var shortenedDeployPath = IOManager.ConcatPath(shortenedPath, DeployDir);
-					await using var buildProcess = ProcessExecutor.LaunchProcess(
+					await using var buildProcess = await ProcessExecutor.LaunchProcess(
 						dotnetPath,
 						shortenedPath,
 						$"run -c Release --project OpenDreamPackageTool -- --tgs -o {shortenedDeployPath}",
+						cancellationToken,
 						null,
 						null,
 						!GeneralConfiguration.OpenDreamSuppressInstallOutput,

src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs

@@ -283,10 +283,11 @@ async ValueTask InstallDirectX(string path, CancellationToken cancellationToken)
 			try
 			{
 				// noShellExecute because we aren't doing runas shennanigans
-				await using var directXInstaller = processExecutor.LaunchProcess(
+				await using var directXInstaller = await processExecutor.LaunchProcess(
 					IOManager.ConcatPath(rbdx, "DXSETUP.exe"),
 					rbdx,
 					"/silent",
+					cancellationToken,
 					noShellExecute: true);
 
 				int exitCode;

src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs

@@ -93,7 +93,7 @@ public GitHubRemoteFeatures(IGitHubServiceFactory gitHubServiceFactory, ILogger<
 				Comment = parameters.Comment,
 				Number = parameters.Number,
 				TargetCommitSha = revisionToUse,
-				Url = pr?.HtmlUrl ?? errorMessage,
+				Url = pr?.HtmlUrl ?? $"https://github.com/{RemoteRepositoryOwner}/{RemoteRepositoryName}/pull/{parameters.Number}",
 			};
 
 			return testMerge;

src/Tgstation.Server.Host/Components/Repository/GitLabRemoteFeatures.cs

@@ -85,7 +85,7 @@ public GitLabRemoteFeatures(ILogger<GitLabRemoteFeatures> logger, Uri remoteUrl)
 					Comment = parameters.Comment,
 					Number = parameters.Number,
 					TargetCommitSha = parameters.TargetCommitSha,
-					Url = ex.Message,
+					Url = $"https://gitlab.com/{RemoteRepositoryOwner}/{RemoteRepositoryName}/-/merge_requests/{parameters.Number}",
 				};
 			}
 		}

src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs

@@ -516,10 +516,11 @@ async ValueTask<IProcess> CreateGameServerProcess(
 					? logFilePath
 					: null);
 
-			var process = processExecutor.LaunchProcess(
+			var process = await processExecutor.LaunchProcess(
 				engineLock.ServerExePath,
 				dmbProvider.Directory,
 				arguments,
+				cancellationToken,
 				environment,
 				logFilePath,
 				engineLock.HasStandardOutput,

src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs

@@ -761,7 +761,7 @@ async ValueTask ExecuteEventScripts(IEnumerable<string?> parameters, bool deploy
 				foreach (var scriptFile in scriptFiles)
 				{
 					logger.LogTrace("Running event script {scriptFile}...", scriptFile);
-					await using (var script = processExecutor.LaunchProcess(
+					await using (var script = await processExecutor.LaunchProcess(
 						ioManager.ConcatPath(resolvedScriptsDir, scriptFile),
 						resolvedScriptsDir,
 						String.Join(
@@ -778,6 +778,7 @@ async ValueTask ExecuteEventScripts(IEnumerable<string?> parameters, bool deploy
 
 								return $"\"{arg}\"";
 							})),
+						cancellationToken,
 						readStandardHandles: true,
 						noShellExecute: true))
 					using (cancellationToken.Register(() => script.Terminate()))

src/Tgstation.Server.Host/Core/Application.cs

@@ -356,6 +356,7 @@ void AddTypedContext<TContext>()
 				services.AddSingleton<IPostWriteHandler, PosixPostWriteHandler>();
 
 				services.AddSingleton<IProcessFeatures, PosixProcessFeatures>();
+				services.AddHostedService<PosixProcessFeatures>();
 
 				// PosixProcessFeatures also needs a IProcessExecutor for gcore
 				services.AddSingleton(x => new Lazy<IProcessExecutor>(() => x.GetRequiredService<IProcessExecutor>(), true));

src/Tgstation.Server.Host/IO/DefaultIOManager.cs

@@ -221,14 +221,7 @@ public Task MoveDirectory(string source, string destination, CancellationToken c
 		/// <inheritdoc />
 		public async ValueTask<byte[]> ReadAllBytes(string path, CancellationToken cancellationToken)
 		{
-			path = ResolvePath(path);
-			await using var file = new FileStream(
-				path,
-				FileMode.Open,
-				FileAccess.Read,
-				FileShare.ReadWrite | FileShare.Delete,
-				DefaultBufferSize,
-				FileOptions.Asynchronous | FileOptions.SequentialScan);
+			await using var file = CreateAsyncSequentialReadStream(path);
 			byte[] buf;
 			buf = new byte[file.Length];
 			await file.ReadAsync(buf, cancellationToken);
@@ -261,6 +254,19 @@ public FileStream CreateAsyncSequentialWriteStream(string path)
 				FileOptions.Asynchronous | FileOptions.SequentialScan);
 		}
 
+		/// <inheritdoc />
+		public FileStream CreateAsyncSequentialReadStream(string path)
+		{
+			path = ResolvePath(path);
+			return new FileStream(
+				path,
+				FileMode.Open,
+				FileAccess.Read,
+				FileShare.ReadWrite | FileShare.Delete,
+				DefaultBufferSize,
+				FileOptions.Asynchronous | FileOptions.SequentialScan);
+		}
+
 		/// <inheritdoc />
 		public Task<IReadOnlyList<string>> GetDirectories(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(
 			() =>

src/Tgstation.Server.Host/IO/IIOManager.cs

@@ -85,6 +85,7 @@ ValueTask CopyDirectory(
 		/// <param name="path">The path of the file to read.</param>
 		/// <param name="cancellationToken">A <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="ValueTask"/> that results in the contents of a file at <paramref name="path"/>.</returns>
+		/// <remarks>This function will fail to read files from the /proc filesystem on Linux.</remarks>
 		ValueTask<byte[]> ReadAllBytes(string path, CancellationToken cancellationToken);
 
 		/// <summary>
@@ -110,6 +111,13 @@ ValueTask CopyDirectory(
 		/// <returns>The open <see cref="FileStream"/>.</returns>
 		FileStream CreateAsyncSequentialWriteStream(string path);
 
+		/// <summary>
+		/// Creates an asynchronous <see cref="FileStream"/> for sequential reading.
+		/// </summary>
+		/// <param name="path">The path of the file to write, will be truncated.</param>
+		/// <returns>The open <see cref="FileStream"/>.</returns>
+		FileStream CreateAsyncSequentialReadStream(string path);
+
 		/// <summary>
 		/// Writes some <paramref name="contents"/> to a file at <paramref name="path"/> overwriting previous content.
 		/// </summary>

src/Tgstation.Server.Host/System/IProcessExecutor.cs

@@ -1,4 +1,6 @@
 using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace Tgstation.Server.Host.System
 {
@@ -13,15 +15,17 @@ interface IProcessExecutor
 		/// <param name="fileName">The full path to the executable file.</param>
 		/// <param name="workingDirectory">The working directory for the <see cref="IProcess"/>.</param>
 		/// <param name="arguments">The arguments for the <see cref="IProcess"/>.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <param name="environment">A <see cref="IReadOnlyDictionary{TKey, TValue}"/> of environment variables to set.</param>
 		/// <param name="fileRedirect">File to write process output and error streams to. Requires <paramref name="readStandardHandles"/> to be <see langword="true"/>.</param>
 		/// <param name="readStandardHandles">If the process output and error streams should be read.</param>
 		/// <param name="noShellExecute">If shell execute should not be used. Must be set if <paramref name="readStandardHandles"/> is set.</param>
-		/// <returns>The new <see cref="IProcess"/>.</returns>
-		IProcess LaunchProcess(
+		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the new <see cref="IProcess"/>.</returns>
+		ValueTask<IProcess> LaunchProcess(
 			string fileName,
 			string workingDirectory,
 			string arguments,
+			CancellationToken cancellationToken,
 			IReadOnlyDictionary<string, string>? environment = null,
 			string? fileRedirect = null,
 			bool readStandardHandles = false,

src/Tgstation.Server.Host/System/IProcessFeatures.cs

@@ -18,23 +18,31 @@ interface IProcessFeatures
 		/// <summary>
 		/// Suspend a given <paramref name="process"/>.
 		/// </summary>
-		/// <param name="process">The <see cref="Process"/> to suspend.</param>
+		/// <param name="process">The <see cref="global::System.Diagnostics.Process"/> to suspend.</param>
 		void SuspendProcess(global::System.Diagnostics.Process process);
 
 		/// <summary>
-		/// Resume a given suspended <see cref="Process"/>.
+		/// Resume a given suspended <see cref="global::System.Diagnostics.Process"/>.
 		/// </summary>
 		/// <param name="process">The <see cref="Process"/> to suspended.</param>
 		void ResumeProcess(global::System.Diagnostics.Process process);
 
 		/// <summary>
 		/// Create a dump file for a given <paramref name="process"/>.
 		/// </summary>
-		/// <param name="process">The <see cref="Process"/> to dump.</param>
+		/// <param name="process">The <see cref="global::System.Diagnostics.Process"/> to dump.</param>
 		/// <param name="outputFile">The full path to the output file.</param>
 		/// <param name="minidump">If a minidump should be taken as opposed to a full dump.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
 		ValueTask CreateDump(global::System.Diagnostics.Process process, string outputFile, bool minidump, CancellationToken cancellationToken);
+
+		/// <summary>
+		/// Run events on starting a process.
+		/// </summary>
+		/// <param name="process">The <see cref="global::System.Diagnostics.Process"/> that was started.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the <paramref name="process"/> ID.</returns>
+		ValueTask<int> HandleProcessStart(global::System.Diagnostics.Process process, CancellationToken cancellationToken);
 	}
 }