Merge pull request #1811 from tgstation/1807-ClientCommandLine [APIDe…

…ploy][NugetDeploy]

Compiler command line arguments and PreCompile reference name

.github/workflows/scripts/rerunFlakyTests.js

@@ -16,10 +16,7 @@ async function getFailedJobsForRun(github, context, workflowRunId, runAttempt) {
 	});
 
 	return jobs
-		.filter((job) => job.conclusion === "failure")
-		.filter((job) =>
-			CONSIDERED_JOBS.some((title) => job.name.startsWith(title))
-		);
+		.filter((job) => job.conclusion === "failure");
 }
 
 export async function rerunFlakyTests({ github, context }) {
@@ -35,12 +32,14 @@ export async function rerunFlakyTests({ github, context }) {
 		return;
 	}
 
-	if (failingJobs.length === 0) {
-		throw new Error(
-			"rerunFlakyTests should not have run on a run with no failing jobs"
-		);
+	const filteredFailingJobs = failingJobs.filter((job) => CONSIDERED_JOBS.some((title) => job.name.startsWith(title)));
+	if (filteredFailingJobs.length === 0) {
+		console.log("Failing jobs are NOT designated flaky. Not rerunning.");
+		return;
 	}
 
+	console.log(`Rerunning job: ${filteredFailingJobs[0].name}`);
+
 	github.rest.actions.reRunWorkflowFailedJobs({
 		owner: context.repo.owner,
 		repo: context.repo.repo,

build/TestCommon.props

@@ -18,9 +18,9 @@
     <!-- Pinned: Be VERY careful about updating https://github.com/moq/moq/issues/1372 -->
     <PackageReference Include="Moq" Version="4.20.70" />
     <!-- Usage: MSTest execution -->
-    <PackageReference Include="MSTest.TestAdapter" Version="3.2.2" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.3.1" />
     <!-- Usage: MSTest asserts etc... -->
-    <PackageReference Include="MSTest.TestFramework" Version="3.2.2" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.3.1" />
   </ItemGroup>
 
 </Project>

build/Version.props

@@ -3,12 +3,12 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="WebpanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>6.4.1</TgsCoreVersion>
+    <TgsCoreVersion>6.5.0</TgsCoreVersion>
     <TgsConfigVersion>5.1.0</TgsConfigVersion>
-    <TgsApiVersion>10.2.0</TgsApiVersion>
+    <TgsApiVersion>10.3.0</TgsApiVersion>
     <TgsCommonLibraryVersion>7.0.0</TgsCommonLibraryVersion>
-    <TgsApiLibraryVersion>13.2.0</TgsApiLibraryVersion>
-    <TgsClientVersion>15.2.0</TgsClientVersion>
+    <TgsApiLibraryVersion>13.3.0</TgsApiLibraryVersion>
+    <TgsClientVersion>15.3.0</TgsClientVersion>
     <TgsDmapiVersion>7.1.2</TgsDmapiVersion>
     <TgsInteropVersion>5.9.0</TgsInteropVersion>
     <TgsHostWatchdogVersion>1.4.1</TgsHostWatchdogVersion>
@@ -17,7 +17,7 @@
     <TgsNugetNetFramework>netstandard2.0</TgsNugetNetFramework>
     <TgsNetMajorVersion>8</TgsNetMajorVersion>
     <!-- Update this frequently with dotnet runtime patches. MAJOR MUST MATCH ABOVE! -->
-    <TgsDotnetRedistUrl>https://download.visualstudio.microsoft.com/download/pr/98ff0a08-a283-428f-8e54-19841d97154c/8c7d5f9600eadf264f04c82c813b7aab/dotnet-hosting-8.0.2-win.exe</TgsDotnetRedistUrl>
+    <TgsDotnetRedistUrl>https://download.visualstudio.microsoft.com/download/pr/00397fee-1bd9-44ef-899b-4504b26e6e96/ab9c73409659f3238d33faee304a8b7c/dotnet-hosting-8.0.4-win.exe</TgsDotnetRedistUrl>
     <TgsMariaDBRedistVersion>10.11.6</TgsMariaDBRedistVersion>
     <TgsYarnVersion>1.22.21</TgsYarnVersion>
   </PropertyGroup>

src/Tgstation.Server.Api/Models/Internal/DreamMakerSettings.cs

@@ -39,5 +39,12 @@ public abstract class DreamMakerSettings
 		/// </summary>
 		[Required]
 		public TimeSpan? Timeout { get; set; }
+
+		/// <summary>
+		/// Additional arguments added to the compiler command line.
+		/// </summary>
+		[StringLength(Limits.MaximumStringLength)]
+		[ResponseOptions]
+		public string? CompilerAdditionalArguments { get; set; }
 	}
 }

src/Tgstation.Server.Api/Rights/DreamMakerRights.cs

@@ -57,5 +57,10 @@ public enum DreamMakerRights : ulong
 		/// User may modify <see cref="Models.Internal.DreamMakerSettings.Timeout"/>.
 		/// </summary>
 		SetTimeout = 1 << 8,
+
+		/// <summary>
+		/// User may modify <see cref="Models.Internal.DreamMakerSettings.CompilerAdditionalArguments"/>.
+		/// </summary>
+		SetCompilerArguments = 1 << 9,
 	}
 }

src/Tgstation.Server.Api/Tgstation.Server.Api.csproj

@@ -27,7 +27,7 @@
     <!-- Usage: HTTP constants reference -->
     <PackageReference Include="Microsoft.AspNetCore.Http.Extensions" Version="2.2.0" />
     <!-- Usage: Decoding the 'nbf' property of JWTs -->
-    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="7.4.1" />
+    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="7.5.1" />
     <!-- Usage: Primary JSON library -->
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <!-- Usage: Data model annotating -->

src/Tgstation.Server.Client/Tgstation.Server.Client.csproj

@@ -11,9 +11,9 @@
 
   <ItemGroup>
     <!-- Usage: Connecting to SignalR hubs in API -->
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.3" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.4" />
     <!-- Usage: Using target JSON serializer for API -->
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="8.0.3" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="8.0.4" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Host/.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "dotnet-ef": {
-      "version": "8.0.3",
+      "version": "8.0.4",
       "commands": [
         "dotnet-ef"
       ]

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -571,6 +571,7 @@ async ValueTask RunCompileJob(
 				progressReporter.StageName = "Copying repository";
 				var resolvedOutputDirectory = ioManager.ResolvePath(outputDirectory);
 				var repoOrigin = repository.Origin;
+				var repoReference = repository.Reference;
 				using (repository)
 					await repository.CopyTo(resolvedOutputDirectory, cancellationToken);
 
@@ -585,6 +586,7 @@ await eventConsumer.HandleEvent(
 						resolvedOutputDirectory,
 						repoOrigin.ToString(),
 						engineLock.Version.ToString(),
+						repoReference,
 					},
 					true,
 					cancellationToken);
@@ -630,7 +632,7 @@ await eventConsumer.HandleEvent(
 
 				// run compiler
 				progressReporter.StageName = "Running Compiler";
-				var compileSuceeded = await RunDreamMaker(engineLock, job, cancellationToken);
+				var compileSuceeded = await RunDreamMaker(engineLock, job, dreamMakerSettings.CompilerAdditionalArguments, cancellationToken);
 
 				// Session takes ownership of the lock and Disposes it so save this for later
 				var engineVersion = engineLock.Version;
@@ -848,12 +850,17 @@ async ValueTask VerifyApi(
 		/// </summary>
 		/// <param name="engineLock">The <see cref="IEngineExecutableLock"/> to use.</param>
 		/// <param name="job">The <see cref="CompileJob"/> for the operation.</param>
+		/// <param name="additionalCompilerArguments">Additional arguments to be added to the compiler.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in <see langword="true"/> if compilation succeeded, <see langword="false"/> otherwise.</returns>
-		async ValueTask<bool> RunDreamMaker(IEngineExecutableLock engineLock, Models.CompileJob job, CancellationToken cancellationToken)
+		async ValueTask<bool> RunDreamMaker(
+			IEngineExecutableLock engineLock,
+			Models.CompileJob job,
+			string? additionalCompilerArguments,
+			CancellationToken cancellationToken)
 		{
 			var environment = await engineLock.LoadEnv(logger, true, cancellationToken);
-			var arguments = engineLock.FormatCompilerArguments($"{job.DmeName}.{DmeExtension}");
+			var arguments = engineLock.FormatCompilerArguments($"{job.DmeName}.{DmeExtension}", additionalCompilerArguments);
 
 			await using var dm = await processExecutor.LaunchProcess(
 				engineLock.CompilerExePath,

src/Tgstation.Server.Host/Components/Engine/ByondInstallation.cs

@@ -147,7 +147,14 @@ public override string FormatServerArguments(
 		}
 
 		/// <inheritdoc />
-		public override string FormatCompilerArguments(string dmePath)
-			=> $"-clean \"{dmePath ?? throw new ArgumentNullException(nameof(dmePath))}\"";
+		public override string FormatCompilerArguments(string dmePath, string? additionalArguments)
+		{
+			if (String.IsNullOrWhiteSpace(additionalArguments))
+				additionalArguments = String.Empty;
+			else
+				additionalArguments = $"{additionalArguments.Trim()} ";
+
+			return $"-clean {additionalArguments}\"{dmePath ?? throw new ArgumentNullException(nameof(dmePath))}\"";
+		}
 	}
 }

src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs

@@ -55,7 +55,7 @@ public string FormatServerArguments(
 				logFilePath);
 
 		/// <inheritdoc />
-		public string FormatCompilerArguments(string dmePath) => Instance.FormatCompilerArguments(dmePath);
+		public string FormatCompilerArguments(string dmePath, string? additionalArguments) => Instance.FormatCompilerArguments(dmePath, additionalArguments);
 
 		/// <inheritdoc />
 		public ValueTask StopServerProcess(ILogger logger, IProcess process, string accessIdentifier, ushort port, CancellationToken cancellationToken)

src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs

@@ -78,7 +78,7 @@ public EngineInstallationBase(IIOManager installationIOManager)
 		}
 
 		/// <inheritdoc />
-		public abstract string FormatCompilerArguments(string dmePath);
+		public abstract string FormatCompilerArguments(string dmePath, string? additionalArguments);
 
 		/// <inheritdoc />
 		public abstract string FormatServerArguments(

src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs

@@ -74,8 +74,9 @@ string FormatServerArguments(
 		/// Return the command line arguments for compiling a given <paramref name="dmePath"/> if compilation is necessary.
 		/// </summary>
 		/// <param name="dmePath">The full path to the .dme to compile.</param>
+		/// <param name="additionalArguments">Optional additional arguments provided to the compiler.</param>
 		/// <returns>The formatted arguments <see cref="string"/>.</returns>
-		string FormatCompilerArguments(string dmePath);
+		string FormatCompilerArguments(string dmePath, string? additionalArguments);
 
 		/// <summary>
 		/// Kills a given engine server <paramref name="process"/>.

src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs

@@ -112,8 +112,15 @@ public override string FormatServerArguments(
 		}
 
 		/// <inheritdoc />
-		public override string FormatCompilerArguments(string dmePath)
-			=> $"--suppress-unimplemented --notices-enabled \"{dmePath ?? throw new ArgumentNullException(nameof(dmePath))}\"";
+		public override string FormatCompilerArguments(string dmePath, string? additionalArguments)
+		{
+			if (String.IsNullOrWhiteSpace(additionalArguments))
+				additionalArguments = String.Empty;
+			else
+				additionalArguments = $"{additionalArguments.Trim()} ";
+
+			return $"--suppress-unimplemented --notices-enabled {additionalArguments}\"{dmePath ?? throw new ArgumentNullException(nameof(dmePath))}\"";
+		}
 
 		/// <inheritdoc />
 		public override async ValueTask StopServerProcess(

src/Tgstation.Server.Host/Components/Events/EventType.cs

@@ -55,7 +55,7 @@ public enum EventType
 		EngineActiveVersionChange,
 
 		/// <summary>
-		/// After the repo is copied, before CodeModifications are applied. Parameters: Game directory path, origin commit sha, engine version string.
+		/// After the repo is copied, before CodeModifications are applied. Parameters: Game directory path, origin commit sha, engine version string, repository reference (or "(no branch)" if there is no reference).
 		/// </summary>
 		[EventScript("PreCompile")]
 		CompileStart,

src/Tgstation.Server.Host/Components/Repository/Repository.cs

@@ -46,6 +46,11 @@ sealed class Repository : DisposeInvoker, IRepository
 		/// </summary>
 		public const string RemoteTemporaryBranchName = "___TGSTempBranch";
 
+		/// <summary>
+		/// The value of <see cref="Reference"/> when not on a reference.
+		/// </summary>
+		public const string NoReference = "(no branch)";
+
 		/// <summary>
 		/// Used when a reference cannot be determined.
 		/// </summary>

src/Tgstation.Server.Host/Controllers/ApiRootController.cs

@@ -357,7 +357,7 @@ public async ValueTask<IActionResult> CreateToken(CancellationToken cancellation
 					var identExpiry = token.ParseJwt().ValidTo;
 					identExpiry += tokenFactory.ValidationParameters.ClockSkew;
 					identExpiry += TimeSpan.FromSeconds(15);
-					identityCache.CacheSystemIdentity(user, systemIdentity!, identExpiry);
+					await identityCache.CacheSystemIdentity(user, systemIdentity!, identExpiry);
 				}
 
 				Logger.LogDebug("Successfully logged in user {userId}!", user.Id);

src/Tgstation.Server.Host/Controllers/DreamMakerController.cs

@@ -171,7 +171,8 @@ await jobManager.RegisterOperation(
 			| DreamMakerRights.SetApiValidationPort
 			| DreamMakerRights.SetSecurityLevel
 			| DreamMakerRights.SetApiValidationRequirement
-			| DreamMakerRights.SetTimeout)]
+			| DreamMakerRights.SetTimeout
+			| DreamMakerRights.SetCompilerArguments)]
 		[ProducesResponseType(typeof(DreamMakerResponse), 200)]
 		[ProducesResponseType(204)]
 		[ProducesResponseType(typeof(ErrorMessageResponse), 410)]
@@ -196,7 +197,8 @@ public async ValueTask<IActionResult> Update([FromBody] DreamMakerRequest model,
 			{
 				if (!dreamMakerRights.HasFlag(DreamMakerRights.SetDme))
 					return Forbid();
-				if (model.ProjectName.Length == 0)
+
+				if (model.ProjectName.Length == 0) // can't use isnullorwhitespace because linux memes
 					hostModel.ProjectName = null;
 				else
 					hostModel.ProjectName = model.ProjectName;
@@ -230,23 +232,38 @@ public async ValueTask<IActionResult> Update([FromBody] DreamMakerRequest model,
 			{
 				if (!dreamMakerRights.HasFlag(DreamMakerRights.SetSecurityLevel))
 					return Forbid();
+
 				hostModel.ApiValidationSecurityLevel = model.ApiValidationSecurityLevel;
 			}
 
 			if (model.RequireDMApiValidation.HasValue)
 			{
 				if (!dreamMakerRights.HasFlag(DreamMakerRights.SetApiValidationRequirement))
 					return Forbid();
+
 				hostModel.RequireDMApiValidation = model.RequireDMApiValidation;
 			}
 
 			if (model.Timeout.HasValue)
 			{
 				if (!dreamMakerRights.HasFlag(DreamMakerRights.SetTimeout))
 					return Forbid();
+
 				hostModel.Timeout = model.Timeout;
 			}
 
+			if (model.CompilerAdditionalArguments != null)
+			{
+				if (!dreamMakerRights.HasFlag(DreamMakerRights.SetCompilerArguments))
+					return Forbid();
+
+				var sanitizedArguments = model.CompilerAdditionalArguments.Trim();
+				if (sanitizedArguments.Length == 0)
+					hostModel.CompilerAdditionalArguments = null;
+				else
+					hostModel.CompilerAdditionalArguments = sanitizedArguments;
+			}
+
 			await DatabaseContext.Save(cancellationToken);
 
 			if (!dreamMakerRights.HasFlag(DreamMakerRights.Read))

src/Tgstation.Server.Host/Controllers/InstanceController.cs

@@ -740,6 +740,7 @@ public async ValueTask<IActionResult> GrantPermissions(long id, CancellationToke
 					ApiValidationSecurityLevel = DreamDaemonSecurity.Safe,
 					RequireDMApiValidation = true,
 					Timeout = TimeSpan.FromHours(1),
+					CompilerAdditionalArguments = null,
 				},
 				Name = initialSettings.Name,
 				Online = false,

src/Tgstation.Server.Host/Database/DatabaseContext.cs

@@ -375,22 +375,22 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
 		/// <summary>
 		/// Used by unit tests to remind us to setup the correct MSSQL migration downgrades.
 		/// </summary>
-		internal static readonly Type MSLatestMigration = typeof(MSAddMinidumpsOption);
+		internal static readonly Type MSLatestMigration = typeof(MSAddCompilerAdditionalArguments);
 
 		/// <summary>
 		/// Used by unit tests to remind us to setup the correct MYSQL migration downgrades.
 		/// </summary>
-		internal static readonly Type MYLatestMigration = typeof(MYAddMinidumpsOption);
+		internal static readonly Type MYLatestMigration = typeof(MYAddCompilerAdditionalArguments);
 
 		/// <summary>
 		/// Used by unit tests to remind us to setup the correct PostgresSQL migration downgrades.
 		/// </summary>
-		internal static readonly Type PGLatestMigration = typeof(PGAddMinidumpsOption);
+		internal static readonly Type PGLatestMigration = typeof(PGAddCompilerAdditionalArguments);
 
 		/// <summary>
 		/// Used by unit tests to remind us to setup the correct SQLite migration downgrades.
 		/// </summary>
-		internal static readonly Type SLLatestMigration = typeof(SLAddMinidumpsOption);
+		internal static readonly Type SLLatestMigration = typeof(SLAddCompilerAdditionalArguments);
 
 		/// <inheritdoc />
 #pragma warning disable CA1502 // Cyclomatic complexity
@@ -419,6 +419,16 @@ public async ValueTask SchemaDowngradeForServerVersion(
 
 			string BadDatabaseType() => throw new ArgumentException($"Invalid DatabaseType: {currentDatabaseType}", nameof(currentDatabaseType));
 
+			if (targetVersion < new Version(6, 5, 0))
+				targetMigration = currentDatabaseType switch
+				{
+					DatabaseType.MySql => nameof(MYAddMinidumpsOption),
+					DatabaseType.PostgresSql => nameof(PGAddMinidumpsOption),
+					DatabaseType.SqlServer => nameof(MSAddMinidumpsOption),
+					DatabaseType.Sqlite => nameof(SLAddMinidumpsOption),
+					_ => BadDatabaseType(),
+				};
+
 			if (targetVersion < new Version(6, 2, 0))
 				targetMigration = currentDatabaseType switch
 				{