Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fallback behavior to ensure in-proc payload compatibility with dotnet-isolated as the FUNCTIONS_WORKER_RUNTIME value #10439

Merged
merged 12 commits into from
Sep 3, 2024
Merged
1 change: 1 addition & 0 deletions release_notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,6 @@
- Worker termination path updated with sanitized logging (#10367)
- Avoid redundant DiagnosticEvents error message (#10395)
- Added logic to shim older versions of the .NET Worker JsonFunctionProvider to ensure backwards compatibility (#10410)
- Added fallback behavior to ensure in-proc payload compatibility with "dotnet-isolated" as the `FUNCTIONS_WORKER_RUNTIME` value (#10439)
- Migrated Scale Metrics to use `Azure.Data.Tables` SDK (#10276)
- Added support for Identity-based connections
8 changes: 8 additions & 0 deletions src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,14 @@ internal bool ThrowOnMissingFunctionsWorkerRuntime
}
}

internal bool ThrowOnFunctionsWorkerRuntimeMismatchWithMetadataFromPayload
{
get
{
return GetFeatureAsBooleanOrDefault(RpcWorkerConstants.ThrowOnFunctionsWorkerRuntimeMismatchWithMetadataFromPayload, false);
}
}

/// <summary>
/// Gets feature by name.
/// </summary>
Expand Down
3 changes: 3 additions & 0 deletions src/WebJobs.Script/Diagnostics/DiagnosticEventConstants.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,8 @@ internal static class DiagnosticEventConstants

public const string NonHISSecretLoaded = "AZFD0012";
public const string NonHISSecretLoadedHelpLink = "https://aka.ms/functions-non-his-secrets";

public const string WorkerRuntimeDoesNotMatchWithFunctionMetadataErrorCode = "AZFD0013";
public const string WorkerRuntimeDoesNotMatchWithFunctionMetadataHelpLink = "https://aka.ms/functions-invalid-worker-runtime";
kshyju marked this conversation as resolved.
Show resolved Hide resolved
}
}
48 changes: 47 additions & 1 deletion src/WebJobs.Script/Host/ScriptHost.cs
Original file line number Diff line number Diff line change
Expand Up @@ -773,14 +773,49 @@ private void TrySetDirectType(FunctionMetadata metadata)
}
}

// Ensure customer deployed application payload matches with the worker runtime configured for the function app and log a warning if not.
// If a customer has "dotnet-isolated" worker runtime configured for the function app, and then they deploy an in-proc app payload, this will warn/error
// If there is a mismatch, the method will return false, else true.
private static bool ValidateAndLogRuntimeMismatch(IEnumerable<FunctionMetadata> functionMetadata, string workerRuntime, IOptions<FunctionsHostingConfigOptions> hostingConfigOptions, ILogger logger)
{
kshyju marked this conversation as resolved.
Show resolved Hide resolved
if (functionMetadata != null && functionMetadata.Any() && !Utility.ContainsAnyFunctionMatchingWorkerRuntime(functionMetadata, workerRuntime))
{
string baseMessage = $"The '{EnvironmentSettingNames.FunctionWorkerRuntime}' is set to '{workerRuntime}', which does not match the worker runtime metadata found in the deployed function app artifacts. See {DiagnosticEventConstants.WorkerRuntimeDoesNotMatchWithFunctionMetadataHelpLink} for more information.";

if (hostingConfigOptions.Value.ThrowOnFunctionsWorkerRuntimeMismatchWithMetadataFromPayload)
{
logger.LogDiagnosticEventError(DiagnosticEventConstants.WorkerRuntimeDoesNotMatchWithFunctionMetadataErrorCode, baseMessage, DiagnosticEventConstants.WorkerRuntimeDoesNotMatchWithFunctionMetadataHelpLink, null);
mattchenderson marked this conversation as resolved.
Show resolved Hide resolved
throw new HostInitializationException(baseMessage);
}

string warningMessage = baseMessage + " The application will continue to run, but may throw an exception in the future.";
logger.LogDiagnosticEventWarning(DiagnosticEventConstants.WorkerRuntimeDoesNotMatchWithFunctionMetadataErrorCode, warningMessage, DiagnosticEventConstants.WorkerRuntimeDoesNotMatchWithFunctionMetadataHelpLink, null);
return false;
}

return true;
}

internal async Task<Collection<FunctionDescriptor>> GetFunctionDescriptorsAsync(IEnumerable<FunctionMetadata> functions, IEnumerable<FunctionDescriptorProvider> descriptorProviders, string workerRuntime, CancellationToken cancellationToken)
{
Collection<FunctionDescriptor> functionDescriptors = new Collection<FunctionDescriptor>();
if (!cancellationToken.IsCancellationRequested)
{
bool throwOnWorkerRuntimeAndPayloadMetadataMismatch = true;
kshyju marked this conversation as resolved.
Show resolved Hide resolved
// this dotnet isolated specific logic is temporary to ensure in-proc payload compatibility with "dotnet-isolated" as the FUNCTIONS_WORKER_RUNTIME value.
if (string.Equals(workerRuntime, RpcWorkerConstants.DotNetIsolatedLanguageWorkerName, StringComparison.OrdinalIgnoreCase))
kshyju marked this conversation as resolved.
Show resolved Hide resolved
{
bool payloadMatchesWorkerRuntime = ValidateAndLogRuntimeMismatch(functions, workerRuntime, _hostingConfigOptions, _logger);
if (!payloadMatchesWorkerRuntime)
{
UpdateFunctionMetadataLanguageForDotnetAssembly(functions, workerRuntime);
throwOnWorkerRuntimeAndPayloadMetadataMismatch = false; // we do not want to throw an exception in this case
}
}

var httpFunctions = new Dictionary<string, HttpTriggerAttribute>();

Utility.VerifyFunctionsMatchSpecifiedLanguage(functions, workerRuntime, _environment.IsPlaceholderModeEnabled(), _isHttpWorker, cancellationToken);
Utility.VerifyFunctionsMatchSpecifiedLanguage(functions, workerRuntime, _environment.IsPlaceholderModeEnabled(), _isHttpWorker, cancellationToken, throwOnMismatch: throwOnWorkerRuntimeAndPayloadMetadataMismatch);

foreach (FunctionMetadata metadata in functions)
{
Expand Down Expand Up @@ -819,6 +854,17 @@ internal async Task<Collection<FunctionDescriptor>> GetFunctionDescriptorsAsync(
return functionDescriptors;
}

private static void UpdateFunctionMetadataLanguageForDotnetAssembly(IEnumerable<FunctionMetadata> functions, string workerRuntime)
{
foreach (var function in functions)
{
if (function.Language == DotNetScriptTypes.DotNetAssembly)
{
function.Language = workerRuntime;
satvu marked this conversation as resolved.
Show resolved Hide resolved
}
}
}

internal static void ValidateFunction(FunctionDescriptor function, Dictionary<string, HttpTriggerAttribute> httpFunctions, IEnvironment environment)
{
var httpTrigger = function.HttpTriggerAttribute;
Expand Down
17 changes: 15 additions & 2 deletions src/WebJobs.Script/Utility.cs
Original file line number Diff line number Diff line change
Expand Up @@ -629,7 +629,7 @@ internal static bool TryReadFunctionConfig(string scriptDir, out string json, IF
return true;
}

internal static void VerifyFunctionsMatchSpecifiedLanguage(IEnumerable<FunctionMetadata> functions, string workerRuntime, bool isPlaceholderMode, bool isHttpWorker, CancellationToken cancellationToken)
internal static void VerifyFunctionsMatchSpecifiedLanguage(IEnumerable<FunctionMetadata> functions, string workerRuntime, bool isPlaceholderMode, bool isHttpWorker, CancellationToken cancellationToken, bool throwOnMismatch = true)
{
cancellationToken.ThrowIfCancellationRequested();

Expand All @@ -646,7 +646,10 @@ internal static void VerifyFunctionsMatchSpecifiedLanguage(IEnumerable<FunctionM
}
else
kshyju marked this conversation as resolved.
Show resolved Hide resolved
{
throw new HostInitializationException($"Did not find functions with language [{workerRuntime}].");
if (throwOnMismatch)
{
throw new HostInitializationException($"Did not find functions with language [{workerRuntime}].");
}
}
}
}
Expand Down Expand Up @@ -748,10 +751,20 @@ private static bool ContainsFunctionWithWorkerRuntime(IEnumerable<FunctionMetada
{
return functions.Any(f => dotNetLanguages.Any(l => l.Equals(f.Language, StringComparison.OrdinalIgnoreCase)));
}

return ContainsAnyFunctionMatchingWorkerRuntime(functions, workerRuntime);
}

/// <summary>
/// Inspect the functions metadata to determine if at least one function is of the specified worker runtime.
/// </summary>
internal static bool ContainsAnyFunctionMatchingWorkerRuntime(IEnumerable<FunctionMetadata> functions, string workerRuntime)
{
if (functions != null && functions.Any())
{
return functions.Any(f => !string.IsNullOrEmpty(f.Language) && f.Language.Equals(workerRuntime, StringComparison.OrdinalIgnoreCase));
}

return false;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ internal class RpcFunctionInvocationDispatcher : IFunctionInvocationDispatcher
private readonly Lazy<Task<int>> _maxProcessCount;
private readonly IOptions<FunctionsHostingConfigOptions> _hostingConfigOptions;
private readonly IHostMetrics _hostMetrics;
private readonly TimeSpan _defaultProcessStartupInterval = TimeSpan.FromSeconds(5);
private readonly TimeSpan _defaultProcessRestartInterval = TimeSpan.FromSeconds(5);
private readonly TimeSpan _defaultProcessShutdownInterval = TimeSpan.FromSeconds(5);

private IScriptEventManager _eventManager;
private IWebHostRpcWorkerChannelManager _webHostLanguageWorkerChannelManager;
Expand Down Expand Up @@ -308,9 +311,9 @@ public async Task InitializeAsync(IEnumerable<FunctionMetadata> functions, Cance
}
else
{
_processStartupInterval = workerConfig.CountOptions.ProcessStartupInterval;
_restartWait = workerConfig.CountOptions.ProcessRestartInterval;
_shutdownTimeout = workerConfig.CountOptions.ProcessShutdownTimeout;
_processStartupInterval = workerConfig?.CountOptions?.ProcessStartupInterval ?? _defaultProcessStartupInterval;
kshyju marked this conversation as resolved.
Show resolved Hide resolved
_restartWait = workerConfig?.CountOptions.ProcessRestartInterval ?? _defaultProcessRestartInterval;
_shutdownTimeout = workerConfig?.CountOptions.ProcessShutdownTimeout ?? _defaultProcessShutdownInterval;
}
ErrorEventsThreshold = 3 * await _maxProcessCount.Value;

Expand Down
1 change: 1 addition & 0 deletions src/WebJobs.Script/Workers/Rpc/RpcWorkerConstants.cs
Original file line number Diff line number Diff line change
Expand Up @@ -93,5 +93,6 @@ public static class RpcWorkerConstants
public const string RevertWorkerShutdownBehavior = "REVERT_WORKER_SHUTDOWN_BEHAVIOR";
public const string ShutdownWebhostWorkerChannelsOnHostShutdown = "ShutdownWebhostWorkerChannelsOnHostShutdown";
public const string ThrowOnMissingFunctionsWorkerRuntime = "THROW_ON_MISSING_FUNCTIONS_WORKER_RUNTIME";
public const string ThrowOnFunctionsWorkerRuntimeMismatchWithMetadataFromPayload = "THROW_ON_FUNCTIONS_WORKER_RUNTIME_MISMATCH_WITH_PAYLOAD";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Net;
using System.Threading.Tasks;
using Microsoft.Azure.WebJobs.Logging;
using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
using Microsoft.Extensions.DependencyInjection;
using Xunit;

Expand Down Expand Up @@ -45,6 +46,27 @@ public async Task ExternalStartup_Succeeds()
}
}

[Fact]
public async Task InProcAppsWorkWithDotnetIsolatedAsFunctionWorkerRuntimeValue()
kshyju marked this conversation as resolved.
Show resolved Hide resolved
{
// test uses an in-proc app, but we are setting "dotnet-isolated" as functions worker runtime value.
var fixture = new CSharpPrecompiledEndToEndTestFixture(_projectName, _envVars, functionWorkerRuntime: RpcWorkerConstants.DotNetIsolatedLanguageWorkerName);
try
{
await fixture.InitializeAsync();
var client = fixture.Host.HttpClient;

var response = await client.GetAsync($"api/Function1");

// The function does all the validation internally.
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
finally
{
await fixture.DisposeAsync();
}
}

[Fact]
public async Task ExternalStartup_InvalidOverwrite_StopsHost()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,8 @@ public void Property_Validation()

(nameof(FunctionsHostingConfigOptions.ThrowOnMissingFunctionsWorkerRuntime), "THROW_ON_MISSING_FUNCTIONS_WORKER_RUNTIME=1", true),
(nameof(FunctionsHostingConfigOptions.WorkerIndexingDisabledApps), "WORKER_INDEXING_DISABLED_APPS=teststring", "teststring"),
(nameof(FunctionsHostingConfigOptions.WorkerIndexingEnabled), "WORKER_INDEXING_ENABLED=1", true)
(nameof(FunctionsHostingConfigOptions.WorkerIndexingEnabled), "WORKER_INDEXING_ENABLED=1", true),
(nameof(FunctionsHostingConfigOptions.ThrowOnFunctionsWorkerRuntimeMismatchWithMetadataFromPayload), "THROW_ON_FUNCTIONS_WORKER_RUNTIME_MISMATCH_WITH_PAYLOAD=1", true)
};

// use reflection to ensure that we have a test that uses every value exposed on FunctionsHostingConfigOptions
Expand Down