Skip to content

Commit

Permalink
Add token credential auth support (#17308)
Browse files Browse the repository at this point in the history
  • Loading branch information
pakrym authored Dec 3, 2020
1 parent 3812cd6 commit 967af1a
Show file tree
Hide file tree
Showing 20 changed files with 576 additions and 395 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Azure.WebJobs.Ext
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Azure.Messaging.EventHubs", "..\Azure.Messaging.EventHubs\src\Azure.Messaging.EventHubs.csproj", "{B51ECD35-11DA-46D2-89D7-9DE3888CF896}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Extensions.Azure", "..\..\extensions\Microsoft.Extensions.Azure\src\Microsoft.Extensions.Azure.csproj", "{E772769A-7CE1-4FBC-A084-C0936EB2C766}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -72,5 +74,17 @@ Global
{B51ECD35-11DA-46D2-89D7-9DE3888CF896}.Release|x64.Build.0 = Release|Any CPU
{B51ECD35-11DA-46D2-89D7-9DE3888CF896}.Release|x86.ActiveCfg = Release|Any CPU
{B51ECD35-11DA-46D2-89D7-9DE3888CF896}.Release|x86.Build.0 = Release|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Debug|x64.ActiveCfg = Debug|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Debug|x64.Build.0 = Debug|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Debug|x86.ActiveCfg = Debug|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Debug|x86.Build.0 = Debug|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Release|Any CPU.Build.0 = Release|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Release|x64.ActiveCfg = Release|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Release|x64.Build.0 = Release|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Release|x86.ActiveCfg = Release|Any CPU
{E772769A-7CE1-4FBC-A084-C0936EB2C766}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,18 @@ namespace Microsoft.Azure.WebJobs
public sealed partial class EventHubAttribute : System.Attribute
{
public EventHubAttribute(string eventHubName) { }
[Microsoft.Azure.WebJobs.Description.ConnectionStringAttribute]
public string Connection { get { throw null; } set { } }
[Microsoft.Azure.WebJobs.Description.AutoResolveAttribute]
public string Connection { get { throw null; } set { } }
public string EventHubName { get { throw null; } }
}
[Microsoft.Azure.WebJobs.Description.BindingAttribute]
[System.AttributeUsageAttribute(System.AttributeTargets.Parameter)]
public sealed partial class EventHubTriggerAttribute : System.Attribute
{
public EventHubTriggerAttribute(string eventHubName) { }
[Microsoft.Azure.WebJobs.Description.AutoResolveAttribute]
public string Connection { get { throw null; } set { } }
[Microsoft.Azure.WebJobs.Description.AutoResolveAttribute]
public string ConsumerGroup { get { throw null; } set { } }
public string EventHubName { get { throw null; } }
}
Expand All @@ -30,8 +31,6 @@ public EventHubOptions() { }
public bool InvokeProcessorAfterReceiveTimeout { get { throw null; } set { } }
public string LeaseContainerName { get { throw null; } set { } }
public int MaxBatchSize { get { throw null; } set { } }
public void AddEventHubProducerClient(Azure.Messaging.EventHubs.Producer.EventHubProducerClient client) { }
public void AddEventHubProducerClient(string eventHubName, Azure.Messaging.EventHubs.Producer.EventHubProducerClient client) { }
public void AddReceiver(string eventHubName, string receiverConnectionString) { }
public void AddReceiver(string eventHubName, string receiverConnectionString, string storageConnectionString) { }
public void AddSender(string eventHubName, string sendConnectionString) { }
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;
using System.Collections.Concurrent;
using Azure.Core;
using Azure.Messaging.EventHubs.Consumer;
using Azure.Messaging.EventHubs.Producer;
using Azure.Storage.Blobs;
using Microsoft.Azure.WebJobs.EventHubs.Processor;
using Microsoft.Azure.WebJobs.Extensions.Clients.Shared;
using Microsoft.Azure.WebJobs.Host;
using Microsoft.Extensions.Azure;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Options;

namespace Microsoft.Azure.WebJobs.EventHubs
{
internal class EventHubClientFactory
{
private readonly IConfiguration _configuration;
private readonly AzureComponentFactory _componentFactory;
private readonly EventHubOptions _options;
private readonly INameResolver _nameResolver;
private readonly ConcurrentDictionary<string, EventHubProducerClient> _producerCache;
private readonly ConcurrentDictionary<string, IEventHubConsumerClient> _consumerCache = new ();

public EventHubClientFactory(
IConfiguration configuration,
AzureComponentFactory componentFactory,
IOptions<EventHubOptions> options,
INameResolver nameResolver)
{
_configuration = configuration;
_componentFactory = componentFactory;
_options = options.Value;
_nameResolver = nameResolver;
_producerCache = new ConcurrentDictionary<string, EventHubProducerClient>(_options.RegisteredProducers);
}

internal EventHubProducerClient GetEventHubProducerClient(string eventHubName, string connection)
{
eventHubName = _nameResolver.ResolveWholeString(eventHubName);

return _producerCache.GetOrAdd(eventHubName, key =>
{
if (!string.IsNullOrWhiteSpace(connection))
{
var info = ResolveConnectionInformation(connection);

if (info.FullyQualifiedEndpoint != null &&
info.TokenCredential != null)
{
return new EventHubProducerClient(info.FullyQualifiedEndpoint, eventHubName, info.TokenCredential);
}

return new EventHubProducerClient(NormalizeConnectionString(info.ConnectionString, eventHubName));
}

throw new InvalidOperationException("No event hub sender named " + eventHubName);
});
}

internal EventProcessorHost GetEventProcessorHost(string eventHubName, string connection, string consumerGroup)
{
eventHubName = _nameResolver.ResolveWholeString(eventHubName);
consumerGroup ??= EventHubConsumerClient.DefaultConsumerGroupName;

if (_options.RegisteredConsumerCredentials.TryGetValue(eventHubName, out var creds))
{
return new EventProcessorHost(consumerGroup: consumerGroup,
connectionString: creds.EventHubConnectionString,
eventHubName: eventHubName,
options: _options.EventProcessorOptions,
eventBatchMaximumCount: _options.MaxBatchSize,
invokeProcessorAfterReceiveTimeout: _options.InvokeProcessorAfterReceiveTimeout,
exceptionHandler: _options.ExceptionHandler);
}
else if (!string.IsNullOrEmpty(connection))
{
var info = ResolveConnectionInformation(connection);

if (info.FullyQualifiedEndpoint != null &&
info.TokenCredential != null)
{
return new EventProcessorHost(consumerGroup: consumerGroup,
fullyQualifiedNamespace: info.FullyQualifiedEndpoint,
eventHubName: eventHubName,
credential: info.TokenCredential,
options: _options.EventProcessorOptions,
eventBatchMaximumCount: _options.MaxBatchSize,
invokeProcessorAfterReceiveTimeout: _options.InvokeProcessorAfterReceiveTimeout,
exceptionHandler: _options.ExceptionHandler);
}

return new EventProcessorHost(consumerGroup: consumerGroup,
connectionString: NormalizeConnectionString(info.ConnectionString, eventHubName),
eventHubName: eventHubName,
options: _options.EventProcessorOptions,
eventBatchMaximumCount: _options.MaxBatchSize,
invokeProcessorAfterReceiveTimeout: _options.InvokeProcessorAfterReceiveTimeout,
exceptionHandler: _options.ExceptionHandler);
}

throw new InvalidOperationException("No event hub receiver named " + eventHubName);
}

internal IEventHubConsumerClient GetEventHubConsumerClient(string eventHubName, string connection, string consumerGroup)
{
eventHubName = _nameResolver.ResolveWholeString(eventHubName);
consumerGroup ??= EventHubConsumerClient.DefaultConsumerGroupName;

return _consumerCache.GetOrAdd(eventHubName, name =>
{
EventHubConsumerClient client = null;
if (_options.RegisteredConsumerCredentials.TryGetValue(eventHubName, out var creds))
{
client = new EventHubConsumerClient(consumerGroup, creds.EventHubConnectionString, eventHubName);
}
else if (!string.IsNullOrEmpty(connection))
{
var info = ResolveConnectionInformation(connection);

if (info.FullyQualifiedEndpoint != null &&
info.TokenCredential != null)
{
client = new EventHubConsumerClient(consumerGroup, info.FullyQualifiedEndpoint, eventHubName, info.TokenCredential);
}
else
{
client = new EventHubConsumerClient(consumerGroup, NormalizeConnectionString(info.ConnectionString, eventHubName));
}
}

if (client != null)
{
return new EventHubConsumerClientImpl(client);
}

throw new InvalidOperationException("No event hub receiver named " + eventHubName);
});
}

internal BlobContainerClient GetCheckpointStoreClient(string eventHubName)
{
string storageConnectionString = null;
if (_options.RegisteredConsumerCredentials.TryGetValue(eventHubName, out var creds))
{
storageConnectionString = creds.StorageConnectionString;
}

// Fall back to default if not explicitly registered
return new BlobContainerClient(storageConnectionString ?? _configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage), _options.LeaseContainerName);
}

internal static string NormalizeConnectionString(string originalConnectionString, string eventHubName)
{
var connectionString = ConnectionString.Parse(originalConnectionString);

if (!connectionString.ContainsSegmentKey("EntityPath"))
{
connectionString.Add("EntityPath", eventHubName);
}

return connectionString.ToString();
}

private EventHubsConnectionInformation ResolveConnectionInformation(string connection)
{
IConfigurationSection connectionSection = _configuration.GetWebJobsConnectionStringSection(connection);
if (!connectionSection.Exists())
{
// Not found
throw new InvalidOperationException($"EventHub account connection string '{connection}' does not exist." +
$"Make sure that it is a defined App Setting.");
}

if (!string.IsNullOrWhiteSpace(connectionSection.Value))
{
return new EventHubsConnectionInformation(connectionSection.Value);
}

var fullyQualifiedNamespace = connectionSection["fullyQualifiedNamespace"];
if (string.IsNullOrWhiteSpace(fullyQualifiedNamespace))
{
// Not found
throw new InvalidOperationException($"Connection should have an 'fullyQualifiedNamespace' property or be a string representing a connection string.");
}

var credential = _componentFactory.CreateTokenCredential(connectionSection);

return new EventHubsConnectionInformation(fullyQualifiedNamespace, credential);
}

private record EventHubsConnectionInformation
{
public EventHubsConnectionInformation(string connectionString)
{
ConnectionString = connectionString;
}

public EventHubsConnectionInformation(string fullyQualifiedEndpoint, TokenCredential tokenCredential)
{
FullyQualifiedEndpoint = fullyQualifiedEndpoint;
TokenCredential = tokenCredential;
}

public string ConnectionString { get; }
public string FullyQualifiedEndpoint { get; }
public TokenCredential TokenCredential { get; }
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,22 +22,24 @@ namespace Microsoft.Azure.WebJobs.EventHubs
[Extension("EventHubs", configurationSection: "EventHubs")]
internal class EventHubExtensionConfigProvider : IExtensionConfigProvider
{
private IConfiguration _config;
private readonly IOptions<EventHubOptions> _options;
private readonly ILoggerFactory _loggerFactory;
private readonly IConverterManager _converterManager;
private readonly INameResolver _nameResolver;
private readonly IWebJobsExtensionConfiguration<EventHubExtensionConfigProvider> _configuration;

public EventHubExtensionConfigProvider(IConfiguration config, IOptions<EventHubOptions> options, ILoggerFactory loggerFactory,
IConverterManager converterManager, INameResolver nameResolver, IWebJobsExtensionConfiguration<EventHubExtensionConfigProvider> configuration)
private readonly EventHubClientFactory _clientFactory;

public EventHubExtensionConfigProvider(
IOptions<EventHubOptions> options,
ILoggerFactory loggerFactory,
IConverterManager converterManager,
IWebJobsExtensionConfiguration<EventHubExtensionConfigProvider> configuration,
EventHubClientFactory clientFactory)
{
_config = config;
_options = options;
_loggerFactory = loggerFactory;
_converterManager = converterManager;
_nameResolver = nameResolver;
_configuration = configuration;
_clientFactory = clientFactory;
}

internal Action<ExceptionReceivedEventArgs> ExceptionHandler { get; set; }
Expand All @@ -54,7 +56,7 @@ public void Initialize(ExtensionConfigContext context)
throw new ArgumentNullException(nameof(context));
}

_options.Value.SetExceptionHandler(ExceptionReceivedHandler);
_options.Value.ExceptionHandler = ExceptionReceivedHandler;
_configuration.ConfigurationSection.Bind(_options);

context
Expand All @@ -65,7 +67,7 @@ public void Initialize(ExtensionConfigContext context)
.AddOpenConverter<OpenType.Poco, EventData>(ConvertPocoToEventData);

// register our trigger binding provider
var triggerBindingProvider = new EventHubTriggerAttributeBindingProvider(_config, _nameResolver, _converterManager, _options, _loggerFactory);
var triggerBindingProvider = new EventHubTriggerAttributeBindingProvider(_converterManager, _options, _loggerFactory, _clientFactory);
context.AddBindingRule<EventHubTriggerAttribute>()
.BindToTrigger(triggerBindingProvider);

Expand All @@ -74,10 +76,7 @@ public void Initialize(ExtensionConfigContext context)
.BindToCollector(BuildFromAttribute);

context.AddBindingRule<EventHubAttribute>()
.BindToInput(attribute =>
{
return _options.Value.GetEventHubProducerClient(attribute.EventHubName, attribute.Connection);
});
.BindToInput(attribute => _clientFactory.GetEventHubProducerClient(attribute.EventHubName, attribute.Connection));

ExceptionHandler = (e =>
{
Expand All @@ -93,26 +92,9 @@ internal static void LogExceptionReceivedEvent(ExceptionReceivedEventArgs e, ILo
Utility.LogException(e.Exception, message, logger);
}

private static LogLevel GetLogLevel(Exception ex)
{
var ehex = ex as EventHubsException;
if (!(ex is OperationCanceledException) && (ehex == null || !ehex.IsTransient))
{
// any non-transient exceptions or unknown exception types
// we want to log as errors
return LogLevel.Error;
}
else
{
// transient messaging errors we log as info so we have a record
// of them, but we don't treat them as actual errors
return LogLevel.Information;
}
}

private IAsyncCollector<EventData> BuildFromAttribute(EventHubAttribute attribute)
{
EventHubProducerClient client = _options.Value.GetEventHubProducerClient(attribute.EventHubName, attribute.Connection);
EventHubProducerClient client = _clientFactory.GetEventHubProducerClient(attribute.EventHubName, attribute.Connection);
return new EventHubAsyncCollector(new EventHubProducerClientImpl(client, _loggerFactory));
}

Expand Down
Loading

0 comments on commit 967af1a

Please sign in to comment.