From 176bd0655ee541bcde5a8e4c5a93edee1edcaa8a Mon Sep 17 00:00:00 2001 From: Einar Date: Wed, 8 Nov 2023 11:28:18 +0100 Subject: [PATCH] Better remote configuration (#320) Create a config file that can be used for remote config. This also makes a few changes to make that more possible. - Allow scopes to be configured either as a list of strings, or as space separated strings. - Allow an empty "certificate" block - Improve handling of empty values. --- Cognite.Common/ListOrSpaceSeparated.cs | 30 ++++++++++ Cognite.Config/Configuration.cs | 59 +++++++++++++++++- Cognite.Extensions/Authenticator.cs | 18 +++--- Cognite.Extensions/MsalAuthenticator.cs | 24 ++++---- ExtractorUtils.Test/unit/ConfigurationTest.cs | 60 +++++++++++++++++++ ExtractorUtils/Cognite/DestinationUtils.cs | 4 +- ExtractorUtils/config/remote_config.yml | 39 ++++++++++++ 7 files changed, 211 insertions(+), 23 deletions(-) create mode 100644 Cognite.Common/ListOrSpaceSeparated.cs create mode 100644 ExtractorUtils/config/remote_config.yml diff --git a/Cognite.Common/ListOrSpaceSeparated.cs b/Cognite.Common/ListOrSpaceSeparated.cs new file mode 100644 index 00000000..ca49d3f2 --- /dev/null +++ b/Cognite.Common/ListOrSpaceSeparated.cs @@ -0,0 +1,30 @@ +using System; + +namespace Cognite.Common +{ + /// + /// A wrapper for yaml types that are serialized either as lists or as space separated values. + /// + public class ListOrSpaceSeparated + { + /// + /// Inner values + /// + public string[] Values { get; } + + /// + /// Constructor + /// + /// List of values, will always be serialized as a list + public ListOrSpaceSeparated(params string[] values) + { + Values = values; + } + + /// + /// Explicit conversion to string array + /// + /// List of values + public static implicit operator string[](ListOrSpaceSeparated list) => list?.Values ?? throw new ArgumentNullException(nameof(list)); + } +} \ No newline at end of file diff --git a/Cognite.Config/Configuration.cs b/Cognite.Config/Configuration.cs index ae175353..50e39360 100644 --- a/Cognite.Config/Configuration.cs +++ b/Cognite.Config/Configuration.cs @@ -4,7 +4,9 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; using System.Text.RegularExpressions; +using Cognite.Common; using Cognite.Extractor.Common; using Microsoft.Extensions.DependencyInjection; using YamlDotNet.Core; @@ -24,12 +26,14 @@ public static class ConfigurationUtils { private static DeserializerBuilder builder = new DeserializerBuilder() .WithNamingConvention(HyphenatedNamingConvention.Instance) + .WithTypeConverter(new ListOrStringConverter()) .WithNodeDeserializer(new TemplatedValueDeserializer()); private static IDeserializer deserializer = builder .Build(); private static DeserializerBuilder ignoreUnmatchedBuilder = new DeserializerBuilder() .WithNamingConvention(HyphenatedNamingConvention.Instance) .WithNodeDeserializer(new TemplatedValueDeserializer()) + .WithTypeConverter(new ListOrStringConverter()) .IgnoreUnmatchedProperties(); private static bool ignoreUnmatchedProperties; @@ -286,6 +290,7 @@ public static string ConfigToString( toIgnore, namePrefixes, allowReadOnly)) + .WithTypeConverter(new ListOrStringConverter()) .WithNamingConvention(HyphenatedNamingConvention.Instance) .Build(); @@ -338,14 +343,19 @@ bool INodeDeserializer.Deserialize(IParser parser, Type expectedType, Func(out var scalar) && scalar != null && _envRegex.IsMatch(scalar.Value)) { parser.MoveNext(); - value = _envRegex.Replace(scalar.Value, LookupEnvironment); + value = Replace(scalar.Value); return true; } value = null; return false; } - static string LookupEnvironment(Match match) + public static string Replace(string toReplace) + { + return _envRegex.Replace(toReplace, LookupEnvironment); + } + + private static string LookupEnvironment(Match match) { return Environment.GetEnvironmentVariable(match.Groups[1].Value) ?? ""; } @@ -436,4 +446,49 @@ public override IEnumerable GetProperties(Type type, object return props; } } + + internal class ListOrStringConverter : IYamlTypeConverter + { + private static readonly Regex _whitespaceRegex = new Regex(@"\s", RegexOptions.Compiled | RegexOptions.CultureInvariant); + public bool Accepts(Type type) + { + return type == typeof(ListOrSpaceSeparated); + } + + public object? ReadYaml(IParser parser, Type type) + { + if (parser.TryConsume(out var scalar)) + { + var items = _whitespaceRegex.Split(scalar.Value); + return new ListOrSpaceSeparated(items.Select(TemplatedValueDeserializer.Replace).ToArray()); + } + if (parser.TryConsume(out _)) + { + var items = new List(); + while (!parser.Accept(out _)) + { + var seqScalar = parser.Consume(); + items.Add(seqScalar.Value); + } + + parser.Consume(); + + return new ListOrSpaceSeparated(items.Select(TemplatedValueDeserializer.Replace).ToArray()); + } + + throw new InvalidOperationException("Expected list or value"); + } + + public void WriteYaml(IEmitter emitter, object? value, Type type) + { + emitter.Emit(new SequenceStart(AnchorName.Empty, TagName.Empty, true, + SequenceStyle.Block, Mark.Empty, Mark.Empty)); + var it = value as ListOrSpaceSeparated; + foreach (var elem in it!.Values) + { + emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, elem, ScalarStyle.DoubleQuoted, false, true)); + } + emitter.Emit(new SequenceEnd()); + } + } } diff --git a/Cognite.Extensions/Authenticator.cs b/Cognite.Extensions/Authenticator.cs index daad3215..e87c30a1 100644 --- a/Cognite.Extensions/Authenticator.cs +++ b/Cognite.Extensions/Authenticator.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Cognite.Extractor.Common; +using Cognite.Common; namespace Cognite.Extensions { @@ -37,7 +38,7 @@ public class CertificateConfig public class AuthenticatorConfig { /// - /// Available authenticator implementations + /// DEPRECATED: Available authenticator implementations /// public enum AuthenticatorImplementation { @@ -52,7 +53,7 @@ public enum AuthenticatorImplementation } /// - /// Which implementation to use in the authenticator (optional) + /// DEPRECATED: Which implementation to use in the authenticator (optional) /// public AuthenticatorImplementation Implementation { get; set; } = AuthenticatorImplementation.MSAL; @@ -96,7 +97,7 @@ public enum AuthenticatorImplementation /// Resource scopes /// /// Scope - public IList? Scopes { get; set; } + public ListOrSpaceSeparated? Scopes { get; set; } /// /// Audience @@ -211,7 +212,7 @@ private async Task RequestToken(CancellationToken token = default) { "grant_type", "client_credentials" } }; - if (_config.Scopes != null && _config.Scopes.Count > 0) + if (_config.Scopes != null && _config.Scopes.Values.Length > 0) { form["scope"] = string.Join(" ", _config.Scopes); } @@ -244,13 +245,14 @@ private async Task RequestToken(CancellationToken token = default) } _logger.LogDebug( - "New OIDC token. Expires on {ttl}", + "New OIDC token. Expires on {ttl}", (DateTime.UtcNow + TimeSpan.FromSeconds(tokenResponse.ExpiresIn)).ToISOString()); return tokenResponse; } else { - try { + try + { var error = JsonSerializer.Deserialize(body); if (error == null) { @@ -261,9 +263,9 @@ private async Task RequestToken(CancellationToken token = default) } catch (JsonException ex) { - _logger.LogError("Unable to obtain OIDC token: R{Code} - {Message}", (int) response.StatusCode, response.ReasonPhrase); + _logger.LogError("Unable to obtain OIDC token: R{Code} - {Message}", (int)response.StatusCode, response.ReasonPhrase); throw new CogniteUtilsException( - $"Could not obtain OIDC token: {(int) response.StatusCode} - {response.ReasonPhrase}", + $"Could not obtain OIDC token: {(int)response.StatusCode} - {response.ReasonPhrase}", ex); } } diff --git a/Cognite.Extensions/MsalAuthenticator.cs b/Cognite.Extensions/MsalAuthenticator.cs index baa6149c..d8e60f99 100644 --- a/Cognite.Extensions/MsalAuthenticator.cs +++ b/Cognite.Extensions/MsalAuthenticator.cs @@ -36,13 +36,14 @@ public MsalAuthenticator(AuthenticatorConfig config, ILogger log { _config = config; _logger = logger ?? new NullLogger(); - if (_config != null) { + if (_config != null) + { Uri authorityUrl; - if (_config.Certificate?.AuthorityUrl != null) + if (!string.IsNullOrWhiteSpace(_config.Certificate?.AuthorityUrl)) { - authorityUrl = new Uri(_config.Certificate.AuthorityUrl); + authorityUrl = new Uri(_config.Certificate!.AuthorityUrl); } - else if (_config.Authority != null && _config.Tenant != null) + else if (!string.IsNullOrWhiteSpace(_config.Authority) && !string.IsNullOrWhiteSpace(_config.Tenant)) { var uriBuilder = new UriBuilder(_config.Authority); uriBuilder.Path = $"{_config.Tenant}"; @@ -57,9 +58,8 @@ public MsalAuthenticator(AuthenticatorConfig config, ILogger log .WithHttpClientFactory(new MsalClientFactory(httpClientFactory, authClientName)) .WithAuthority(authorityUrl); - if (_config.Certificate != null) + if (_config.Certificate?.Path != null) { - if (_config.Certificate.Path == null) throw new ConfigurationException("Certificate path is required for certificate authentication"); var ext = Path.GetExtension(_config.Certificate.Path); X509Certificate2 cert; @@ -122,7 +122,8 @@ public MsalAuthenticator(AuthenticatorConfig config, ILogger log /// Thrown when it was not possible to obtain an authentication token. public async Task GetToken(CancellationToken token = default) { - if (_config == null || _app == null) { + if (_config == null || _app == null) + { _logger.LogInformation("OIDC authentication disabled."); return null; } @@ -130,14 +131,15 @@ public MsalAuthenticator(AuthenticatorConfig config, ILogger log await _mutex.WaitAsync(token).ConfigureAwait(false); try { - var result = await _app.AcquireTokenForClient(_config.Scopes) + var result = await _app.AcquireTokenForClient(_config.Scopes?.Values) .ExecuteAsync(token).ConfigureAwait(false); - + // The client application will take care of caching the token and // renewal before expiration - if (result.ExpiresOn != _lastTokenTime) { + if (result.ExpiresOn != _lastTokenTime) + { _logger.LogDebug( - "New OIDC token. Expires on {ttl}", + "New OIDC token. Expires on {ttl}", result.ExpiresOn.UtcDateTime.ToISOString()); _lastTokenTime = result.ExpiresOn; } diff --git a/ExtractorUtils.Test/unit/ConfigurationTest.cs b/ExtractorUtils.Test/unit/ConfigurationTest.cs index a41ad721..054839a6 100644 --- a/ExtractorUtils.Test/unit/ConfigurationTest.cs +++ b/ExtractorUtils.Test/unit/ConfigurationTest.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using Cognite.Common; using Cognite.Extractor.Common; using Cognite.Extractor.Configuration; using Cognite.Extractor.Logging; @@ -307,7 +308,66 @@ public static void TestExtendedConfiguration() Assert.Equal("value", customConfig.SomeValue); File.Delete(path); + } + + class TestListOrSpaceSep + { + public ListOrSpaceSeparated Foo { get; set; } + public ListOrSpaceSeparated Bar { get; set; } + public ListOrSpaceSeparated Baz { get; set; } + } + + [Fact] + public static void TestListOrSpaceSeparated() + { + string input = +@"foo: some space separated strings +bar: + - some + - strings + - in + - list"; + var res = ConfigurationUtils.ReadString(input); + Assert.Equal(4, res.Foo.Values.Count()); + Assert.Equal(4, res.Bar.Values.Count()); + Assert.Equal("list", res.Bar.Values.ElementAt(3)); + Assert.Equal("strings", res.Foo.Values.ElementAt(3)); + string output = +@"foo: +- ""some"" +- ""space"" +- ""separated"" +- ""strings"" +bar: +- ""some"" +- ""strings"" +- ""in"" +- ""list"" +"; + var outRes = ConfigurationUtils.ConfigToString(res, Enumerable.Empty(), Enumerable.Empty(), Enumerable.Empty(), false); + Assert.Equal(output, outRes); + } + [Fact] + public static void TestListOrSpaceSeparatedWithEnv() + { + string input = +@"foo: ${ENV_1} ${ENV_2} +bar: +- ${ENV_1} +- ${ENV_2} +"; + Environment.SetEnvironmentVariable("ENV_1", "v1"); + Environment.SetEnvironmentVariable("ENV_2", "v2"); + + var res = ConfigurationUtils.ReadString(input); + Assert.Equal(2, res.Foo.Values.Count()); + Assert.Equal(2, res.Bar.Values.Count()); + Assert.Equal("v1", res.Bar.Values.ElementAt(0)); + Assert.Equal("v2", res.Foo.Values.ElementAt(1)); + + Environment.SetEnvironmentVariable("ENV_1", null); + Environment.SetEnvironmentVariable("ENV_2", null); } [Fact] diff --git a/ExtractorUtils/Cognite/DestinationUtils.cs b/ExtractorUtils/Cognite/DestinationUtils.cs index 9d711781..2fab5519 100644 --- a/ExtractorUtils/Cognite/DestinationUtils.cs +++ b/ExtractorUtils/Cognite/DestinationUtils.cs @@ -178,8 +178,8 @@ public static void AddCogniteClient(this IServiceCollection services, var logger = provider.GetRequiredService>(); var clientFactory = provider.GetRequiredService(); - if (!string.IsNullOrWhiteSpace(conf.IdpAuthentication.Tenant.TrimToNull()) - || conf.IdpAuthentication.Certificate != null) + if (!string.IsNullOrWhiteSpace(conf.IdpAuthentication.Tenant) + || !string.IsNullOrWhiteSpace(conf.IdpAuthentication.Certificate?.Path)) { return new MsalAuthenticator(conf.IdpAuthentication, logger, clientFactory, authClientName); } diff --git a/ExtractorUtils/config/remote_config.yml b/ExtractorUtils/config/remote_config.yml new file mode 100644 index 00000000..14cd3e20 --- /dev/null +++ b/ExtractorUtils/config/remote_config.yml @@ -0,0 +1,39 @@ +# Sample configuration file for remote config. This can be used as remote config directly. + +cognite: + # The project to connect to in the API + project: ${COGNITE_PROJECT} + # Cognite service url + host: ${COGNITE_BASE_URL} + # Config for authentication if a bearer access token has to be used for authentication. + # Leave empty to disable. + idp-authentication: + # URL to fetch tokens from, either this, or tenant must be present. + token-url: ${COGNITE_TOKEN_URL} + # Identity provider authority endpoint (optional, only used in combination with tenant) + authority: ${COGNITE_AUTHORITY_URL} + # Directory tenant + tenant: ${COGNITE_TENANT} + # Application Id + client-id: ${COGNITE_CLIENT_ID} + # Client secret + secret: ${COGNITE_CLIENT_SECRET} + # List of resource scopes, ex: + # scopes: + # - scopeA + # - scopeB + scopes: ${COGNITE_SCOPES} + # Audience + audience: ${COGNITE_AUDIENCE} + # Minimum time-to-live in seconds for the token (optional) + min-ttl: 30 + # Configuration for certificate based authentication + certificate: + # Path to a .pfx or .pem certificate file (.pem only for .NET 7 extractors) + path: ${COGNITE_CERTIFICATE_PATH} + # Optional certificate key password + password: ${COGNITE_CERTIFICATE_PASSWORD} + # Optional authority URL. If this is not specified, authority + tenant is used. + authority-url: ${COGNITE_CERTIFICATE_AUTHORITY_URL} + extraction-pipeline: + external-id: ${COGNITE_EXTRACTION_PIPELINE} \ No newline at end of file