Better remote configuration (#320)

Create a config file that can be used for remote config. This also makes a few changes to make that more possible.

 - Allow scopes to be configured either as a list of strings, or as space separated strings.
 - Allow an empty "certificate" block
 - Improve handling of empty values.

Cognite.Common/ListOrSpaceSeparated.cs

@@ -0,0 +1,30 @@
+using System;
+
+namespace Cognite.Common
+{
+    /// <summary>
+    /// A wrapper for yaml types that are serialized either as lists or as space separated values.
+    /// </summary>
+    public class ListOrSpaceSeparated
+    {
+        /// <summary>
+        /// Inner values
+        /// </summary>
+        public string[] Values { get; }
+
+        /// <summary>
+        /// Constructor
+        /// </summary>
+        /// <param name="values">List of values, will always be serialized as a list</param>
+        public ListOrSpaceSeparated(params string[] values)
+        {
+            Values = values;
+        }
+
+        /// <summary>
+        /// Explicit conversion to string array
+        /// </summary>
+        /// <param name="list">List of values</param>
+        public static implicit operator string[](ListOrSpaceSeparated list) => list?.Values ?? throw new ArgumentNullException(nameof(list));
+    }
+}

Cognite.Config/Configuration.cs

@@ -4,7 +4,9 @@
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Runtime.CompilerServices;
 using System.Text.RegularExpressions;
+using Cognite.Common;
 using Cognite.Extractor.Common;
 using Microsoft.Extensions.DependencyInjection;
 using YamlDotNet.Core;
@@ -24,12 +26,14 @@ public static class ConfigurationUtils
     {
         private static DeserializerBuilder builder = new DeserializerBuilder()
             .WithNamingConvention(HyphenatedNamingConvention.Instance)
+            .WithTypeConverter(new ListOrStringConverter())
             .WithNodeDeserializer(new TemplatedValueDeserializer());
         private static IDeserializer deserializer = builder
             .Build();
         private static DeserializerBuilder ignoreUnmatchedBuilder = new DeserializerBuilder()
             .WithNamingConvention(HyphenatedNamingConvention.Instance)
             .WithNodeDeserializer(new TemplatedValueDeserializer())
+            .WithTypeConverter(new ListOrStringConverter())
             .IgnoreUnmatchedProperties();
 
         private static bool ignoreUnmatchedProperties;
@@ -286,6 +290,7 @@ public static string ConfigToString<T>(
                     toIgnore,
                     namePrefixes,
                     allowReadOnly))
+                .WithTypeConverter(new ListOrStringConverter())
                 .WithNamingConvention(HyphenatedNamingConvention.Instance)
                 .Build();
 
@@ -338,14 +343,19 @@ bool INodeDeserializer.Deserialize(IParser parser, Type expectedType, Func<IPars
             if (parser.Accept<Scalar>(out var scalar) && scalar != null && _envRegex.IsMatch(scalar.Value))
             {
                 parser.MoveNext();
-                value = _envRegex.Replace(scalar.Value, LookupEnvironment);
+                value = Replace(scalar.Value);
                 return true;
             }
             value = null;
             return false;
         }
 
-        static string LookupEnvironment(Match match)
+        public static string Replace(string toReplace)
+        {
+            return _envRegex.Replace(toReplace, LookupEnvironment);
+        }
+
+        private static string LookupEnvironment(Match match)
         {
             return Environment.GetEnvironmentVariable(match.Groups[1].Value) ?? "";
         }
@@ -436,4 +446,49 @@ public override IEnumerable<IPropertyDescriptor> GetProperties(Type type, object
             return props;
         }
     }
+
+    internal class ListOrStringConverter : IYamlTypeConverter
+    {
+        private static readonly Regex _whitespaceRegex = new Regex(@"\s", RegexOptions.Compiled | RegexOptions.CultureInvariant);
+        public bool Accepts(Type type)
+        {
+            return type == typeof(ListOrSpaceSeparated);
+        }
+
+        public object? ReadYaml(IParser parser, Type type)
+        {
+            if (parser.TryConsume<Scalar>(out var scalar))
+            {
+                var items = _whitespaceRegex.Split(scalar.Value);
+                return new ListOrSpaceSeparated(items.Select(TemplatedValueDeserializer.Replace).ToArray());
+            }
+            if (parser.TryConsume<SequenceStart>(out _))
+            {
+                var items = new List<string>();
+                while (!parser.Accept<SequenceEnd>(out _))
+                {
+                    var seqScalar = parser.Consume<Scalar>();
+                    items.Add(seqScalar.Value);
+                }
+
+                parser.Consume<SequenceEnd>();
+
+                return new ListOrSpaceSeparated(items.Select(TemplatedValueDeserializer.Replace).ToArray());
+            }
+
+            throw new InvalidOperationException("Expected list or value");
+        }
+
+        public void WriteYaml(IEmitter emitter, object? value, Type type)
+        {
+            emitter.Emit(new SequenceStart(AnchorName.Empty, TagName.Empty, true,
+                SequenceStyle.Block, Mark.Empty, Mark.Empty));
+            var it = value as ListOrSpaceSeparated;
+            foreach (var elem in it!.Values)
+            {
+                emitter.Emit(new Scalar(AnchorName.Empty, TagName.Empty, elem, ScalarStyle.DoubleQuoted, false, true));
+            }
+            emitter.Emit(new SequenceEnd());
+        }
+    }
 }

Cognite.Extensions/Authenticator.cs

@@ -7,6 +7,7 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Cognite.Extractor.Common;
+using Cognite.Common;
 
 namespace Cognite.Extensions
 {
@@ -37,7 +38,7 @@ public class CertificateConfig
     public class AuthenticatorConfig
     {
         /// <summary>
-        /// Available authenticator implementations 
+        /// DEPRECATED: Available authenticator implementations 
         /// </summary>
         public enum AuthenticatorImplementation
         {
@@ -52,7 +53,7 @@ public enum AuthenticatorImplementation
         }
 
         /// <summary>
-        /// Which implementation to use in the authenticator (optional)
+        /// DEPRECATED: Which implementation to use in the authenticator (optional)
         /// </summary>
         public AuthenticatorImplementation Implementation { get; set; } = AuthenticatorImplementation.MSAL;
 
@@ -96,7 +97,7 @@ public enum AuthenticatorImplementation
         /// Resource scopes
         /// </summary>
         /// <value>Scope</value>
-        public IList<string>? Scopes { get; set; }
+        public ListOrSpaceSeparated? Scopes { get; set; }
 
         /// <summary>
         /// Audience
@@ -211,7 +212,7 @@ private async Task<ResponseDTO> RequestToken(CancellationToken token = default)
                 { "grant_type", "client_credentials" }
             };
 
-            if (_config.Scopes != null && _config.Scopes.Count > 0)
+            if (_config.Scopes != null && _config.Scopes.Values.Length > 0)
             {
                 form["scope"] = string.Join(" ", _config.Scopes);
             }
@@ -244,13 +245,14 @@ private async Task<ResponseDTO> RequestToken(CancellationToken token = default)
                     }
 
                     _logger.LogDebug(
-                        "New OIDC token. Expires on {ttl}", 
+                        "New OIDC token. Expires on {ttl}",
                         (DateTime.UtcNow + TimeSpan.FromSeconds(tokenResponse.ExpiresIn)).ToISOString());
                     return tokenResponse;
                 }
                 else
                 {
-                    try {
+                    try
+                    {
                         var error = JsonSerializer.Deserialize<ErrorDTO>(body);
                         if (error == null)
                         {
@@ -261,9 +263,9 @@ private async Task<ResponseDTO> RequestToken(CancellationToken token = default)
                     }
                     catch (JsonException ex)
                     {
-                        _logger.LogError("Unable to obtain OIDC token: R{Code} - {Message}", (int) response.StatusCode, response.ReasonPhrase);
+                        _logger.LogError("Unable to obtain OIDC token: R{Code} - {Message}", (int)response.StatusCode, response.ReasonPhrase);
                         throw new CogniteUtilsException(
-                            $"Could not obtain OIDC token: {(int) response.StatusCode} - {response.ReasonPhrase}",
+                            $"Could not obtain OIDC token: {(int)response.StatusCode} - {response.ReasonPhrase}",
                             ex);
                     }
                 }

Cognite.Extensions/MsalAuthenticator.cs

@@ -36,13 +36,14 @@ public MsalAuthenticator(AuthenticatorConfig config, ILogger<IAuthenticator> log
         {
             _config = config;
             _logger = logger ?? new NullLogger<MsalAuthenticator>();
-            if (_config != null) {
+            if (_config != null)
+            {
                 Uri authorityUrl;
-                if (_config.Certificate?.AuthorityUrl != null)
+                if (!string.IsNullOrWhiteSpace(_config.Certificate?.AuthorityUrl))
                 {
-                    authorityUrl = new Uri(_config.Certificate.AuthorityUrl);
+                    authorityUrl = new Uri(_config.Certificate!.AuthorityUrl);
                 }
-                else if (_config.Authority != null && _config.Tenant != null)
+                else if (!string.IsNullOrWhiteSpace(_config.Authority) && !string.IsNullOrWhiteSpace(_config.Tenant))
                 {
                     var uriBuilder = new UriBuilder(_config.Authority);
                     uriBuilder.Path = $"{_config.Tenant}";
@@ -57,9 +58,8 @@ public MsalAuthenticator(AuthenticatorConfig config, ILogger<IAuthenticator> log
                     .WithHttpClientFactory(new MsalClientFactory(httpClientFactory, authClientName))
                     .WithAuthority(authorityUrl);
 
-                if (_config.Certificate != null)
+                if (_config.Certificate?.Path != null)
                 {
-                    if (_config.Certificate.Path == null) throw new ConfigurationException("Certificate path is required for certificate authentication");
                     var ext = Path.GetExtension(_config.Certificate.Path);
 
                     X509Certificate2 cert;
@@ -122,22 +122,24 @@ public MsalAuthenticator(AuthenticatorConfig config, ILogger<IAuthenticator> log
         /// <exception cref="CogniteUtilsException">Thrown when it was not possible to obtain an authentication token.</exception>
         public async Task<string?> GetToken(CancellationToken token = default)
         {
-            if (_config == null || _app == null) {
+            if (_config == null || _app == null)
+            {
                 _logger.LogInformation("OIDC authentication disabled.");
                 return null;
             }
 
             await _mutex.WaitAsync(token).ConfigureAwait(false);
             try
             {
-                var result = await _app.AcquireTokenForClient(_config.Scopes)
+                var result = await _app.AcquireTokenForClient(_config.Scopes?.Values)
                     .ExecuteAsync(token).ConfigureAwait(false);
-                
+
                 // The client application will take care of caching the token and 
                 // renewal before expiration
-                if (result.ExpiresOn != _lastTokenTime) {
+                if (result.ExpiresOn != _lastTokenTime)
+                {
                     _logger.LogDebug(
-                        "New OIDC token. Expires on {ttl}", 
+                        "New OIDC token. Expires on {ttl}",
                         result.ExpiresOn.UtcDateTime.ToISOString());
                     _lastTokenTime = result.ExpiresOn;
                 }

ExtractorUtils.Test/unit/ConfigurationTest.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Cognite.Common;
 using Cognite.Extractor.Common;
 using Cognite.Extractor.Configuration;
 using Cognite.Extractor.Logging;
@@ -307,7 +308,66 @@ public static void TestExtendedConfiguration()
             Assert.Equal("value", customConfig.SomeValue);
 
             File.Delete(path);
+        }
+
+        class TestListOrSpaceSep
+        {
+            public ListOrSpaceSeparated Foo { get; set; }
+            public ListOrSpaceSeparated Bar { get; set; }
+            public ListOrSpaceSeparated Baz { get; set; }
+        }
+
+        [Fact]
+        public static void TestListOrSpaceSeparated()
+        {
+            string input =
+@"foo: some space separated strings
+bar:
+    - some
+    - strings
+    - in
+    - list";
+            var res = ConfigurationUtils.ReadString<TestListOrSpaceSep>(input);
+            Assert.Equal(4, res.Foo.Values.Count());
+            Assert.Equal(4, res.Bar.Values.Count());
+            Assert.Equal("list", res.Bar.Values.ElementAt(3));
+            Assert.Equal("strings", res.Foo.Values.ElementAt(3));
+            string output =
+@"foo:
+- ""some""
+- ""space""
+- ""separated""
+- ""strings""
+bar:
+- ""some""
+- ""strings""
+- ""in""
+- ""list""
+";
+            var outRes = ConfigurationUtils.ConfigToString(res, Enumerable.Empty<string>(), Enumerable.Empty<string>(), Enumerable.Empty<string>(), false);
+            Assert.Equal(output, outRes);
+        }
 
+        [Fact]
+        public static void TestListOrSpaceSeparatedWithEnv()
+        {
+            string input =
+@"foo: ${ENV_1} ${ENV_2}
+bar:
+- ${ENV_1}
+- ${ENV_2}
+";
+            Environment.SetEnvironmentVariable("ENV_1", "v1");
+            Environment.SetEnvironmentVariable("ENV_2", "v2");
+
+            var res = ConfigurationUtils.ReadString<TestListOrSpaceSep>(input);
+            Assert.Equal(2, res.Foo.Values.Count());
+            Assert.Equal(2, res.Bar.Values.Count());
+            Assert.Equal("v1", res.Bar.Values.ElementAt(0));
+            Assert.Equal("v2", res.Foo.Values.ElementAt(1));
+
+            Environment.SetEnvironmentVariable("ENV_1", null);
+            Environment.SetEnvironmentVariable("ENV_2", null);
         }
 
         [Fact]

ExtractorUtils/Cognite/DestinationUtils.cs

@@ -178,8 +178,8 @@ public static void AddCogniteClient(this IServiceCollection services,
                 var logger = provider.GetRequiredService<ILogger<IAuthenticator>>();
                 var clientFactory = provider.GetRequiredService<IHttpClientFactory>();
 
-                if (!string.IsNullOrWhiteSpace(conf.IdpAuthentication.Tenant.TrimToNull())
-                    || conf.IdpAuthentication.Certificate != null)
+                if (!string.IsNullOrWhiteSpace(conf.IdpAuthentication.Tenant)
+                    || !string.IsNullOrWhiteSpace(conf.IdpAuthentication.Certificate?.Path))
                 {
                     return new MsalAuthenticator(conf.IdpAuthentication, logger, clientFactory, authClientName);
                 }

ExtractorUtils/config/remote_config.yml

@@ -0,0 +1,39 @@
+# Sample configuration file for remote config. This can be used as remote config directly.
+
+cognite:
+    # The project to connect to in the API
+    project: ${COGNITE_PROJECT}
+    # Cognite service url
+    host: ${COGNITE_BASE_URL}
+    # Config for authentication if a bearer access token has to be used for authentication.
+    # Leave empty to disable.
+    idp-authentication:
+        # URL to fetch tokens from, either this, or tenant must be present.
+        token-url: ${COGNITE_TOKEN_URL}
+        # Identity provider authority endpoint (optional, only used in combination with tenant)
+        authority: ${COGNITE_AUTHORITY_URL}
+        # Directory tenant
+        tenant: ${COGNITE_TENANT}
+        # Application Id
+        client-id: ${COGNITE_CLIENT_ID}
+        # Client secret
+        secret: ${COGNITE_CLIENT_SECRET}
+        # List of resource scopes, ex:
+        # scopes:
+        #   - scopeA
+        #   - scopeB
+        scopes: ${COGNITE_SCOPES}
+        # Audience
+        audience: ${COGNITE_AUDIENCE}
+        # Minimum time-to-live in seconds for the token (optional)
+        min-ttl: 30
+        # Configuration for certificate based authentication
+        certificate:
+            # Path to a .pfx or .pem certificate file (.pem only for .NET 7 extractors)
+            path: ${COGNITE_CERTIFICATE_PATH}
+            # Optional certificate key password
+            password: ${COGNITE_CERTIFICATE_PASSWORD}
+            # Optional authority URL. If this is not specified, authority + tenant is used.
+            authority-url: ${COGNITE_CERTIFICATE_AUTHORITY_URL}
+    extraction-pipeline:
+        external-id: ${COGNITE_EXTRACTION_PIPELINE}