diff --git a/Directory.Packages.props b/Directory.Packages.props index a346e27eca..7922a65d9a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,6 +4,7 @@ + diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index 1bae3a7d84..b0b29f483d 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -9,6 +9,7 @@ using System.Net; using System.Reflection; using System.Security.Cryptography.X509Certificates; +using Azure.Identity; using CommandLine; using Garnet.server; using Garnet.server.Auth.Aad; @@ -315,6 +316,13 @@ internal sealed class Options [Option("use-azure-storage", Required = false, HelpText = "Use Azure Page Blobs for storage instead of local storage.")] public bool? UseAzureStorage { get; set; } + [HttpsUrlValidation] + [Option("storage-service-uri", Required = false, HelpText = "The URI to use when establishing connection to Azure Blobs Storage.")] + public string AzureStorageServiceUri { get; set; } + + [Option("storage-managed-identity", Required = false, HelpText = "The managed identity to use when establishing connection to Azure Blobs Storage.")] + public string AzureStorageManagedIdentity { get; set; } + [Option("storage-string", Required = false, HelpText = "The connection string to use when establishing connection to Azure Blobs Storage.")] public string AzureStorageConnectionString { get; set; } @@ -511,8 +519,17 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) var enableStorageTier = EnableStorageTier.GetValueOrDefault(); var enableRevivification = EnableRevivification.GetValueOrDefault(); - if (useAzureStorage && string.IsNullOrEmpty(AzureStorageConnectionString)) - throw new Exception("Cannot enable use-azure-storage without supplying storage-string."); + if (useAzureStorage && ( + string.IsNullOrEmpty(AzureStorageConnectionString) + && (string.IsNullOrEmpty(AzureStorageServiceUri) || string.IsNullOrEmpty(AzureStorageManagedIdentity)))) + { + throw new InvalidAzureConfiguration("Cannot enable use-azure-storage without supplying storage-string or storage-service-uri & storage-managed-identity"); + } + if (useAzureStorage && !string.IsNullOrEmpty(AzureStorageConnectionString) + && (!string.IsNullOrEmpty(AzureStorageServiceUri) || !string.IsNullOrEmpty(AzureStorageManagedIdentity))) + { + throw new InvalidAzureConfiguration("Cannot enable use-azure-storage with both storage-string and storage-service-uri or storage-managed-identity"); + } var logDir = LogDir; if (!useAzureStorage && enableStorageTier) logDir = new DirectoryInfo(string.IsNullOrEmpty(logDir) ? "." : logDir).FullName; @@ -565,6 +582,10 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) CompactionForceDelete = true; } + Func azureFactoryCreator = string.IsNullOrEmpty(AzureStorageConnectionString) + ? () => new AzureStorageNamedDeviceFactory(AzureStorageServiceUri, new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = AzureStorageManagedIdentity }), logger) + : () => new AzureStorageNamedDeviceFactory(AzureStorageConnectionString, logger); + return new GarnetServerOptions(logger) { Port = Port, @@ -633,9 +654,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) QuietMode = QuietMode.GetValueOrDefault(), ThreadPoolMinThreads = ThreadPoolMinThreads, ThreadPoolMaxThreads = ThreadPoolMaxThreads, - DeviceFactoryCreator = useAzureStorage - ? () => new AzureStorageNamedDeviceFactory(AzureStorageConnectionString, logger) - : () => new LocalStorageNamedDeviceFactory(useNativeDeviceLinux: UseNativeDeviceLinux.GetValueOrDefault(), logger: logger), + DeviceFactoryCreator = useAzureStorage ? azureFactoryCreator : () => new LocalStorageNamedDeviceFactory(useNativeDeviceLinux: UseNativeDeviceLinux.GetValueOrDefault(), logger: logger), CheckpointThrottleFlushDelayMs = CheckpointThrottleFlushDelayMs, EnableScatterGatherGet = EnableScatterGatherGet.GetValueOrDefault(), ReplicaSyncDelayMs = ReplicaSyncDelayMs, @@ -694,4 +713,9 @@ internal enum ConfigFileType // Redis.conf file format RedisConf = 1, } + + public class InvalidAzureConfiguration : Exception + { + public InvalidAzureConfiguration(string message) : base(message) { } + } } \ No newline at end of file diff --git a/libs/host/Configuration/OptionsValidators.cs b/libs/host/Configuration/OptionsValidators.cs index ea7f64682e..5c1974eb4d 100644 --- a/libs/host/Configuration/OptionsValidators.cs +++ b/libs/host/Configuration/OptionsValidators.cs @@ -491,8 +491,8 @@ internal LogDirValidationAttribute(bool mustExist, bool isRequired) : base(mustE } /// - /// Validation logic for Log Directory, valid if UseAzureStorage is specified or if EnableStorageTier is not specified in parent Options object - /// If neither applies, reverts to OptionValidationAttribute validation + /// Validation logic for Log Directory, valid if is specified or if is not specified in parent Options object + /// If neither applies, reverts to validation /// /// Value of Log Directory /// Validation context @@ -518,13 +518,12 @@ internal CheckpointDirValidationAttribute(bool mustExist, bool isRequired) : bas } /// - /// Validation logic for Checkpoint Directory, valid if UseAzureStorage is specified in parent Options object - /// If not, reverts to OptionValidationAttribute validation + /// Validation logic for , valid if is specified in parent Options object + /// If not, reverts to validation /// /// Value of Log Directory /// Validation context /// Validation result - /// protected override ValidationResult IsValid(object value, ValidationContext validationContext) { var options = (Options)validationContext.ObjectInstance; @@ -536,7 +535,7 @@ protected override ValidationResult IsValid(object value, ValidationContext vali } /// - /// Validation logic for CertFileName + /// Validation logic for /// [AttributeUsage(AttributeTargets.Property)] internal sealed class CertFileValidationAttribute : FilePathValidationAttribute @@ -553,7 +552,6 @@ internal CertFileValidationAttribute(bool fileMustExist, bool directoryMustExist /// Value of CertFileName /// Validation context /// Validation result - /// protected override ValidationResult IsValid(object value, ValidationContext validationContext) { var options = (Options)validationContext.ObjectInstance; @@ -563,4 +561,34 @@ protected override ValidationResult IsValid(object value, ValidationContext vali return base.IsValid(value, validationContext); } } + + /// + /// Represents an attribute used for validating HTTPS URLs as options. + /// + [AttributeUsage(AttributeTargets.Property)] + internal sealed class HttpsUrlValidationAttribute : OptionValidationAttribute + { + internal HttpsUrlValidationAttribute(bool isRequired = false) : base(isRequired) + { + } + + /// + /// HTTPS URLs validation logic, checks if string is a valid HTTPS URL. + /// + /// URL string + /// Validation Logic + /// Validation result + protected override ValidationResult IsValid(object value, ValidationContext validationContext) + { + if (TryInitialValidation(value, validationContext, out var initValidationResult, out var url)) + return initValidationResult; + + if (Uri.TryCreate(url, UriKind.Absolute, out var uri) && uri.Scheme == Uri.UriSchemeHttps) + return ValidationResult.Success; + + var baseError = validationContext.MemberName != null ? base.FormatErrorMessage(validationContext.MemberName) : string.Empty; + var errorMessage = $"{baseError} Expected string in URI format. Actual value: {url}"; + return new ValidationResult(errorMessage, [validationContext.MemberName]); + } + } } \ No newline at end of file diff --git a/libs/host/Garnet.host.csproj b/libs/host/Garnet.host.csproj index 2a82a39535..0ac864c493 100644 --- a/libs/host/Garnet.host.csproj +++ b/libs/host/Garnet.host.csproj @@ -19,6 +19,7 @@ + diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf index 27e700c966..3ee1c9ff58 100644 --- a/libs/host/defaults.conf +++ b/libs/host/defaults.conf @@ -236,6 +236,12 @@ /* The connection string to use when establishing connection to Azure Blobs Storage. */ "AzureStorageConnectionString" : null, + /* The URI to use when establishing connection to Azure Blobs Storage. */ + "AzureStorageServiceUri": null, + + /* The managed identity to use when establishing connection to Azure Blobs Storage. */ + "AzureStorageManagedIdentity": null, + /* Whether and by how much should we throttle the disk IO for checkpoints: -1 - disable throttling; >= 0 - run checkpoint flush in separate task, sleep for specified time after each WriteAsync */ "CheckpointThrottleFlushDelayMs" : 0, diff --git a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureStorageNamedDeviceFactory.cs b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureStorageNamedDeviceFactory.cs index 2bf672f308..a134471c05 100644 --- a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureStorageNamedDeviceFactory.cs +++ b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureStorageNamedDeviceFactory.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Azure.Core; using Microsoft.Extensions.Logging; using Tsavorite.core; @@ -29,6 +30,17 @@ public AzureStorageNamedDeviceFactory(string connectionString, ILogger logger = { } + /// + /// Create instance of factory for Azure devices + /// + /// + /// + /// + public AzureStorageNamedDeviceFactory(string serviceUri, TokenCredential credential, ILogger logger = null) + : this(BlobUtilsV12.GetServiceClients(serviceUri, credential), logger) + { + } + /// /// Create instance of factory for Azure devices /// diff --git a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobUtilsV12.cs b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobUtilsV12.cs index eb04700bc5..a9acd769f9 100644 --- a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobUtilsV12.cs +++ b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobUtilsV12.cs @@ -37,6 +37,31 @@ public struct ServiceClients } internal static ServiceClients GetServiceClients(string connectionString) + { + var (aggressiveOptions, defaultOptions, withRetriesOptions) = GetBlobClientOptions(); + + return new ServiceClients() + { + Default = new BlobServiceClient(connectionString, defaultOptions), + Aggressive = new BlobServiceClient(connectionString, aggressiveOptions), + WithRetries = new BlobServiceClient(connectionString, withRetriesOptions), + }; + } + + internal static ServiceClients GetServiceClients(string serviceUrl, TokenCredential credential) + { + var (aggressiveOptions, defaultOptions, withRetriesOptions) = GetBlobClientOptions(); + var serviceUri = new Uri(serviceUrl); + + return new ServiceClients() + { + Default = new BlobServiceClient(serviceUri, credential, defaultOptions), + Aggressive = new BlobServiceClient(serviceUri, credential, aggressiveOptions), + WithRetries = new BlobServiceClient(serviceUri, credential, withRetriesOptions), + }; + } + + private static (BlobClientOptions aggressiveOptions, BlobClientOptions defaultOptions, BlobClientOptions withRetriesOptions) GetBlobClientOptions() { var aggressiveOptions = new BlobClientOptions(); aggressiveOptions.Retry.MaxRetries = 0; @@ -54,12 +79,7 @@ internal static ServiceClients GetServiceClients(string connectionString) withRetriesOptions.Retry.Delay = TimeSpan.FromSeconds(1); withRetriesOptions.Retry.MaxDelay = TimeSpan.FromSeconds(30); - return new ServiceClients() - { - Default = new BlobServiceClient(connectionString, defaultOptions), - Aggressive = new BlobServiceClient(connectionString, aggressiveOptions), - WithRetries = new BlobServiceClient(connectionString, withRetriesOptions), - }; + return (aggressiveOptions, defaultOptions, withRetriesOptions); } public struct ContainerClients diff --git a/test/Garnet.test/GarnetServerConfigTests.cs b/test/Garnet.test/GarnetServerConfigTests.cs index 318a751624..ea21345632 100644 --- a/test/Garnet.test/GarnetServerConfigTests.cs +++ b/test/Garnet.test/GarnetServerConfigTests.cs @@ -191,41 +191,96 @@ public void ImportExportRedisConfigLocal() [Test] public void ImportExportConfigAzure() { + if (!TestUtils.IsRunningAzureTests) + { + Assert.Ignore("Azure tests are disabled."); + } + var AzureTestDirectory = $"{TestContext.CurrentContext.Test.MethodName.ToLowerInvariant()}"; var configPath = $"{AzureTestDirectory}/test1.config"; var AzureEmulatedStorageString = "UseDevelopmentStorage=true;"; - if (TestUtils.IsRunningAzureTests) - { - // Delete blob if exists - var deviceFactory = new AzureStorageNamedDeviceFactory(AzureEmulatedStorageString, default); - deviceFactory.Initialize(AzureTestDirectory); - deviceFactory.Delete(new FileDescriptor { directoryName = "" }); - - var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(null, out var options, out var invalidOptions); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.IsTrue(options.PageSize == "32m"); - ClassicAssert.IsTrue(options.MemorySize == "16g"); - - var args = new string[] { "--storage-string", AzureEmulatedStorageString, "--use-azure-storage-for-config-export", "true", "--config-export-path", configPath, "-p", "4m", "-m", "128m" }; - parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.IsTrue(options.PageSize == "4m"); - ClassicAssert.IsTrue(options.MemorySize == "128m"); - - args = ["--storage-string", AzureEmulatedStorageString, "--use-azure-storage-for-config-import", "true", "--config-import-path", configPath]; - parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.IsTrue(options.PageSize == "4m"); - ClassicAssert.IsTrue(options.MemorySize == "128m"); - - // Delete blob - deviceFactory.Initialize(AzureTestDirectory); - deviceFactory.Delete(new FileDescriptor { directoryName = "" }); - } + // Delete blob if exists + var deviceFactory = new AzureStorageNamedDeviceFactory(AzureEmulatedStorageString, default); + deviceFactory.Initialize(AzureTestDirectory); + deviceFactory.Delete(new FileDescriptor { directoryName = "" }); + + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(null, out var options, out var invalidOptions); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsEmpty(invalidOptions); + ClassicAssert.IsTrue(options.PageSize == "32m"); + ClassicAssert.IsTrue(options.MemorySize == "16g"); + + var args = new string[] { "--storage-string", AzureEmulatedStorageString, "--use-azure-storage-for-config-export", "true", "--config-export-path", configPath, "-p", "4m", "-m", "128m" }; + parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsEmpty(invalidOptions); + ClassicAssert.IsTrue(options.PageSize == "4m"); + ClassicAssert.IsTrue(options.MemorySize == "128m"); + + args = ["--storage-string", AzureEmulatedStorageString, "--use-azure-storage-for-config-import", "true", "--config-import-path", configPath]; + parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsEmpty(invalidOptions); + ClassicAssert.IsTrue(options.PageSize == "4m"); + ClassicAssert.IsTrue(options.MemorySize == "128m"); + + args = ["--use-azure-storage", "--storage-string", AzureEmulatedStorageString]; + parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsEmpty(invalidOptions); + ClassicAssert.AreEqual(AzureEmulatedStorageString, options.AzureStorageConnectionString); + + // Delete blob + deviceFactory.Initialize(AzureTestDirectory); + deviceFactory.Delete(new FileDescriptor { directoryName = "" }); + } + + [Test] + public void AzureStorageConfiguration() + { + // missing both storage-string and managed-identity + var args = new string[] { "--use-azure-storage" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsEmpty(invalidOptions); + Assert.Throws(() => options.GetServerOptions()); + + // valid storage-string + args = ["--use-azure-storage", "--storage-string", "UseDevelopmentStorage=true;"]; + parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsEmpty(invalidOptions); + Assert.DoesNotThrow(() => options.GetServerOptions()); + + // insecure service-uri + args = ["--use-azure-storage", "--storage-service-uri", "http://demo.blob.core.windows.net"]; + parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions); + ClassicAssert.IsFalse(parseSuccessful); + ClassicAssert.AreEqual(invalidOptions.Count, 1); + ClassicAssert.AreEqual(invalidOptions[0], nameof(Options.AzureStorageServiceUri)); + + // secure service-uri but missing managed-identity + args = ["--use-azure-storage", "--storage-service-uri", "https://demo.blob.core.windows.net"]; + parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsEmpty(invalidOptions); + Assert.Throws(() => options.GetServerOptions()); + + // secure service-uri with managed-identity + args = ["--use-azure-storage", "--storage-service-uri", "https://demo.blob.core.windows.net", "--storage-managed-identity", "demo"]; + parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsEmpty(invalidOptions); + var serverOptions = options.GetServerOptions(); + Assert.DoesNotThrow(() => options.GetServerOptions()); + + // both storage-string and managed-identity + args = ["--use-azure-storage", "--storage-string", "UseDevelopmentStorage", "--storage-managed-identity", "demo", "--storage-service-uri", "https://demo.blob.core.windows.net"]; + parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsEmpty(invalidOptions); + Assert.Throws(() => options.GetServerOptions()); } } } \ No newline at end of file diff --git a/website/docs/getting-started/configuration.md b/website/docs/getting-started/configuration.md index 04c7836491..c09b65f4bd 100644 --- a/website/docs/getting-started/configuration.md +++ b/website/docs/getting-started/configuration.md @@ -138,6 +138,8 @@ For all available command line settings, run `GarnetServer.exe -h` or `GarnetSer | **ThreadPoolMaxThreads** | ```--maxthreads``` | ```int``` | Integer in range:
[0, MaxValue] | Maximum worker and completion threads in thread pool, 0 uses the system default. | | **UseAzureStorage** | ```--use-azure-storage``` | ```bool``` | | Use Azure Page Blobs for storage instead of local storage. | | **AzureStorageConnectionString** | ```--storage-string``` | ```string``` | | The connection string to use when establishing connection to Azure Blobs Storage. | +| **AzureStorageServiceUri** | ```--storage-service-uri``` | ```string``` | | The service URI to use when establishing connection to Azure Blobs Storage. | +| **AzureStorageManagedIdentity** | ```--storage-managed-identity``` | ```string``` | | The managed identity to use when establishing connection to Azure Blobs Storage. | | **CheckpointThrottleFlushDelayMs** | ```--checkpoint-throttle-delay``` | ```int``` | Integer in range:
[-1, MaxValue] | Whether and by how much should we throttle the disk IO for checkpoints: -1 - disable throttling; >= 0 - run checkpoint flush in separate task, sleep for specified time after each WriteAsync | | **EnableFastCommit** | ```--fast-commit``` | ```bool``` | | Use FastCommit when writing AOF. | | **FastCommitThrottleFreq** | ```--fast-commit-throttle``` | ```int``` | Integer in range:
[0, MaxValue] | Throttle FastCommit to write metadata once every K commits. |