diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index 0910af15a2391..3743abb92ce1e 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -19,7 +19,11 @@ bin/elasticsearch-keystore add azure.client.default.account bin/elasticsearch-keystore add azure.client.default.key ---------------------------------------------------------------- -Where `account` is the azure account name and `key` the azure secret key. +Where `account` is the azure account name and `key` the azure secret key. Instead of an azure secret key under `key`, you can alternatively +define a shared access signatures (SAS) token under `sas_token` to use for authentication instead. When using an SAS token instead of an +account key, the SAS token must have read (r), write (w), list (l), and delete (d) permissions for the repository base path and +all its contents. These permissions need to be granted for the blob service (b) and apply to resource types service (s), container (c), and +object (o). These settings are used by the repository's internal azure client. Note that you can also define more than one account: @@ -29,14 +33,14 @@ Note that you can also define more than one account: bin/elasticsearch-keystore add azure.client.default.account bin/elasticsearch-keystore add azure.client.default.key bin/elasticsearch-keystore add azure.client.secondary.account -bin/elasticsearch-keystore add azure.client.secondary.key +bin/elasticsearch-keystore add azure.client.secondary.sas_token ---------------------------------------------------------------- `default` is the default account name which will be used by a repository, unless you set an explicit one in the <>. -Both `account` and `key` storage settings are +The `account`, `key`, and `sas_token` storage settings are {ref}/secure-settings.html#reloadable-secure-settings[reloadable]. After you reload the settings, the internal azure clients, which are used to transfer the snapshot, will utilize the latest settings from the keystore. diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 2669e4bf6092a..47ee57d3a3e78 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -76,6 +76,7 @@ String azureAccount = System.getenv("azure_storage_account") String azureKey = System.getenv("azure_storage_key") String azureContainer = System.getenv("azure_storage_container") String azureBasePath = System.getenv("azure_storage_base_path") +String azureSasToken = System.getenv("azure_storage_sas_token") test { exclude '**/AzureStorageCleanupThirdPartyTests.class' @@ -85,10 +86,11 @@ task thirdPartyTest(type: Test) { include '**/AzureStorageCleanupThirdPartyTests.class' systemProperty 'test.azure.account', azureAccount ? azureAccount : "" systemProperty 'test.azure.key', azureKey ? azureKey : "" + systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : "" systemProperty 'test.azure.container', azureContainer ? azureContainer : "" systemProperty 'test.azure.base', azureBasePath ? azureBasePath : "" } -if (azureAccount || azureKey || azureContainer || azureBasePath) { +if (azureAccount || azureKey || azureContainer || azureBasePath || azureSasToken) { check.dependsOn(thirdPartyTest) } diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle index 8d8aaebac9f3f..decef58aff24d 100644 --- a/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle +++ b/plugins/repository-azure/qa/microsoft-azure-storage/build.gradle @@ -29,12 +29,14 @@ String azureAccount = System.getenv("azure_storage_account") String azureKey = System.getenv("azure_storage_key") String azureContainer = System.getenv("azure_storage_container") String azureBasePath = System.getenv("azure_storage_base_path") +String azureSasToken = System.getenv("azure_storage_sas_token") -if (!azureAccount && !azureKey && !azureContainer && !azureBasePath) { +if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) { azureAccount = 'azure_integration_test_account' azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64 azureContainer = 'container_test' azureBasePath = 'integration_test' + azureSasToken = '' useFixture = true } @@ -63,7 +65,14 @@ integTest { testClusters.integTest { plugin file(project(':plugins:repository-azure').bundlePlugin.archiveFile) keystore 'azure.client.integration_test.account', azureAccount - keystore 'azure.client.integration_test.key', azureKey + if (azureKey != null && azureKey.isEmpty() == false) { + println "Using access key in external service tests." + keystore 'azure.client.integration_test.key', azureKey + } + if (azureSasToken != null && azureSasToken.isEmpty() == false) { + println "Using SAS token in external service tests." + keystore 'azure.client.integration_test.sas_token', azureSasToken + } if (useFixture) { tasks.integTest.dependsOn azureStorageFixture diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java index ab48cf1314ec5..21d041a2d7a30 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java @@ -60,6 +60,7 @@ public List> getSettings() { return Arrays.asList( AzureStorageSettings.ACCOUNT_SETTING, AzureStorageSettings.KEY_SETTING, + AzureStorageSettings.SAS_TOKEN_SETTING, AzureStorageSettings.ENDPOINT_SUFFIX_SETTING, AzureStorageSettings.TIMEOUT_SETTING, AzureStorageSettings.MAX_RETRIES_SETTING, diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java index eda159e933860..ee00551849d51 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java @@ -117,7 +117,7 @@ private static CloudBlobClient buildClient(AzureStorageSettings azureStorageSett } private static CloudBlobClient createClient(AzureStorageSettings azureStorageSettings) throws InvalidKeyException, URISyntaxException { - final String connectionString = azureStorageSettings.buildConnectionString(); + final String connectionString = azureStorageSettings.getConnectString(); return CloudStorageAccount.parse(connectionString).createCloudBlobClient(); } diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java index 508d1ef317832..71c78d7f6f931 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java @@ -21,6 +21,7 @@ import com.microsoft.azure.storage.LocationMode; import com.microsoft.azure.storage.RetryPolicy; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.settings.SecureSetting; @@ -53,6 +54,10 @@ final class AzureStorageSettings { public static final AffixSetting KEY_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "key", key -> SecureSetting.secureString(key, null)); + /** Azure SAS token */ + public static final AffixSetting SAS_TOKEN_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "sas_token", + key -> SecureSetting.secureString(key, null)); + /** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */ public static final Setting MAX_RETRIES_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "max_retries", @@ -82,7 +87,7 @@ final class AzureStorageSettings { PROXY_HOST_SETTING); private final String account; - private final String key; + private final String connectString; private final String endpointSuffix; private final TimeValue timeout; private final int maxRetries; @@ -90,10 +95,10 @@ final class AzureStorageSettings { private final LocationMode locationMode; // copy-constructor - private AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries, Proxy proxy, - LocationMode locationMode) { + private AzureStorageSettings(String account, String connectString, String endpointSuffix, TimeValue timeout, int maxRetries, + Proxy proxy, LocationMode locationMode) { this.account = account; - this.key = key; + this.connectString = connectString; this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.maxRetries = maxRetries; @@ -101,10 +106,10 @@ private AzureStorageSettings(String account, String key, String endpointSuffix, this.locationMode = locationMode; } - AzureStorageSettings(String account, String key, String endpointSuffix, TimeValue timeout, int maxRetries, - Proxy.Type proxyType, String proxyHost, Integer proxyPort) { + private AzureStorageSettings(String account, String key, String sasToken, String endpointSuffix, TimeValue timeout, int maxRetries, + Proxy.Type proxyType, String proxyHost, Integer proxyPort) { this.account = account; - this.key = key; + this.connectString = buildConnectString(account, key, sasToken, endpointSuffix); this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.maxRetries = maxRetries; @@ -145,13 +150,26 @@ public Proxy getProxy() { return proxy; } - public String buildConnectionString() { + public String getConnectString() { + return connectString; + } + + private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) { + final boolean hasSasToken = Strings.hasText(sasToken); + final boolean hasKey = Strings.hasText(key); + if (hasSasToken == false && hasKey == false) { + throw new SettingsException("Neither a secret key nor a shared access token was set."); + } + if (hasSasToken && hasKey) { + throw new SettingsException("Both a secret as well as a shared access token were set."); + } final StringBuilder connectionStringBuilder = new StringBuilder(); - connectionStringBuilder.append("DefaultEndpointsProtocol=https") - .append(";AccountName=") - .append(account) - .append(";AccountKey=") - .append(key); + connectionStringBuilder.append("DefaultEndpointsProtocol=https").append(";AccountName=").append(account); + if (hasKey) { + connectionStringBuilder.append(";AccountKey=").append(key); + } else { + connectionStringBuilder.append(";SharedAccessSignature=").append(sasToken); + } if (Strings.hasText(endpointSuffix)) { connectionStringBuilder.append(";EndpointSuffix=").append(endpointSuffix); } @@ -166,7 +184,6 @@ public LocationMode getLocationMode() { public String toString() { final StringBuilder sb = new StringBuilder("AzureStorageSettings{"); sb.append("account='").append(account).append('\''); - sb.append(", key='").append(key).append('\''); sb.append(", timeout=").append(timeout); sb.append(", endpointSuffix='").append(endpointSuffix).append('\''); sb.append(", maxRetries=").append(maxRetries); @@ -201,8 +218,9 @@ public static Map load(Settings settings) { /** Parse settings for a single client. */ private static AzureStorageSettings getClientSettings(Settings settings, String clientName) { try (SecureString account = getConfigValue(settings, clientName, ACCOUNT_SETTING); - SecureString key = getConfigValue(settings, clientName, KEY_SETTING)) { - return new AzureStorageSettings(account.toString(), key.toString(), + SecureString key = getConfigValue(settings, clientName, KEY_SETTING); + SecureString sasToken = getConfigValue(settings, clientName, SAS_TOKEN_SETTING)) { + return new AzureStorageSettings(account.toString(), key.toString(), sasToken.toString(), getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING), getValue(settings, clientName, TIMEOUT_SETTING), getValue(settings, clientName, MAX_RETRIES_SETTING), @@ -228,10 +246,9 @@ static Map overrideLocationMode(Map mapBuilder = new MapBuilder<>(); for (final Map.Entry entry : clientsSettings.entrySet()) { - final AzureStorageSettings azureSettings = new AzureStorageSettings(entry.getValue().account, entry.getValue().key, - entry.getValue().endpointSuffix, entry.getValue().timeout, entry.getValue().maxRetries, entry.getValue().proxy, - locationMode); - mapBuilder.put(entry.getKey(), azureSettings); + mapBuilder.put(entry.getKey(), + new AzureStorageSettings(entry.getValue().account, entry.getValue().connectString, entry.getValue().endpointSuffix, + entry.getValue().timeout, entry.getValue().maxRetries, entry.getValue().proxy, locationMode)); } return mapBuilder.immutableMap(); } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java index 596fdf73342eb..1c5c2dd39fae6 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.repositories.azure; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureSettings; import org.elasticsearch.common.settings.Settings; @@ -42,13 +43,22 @@ protected Collection> getPlugins() { @Override protected SecureSettings credentials() { assertThat(System.getProperty("test.azure.account"), not(blankOrNullString())); - assertThat(System.getProperty("test.azure.key"), not(blankOrNullString())); + final boolean hasSasToken = Strings.hasText(System.getProperty("test.azure.sas_token")); + if (hasSasToken == false) { + assertThat(System.getProperty("test.azure.key"), not(blankOrNullString())); + } else { + assertThat(System.getProperty("test.azure.key"), blankOrNullString()); + } assertThat(System.getProperty("test.azure.container"), not(blankOrNullString())); assertThat(System.getProperty("test.azure.base"), not(blankOrNullString())); MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("azure.client.default.account", System.getProperty("test.azure.account")); - secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key")); + if (hasSasToken) { + secureSettings.setString("azure.client.default.sas_token", System.getProperty("test.azure.sas_token")); + } else { + secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key")); + } return secureSettings; } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java index f7b49bd24adf6..128e0e0a2140e 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceTests.java @@ -158,15 +158,21 @@ public void testReinitClientWrongSettings() throws IOException { secureSettings2.setString("azure.client.azure1.account", "myaccount1"); // missing key final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build(); + final MockSecureSettings secureSettings3 = new MockSecureSettings(); + secureSettings3.setString("azure.client.azure1.account", "myaccount3"); + secureSettings3.setString("azure.client.azure1.key", encodeKey("mykey33")); + secureSettings3.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33")); + final Settings settings3 = Settings.builder().setSecureSettings(secureSettings3).build(); try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) { final AzureStorageService azureStorageService = plugin.azureStoreService; final CloudBlobClient client11 = azureStorageService.client("azure1").v1(); assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net")); - plugin.reload(settings2); + final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2)); + assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set.")); + final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3)); + assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set.")); // existing client untouched assertThat(client11.getEndpoint().toString(), equalTo("https://myaccount1.blob.core.windows.net")); - final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure1")); - assertThat(e.getMessage(), is("Invalid azure client settings with name [azure1]")); } }