From bc01679f835000ae64d0d23c887f6c3ffc885001 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sun, 6 Aug 2023 15:33:27 +0200 Subject: [PATCH 01/84] Connect to Minio with temporary credentials from STS obtained with OpenID Connect token from Keycloak (#14893) Co-authored-by: Ismail Cadaroski Co-authored-by: Sherom Sandmeier --- .github/workflows/build.yml | 12 +- .../ch/cyberduck/core/AbstractProtocol.java | 5 + .../core/DefaultHostPasswordStore.java | 17 +- .../java/ch/cyberduck/core/OAuthTokens.java | 12 +- .../main/java/ch/cyberduck/core/Profile.java | 9 + .../main/java/ch/cyberduck/core/Protocol.java | 5 + .../ch/cyberduck/test/TestcontainerTest.java | 19 + .../src/main/resources/default.properties | 7 + .../oauth/OAuth2AuthorizationService.java | 44 +- .../core/oauth/OAuth2RequestInterceptor.java | 8 +- pom.xml | 9 +- s3/pom.xml | 12 + .../java/ch/cyberduck/core/s3/S3Protocol.java | 10 +- .../java/ch/cyberduck/core/s3/S3Session.java | 31 +- ...entityTokenExpiredResponseInterceptor.java | 130 + .../sts/STSCredentialsRequestInterceptor.java | 196 ++ .../cyberduck/core/sts/AbstractOidcTest.java | 81 + .../core/sts/OidcAuthenticationTest.java | 168 ++ .../core/sts/OidcAuthorizationTest.java | 103 + .../cyberduck/core/sts/S3OidcProfileTest.java | 43 + .../S3-OIDC-Google-Testing.cyberduckprofile | 67 + .../S3-OIDC-Testing.cyberduckprofile | 66 + .../oidcTestcontainer/docker-compose.yml | 89 + .../oidcTestcontainer/keycloak/Dockerfile | 1 + .../keycloak/keycloak-realm.json | 2260 +++++++++++++++++ .../oidcTestcontainer/minio/Dockerfile | 1 + .../testcontainers.properties | 14 + .../resources/oidcTestcontainer/testfile.txt | 1 + 28 files changed, 3384 insertions(+), 36 deletions(-) create mode 100644 core/src/test/java/ch/cyberduck/test/TestcontainerTest.java create mode 100644 s3/src/main/java/ch/cyberduck/core/sts/S3WebIdentityTokenExpiredResponseInterceptor.java create mode 100644 s3/src/main/java/ch/cyberduck/core/sts/STSCredentialsRequestInterceptor.java create mode 100644 s3/src/test/java/ch/cyberduck/core/sts/AbstractOidcTest.java create mode 100644 s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java create mode 100644 s3/src/test/java/ch/cyberduck/core/sts/OidcAuthorizationTest.java create mode 100644 s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java create mode 100644 s3/src/test/resources/S3-OIDC-Google-Testing.cyberduckprofile create mode 100644 s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile create mode 100644 s3/src/test/resources/oidcTestcontainer/docker-compose.yml create mode 100644 s3/src/test/resources/oidcTestcontainer/keycloak/Dockerfile create mode 100644 s3/src/test/resources/oidcTestcontainer/keycloak/keycloak-realm.json create mode 100644 s3/src/test/resources/oidcTestcontainer/minio/Dockerfile create mode 100644 s3/src/test/resources/oidcTestcontainer/testcontainers.properties create mode 100644 s3/src/test/resources/oidcTestcontainer/testfile.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2ac6b9ec022..ee29b110726 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,12 +19,16 @@ jobs: with: distribution: 'temurin' java-version: 17 + - name: Disable Testcontainers for Windows and MacOS + run: echo "args=-P=no-testcontainers" >> "$GITHUB_ENV" + shell: bash + if: runner.os == 'windows' || runner.os == 'macos' - run: choco install bonjour - if: runner.os == 'Windows' + if: runner.os == 'windows' - uses: ilammy/msvc-dev-cmd@v1 - if: runner.os == 'Windows' + if: runner.os == 'windows' - uses: microsoft/setup-msbuild@v1.3 - if: runner.os == 'Windows' + if: runner.os == 'windows' with: msbuild-architecture: x64 - name: Cache local Maven repository @@ -35,4 +39,4 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - name: Build with Maven - run: mvn --no-transfer-progress verify -DskipITs -DskipSign --batch-mode -Drevision=0 + run: mvn --no-transfer-progress verify -DskipITs -DskipSign ${{ env.args }} --batch-mode -Drevision=0 \ No newline at end of file diff --git a/core/src/main/java/ch/cyberduck/core/AbstractProtocol.java b/core/src/main/java/ch/cyberduck/core/AbstractProtocol.java index 5d8bac951d1..f27eee6ecbb 100644 --- a/core/src/main/java/ch/cyberduck/core/AbstractProtocol.java +++ b/core/src/main/java/ch/cyberduck/core/AbstractProtocol.java @@ -211,6 +211,11 @@ public boolean isOAuthPKCE() { return true; } + @Override + public String getSTSEndpoint() { + return null; + } + @Override public String getDefaultHostname() { // Blank by default diff --git a/core/src/main/java/ch/cyberduck/core/DefaultHostPasswordStore.java b/core/src/main/java/ch/cyberduck/core/DefaultHostPasswordStore.java index 537372f1626..0c3e2470f70 100644 --- a/core/src/main/java/ch/cyberduck/core/DefaultHostPasswordStore.java +++ b/core/src/main/java/ch/cyberduck/core/DefaultHostPasswordStore.java @@ -18,8 +18,6 @@ */ import ch.cyberduck.core.exception.LocalAccessDeniedException; -import ch.cyberduck.core.preferences.Preferences; -import ch.cyberduck.core.preferences.PreferencesFactory; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -30,8 +28,6 @@ public abstract class DefaultHostPasswordStore implements HostPasswordStore { private static final Logger log = LogManager.getLogger(DefaultHostPasswordStore.class); - private final Preferences preferences = PreferencesFactory.get(); - /** * Find password for login * @@ -161,7 +157,9 @@ public OAuthTokens findOAuthTokens(final Host bookmark) { String.format("%s OAuth2 Access Token", prefix)), this.getPassword(bookmark.getProtocol().getScheme(), bookmark.getPort(), hostname, String.format("%s OAuth2 Refresh Token", prefix)), - expiry != null ? Long.parseLong(expiry) : -1L)); + expiry != null ? Long.parseLong(expiry) : -1L, + this.getPassword(bookmark.getProtocol().getScheme(), bookmark.getPort(), hostname, + String.format("%s OIDC Id Token", prefix)))); } catch(LocalAccessDeniedException e) { log.warn(String.format("Failure %s searching in keychain", e)); @@ -232,6 +230,11 @@ public void save(final Host bookmark) throws LocalAccessDeniedException { this.addPassword(this.getOAuthHostname(bookmark), String.format("%s OAuth2 Token Expiry", prefix), String.valueOf(credentials.getOauth().getExpiryInMilliseconds())); } + if(StringUtils.isNotBlank(credentials.getOauth().getIdToken())) { + this.addPassword(bookmark.getProtocol().getScheme(), + bookmark.getPort(), this.getOAuthHostname(bookmark), + String.format("%s OIDC Id Token", prefix), credentials.getOauth().getRefreshToken()); + } } } @@ -272,6 +275,10 @@ public void delete(final Host bookmark) throws LocalAccessDeniedException { if(credentials.getOauth().getExpiryInMilliseconds() != null) { this.deletePassword(this.getOAuthHostname(bookmark), String.format("%s OAuth2 Token Expiry", prefix)); } + if(StringUtils.isNotBlank(credentials.getOauth().getIdToken())) { + this.deletePassword(protocol.getScheme(), bookmark.getPort(), this.getOAuthHostname(bookmark), + String.format("%s OIDC Id Token", prefix)); + } } } } diff --git a/core/src/main/java/ch/cyberduck/core/OAuthTokens.java b/core/src/main/java/ch/cyberduck/core/OAuthTokens.java index 6c7a7784055..aee3c1badba 100644 --- a/core/src/main/java/ch/cyberduck/core/OAuthTokens.java +++ b/core/src/main/java/ch/cyberduck/core/OAuthTokens.java @@ -18,16 +18,22 @@ import org.apache.commons.lang3.StringUtils; public final class OAuthTokens { - public static final OAuthTokens EMPTY = new OAuthTokens(null, null, Long.MAX_VALUE); + public static final OAuthTokens EMPTY = new OAuthTokens(null, null, Long.MAX_VALUE, null); private final String accessToken; private final String refreshToken; private final Long expiryInMilliseconds; + private final String idToken; public OAuthTokens(final String accessToken, final String refreshToken, final Long expiryInMilliseconds) { + this(accessToken, refreshToken, expiryInMilliseconds, null); + } + + public OAuthTokens(final String accessToken, final String refreshToken, final Long expiryInMilliseconds, final String idToken) { this.accessToken = accessToken; this.refreshToken = refreshToken; this.expiryInMilliseconds = expiryInMilliseconds; + this.idToken = idToken; } public boolean validate() { @@ -46,6 +52,10 @@ public Long getExpiryInMilliseconds() { return expiryInMilliseconds; } + public String getIdToken() { + return idToken; + } + public boolean isExpired() { return System.currentTimeMillis() >= expiryInMilliseconds; } diff --git a/core/src/main/java/ch/cyberduck/core/Profile.java b/core/src/main/java/ch/cyberduck/core/Profile.java index bdfa9db6b6a..90201b5c644 100644 --- a/core/src/main/java/ch/cyberduck/core/Profile.java +++ b/core/src/main/java/ch/cyberduck/core/Profile.java @@ -545,6 +545,15 @@ public boolean isOAuthPKCE() { return this.bool("OAuth PKCE"); } + @Override + public String getSTSEndpoint() { + final String v = this.value("STS Endpoint"); + if(StringUtils.isBlank(v)) { + return parent.getSTSEndpoint(); + } + return v; + } + @Override public Map getProperties() { final List properties = this.list("Properties"); diff --git a/core/src/main/java/ch/cyberduck/core/Protocol.java b/core/src/main/java/ch/cyberduck/core/Protocol.java index 02435d12cde..5a9e9f8ea54 100644 --- a/core/src/main/java/ch/cyberduck/core/Protocol.java +++ b/core/src/main/java/ch/cyberduck/core/Protocol.java @@ -179,6 +179,11 @@ public interface Protocol extends Comparable, Serializable { */ String[] getSchemes(); + /** + * @return Default STS Endpoint URL. + */ + String getSTSEndpoint(); + /** * @return Default hostname for server */ diff --git a/core/src/test/java/ch/cyberduck/test/TestcontainerTest.java b/core/src/test/java/ch/cyberduck/test/TestcontainerTest.java new file mode 100644 index 00000000000..b0a3959e965 --- /dev/null +++ b/core/src/test/java/ch/cyberduck/test/TestcontainerTest.java @@ -0,0 +1,19 @@ +package ch.cyberduck.test; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +public @interface TestcontainerTest { +} diff --git a/defaults/src/main/resources/default.properties b/defaults/src/main/resources/default.properties index 61b322fc02c..dd6a6178c56 100644 --- a/defaults/src/main/resources/default.properties +++ b/defaults/src/main/resources/default.properties @@ -296,6 +296,13 @@ s3.endpoint.format.ipv6=s3.dualstack.%s.amazonaws.com s3.acl.default=private +# STS Assume Role request parameters +s3.assumerole.durationseconds=0 +s3.assumerole.policy= +s3.assumerole.rolearn= +s3.assumerole.rolesessionname= + + # Default redundancy level s3.storage.class=STANDARD s3.storage.class.options=STANDARD diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java index 87497d3fb08..2f42b8caabc 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java @@ -52,8 +52,8 @@ import com.google.api.client.auth.oauth2.Credential; import com.google.api.client.auth.oauth2.PasswordTokenRequest; import com.google.api.client.auth.oauth2.RefreshTokenRequest; -import com.google.api.client.auth.oauth2.TokenResponse; import com.google.api.client.auth.oauth2.TokenResponseException; +import com.google.api.client.auth.openidconnect.IdTokenResponse; import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpTransport; @@ -138,25 +138,30 @@ public OAuthTokens authorize(final Host bookmark, final LoginCallback prompt, fi if(log.isDebugEnabled()) { log.debug(String.format("Start new OAuth flow for %s with missing access token", bookmark)); } - final TokenResponse response; + + final IdTokenResponse response; + // Save access token, refresh token and id token switch(flowType) { case AuthorizationCode: response = this.authorizeWithCode(bookmark, prompt); - break; + return credentials.withOauth(new OAuthTokens( + response.getAccessToken(), response.getRefreshToken(), + null == response.getExpiresInSeconds() ? System.currentTimeMillis() : + System.currentTimeMillis() + response.getExpiresInSeconds() * 1000, response.getIdToken())) + .withSaved(new LoginOptions().keychain).getOauth(); case PasswordGrant: response = this.authorizeWithPassword(credentials); - break; + return credentials.withOauth(new OAuthTokens( + response.getAccessToken(), response.getRefreshToken(), + null == response.getExpiresInSeconds() ? Long.MAX_VALUE : + System.currentTimeMillis() + response.getExpiresInSeconds() * 1000, response.getIdToken())) + .withSaved(new LoginOptions().keychain).getOauth(); default: throw new LoginCanceledException(); } - // Save access key and refresh key - return credentials.withOauth(new OAuthTokens( - response.getAccessToken(), response.getRefreshToken(), - null == response.getExpiresInSeconds() ? Long.MAX_VALUE : - System.currentTimeMillis() + response.getExpiresInSeconds() * 1000)).withSaved(new LoginOptions().keychain).getOauth(); } - private TokenResponse authorizeWithCode(final Host bookmark, final LoginCallback prompt) throws BackgroundException { + private IdTokenResponse authorizeWithCode(final Host bookmark, final LoginCallback prompt) throws BackgroundException { if(PreferencesFactory.get().getBoolean("oauth.browser.open.warn")) { prompt.warn(bookmark, LocaleFactory.localizedString("Provide additional login credentials", "Credentials"), @@ -219,7 +224,7 @@ private TokenResponse authorizeWithCode(final Host bookmark, final LoginCallback } } - private TokenResponse authorizeWithPassword(final Credentials credentials) throws BackgroundException { + private IdTokenResponse authorizeWithPassword(final Credentials credentials) throws BackgroundException { try { if(log.isDebugEnabled()) { log.debug(String.format("Request tokens for user %s", credentials.getUsername())); @@ -258,7 +263,7 @@ public OAuthTokens refresh(final OAuthTokens tokens) throws BackgroundException log.debug(String.format("Refresh expired tokens %s", tokens)); } try { - final TokenResponse response = new RefreshTokenRequest(transport, json, new GenericUrl(tokenServerUrl), + final IdTokenResponse response = new RefreshTokenRequest(transport, json, new GenericUrl(tokenServerUrl), tokens.getRefreshToken()) .setScopes(scopes.isEmpty() ? null : scopes) .setRequestInitializer(new UserAgentHttpRequestInitializer(new PreferencesUseragentProvider())) @@ -266,9 +271,9 @@ public OAuthTokens refresh(final OAuthTokens tokens) throws BackgroundException .executeUnparsed().parseAs(PermissiveTokenResponse.class).toTokenResponse(); final long expiryInMilliseconds = System.currentTimeMillis() + response.getExpiresInSeconds() * 1000; if(StringUtils.isBlank(response.getRefreshToken())) { - return new OAuthTokens(response.getAccessToken(), tokens.getRefreshToken(), expiryInMilliseconds); + return new OAuthTokens(response.getAccessToken(), tokens.getRefreshToken(), expiryInMilliseconds, response.getIdToken()); } - return new OAuthTokens(response.getAccessToken(), response.getRefreshToken(), expiryInMilliseconds); + return new OAuthTokens(response.getAccessToken(), response.getRefreshToken(), expiryInMilliseconds, response.getIdToken()); } catch(TokenResponseException e) { throw new OAuthExceptionMappingService().map(e); @@ -308,6 +313,7 @@ public enum FlowType { } public static final class PermissiveTokenResponse extends GenericJson { + private String idToken; private String accessToken; private String tokenType; private Long expiresInSeconds; @@ -316,6 +322,9 @@ public static final class PermissiveTokenResponse extends GenericJson { @Override public PermissiveTokenResponse set(final String fieldName, final Object value) { + if("id_token".equals(fieldName)) { + idToken = (String) value; + } if("access_token".equals(fieldName)) { accessToken = (String) value; } @@ -350,13 +359,14 @@ else if(value instanceof Number) { return this; } - public TokenResponse toTokenResponse() { - return new TokenResponse() + public IdTokenResponse toTokenResponse() { + return new IdTokenResponse() .setTokenType(tokenType) .setScope(scope) .setExpiresInSeconds(expiresInSeconds) .setAccessToken(accessToken) - .setRefreshToken(refreshToken); + .setRefreshToken(refreshToken) + .setIdToken(idToken); } } diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2RequestInterceptor.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2RequestInterceptor.java index e2871e8a93d..9cb9e53317a 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2RequestInterceptor.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2RequestInterceptor.java @@ -49,10 +49,10 @@ public class OAuth2RequestInterceptor extends OAuth2AuthorizationService impleme /** * Currently valid tokens */ - private OAuthTokens tokens = OAuthTokens.EMPTY; + protected OAuthTokens tokens = OAuthTokens.EMPTY; private final HostPasswordStore store = PasswordStoreFactory.get(); - private final Host host; + protected final Host host; public OAuth2RequestInterceptor(final HttpClient client, final Host host, final LoginCallback prompt) throws LoginCanceledException { this(client, host, @@ -147,4 +147,8 @@ public OAuth2RequestInterceptor withParameter(final String key, final String val super.withParameter(key, value); return this; } + + public OAuthTokens getTokens() { + return tokens; + } } diff --git a/pom.xml b/pom.xml index 247fff60808..9ab8118d9c3 100644 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,7 @@ 8.0.312.b07-8 8u312b07 + ch.cyberduck.test.IntegrationTest @@ -508,7 +509,7 @@ ${project.build.directory} - ch.cyberduck.test.IntegrationTest + ${excludedGroups} false @@ -934,5 +935,11 @@ ${maven.multiModuleProjectDirectory} + + no-testcontainers + + ch.cyberduck.test.IntegrationTest,ch.cyberduck.test.TestcontainerTest + + diff --git a/s3/pom.xml b/s3/pom.xml index 900376f1184..3c89a68fc43 100644 --- a/s3/pom.xml +++ b/s3/pom.xml @@ -71,6 +71,18 @@ ${project.version} test + + ch.cyberduck + oauth + ${project.version} + compile + + + org.testcontainers + testcontainers + 1.18.3 + test + diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Protocol.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Protocol.java index acfcd27d93a..66aecc128d2 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Protocol.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Protocol.java @@ -47,10 +47,10 @@ public class S3Protocol extends AbstractProtocol { private static final Logger log = LogManager.getLogger(S3Protocol.class); private final AWSCredentialsConfigurator credentials = new AWSCredentialsConfigurator( - new AWSCredentialsProviderChain( - new ProfileCredentialsProvider(), - new EnvironmentVariableCredentialsProvider() - ) + new AWSCredentialsProviderChain( + new ProfileCredentialsProvider(), + new EnvironmentVariableCredentialsProvider() + ) ); @Override @@ -145,7 +145,7 @@ public static AuthenticationHeaderSignatureVersion getDefault(final Protocol pro catch(IllegalArgumentException e) { log.warn(String.format("Unsupported authentication context %s", protocol.getAuthorization())); return S3Protocol.AuthenticationHeaderSignatureVersion.valueOf( - PreferencesFactory.get().getProperty("s3.signature.version")); + PreferencesFactory.get().getProperty("s3.signature.version")); } } diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 78e0dd9b0eb..ab4c447e7f9 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -48,9 +48,11 @@ import ch.cyberduck.core.features.*; import ch.cyberduck.core.http.HttpSession; import ch.cyberduck.core.kms.KMSEncryptionFeature; +import ch.cyberduck.core.oauth.OAuth2AuthorizationService; import ch.cyberduck.core.preferences.HostPreferences; import ch.cyberduck.core.preferences.PreferencesReader; import ch.cyberduck.core.proxy.Proxy; +import ch.cyberduck.core.proxy.ProxyFactory; import ch.cyberduck.core.restore.Glacier; import ch.cyberduck.core.shared.DefaultHomeFinderService; import ch.cyberduck.core.shared.DefaultPathHomeFeature; @@ -63,6 +65,8 @@ import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.sts.AWSProfileSTSCredentialsConfigurator; +import ch.cyberduck.core.sts.S3WebIdentityTokenExpiredResponseInterceptor; +import ch.cyberduck.core.sts.STSCredentialsRequestInterceptor; import ch.cyberduck.core.threading.BackgroundExceptionCallable; import ch.cyberduck.core.threading.CancelCallback; import ch.cyberduck.core.transfer.TransferStatus; @@ -116,6 +120,8 @@ public class S3Session extends HttpSession { private final PreferencesReader preferences = new HostPreferences(host); + private STSCredentialsRequestInterceptor authorizationService; + private final S3AccessControlListFeature acl = new S3AccessControlListFeature(this); private final Versioning versioning = preferences.getBoolean("s3.versioning.enable") @@ -203,6 +209,17 @@ protected String getRestMetadataPrefix() { @Override protected RequestEntityRestStorageService connect(final Proxy proxy, final HostKeyCallback hostkey, final LoginCallback prompt, final CancelCallback cancel) { final HttpClientBuilder configuration = builder.build(proxy, this, prompt); + + if((host.getProtocol().isOAuthConfigurable())) { + authorizationService = new STSCredentialsRequestInterceptor(builder.build(ProxyFactory.get() + .find(host.getProtocol().getSTSEndpoint()), this, prompt).build(), host, trust, key, prompt, this) + .withRedirectUri(host.getProtocol().getOAuthRedirectUrl()) + .withFlowType(OAuth2AuthorizationService.FlowType.valueOf(host.getProtocol().getAuthorization())); + + configuration.addInterceptorLast(authorizationService); + configuration.setServiceUnavailableRetryStrategy(new S3WebIdentityTokenExpiredResponseInterceptor(this, prompt, authorizationService)); + } + // Only for AWS if(S3Session.isAwsHostname(host.getHostname())) { configuration.setServiceUnavailableRetryStrategy(new S3TokenExpiredResponseInterceptor(this, @@ -406,6 +423,10 @@ private Map getHeadersAsObject(final HttpRequest request) { @Override public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { + if(host.getProtocol().isOAuthConfigurable()) { + authorizationService.authorize(host, prompt, cancel); + } + if(Scheme.isURL(host.getProtocol().getContext())) { try { final Credentials temporary = new AWSSessionCredentialsRetriever(trust, key, this, host.getProtocol().getContext()).get(); @@ -420,8 +441,12 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal } else { final Credentials credentials; + // get temporary STS credentials with oAuth token + if(host.getProtocol().isOAuthConfigurable()) { + credentials = authorizationService.assumeRoleWithWebIdentity(); + } // Only for AWS - if(isAwsHostname(host.getHostname())) { + else if(isAwsHostname(host.getHostname())) { // Try auto-configure credentials = new AWSProfileSTSCredentialsConfigurator( new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt).configure(host); @@ -486,6 +511,10 @@ public static boolean isAwsHostname(final String hostname, boolean cn) { return hostname.matches("([a-z0-9\\-]+\\.)?s3(\\.dualstack)?(\\.[a-z0-9\\-]+)?(\\.vpce)?\\.amazonaws\\.com"); } + public STSCredentialsRequestInterceptor getAuthorizationService() { + return authorizationService; + } + @Override @SuppressWarnings("unchecked") public T _getFeature(final Class type) { diff --git a/s3/src/main/java/ch/cyberduck/core/sts/S3WebIdentityTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/S3WebIdentityTokenExpiredResponseInterceptor.java new file mode 100644 index 00000000000..432d522c60e --- /dev/null +++ b/s3/src/main/java/ch/cyberduck/core/sts/S3WebIdentityTokenExpiredResponseInterceptor.java @@ -0,0 +1,130 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.DisabledCancelCallback; +import ch.cyberduck.core.Host; +import ch.cyberduck.core.LoginCallback; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.ExpiredTokenException; +import ch.cyberduck.core.exception.InteroperabilityException; +import ch.cyberduck.core.exception.LoginFailureException; +import ch.cyberduck.core.http.DisabledServiceUnavailableRetryStrategy; +import ch.cyberduck.core.s3.S3ExceptionMappingService; +import ch.cyberduck.core.s3.S3Session; + +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.entity.BufferedHttpEntity; +import org.apache.http.protocol.HttpContext; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jets3t.service.S3ServiceException; +import org.jets3t.service.security.AWSSessionCredentials; + +import java.io.IOException; + +public class S3WebIdentityTokenExpiredResponseInterceptor extends DisabledServiceUnavailableRetryStrategy { + private static final Logger log = LogManager.getLogger(S3WebIdentityTokenExpiredResponseInterceptor.class); + + private static final int MAX_RETRIES = 1; + + private final S3Session session; + private final Host host; + private final STSCredentialsRequestInterceptor authorizationService; + private final LoginCallback prompt; + + public S3WebIdentityTokenExpiredResponseInterceptor(final S3Session session, final LoginCallback prompt, + STSCredentialsRequestInterceptor authorizationService) { + this.session = session; + this.host = session.getHost(); + this.authorizationService = authorizationService; + this.prompt = prompt; + } + + @Override + public boolean retryRequest(final HttpResponse response, final int executionCount, final HttpContext context) { + if(executionCount <= MAX_RETRIES) { + switch(response.getStatusLine().getStatusCode()) { + case HttpStatus.SC_BAD_REQUEST: + case HttpStatus.SC_FORBIDDEN: + try { + final S3ServiceException failure; + if(null != response.getEntity()) { + EntityUtils.updateEntity(response, new BufferedHttpEntity(response.getEntity())); + failure = new S3ServiceException(response.getStatusLine().getReasonPhrase(), + EntityUtils.toString(response.getEntity())); + } + // In case of a http HEAD request minio packs the error code and description in the response header + else { + failure = new S3ServiceException(response.getStatusLine().getReasonPhrase()); + if(response.containsHeader("x-minio-error-code")) { + failure.setErrorCode(response.getFirstHeader("x-minio-error-code").getValue()); + } + if(response.containsHeader("x-minio-error-desc")) { + failure.setErrorMessage(response.getFirstHeader("x-minio-error-desc").getValue()); + } + } + + failure.setResponseCode(response.getStatusLine().getStatusCode()); + BackgroundException s3exception = new S3ExceptionMappingService().map(failure); + + if(failure.getErrorCode().equals("InvalidAccessKeyId") || s3exception instanceof ExpiredTokenException) { + refreshOAuthAndSTS(); + } + return true; + } + catch(IOException e) { + log.warn(String.format("Failure parsing response entity from %s", response)); + } + } + } + + else { + if(log.isWarnEnabled()) { + log.warn(String.format("Skip retry for response %s after %d executions", response, executionCount)); + } + } + return false; + } + + private void refreshOAuthAndSTS() { + try { + try { + authorizationService.save(authorizationService.refresh()); + log.debug("OAuth refreshed. Refreshing STS token."); + } + catch(InteroperabilityException | LoginFailureException e3) { + log.warn(String.format("Failure %s refreshing OAuth tokens", e3)); + authorizationService.save(authorizationService.authorize(host, prompt, new DisabledCancelCallback())); + } + try { + Credentials credentials = authorizationService.assumeRoleWithWebIdentity(); + session.getClient().setProviderCredentials(credentials.isAnonymousLogin() ? null : + new AWSSessionCredentials(credentials.getUsername(), credentials.getPassword(), + credentials.getToken())); + } + catch(BackgroundException e) { + log.warn(String.format("Failure %s fetching temporary STS credentials with oAuth token", e)); + } + } + catch(BackgroundException e) { + log.warn(String.format("Failure %s refreshing OAuth tokens", e)); + } + } +} \ No newline at end of file diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSCredentialsRequestInterceptor.java new file mode 100644 index 00000000000..133fdf81edb --- /dev/null +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSCredentialsRequestInterceptor.java @@ -0,0 +1,196 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2019 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.DisabledCancelCallback; +import ch.cyberduck.core.Host; +import ch.cyberduck.core.LoginCallback; +import ch.cyberduck.core.aws.CustomClientConfiguration; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.InteroperabilityException; +import ch.cyberduck.core.exception.LoginFailureException; +import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; +import ch.cyberduck.core.preferences.HostPreferences; +import ch.cyberduck.core.s3.S3Session; +import ch.cyberduck.core.ssl.ThreadLocalHostnameDelegatingTrustManager; +import ch.cyberduck.core.ssl.X509KeyManager; +import ch.cyberduck.core.ssl.X509TrustManager; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpException; +import org.apache.http.client.HttpClient; +import org.apache.http.protocol.HttpContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jets3t.service.security.AWSSessionCredentials; + +import java.io.IOException; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.securitytoken.AWSSecurityTokenService; +import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; +import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException; +import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityRequest; +import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityResult; +import com.google.api.client.auth.oauth2.Credential; + +public class STSCredentialsRequestInterceptor extends OAuth2RequestInterceptor { + private static final Logger log = LogManager.getLogger(STSCredentialsRequestInterceptor.class); + + private final X509TrustManager trust; + private final X509KeyManager key; + private final LoginCallback prompt; + private final S3Session session; + + private long stsExpiryInMilliseconds; + + public STSCredentialsRequestInterceptor(HttpClient client, Host host, final X509TrustManager trust, final X509KeyManager key, + LoginCallback prompt, S3Session session) { + super(client, host); + this.trust = trust; + this.key = key; + this.prompt = prompt; + this.session = session; + } + + @Override + public void process(final org.apache.http.HttpRequest request, final HttpContext context) throws HttpException, IOException { + if(System.currentTimeMillis() >= stsExpiryInMilliseconds) { + try { + if(tokens.isExpired()) { + try { + this.save(this.refresh(tokens)); + } + catch(InteroperabilityException | LoginFailureException e3) { + log.warn(String.format("Failure %s refreshing OAuth tokens", e3)); + try { + this.save(this.authorize(host, prompt, new DisabledCancelCallback())); + } + catch(BackgroundException e) { + log.warn(String.format("Failure %s OAuth authentication", e)); + } + } + } + try { + Credentials credentials = assumeRoleWithWebIdentity(); + session.getClient().setProviderCredentials(credentials.isAnonymousLogin() ? null : + new AWSSessionCredentials(credentials.getUsername(), credentials.getPassword(), + credentials.getToken())); + } + catch(BackgroundException e) { + log.warn(String.format("Failure %s to fetch temporary sts credentials", e)); + // Follow-up error 400 or 403 handled in web identity token expired interceptor + } + } + catch(BackgroundException e) { + log.warn(String.format("Failure %s getting web identity", e)); + } + } + } + + public Credentials assumeRoleWithWebIdentity() throws BackgroundException { + AWSSecurityTokenService service = this.getTokenService(host); + + AssumeRoleWithWebIdentityRequest webIdReq = new AssumeRoleWithWebIdentityRequest(); + if(StringUtils.isNotBlank(tokens.getIdToken())) { + webIdReq.withWebIdentityToken(tokens.getIdToken()); + } + else { + webIdReq.withWebIdentityToken(tokens.getAccessToken()); + } + + + if(new HostPreferences(host).getInteger("s3.assumerole.durationseconds") != 0) { + webIdReq.withDurationSeconds(new HostPreferences(host).getInteger("s3.assumerole.durationseconds")); + } + + if(StringUtils.isNotBlank(new HostPreferences(host).getProperty("s3.assumerole.policy"))) { + webIdReq.withPolicy(new HostPreferences(host).getProperty("s3.assumerole.policy")); + } + + if(StringUtils.isNotBlank(new HostPreferences(host).getProperty("s3.assumerole.rolearn"))) { + webIdReq.withRoleArn(new HostPreferences(host).getProperty("s3.assumerole.rolearn")); + } + + if(StringUtils.isNotBlank(new HostPreferences(host).getProperty("s3.assumerole.rolesessionname"))) { + webIdReq.withRoleSessionName(new HostPreferences(host).getProperty("s3.assumerole.rolesessionname")); + } + + + Credentials credentials = new Credentials(); + try { + AssumeRoleWithWebIdentityResult result = service.assumeRoleWithWebIdentity(webIdReq); + com.amazonaws.services.securitytoken.model.Credentials cred = result.getCredentials(); + + if(log.isDebugEnabled()) { + log.debug(cred.toString()); + } + + stsExpiryInMilliseconds = cred.getExpiration().getTime(); + + credentials.setUsername(cred.getAccessKeyId()); + credentials.setPassword(cred.getSecretAccessKey()); + credentials.setToken(cred.getSessionToken()); + } + catch(AWSSecurityTokenServiceException e) { + throw new LoginFailureException(e.getMessage(), e); + } + return credentials; + } + + private AWSSecurityTokenService getTokenService(final Host host) { + final ClientConfiguration configuration = new CustomClientConfiguration(host, + new ThreadLocalHostnameDelegatingTrustManager(trust, host.getProtocol().getSTSEndpoint()), key); + return AWSSecurityTokenServiceClientBuilder + .standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(host.getProtocol().getSTSEndpoint(), null)) + .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) + .withClientConfiguration(configuration) + .build(); + } + + @Override + public STSCredentialsRequestInterceptor withMethod(final Credential.AccessMethod method) { + super.withMethod(method); + return this; + } + + @Override + public STSCredentialsRequestInterceptor withRedirectUri(final String redirectUri) { + super.withRedirectUri(redirectUri); + return this; + } + + @Override + public STSCredentialsRequestInterceptor withFlowType(final FlowType flowType) { + super.withFlowType(flowType); + return this; + } + + @Override + public STSCredentialsRequestInterceptor withParameter(final String key, final String value) { + super.withParameter(key, value); + return this; + } + + public long getStsExpiryInMilliseconds() { + return stsExpiryInMilliseconds; + } +} diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AbstractOidcTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AbstractOidcTest.java new file mode 100644 index 00000000000..02eb3d73f23 --- /dev/null +++ b/s3/src/test/java/ch/cyberduck/core/sts/AbstractOidcTest.java @@ -0,0 +1,81 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + + +import ch.cyberduck.core.Profile; +import ch.cyberduck.core.ProtocolFactory; +import ch.cyberduck.core.exception.AccessDeniedException; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.s3.S3Protocol; +import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; +import ch.cyberduck.test.TestcontainerTest; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.io.File; +import java.util.Collections; +import java.util.HashSet; + +@Category(TestcontainerTest.class) +public abstract class AbstractOidcTest { + + protected static final Logger log = LogManager.getLogger(AbstractOidcTest.class); + protected static Profile profile = null; + private static Network network; + private static final DockerComposeContainer compose; + + static { + compose = new DockerComposeContainer<>( + new File("src/test/resources/oidcTestcontainer/docker-compose.yml")) + .withPull(false) + .withLocalCompose(true) + .withOptions("--compatibility") + .withExposedService("keycloak_1", 8080, Wait.forListeningPort()) + .withExposedService("minio_1", 9000, Wait.forListeningPort()); + } + + @BeforeClass + public static void beforeAll() { + compose.start(); + } + + @Before + public void setup() throws BackgroundException { + profile = readProfile(); + } + + private Profile readProfile() throws AccessDeniedException { + final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); + return new ProfilePlistReader(factory).read( + this.getClass().getResourceAsStream("/S3-OIDC-Testing.cyberduckprofile")); + } + + @AfterClass + public static void disconnect() { + if(compose == null && network != null) { + network.close(); + } + } +} \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java new file mode 100644 index 00000000000..561dcb044ce --- /dev/null +++ b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java @@ -0,0 +1,168 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.DisabledCancelCallback; +import ch.cyberduck.core.DisabledHostKeyCallback; +import ch.cyberduck.core.DisabledLoginCallback; +import ch.cyberduck.core.Host; +import ch.cyberduck.core.Path; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.LoginFailureException; +import ch.cyberduck.core.preferences.HostPreferences; +import ch.cyberduck.core.proxy.Proxy; +import ch.cyberduck.core.s3.S3AccessControlListFeature; +import ch.cyberduck.core.s3.S3FindFeature; +import ch.cyberduck.core.s3.S3Session; +import ch.cyberduck.test.TestcontainerTest; + +import org.jets3t.service.security.AWSSessionCredentials; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.testcontainers.shaded.org.apache.commons.lang3.StringUtils; + +import java.util.EnumSet; +import java.util.Optional; + +import static org.junit.Assert.*; + +@Category(TestcontainerTest.class) +public class OidcAuthenticationTest extends AbstractOidcTest { + @Test + public void testSuccessfulLoginViaOidc() throws BackgroundException { + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + + Credentials creds = host.getCredentials(); + assertNotEquals(StringUtils.EMPTY, creds.getUsername()); + assertNotEquals(StringUtils.EMPTY, creds.getPassword()); + // credentials from STS are written to the S3Session's client object and not into the credential object from the Host. + assertTrue(creds.getToken().isEmpty()); + assertNotNull(creds.getOauth().getIdToken()); + assertNotNull(creds.getOauth().getRefreshToken()); + assertNotEquals(Optional.of(Long.MAX_VALUE).get(), creds.getOauth().getExpiryInMilliseconds()); + session.close(); + } + + @Test(expected = LoginFailureException.class) + public void testInvalidUserName() throws BackgroundException { + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("WrongUsername", "rouser")); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.close(); + } + + @Test(expected = LoginFailureException.class) + public void testInvalidPassword() throws BackgroundException { + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "invalidPassword")); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.close(); + } + + @Test + public void testTokenRefresh() throws BackgroundException, InterruptedException { + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + + String firstAccessToken = session.getAuthorizationService().getTokens().getIdToken(); + String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); + String firstSessionToken = ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken(); + + Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + Thread.sleep(1100 * 30); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + + assertNotEquals(firstAccessToken, session.getAuthorizationService().getTokens().getIdToken()); + assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); + assertNotEquals(firstSessionToken, ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken()); + session.close(); + } + + @Test + public void testSTSCredentialExpiryTimeIsBoundToOAuthExpiryTime() throws BackgroundException { + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); + host.setProperty("s3.assumerole.durationseconds", "900"); + assertEquals(new HostPreferences(host).getInteger("s3.assumerole.durationseconds"), 900); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + + // assert that STS credentials expires with the oAuth token even though the duration seconds is valid for longer + STSCredentialsRequestInterceptor authorizationService = session.getAuthorizationService(); + assertTrue(40 > ((authorizationService.getStsExpiryInMilliseconds() - System.currentTimeMillis()) / 1000)); + assertEquals(Optional.of(authorizationService.getStsExpiryInMilliseconds()).get() / 1000, authorizationService.getTokens().getExpiryInMilliseconds() / 1000); + + session.close(); + } + + /** only use with the below specified changes in the keycloak config json file and run as separate test + * set config keycloak-realm.json: + * "access.token.lifespan": "930" + * "ssoSessionMaxLifespan": 1100, + */ + /*@Test + public void testSTSCredentialsExpiredValidOAuthToken() throws BackgroundException, InterruptedException { + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); + host.setProperty("s3.assumerole.durationseconds", "900"); + assertEquals(new HostPreferences(host).getInteger("s3.assumerole.durationseconds"), 900); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + + STSCredentialsRequestInterceptor authorizationService = session.getAuthorizationService(); + String firstAccessToken = authorizationService.getTokens().getAccessToken(); + String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); + Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + Thread.sleep(1000 * 910); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); + assertEquals(firstAccessToken, authorizationService.getTokens().getAccessToken()); + }*/ + + /** + * This test fails if the x-minio Headers are not read because of InvalidAccessKeyId error code which has no response body. + * Adjust the sleep time according to the network latency + */ +// @Test +// public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws BackgroundException, InterruptedException { +// final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); +// final S3Session session = new S3Session(host); +// session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); +// session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); +// +// String firstAccessToken = session.getAuthorizationService().getTokens().getIdToken(); +// String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); +// +// // Time of latency may vary and so the time needs to be adjusted accordingly +// Thread.sleep(28820); +// Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); +// assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); +// +// assertNotEquals(firstAccessToken, session.getAuthorizationService().getTokens().getIdToken()); +// assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); +// session.close(); +// } +} \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthorizationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthorizationTest.java new file mode 100644 index 00000000000..2237bb3fe8a --- /dev/null +++ b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthorizationTest.java @@ -0,0 +1,103 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.AlphanumericRandomStringService; +import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.DisabledCancelCallback; +import ch.cyberduck.core.DisabledConnectionCallback; +import ch.cyberduck.core.DisabledHostKeyCallback; +import ch.cyberduck.core.DisabledLoginCallback; +import ch.cyberduck.core.Host; +import ch.cyberduck.core.Path; +import ch.cyberduck.core.exception.AccessDeniedException; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.features.Delete; +import ch.cyberduck.core.proxy.Proxy; +import ch.cyberduck.core.s3.S3AccessControlListFeature; +import ch.cyberduck.core.s3.S3DefaultDeleteFeature; +import ch.cyberduck.core.s3.S3FindFeature; +import ch.cyberduck.core.s3.S3ReadFeature; +import ch.cyberduck.core.s3.S3Session; +import ch.cyberduck.core.s3.S3TouchFeature; +import ch.cyberduck.core.transfer.TransferStatus; +import ch.cyberduck.test.TestcontainerTest; + +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.Collections; +import java.util.EnumSet; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@Category(TestcontainerTest.class) +public class OidcAuthorizationTest extends AbstractOidcTest { + + @Test + public void testAuthorizationFindBucket() throws BackgroundException { + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + final Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + session.close(); + } + + @Test + public void testAuthorizationUserReadAccessOnBucket() throws BackgroundException { + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + final TransferStatus status = new TransferStatus(); + final Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); + new S3ReadFeature(session).read(new Path(container, "testfile.txt", EnumSet.of(Path.Type.file)), status, new DisabledConnectionCallback()); + session.close(); + } + + @Test + public void testAuthorizationWritePermissionOnBucket() throws BackgroundException { + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + final Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); + final Path test = new Path(container, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)); + new S3TouchFeature(session, new S3AccessControlListFeature(session)).touch(test, new TransferStatus()); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(test)); + new S3DefaultDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); + assertFalse(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(test)); + session.close(); + } + + @Test(expected = AccessDeniedException.class) + public void testAuthorizationNoWritePermissionOnBucket() throws BackgroundException { + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + final Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); + final Path test = new Path(container, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)); + new S3TouchFeature(session, new S3AccessControlListFeature(session)).touch(test, new TransferStatus()); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(test)); + new S3DefaultDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); + assertFalse(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(test)); + session.close(); + } +} \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java b/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java new file mode 100644 index 00000000000..598d1cf049d --- /dev/null +++ b/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java @@ -0,0 +1,43 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Profile; +import ch.cyberduck.core.ProtocolFactory; +import ch.cyberduck.core.s3.S3Protocol; +import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; + +import org.junit.Test; + +import java.util.Collections; +import java.util.HashSet; + +import static org.junit.Assert.*; + +public class S3OidcProfileTest { + + @Test + public void testDefaultProfile() throws Exception { + final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); + final Profile profile = new ProfilePlistReader(factory).read( + this.getClass().getResourceAsStream("/S3-OIDC-Testing.cyberduckprofile")); + assertEquals("minio", profile.getOAuthClientId()); + assertEquals("password", profile.getOAuthClientSecret()); + assertNotNull(profile.getOAuthAuthorizationUrl()); + assertNotNull(profile.getOAuthTokenUrl()); + assertFalse(profile.getOAuthScopes().isEmpty()); + } +} diff --git a/s3/src/test/resources/S3-OIDC-Google-Testing.cyberduckprofile b/s3/src/test/resources/S3-OIDC-Google-Testing.cyberduckprofile new file mode 100644 index 00000000000..08ef3974852 --- /dev/null +++ b/s3/src/test/resources/S3-OIDC-Google-Testing.cyberduckprofile @@ -0,0 +1,67 @@ + + + + + + + + + Protocol + s3 + Vendor + s3-sts + Bundled + + Description + S3 STS AssumeRoleWithWebIdentity + Default Hostname + testminiogoogle.duckdns.org + Default Nickname + MinIO-Google-Webbased-Test + Scheme + https + Authorization + AuthorizationCode + OAuth Authorization Url + https://accounts.google.com/o/oauth2/v2/auth + OAuth Token Url + https://oauth2.googleapis.com/token + OAuth Client ID + Your-Google-Client-ID + OAuth Client Secret + Your-Google-CLient-Secret + OAuth Redirect Url + https://cyberduck.io/oauth/ + STS Endpoint + https://testminiogoogle.duckdns.org + Scopes + + openid + profile + + Password Configurable + + Username Configurable + + Path Configurable + + Properties + + s3.bucket.virtualhost.disable=true + s3.assumerole.rolearn=arn:minio:iam:::role/Your-Configured-Role-ID + + + diff --git a/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile b/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile new file mode 100644 index 00000000000..5790f70cb4d --- /dev/null +++ b/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile @@ -0,0 +1,66 @@ + + + + + + + Protocol + s3 + Vendor + s3-sts + Bundled + + Description + S3 STS AssumeRoleWithWebIdentity + Default Hostname + localhost + Default Port + 9000 + Default Nickname + MinIO + Scheme + http + Authorization + PasswordGrant + OAuth Authorization Url + http://localhost:8080/realms/cyberduckrealm/protocol/openid-connect/auth + OAuth Token Url + http://localhost:8080/realms/cyberduckrealm/protocol/openid-connect/token + OAuth Client ID + minio + OAuth Client Secret + password + OAuth Redirect Url + x-cyberduck-action:oauth + STS Endpoint + http://localhost:9000 + Scopes + + openid + minio-authorization + + Password Configurable + + Username Configurable + + Path Configurable + + Properties + + s3.bucket.virtualhost.disable=true + + + diff --git a/s3/src/test/resources/oidcTestcontainer/docker-compose.yml b/s3/src/test/resources/oidcTestcontainer/docker-compose.yml new file mode 100644 index 00000000000..6a44e2c183c --- /dev/null +++ b/s3/src/test/resources/oidcTestcontainer/docker-compose.yml @@ -0,0 +1,89 @@ +version: '3' + +services: + + keycloak: + hostname: keycloak + build: keycloak + ports: + - "8080:8080" + environment: + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + KEYCLOAK_IMPORT: /tmp/keycloak-realm.json + PROXY_ADDRESS_FORWARDING: "true" + KEYCLOAK_LOGLEVEL: DEBUG + DB_VENDOR: h2 + KC_HEALTH_ENABLED: "true" + KC_METRICS_ENABLED: "true" + volumes: + - ./keycloak/keycloak-realm.json:/opt/keycloak/data/import/keycloak-realm.json + command: start-dev --import-realm --db=dev-mem --health-enabled=true + networks: + - testContainerNetwork + + healthcheck: + hostname: healthcheck + image: busybox + depends_on: + keycloak: + condition: service_started + command: sh -c "until wget -q -O- http://keycloak:8080/realms/cyberduckrealm/.well-known/openid-configuration >/dev/null 2>&1; do sleep 1; done" + healthcheck: + test: ["CMD-SHELL", "exit 0"] + interval: 1s + timeout: 1s + retries: 5 + networks: + - testContainerNetwork + + minio: + hostname: minio + build: minio + depends_on: + healthcheck: + condition: service_completed_successfully + restart: on-failure + ports: + - "9000:9000" + - "9001:9001" + volumes: + - ./data:/data + environment: + MINIO_ROOT_USER: cyberduckAccessKey + MINIO_ROOT_PASSWORD: cyberduckSecretKey + MINIO_IDENTITY_OPENID_CONFIG_URL: http://keycloak:8080/realms/cyberduckrealm/.well-known/openid-configuration + MINIO_IDENTITY_OPENID_CLIENT_ID: minio + MINIO_IDENTITY_OPENID_CLIENT_SECRET: password + MINIO_IDENTITY_OPENID_DISPLAY_NAME: SSO_Keycloak + MINIO_IDENTITY_OPENID_SCOPES: openid,minio-authorization + MINIO_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC: "on" + healthcheck: + test: [ "CMD-SHELL", "curl --fail http://minio:9001/login || exit 1" ] + interval: 10s + retries: 5 + command: server /data --console-address :9001 + networks: + - testContainerNetwork + + createbuckets: + hostname: bucketcreator + image: minio/mc + depends_on: + - minio + volumes: + - ./testfile.txt:/testfile.txt + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 cyberduckAccessKey cyberduckSecretKey; + /usr/bin/mc mb myminio/cyberduckbucket; + /usr/bin/mc policy set public myminio/cyberduckbucket; + /usr/bin/mc share upload --recursive myminio/mydata; + /usr/bin/mc cp /testfile.txt myminio/cyberduckbucket/testfile.txt; + exit 0; + " + networks: + - testContainerNetwork + +networks: + testContainerNetwork: \ No newline at end of file diff --git a/s3/src/test/resources/oidcTestcontainer/keycloak/Dockerfile b/s3/src/test/resources/oidcTestcontainer/keycloak/Dockerfile new file mode 100644 index 00000000000..d5cbb0ca507 --- /dev/null +++ b/s3/src/test/resources/oidcTestcontainer/keycloak/Dockerfile @@ -0,0 +1 @@ +FROM quay.io/keycloak/keycloak:21.1.1 \ No newline at end of file diff --git a/s3/src/test/resources/oidcTestcontainer/keycloak/keycloak-realm.json b/s3/src/test/resources/oidcTestcontainer/keycloak/keycloak-realm.json new file mode 100644 index 00000000000..bc77ca959ff --- /dev/null +++ b/s3/src/test/resources/oidcTestcontainer/keycloak/keycloak-realm.json @@ -0,0 +1,2260 @@ +{ + "id": "e1afe7e1-b8f5-4032-b161-bac1d4c6455c", + "realm": "cyberduckrealm", + "notBefore": 0, + "defaultSignatureAlgorithm": "RS256", + "revokeRefreshToken": false, + "refreshTokenMaxReuse": 0, + "accessTokenLifespan": 300, + "accessTokenLifespanForImplicitFlow": 900, + "ssoSessionIdleTimeout": 180, + "ssoSessionMaxLifespan": 300, + "ssoSessionIdleTimeoutRememberMe": 0, + "ssoSessionMaxLifespanRememberMe": 0, + "offlineSessionIdleTimeout": 2592000, + "offlineSessionMaxLifespanEnabled": false, + "offlineSessionMaxLifespan": 5184000, + "clientSessionIdleTimeout": 0, + "clientSessionMaxLifespan": 0, + "clientOfflineSessionIdleTimeout": 0, + "clientOfflineSessionMaxLifespan": 0, + "accessCodeLifespan": 60, + "accessCodeLifespanUserAction": 300, + "accessCodeLifespanLogin": 1800, + "actionTokenGeneratedByAdminLifespan": 43200, + "actionTokenGeneratedByUserLifespan": 300, + "oauth2DeviceCodeLifespan": 600, + "oauth2DevicePollingInterval": 5, + "enabled": true, + "sslRequired": "external", + "registrationAllowed": false, + "registrationEmailAsUsername": false, + "rememberMe": false, + "verifyEmail": false, + "loginWithEmailAllowed": true, + "duplicateEmailsAllowed": false, + "resetPasswordAllowed": false, + "editUsernameAllowed": false, + "bruteForceProtected": false, + "permanentLockout": false, + "maxFailureWaitSeconds": 900, + "minimumQuickLoginWaitSeconds": 60, + "waitIncrementSeconds": 60, + "quickLoginCheckMilliSeconds": 1000, + "maxDeltaTimeSeconds": 43200, + "failureFactor": 30, + "roles": { + "realm": [ + { + "id": "8194a695-2e25-4f0f-b327-72ff146af1d3", + "name": "default-roles-cyberduckrealm", + "description": "${role_default-roles}", + "composite": true, + "composites": { + "realm": [ + "offline_access", + "uma_authorization" + ], + "client": { + "account": [ + "manage-account", + "view-profile" + ] + } + }, + "clientRole": false, + "containerId": "e1afe7e1-b8f5-4032-b161-bac1d4c6455c", + "attributes": {} + }, + { + "id": "f482863a-6db4-4474-8f74-113fbd1e1936", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "e1afe7e1-b8f5-4032-b161-bac1d4c6455c", + "attributes": {} + }, + { + "id": "68b4bf91-1719-47d5-96e7-d61c23861ff3", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "e1afe7e1-b8f5-4032-b161-bac1d4c6455c", + "attributes": {} + } + ], + "client": { + "realm-management": [ + { + "id": "0b92f964-d7ac-406e-9b1b-686a4c9294f6", + "name": "query-clients", + "description": "${role_query-clients}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "265eafec-47c8-4c3d-9943-5ba90cbb89b5", + "name": "view-authorization", + "description": "${role_view-authorization}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "052b711b-98cc-4d5e-b82a-4ae771376872", + "name": "view-realm", + "description": "${role_view-realm}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "0e793a88-ec20-4f49-9618-7d15b209991c", + "name": "realm-admin", + "description": "${role_realm-admin}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients", + "view-authorization", + "view-realm", + "create-client", + "manage-users", + "manage-events", + "manage-authorization", + "query-realms", + "view-identity-providers", + "query-users", + "view-clients", + "manage-realm", + "view-users", + "query-groups", + "impersonation", + "manage-identity-providers", + "manage-clients", + "view-events" + ] + } + }, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "49fb3abc-a339-454a-aa7f-0645c8d63fa9", + "name": "create-client", + "description": "${role_create-client}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "6f0a4e65-02ac-4dbf-961a-a038ee4a80a1", + "name": "manage-users", + "description": "${role_manage-users}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "6d5f8beb-2257-4112-b0a9-9dadd8a5bb85", + "name": "manage-events", + "description": "${role_manage-events}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "2dc59d62-411f-4d5a-94af-a65924e0a1c1", + "name": "manage-authorization", + "description": "${role_manage-authorization}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "e4d3c86a-6d45-406c-b9e7-530b427ff4a1", + "name": "query-realms", + "description": "${role_query-realms}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "fe3c4b94-25c4-4ee4-947e-d6ebae4a2432", + "name": "view-identity-providers", + "description": "${role_view-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "7555375d-6bb7-41de-8acf-c43fa059a1ba", + "name": "query-users", + "description": "${role_query-users}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "f915e51e-9887-4896-822f-6b73c1033de3", + "name": "view-clients", + "description": "${role_view-clients}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients" + ] + } + }, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "be4ad746-7105-4f30-8c8a-e3a6eafb6f0a", + "name": "manage-realm", + "description": "${role_manage-realm}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "0b361b2e-f79c-4c1f-8f75-4b077b50eb07", + "name": "view-users", + "description": "${role_view-users}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-users", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "8180252b-b94f-4f0a-8755-6ca517bbdcb1", + "name": "impersonation", + "description": "${role_impersonation}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "09f264ef-8c4f-4e51-8051-c1364adf8bec", + "name": "query-groups", + "description": "${role_query-groups}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "06b213ed-9bca-4071-9c9f-c41ccecc81d9", + "name": "manage-identity-providers", + "description": "${role_manage-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "2f6f693a-ddcd-45a4-bac8-b4ab83573399", + "name": "manage-clients", + "description": "${role_manage-clients}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + }, + { + "id": "e44f8384-814b-4fdf-90e2-246472751829", + "name": "view-events", + "description": "${role_view-events}", + "composite": false, + "clientRole": true, + "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "attributes": {} + } + ], + "security-admin-console": [], + "admin-cli": [], + "minio": [], + "account-console": [], + "broker": [ + { + "id": "c6d3cbe9-7852-44d6-b956-03208cdf590b", + "name": "read-token", + "description": "${role_read-token}", + "composite": false, + "clientRole": true, + "containerId": "40e2c9c5-ca68-43e6-bdca-1bec70485771", + "attributes": {} + } + ], + "account": [ + { + "id": "2010e2cc-7804-45f4-9ffe-8ce50460366d", + "name": "view-consent", + "description": "${role_view-consent}", + "composite": false, + "clientRole": true, + "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", + "attributes": {} + }, + { + "id": "7cc96e86-6f91-44d8-b3d2-6a92b26e23b9", + "name": "view-applications", + "description": "${role_view-applications}", + "composite": false, + "clientRole": true, + "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", + "attributes": {} + }, + { + "id": "68c10534-629e-403b-bba2-09bd9de3f56d", + "name": "manage-consent", + "description": "${role_manage-consent}", + "composite": true, + "composites": { + "client": { + "account": [ + "view-consent" + ] + } + }, + "clientRole": true, + "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", + "attributes": {} + }, + { + "id": "413c38d0-50bf-4733-8320-6b3c004e3163", + "name": "manage-account", + "description": "${role_manage-account}", + "composite": true, + "composites": { + "client": { + "account": [ + "manage-account-links" + ] + } + }, + "clientRole": true, + "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", + "attributes": {} + }, + { + "id": "0439c628-4012-4d0a-a51c-522d51b204e6", + "name": "view-profile", + "description": "${role_view-profile}", + "composite": false, + "clientRole": true, + "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", + "attributes": {} + }, + { + "id": "a697a2f8-0517-4658-a5aa-89e79705ce2b", + "name": "manage-account-links", + "description": "${role_manage-account-links}", + "composite": false, + "clientRole": true, + "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", + "attributes": {} + }, + { + "id": "f7376ccf-182c-4227-bb8c-16f7a101d913", + "name": "delete-account", + "description": "${role_delete-account}", + "composite": false, + "clientRole": true, + "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", + "attributes": {} + } + ] + } + }, + "groups": [], + "defaultRole": { + "id": "8194a695-2e25-4f0f-b327-72ff146af1d3", + "name": "default-roles-cyberduckrealm", + "description": "${role_default-roles}", + "composite": true, + "clientRole": false, + "containerId": "e1afe7e1-b8f5-4032-b161-bac1d4c6455c" + }, + "requiredCredentials": [ + "password" + ], + "otpPolicyType": "totp", + "otpPolicyAlgorithm": "HmacSHA1", + "otpPolicyInitialCounter": 0, + "otpPolicyDigits": 6, + "otpPolicyLookAheadWindow": 1, + "otpPolicyPeriod": 30, + "otpSupportedApplications": [ + "FreeOTP", + "Google Authenticator" + ], + "webAuthnPolicyRpEntityName": "keycloak", + "webAuthnPolicySignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyRpId": "", + "webAuthnPolicyAttestationConveyancePreference": "not specified", + "webAuthnPolicyAuthenticatorAttachment": "not specified", + "webAuthnPolicyRequireResidentKey": "not specified", + "webAuthnPolicyUserVerificationRequirement": "not specified", + "webAuthnPolicyCreateTimeout": 0, + "webAuthnPolicyAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyAcceptableAaguids": [], + "webAuthnPolicyPasswordlessRpEntityName": "keycloak", + "webAuthnPolicyPasswordlessSignatureAlgorithms": [ + "ES256" + ], + "webAuthnPolicyPasswordlessRpId": "", + "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified", + "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", + "webAuthnPolicyPasswordlessRequireResidentKey": "not specified", + "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified", + "webAuthnPolicyPasswordlessCreateTimeout": 0, + "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false, + "webAuthnPolicyPasswordlessAcceptableAaguids": [], + "scopeMappings": [ + { + "clientScope": "offline_access", + "roles": [ + "offline_access" + ] + } + ], + "clientScopeMappings": { + "account": [ + { + "client": "account-console", + "roles": [ + "manage-account" + ] + } + ] + }, + "clients": [ + { + "id": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", + "clientId": "account", + "name": "${client_account}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/cyberduckrealm/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/cyberduckrealm/account/*" + ], + "webOrigins": ["*"], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "dd489003-815a-4c63-acf4-0a7f8b41bd79", + "clientId": "account-console", + "name": "${client_account-console}", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/cyberduckrealm/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/realms/cyberduckrealm/account/*" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "daf73eea-eebe-4823-ae33-e1276b9a6ddc", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "c902977d-6c6a-4bf0-8949-565ab2bfb297", + "clientId": "admin-cli", + "name": "${client_admin-cli}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": false, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "40e2c9c5-ca68-43e6-bdca-1bec70485771", + "clientId": "broker", + "name": "${client_broker}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "8af8c8a6-7778-4a39-bd1f-9a2879fa00c1", + "clientId": "minio", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/cyberduckrealm/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": true, + "clientAuthenticatorType": "client-secret", + "secret": "password", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "http://minio:9000" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "access.token.lifespan": "30", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "oauth2.device.authorization.grant.enabled": "false", + "use.jwks.url": "true", + "backchannel.logout.revoke.offline.tokens": "false", + "s3IdentityProviderEndpoint": "http://minio:9000", + "s3IdentityProviderSecretKey": "cyberduckSecretKey", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "oidc.ciba.grant.enabled": "false", + "use.jwks.string": "false", + "backchannel.logout.session.required": "false", + "client_credentials.use_refresh_token": "false", + "require.pushed.authorization.requests": "false", + "saml.client.signature": "false", + "s3IdentityProviderAccessKey": "cyberduckAccessKey", + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.encrypt": "false", + "saml.server.signature": "false", + "exclude.session.state.from.auth.response": "false", + "saml.artifact.binding": "false", + "saml_force_name_id_format": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "false", + "s3IdentityProviderEnabled": "true", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "minio-authorization", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "82124274-9a44-4ae4-8a26-f9f938cbcadc", + "clientId": "realm-management", + "name": "${client_realm-management}", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": true, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": {}, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + }, + { + "id": "70c5b895-1dbd-4b32-964a-db47c5f5afa9", + "clientId": "security-admin-console", + "name": "${client_security-admin-console}", + "rootUrl": "${authAdminUrl}", + "baseUrl": "/admin/cyberduckrealm/console/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "/admin/cyberduckrealm/console/*" + ], + "webOrigins": [ + "+" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": true, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "pkce.code.challenge.method": "S256" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": false, + "nodeReRegistrationTimeout": 0, + "protocolMappers": [ + { + "id": "a4779ad7-c6ac-4050-924e-5e851c6580fd", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + } + ], + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ] + } + ], + "clientScopes": [ + { + "id": "0941c887-88c7-455a-9dae-b0982c4a8ba0", + "name": "phone", + "description": "OpenID Connect built-in scope: phone", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${phoneScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "cb825f68-9f26-4feb-a111-5cf2d214f77c", + "name": "phone number verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumberVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number_verified", + "jsonType.label": "boolean" + } + }, + { + "id": "becf72e9-7bbf-4737-a937-64ea66603f4b", + "name": "phone number", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "phoneNumber", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "phone_number", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "23647c33-d481-4b6a-a822-85d93b2950be", + "name": "web-origins", + "description": "OpenID Connect scope for add allowed web origins to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "false", + "consent.screen.text": "" + }, + "protocolMappers": [ + { + "id": "868104af-32f4-4f67-8652-89f7bc097044", + "name": "allowed web origins", + "protocol": "openid-connect", + "protocolMapper": "oidc-allowed-origins-mapper", + "consentRequired": false, + "config": {} + } + ] + }, + { + "id": "52399e9d-b715-478f-b3d5-3c4c41a19492", + "name": "roles", + "description": "OpenID Connect scope for add user roles to the access token", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false", + "display.on.consent.screen": "true", + "consent.screen.text": "${rolesScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "27648e0c-ff42-4fe8-85a1-3a13b3fc4187", + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String", + "multivalued": "true" + } + }, + { + "id": "9b824cc3-1685-49b3-8bc2-a3ffaecb1b71", + "name": "audience resolve", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-resolve-mapper", + "consentRequired": false, + "config": {} + }, + { + "id": "0bbebb2f-d53a-41d8-9f81-fe85b7dbfef1", + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "user.attribute": "foo", + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String", + "multivalued": "true" + } + } + ] + }, + { + "id": "adf7c33f-01cb-4d63-9186-646ba28daabd", + "name": "address", + "description": "OpenID Connect built-in scope: address", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${addressScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "ff1c2e7f-6170-4915-9a15-a2f796665429", + "name": "address", + "protocol": "openid-connect", + "protocolMapper": "oidc-address-mapper", + "consentRequired": false, + "config": { + "user.attribute.formatted": "formatted", + "user.attribute.country": "country", + "user.attribute.postal_code": "postal_code", + "userinfo.token.claim": "true", + "user.attribute.street": "street", + "id.token.claim": "true", + "user.attribute.region": "region", + "access.token.claim": "true", + "user.attribute.locality": "locality" + } + } + ] + }, + { + "id": "c1326a37-f3b3-4d45-9c0e-d92e3d21b00f", + "name": "role_list", + "description": "SAML role list", + "protocol": "saml", + "attributes": { + "consent.screen.text": "${samlRoleListScopeConsentText}", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "a0cbecae-fa99-493e-b840-109e8daeadb5", + "name": "role list", + "protocol": "saml", + "protocolMapper": "saml-role-list-mapper", + "consentRequired": false, + "config": { + "single": "false", + "attribute.nameformat": "Basic", + "attribute.name": "Role" + } + } + ] + }, + { + "id": "736f4ffd-516a-4608-bc7c-dead9baf4d4c", + "name": "microprofile-jwt", + "description": "Microprofile - JWT built-in scope", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "id": "d45c912b-c4df-49c9-afd9-7b80451f52fd", + "name": "upn", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "upn", + "jsonType.label": "String" + } + }, + { + "id": "303703a4-adbb-4f7a-a16b-41bdacad23fa", + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "multivalued": "true", + "user.attribute": "foo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "7169650f-742e-4f08-9e47-ea57342bea74", + "name": "email", + "description": "OpenID Connect built-in scope: email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${emailScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "dce18712-0535-475c-97b7-d35b9254a2f1", + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "email", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email", + "jsonType.label": "String" + } + }, + { + "id": "bd1489c6-68d7-4311-9e95-875d18376c15", + "name": "email verified", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "emailVerified", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "email_verified", + "jsonType.label": "boolean" + } + } + ] + }, + { + "id": "26b0e632-2359-4570-95b7-573fa84aa01a", + "name": "offline_access", + "description": "OpenID Connect built-in scope: offline_access", + "protocol": "openid-connect", + "attributes": { + "consent.screen.text": "${offlineAccessScopeConsentText}", + "display.on.consent.screen": "true" + } + }, + { + "id": "5df385b5-e928-4c70-ad48-817680493427", + "name": "profile", + "description": "OpenID Connect built-in scope: profile", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true", + "consent.screen.text": "${profileScopeConsentText}" + }, + "protocolMappers": [ + { + "id": "341891af-ee4c-4c39-b5f1-e774a8665835", + "name": "profile", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "profile", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "profile", + "jsonType.label": "String" + } + }, + { + "id": "d9141fff-4bb4-414c-a98e-773e195d45fd", + "name": "locale", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "locale", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "locale", + "jsonType.label": "String" + } + }, + { + "id": "02381dc5-699e-4763-abee-156c77d3bb10", + "name": "nickname", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "nickname", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "nickname", + "jsonType.label": "String" + } + }, + { + "id": "897f4d28-b959-4154-8cc8-45331d3a8702", + "name": "zoneinfo", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "zoneinfo", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "zoneinfo", + "jsonType.label": "String" + } + }, + { + "id": "e0fe20ed-dbe1-4c87-8ecb-0e986461036d", + "name": "full name", + "protocol": "openid-connect", + "protocolMapper": "oidc-full-name-mapper", + "consentRequired": false, + "config": { + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "id": "ed3bdec1-29ff-4c46-90fb-8dfaa602b5c0", + "name": "birthdate", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "birthdate", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "birthdate", + "jsonType.label": "String" + } + }, + { + "id": "423ab9b6-a48a-436c-aad1-43875b1ab9f5", + "name": "picture", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "picture", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "picture", + "jsonType.label": "String" + } + }, + { + "id": "4e8de822-abfd-4194-a30e-c6c50b337abd", + "name": "updated at", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "updatedAt", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "updated_at", + "jsonType.label": "String" + } + }, + { + "id": "7b937773-ec5c-4137-be86-f85d252aea2d", + "name": "middle name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "middleName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "middle_name", + "jsonType.label": "String" + } + }, + { + "id": "c9ce2856-dc8a-4fa6-a3eb-2b0376aabfab", + "name": "given name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "firstName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "given_name", + "jsonType.label": "String" + } + }, + { + "id": "1ce2cd9a-174c-4f0e-b1e3-14b9172c65d5", + "name": "family name", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "lastName", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "family_name", + "jsonType.label": "String" + } + }, + { + "id": "c29dccad-6e2a-4952-9c54-0693640acf3b", + "name": "gender", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "gender", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "gender", + "jsonType.label": "String" + } + }, + { + "id": "adbe68c1-ae8f-46e3-a06e-6efb18e8878c", + "name": "username", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "username", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "preferred_username", + "jsonType.label": "String" + } + }, + { + "id": "f8254933-a164-4407-9a7c-079f88e47f57", + "name": "website", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "userinfo.token.claim": "true", + "user.attribute": "website", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "website", + "jsonType.label": "String" + } + } + ] + }, + { + "id": "00e3796f-ae81-403d-a6e3-757a4481e050", + "name": "minio-authorization", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "id": "bd14ce8f-e019-4094-92e1-0b2ea1ba84e1", + "name": "minio-policy-mapper", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "aggregate.attrs": "true", + "multivalued": "true", + "userinfo.token.claim": "true", + "user.attribute": "policy", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "policy", + "jsonType.label": "String" + } + } + ] + } + ], + "defaultDefaultClientScopes": [ + "web-origins", + "roles", + "profile", + "email", + "role_list" + ], + "defaultOptionalClientScopes": [ + "phone", + "offline_access", + "microprofile-jwt", + "address" + ], + "browserSecurityHeaders": { + "contentSecurityPolicyReportOnly": "", + "xContentTypeOptions": "nosniff", + "xRobotsTag": "none", + "xFrameOptions": "SAMEORIGIN", + "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';", + "xXSSProtection": "1; mode=block", + "strictTransportSecurity": "max-age=31536000; includeSubDomains" + }, + "smtpServer": {}, + "eventsEnabled": false, + "eventsListeners": [ + "jboss-logging" + ], + "enabledEventTypes": [], + "adminEventsEnabled": false, + "adminEventsDetailsEnabled": false, + "identityProviders": [], + "identityProviderMappers": [], + "components": { + "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [ + { + "id": "cf127fb5-bcf4-461e-849a-4caf70fdb100", + "name": "Max Clients Limit", + "providerId": "max-clients", + "subType": "anonymous", + "subComponents": {}, + "config": { + "max-clients": [ + "200" + ] + } + }, + { + "id": "ad413384-9c48-4bfa-b71f-312893f50689", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "oidc-address-mapper", + "oidc-full-name-mapper", + "saml-user-attribute-mapper", + "saml-role-list-mapper", + "oidc-usermodel-property-mapper", + "saml-user-property-mapper", + "oidc-sha256-pairwise-sub-mapper", + "oidc-usermodel-attribute-mapper" + ] + } + }, + { + "id": "18a947be-6679-4bcf-98be-afed9b0cf80e", + "name": "Full Scope Disabled", + "providerId": "scope", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "fb98c161-3ea2-4d75-b4ac-bbf8e4f95060", + "name": "Consent Required", + "providerId": "consent-required", + "subType": "anonymous", + "subComponents": {}, + "config": {} + }, + { + "id": "3e76db0d-ba83-4fd9-b46f-60c76869791f", + "name": "Allowed Protocol Mapper Types", + "providerId": "allowed-protocol-mappers", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allowed-protocol-mapper-types": [ + "saml-user-attribute-mapper", + "oidc-address-mapper", + "oidc-usermodel-property-mapper", + "saml-role-list-mapper", + "oidc-usermodel-attribute-mapper", + "saml-user-property-mapper", + "oidc-full-name-mapper", + "oidc-sha256-pairwise-sub-mapper" + ] + } + }, + { + "id": "5777b57a-f847-4fa9-9ea0-3776bb98ba81", + "name": "Trusted Hosts", + "providerId": "trusted-hosts", + "subType": "anonymous", + "subComponents": {}, + "config": { + "host-sending-registration-request-must-match": [ + "true" + ], + "client-uris-must-match": [ + "true" + ] + } + }, + { + "id": "57ffa16e-0cc8-4144-96f9-811bdcc68095", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "anonymous", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + }, + { + "id": "7e5883b7-051f-4aaf-aca4-1b3f90dc5d1f", + "name": "Allowed Client Scopes", + "providerId": "allowed-client-templates", + "subType": "authenticated", + "subComponents": {}, + "config": { + "allow-default-scopes": [ + "true" + ] + } + } + ], + "org.keycloak.userprofile.UserProfileProvider": [ + { + "id": "5cf69331-9fe2-4b0c-8632-98fd2deb393b", + "providerId": "declarative-user-profile", + "subComponents": {}, + "config": {} + } + ], + "org.keycloak.keys.KeyProvider": [ + { + "id": "f678f09a-8dcf-44f2-bb92-6e6a8a215d81", + "name": "hmac-generated", + "providerId": "hmac-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ], + "algorithm": [ + "HS256" + ] + } + }, + { + "id": "81086736-cd47-4b8e-8f68-8d7e27814537", + "name": "rsa-generated", + "providerId": "rsa-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + }, + { + "id": "028d8122-2150-40c9-9770-144dc7516792", + "name": "rsa-enc-generated", + "providerId": "rsa-enc-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ], + "algorithm": [ + "RSA-OAEP" + ] + } + }, + { + "id": "d607e956-c8d0-424f-ae9b-476caac12f3a", + "name": "aes-generated", + "providerId": "aes-generated", + "subComponents": {}, + "config": { + "priority": [ + "100" + ] + } + } + ] + }, + "internationalizationEnabled": false, + "supportedLocales": [], + "authenticationFlows": [ + { + "id": "8f395abd-02f0-4fca-9142-73120a6af711", + "alias": "Account verification options", + "description": "Method with which to verity the existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-email-verification", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Verify Existing Account by Re-authentication", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "7a7f2eff-b041-484b-b93d-1df4cd916496", + "alias": "Authentication Options", + "description": "Authentication options.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "basic-auth", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "basic-auth-otp", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "98f32dcd-203e-4179-8000-b3d6fc42dd35", + "alias": "Browser - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "95b7e6df-f963-4930-950e-46c6eff90480", + "alias": "Direct Grant - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "9e55e789-b7a3-4c32-a843-60a2ecf0518b", + "alias": "First broker login - Conditional OTP", + "description": "Flow to determine if the OTP is required for the authentication", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-otp-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "c2c7436f-b39a-4af4-a596-b1c0c343dd18", + "alias": "Handle Existing Account", + "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-confirm-link", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Account verification options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "5a707131-58e2-4cc3-8d62-ddfaa12a2ee8", + "alias": "Reset - Conditional OTP", + "description": "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "conditional-user-configured", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-otp", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "4612185d-8e99-4fb9-b772-4251c96af818", + "alias": "User creation or linking", + "description": "Flow for the existing/non-existing user alternatives", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "create unique user config", + "authenticator": "idp-create-user-if-unique", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 20, + "flowAlias": "Handle Existing Account", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "6d129dc6-2fd3-4cf6-8bcd-5e207cf832d6", + "alias": "Verify Existing Account by Re-authentication", + "description": "Reauthentication of existing account", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "idp-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "First broker login - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "259142b1-b8ad-4fb8-a464-2d53b999a0cd", + "alias": "browser", + "description": "browser based authentication", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-cookie", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "auth-spnego", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "identity-provider-redirector", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 25, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "ALTERNATIVE", + "priority": 30, + "flowAlias": "forms", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "45be8796-9ea8-4b47-91d0-b3ed012b1030", + "alias": "clients", + "description": "Base authentication for clients", + "providerId": "client-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "client-secret", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-secret-jwt", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "client-x509", + "authenticatorFlow": false, + "requirement": "ALTERNATIVE", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "728be498-3ab4-4e6f-9791-79b720abc8ad", + "alias": "direct grant", + "description": "OpenID Connect Resource Owner Grant", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "direct-grant-validate-username", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "direct-grant-validate-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 30, + "flowAlias": "Direct Grant - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "b14135ca-f14d-46ac-9421-fca779668d0d", + "alias": "docker auth", + "description": "Used by Docker clients to authenticate against the IDP", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "docker-http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "551ea373-9762-4327-affb-41fc55860b9b", + "alias": "first broker login", + "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticatorConfig": "review profile config", + "authenticator": "idp-review-profile", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "User creation or linking", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "01570967-c01b-4631-a0d4-99b10c4382b3", + "alias": "forms", + "description": "Username, password, otp and other auth forms.", + "providerId": "basic-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "auth-username-password-form", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 20, + "flowAlias": "Browser - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "6a63fd0b-ca42-4658-9d8f-eb7b72341343", + "alias": "http challenge", + "description": "An authentication flow based on challenge-response HTTP Authentication Schemes", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "no-cookie-redirect", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 20, + "flowAlias": "Authentication Options", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "c05fa3bf-5b0d-4fc0-b251-bdef09cad34a", + "alias": "registration", + "description": "registration flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-page-form", + "authenticatorFlow": true, + "requirement": "REQUIRED", + "priority": 10, + "flowAlias": "registration form", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "8b021678-bb93-4514-8a86-3288d5ec0c63", + "alias": "registration form", + "description": "registration form", + "providerId": "form-flow", + "topLevel": false, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "registration-user-creation", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-profile-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 40, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-password-action", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 50, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "registration-recaptcha-action", + "authenticatorFlow": false, + "requirement": "DISABLED", + "priority": 60, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + }, + { + "id": "70720740-95ba-4cce-868a-9ba5d4ed9a8a", + "alias": "reset credentials", + "description": "Reset credentials for a user if they forgot their password or something", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "reset-credentials-choose-user", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-credential-email", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 20, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticator": "reset-password", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 30, + "userSetupAllowed": false, + "autheticatorFlow": false + }, + { + "authenticatorFlow": true, + "requirement": "CONDITIONAL", + "priority": 40, + "flowAlias": "Reset - Conditional OTP", + "userSetupAllowed": false, + "autheticatorFlow": true + } + ] + }, + { + "id": "c42ef4b2-d9ea-4e76-876c-71c05131ea27", + "alias": "saml ecp", + "description": "SAML ECP Profile Authentication Flow", + "providerId": "basic-flow", + "topLevel": true, + "builtIn": true, + "authenticationExecutions": [ + { + "authenticator": "http-basic-authenticator", + "authenticatorFlow": false, + "requirement": "REQUIRED", + "priority": 10, + "userSetupAllowed": false, + "autheticatorFlow": false + } + ] + } + ], + "authenticatorConfig": [ + { + "id": "ff127d3c-27e0-49bc-8500-7241eae7f232", + "alias": "create unique user config", + "config": { + "require.password.update.after.registration": "false" + } + }, + { + "id": "76ffa39a-94d4-480f-8cbb-8ac375865c4b", + "alias": "review profile config", + "config": { + "update.profile.on.first.login": "missing" + } + } + ], + "requiredActions": [ + { + "alias": "CONFIGURE_TOTP", + "name": "Configure OTP", + "providerId": "CONFIGURE_TOTP", + "enabled": true, + "defaultAction": false, + "priority": 10, + "config": {} + }, + { + "alias": "terms_and_conditions", + "name": "Terms and Conditions", + "providerId": "terms_and_conditions", + "enabled": false, + "defaultAction": false, + "priority": 20, + "config": {} + }, + { + "alias": "UPDATE_PASSWORD", + "name": "Update Password", + "providerId": "UPDATE_PASSWORD", + "enabled": true, + "defaultAction": false, + "priority": 30, + "config": {} + }, + { + "alias": "UPDATE_PROFILE", + "name": "Update Profile", + "providerId": "UPDATE_PROFILE", + "enabled": true, + "defaultAction": false, + "priority": 40, + "config": {} + }, + { + "alias": "VERIFY_EMAIL", + "name": "Verify Email", + "providerId": "VERIFY_EMAIL", + "enabled": true, + "defaultAction": false, + "priority": 50, + "config": {} + }, + { + "alias": "delete_account", + "name": "Delete Account", + "providerId": "delete_account", + "enabled": false, + "defaultAction": false, + "priority": 60, + "config": {} + }, + { + "alias": "update_user_locale", + "name": "Update User Locale", + "providerId": "update_user_locale", + "enabled": true, + "defaultAction": false, + "priority": 1000, + "config": {} + } + ], + "browserFlow": "browser", + "registrationFlow": "registration", + "directGrantFlow": "direct grant", + "resetCredentialsFlow": "reset credentials", + "clientAuthenticationFlow": "clients", + "dockerAuthenticationFlow": "docker auth", + "attributes": { + "cibaBackchannelTokenDeliveryMode": "poll", + "cibaExpiresIn": "120", + "cibaAuthRequestedUserHint": "login_hint", + "oauth2DeviceCodeLifespan": "600", + "oauth2DevicePollingInterval": "5", + "clientOfflineSessionMaxLifespan": "0", + "clientSessionIdleTimeout": "0", + "userProfileEnabled": "false", + "parRequestUriLifespan": "60", + "clientSessionMaxLifespan": "0", + "clientOfflineSessionIdleTimeout": "0", + "cibaInterval": "5" + }, + "keycloakVersion": "16.1.0", + "userManagedAccessAllowed": false, + "clientProfiles": { + "profiles": [] + }, + "clientPolicies": { + "policies": [] + }, + "users": [ + { + "username": "rawuser", + "enabled": true, + "email": "readandwrite@test.com", + "attributes": + { + "policy":"readwrite" + }, + "credentials": [ + { + "type": "password", + "value": "rawuser" + } + ], + "realmRoles": [ + "user" + ], + "clientRoles": { + "account": [ + "view-profile", + "manage-account" + ] + } + }, + { + "username": "rouser", + "enabled": true, + "email": "readonly@test.com", + "attributes": + { + "policy":"readonly" + }, + "credentials": [ + { + "type": "password", + "value": "rouser" + } + ], + "realmRoles": [ + "user" + ], + "clientRoles": { + "account": [ + "view-profile", + "manage-account" + ] + } + } + ] +} diff --git a/s3/src/test/resources/oidcTestcontainer/minio/Dockerfile b/s3/src/test/resources/oidcTestcontainer/minio/Dockerfile new file mode 100644 index 00000000000..cdd080d2981 --- /dev/null +++ b/s3/src/test/resources/oidcTestcontainer/minio/Dockerfile @@ -0,0 +1 @@ +FROM minio/minio:latest diff --git a/s3/src/test/resources/oidcTestcontainer/testcontainers.properties b/s3/src/test/resources/oidcTestcontainer/testcontainers.properties new file mode 100644 index 00000000000..6031841261c --- /dev/null +++ b/s3/src/test/resources/oidcTestcontainer/testcontainers.properties @@ -0,0 +1,14 @@ +# +# Copyright (c) 2002-2023 iterate GmbH. All rights reserved. +# https://cyberduck.io/ +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# \ No newline at end of file diff --git a/s3/src/test/resources/oidcTestcontainer/testfile.txt b/s3/src/test/resources/oidcTestcontainer/testfile.txt new file mode 100644 index 00000000000..af27ff4986a --- /dev/null +++ b/s3/src/test/resources/oidcTestcontainer/testfile.txt @@ -0,0 +1 @@ +This is a test file. \ No newline at end of file From e3b46c3a55cf2add2030e9ea0f2537ba271e75f0 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sun, 6 Aug 2023 15:45:48 +0200 Subject: [PATCH 02/84] Review tests. --- .../core/sts/OidcAuthenticationTest.java | 72 ++++++++----------- 1 file changed, 29 insertions(+), 43 deletions(-) diff --git a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java index 561dcb044ce..b6428906563 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java @@ -31,6 +31,7 @@ import ch.cyberduck.test.TestcontainerTest; import org.jets3t.service.security.AWSSessionCredentials; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.testcontainers.shaded.org.apache.commons.lang3.StringUtils; @@ -42,6 +43,7 @@ @Category(TestcontainerTest.class) public class OidcAuthenticationTest extends AbstractOidcTest { + @Test public void testSuccessfulLoginViaOidc() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); @@ -85,7 +87,7 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); - String firstAccessToken = session.getAuthorizationService().getTokens().getIdToken(); + String firstAccessToken = host.getCredentials().getOauth().getIdToken(); String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); String firstSessionToken = ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken(); @@ -94,35 +96,19 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException Thread.sleep(1100 * 30); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - assertNotEquals(firstAccessToken, session.getAuthorizationService().getTokens().getIdToken()); + assertNotEquals(firstAccessToken, host.getCredentials().getOauth().getIdToken()); assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); assertNotEquals(firstSessionToken, ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken()); session.close(); } - @Test - public void testSTSCredentialExpiryTimeIsBoundToOAuthExpiryTime() throws BackgroundException { - final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); - host.setProperty("s3.assumerole.durationseconds", "900"); - assertEquals(new HostPreferences(host).getInteger("s3.assumerole.durationseconds"), 900); - final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); - - // assert that STS credentials expires with the oAuth token even though the duration seconds is valid for longer - STSCredentialsRequestInterceptor authorizationService = session.getAuthorizationService(); - assertTrue(40 > ((authorizationService.getStsExpiryInMilliseconds() - System.currentTimeMillis()) / 1000)); - assertEquals(Optional.of(authorizationService.getStsExpiryInMilliseconds()).get() / 1000, authorizationService.getTokens().getExpiryInMilliseconds() / 1000); - - session.close(); - } - /** only use with the below specified changes in the keycloak config json file and run as separate test * set config keycloak-realm.json: * "access.token.lifespan": "930" * "ssoSessionMaxLifespan": 1100, */ - /*@Test + @Test + @Ignore public void testSTSCredentialsExpiredValidOAuthToken() throws BackgroundException, InterruptedException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); host.setProperty("s3.assumerole.durationseconds", "900"); @@ -131,38 +117,38 @@ public void testSTSCredentialsExpiredValidOAuthToken() throws BackgroundExceptio session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); - STSCredentialsRequestInterceptor authorizationService = session.getAuthorizationService(); - String firstAccessToken = authorizationService.getTokens().getAccessToken(); + String firstAccessToken = host.getCredentials().getOauth().getAccessToken(); String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); Thread.sleep(1000 * 910); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); - assertEquals(firstAccessToken, authorizationService.getTokens().getAccessToken()); - }*/ + assertEquals(firstAccessToken, host.getCredentials().getOauth().getAccessToken()); + } /** * This test fails if the x-minio Headers are not read because of InvalidAccessKeyId error code which has no response body. * Adjust the sleep time according to the network latency */ -// @Test -// public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws BackgroundException, InterruptedException { -// final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); -// final S3Session session = new S3Session(host); -// session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); -// session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); -// -// String firstAccessToken = session.getAuthorizationService().getTokens().getIdToken(); -// String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); -// -// // Time of latency may vary and so the time needs to be adjusted accordingly -// Thread.sleep(28820); -// Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); -// assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); -// -// assertNotEquals(firstAccessToken, session.getAuthorizationService().getTokens().getIdToken()); -// assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); -// session.close(); -// } + @Test + @Ignore + public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws BackgroundException, InterruptedException { + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + + String firstAccessToken = host.getCredentials().getOauth().getIdToken(); + String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); + + // Time of latency may vary and so the time needs to be adjusted accordingly + Thread.sleep(28820); + Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + + assertNotEquals(firstAccessToken, host.getCredentials().getOauth().getIdToken()); + assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); + session.close(); + } } \ No newline at end of file From fb90c10bca2eee31896017499446a4cda2cddaa1 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sun, 6 Aug 2023 23:29:43 +0200 Subject: [PATCH 03/84] Fix null pointer. --- .../cyberduck/core/oauth/OAuth2AuthorizationService.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java index 2f42b8caabc..e255fd7001f 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java @@ -360,13 +360,16 @@ else if(value instanceof Number) { } public IdTokenResponse toTokenResponse() { - return new IdTokenResponse() + final IdTokenResponse response = new IdTokenResponse() .setTokenType(tokenType) .setScope(scope) .setExpiresInSeconds(expiresInSeconds) .setAccessToken(accessToken) - .setRefreshToken(refreshToken) - .setIdToken(idToken); + .setRefreshToken(refreshToken); + if(null == idToken) { + return response; + } + return response.setIdToken(idToken); } } From 7ae0a458ea2c1f45a9689997fa47affcaf94b020 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sun, 6 Aug 2023 23:36:59 +0200 Subject: [PATCH 04/84] Set "Accept" header to request JSON response for interoperability with GitHub. --- .../ch/cyberduck/core/http/UserAgentHttpRequestInitializer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/oauth/src/main/java/ch/cyberduck/core/http/UserAgentHttpRequestInitializer.java b/oauth/src/main/java/ch/cyberduck/core/http/UserAgentHttpRequestInitializer.java index 80394a7f4e2..8bbd8b7f275 100644 --- a/oauth/src/main/java/ch/cyberduck/core/http/UserAgentHttpRequestInitializer.java +++ b/oauth/src/main/java/ch/cyberduck/core/http/UserAgentHttpRequestInitializer.java @@ -31,6 +31,7 @@ public UserAgentHttpRequestInitializer(final UseragentProvider provider) { @Override public void initialize(final HttpRequest request) { request.getHeaders().setUserAgent(provider.get()); + request.getHeaders().setAccept("application/json"); request.setSuppressUserAgentSuffix(true); } } From c209bcd5c2e6af2ca2e5ce68dcf3bf2e9b34c5f5 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sun, 6 Aug 2023 23:37:18 +0200 Subject: [PATCH 05/84] Set default STS endpoint. --- s3/src/main/java/ch/cyberduck/core/s3/S3Protocol.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Protocol.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Protocol.java index 66aecc128d2..02b75b7343b 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Protocol.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Protocol.java @@ -88,6 +88,11 @@ public String getDefaultHostname() { return "s3.amazonaws.com"; } + @Override + public String getSTSEndpoint() { + return "sts.amazonaws.com"; + } + @Override public Set getRegions(final List regions) { return regions.stream().map(S3LocationFeature.S3Region::new).collect(Collectors.toSet()); From 221bad074fbc3086b59e00591673eee020494b4c Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 7 Aug 2023 15:50:15 +0200 Subject: [PATCH 06/84] Set token in request header from configured credentials depending on profile configuration. --- .../java/ch/cyberduck/core/s3/S3Session.java | 26 +++++++++---------- .../S3TokenExpiredResponseInterceptor.java | 16 ++++++++---- 2 files changed, 23 insertions(+), 19 deletions(-) rename s3/src/main/java/ch/cyberduck/core/{s3 => sts}/S3TokenExpiredResponseInterceptor.java (87%) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index ab4c447e7f9..32a57ef892f 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -219,12 +219,6 @@ protected RequestEntityRestStorageService connect(final Proxy proxy, final HostK configuration.addInterceptorLast(authorizationService); configuration.setServiceUnavailableRetryStrategy(new S3WebIdentityTokenExpiredResponseInterceptor(this, prompt, authorizationService)); } - - // Only for AWS - if(S3Session.isAwsHostname(host.getHostname())) { - configuration.setServiceUnavailableRetryStrategy(new S3TokenExpiredResponseInterceptor(this, - new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt)); - } if(preferences.getBoolean("s3.upload.expect-continue")) { final String header = HTTP.EXPECT_DIRECTIVE; if(log.isDebugEnabled()) { @@ -277,15 +271,19 @@ public void process(final HttpRequest request, final HttpContext context) { request.setHeader(S3_ALTERNATE_DATE, SignatureUtils.formatAwsFlavouredISO8601Date(client.getCurrentTimeWithOffset())); } }); - configuration.addInterceptorLast(new HttpRequestInterceptor() { - @Override - public void process(final HttpRequest request, final HttpContext context) { - final ProviderCredentials credentials = client.getProviderCredentials(); - if(credentials instanceof AWSSessionCredentials) { - request.setHeader(SECURITY_TOKEN, ((AWSSessionCredentials) credentials).getSessionToken()); + if(host.getProtocol().isTokenConfigurable()) { + configuration.addInterceptorLast(new HttpRequestInterceptor() { + @Override + public void process(final HttpRequest request, final HttpContext context) { + final ProviderCredentials credentials = client.getProviderCredentials(); + if(credentials instanceof AWSSessionCredentials) { + request.setHeader(SECURITY_TOKEN, ((AWSSessionCredentials) credentials).getSessionToken()); + } } - } - }); + }); + configuration.setServiceUnavailableRetryStrategy(new S3TokenExpiredResponseInterceptor(this, + new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt)); + } configuration.addInterceptorLast(new HttpRequestInterceptor() { @Override public void process(final HttpRequest request, final HttpContext context) throws IOException { diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java similarity index 87% rename from s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java rename to s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java index 5a854b84065..36bd204b489 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java @@ -1,7 +1,7 @@ -package ch.cyberduck.core.s3; +package ch.cyberduck.core.sts; /* - * Copyright (c) 2002-2018 iterate GmbH. All rights reserved. + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify @@ -16,12 +16,14 @@ */ import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.CredentialsConfigurator; import ch.cyberduck.core.LoginCallback; import ch.cyberduck.core.exception.ExpiredTokenException; import ch.cyberduck.core.http.DisabledServiceUnavailableRetryStrategy; +import ch.cyberduck.core.s3.S3ExceptionMappingService; +import ch.cyberduck.core.s3.S3Session; import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; -import ch.cyberduck.core.sts.AWSProfileSTSCredentialsConfigurator; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; @@ -41,11 +43,15 @@ public class S3TokenExpiredResponseInterceptor extends DisabledServiceUnavailabl private static final int MAX_RETRIES = 1; private final S3Session session; - private final AWSProfileSTSCredentialsConfigurator configurator; + private final CredentialsConfigurator configurator; public S3TokenExpiredResponseInterceptor(final S3Session session, final X509TrustManager trust, final X509KeyManager key, final LoginCallback prompt) { + this(session, new AWSProfileSTSCredentialsConfigurator(trust, key, prompt)); + } + + public S3TokenExpiredResponseInterceptor(final S3Session session, final AWSProfileSTSCredentialsConfigurator configurator) { this.session = session; - this.configurator = new AWSProfileSTSCredentialsConfigurator(trust, key, prompt); + this.configurator = configurator; } @Override From da81dff6b0b3e1bf798f74ac0be10878ca06a9dd Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 7 Aug 2023 16:03:18 +0200 Subject: [PATCH 07/84] Delete unused accessor. --- s3/src/main/java/ch/cyberduck/core/s3/S3Session.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 32a57ef892f..9009b863f1e 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -509,10 +509,6 @@ public static boolean isAwsHostname(final String hostname, boolean cn) { return hostname.matches("([a-z0-9\\-]+\\.)?s3(\\.dualstack)?(\\.[a-z0-9\\-]+)?(\\.vpce)?\\.amazonaws\\.com"); } - public STSCredentialsRequestInterceptor getAuthorizationService() { - return authorizationService; - } - @Override @SuppressWarnings("unchecked") public T _getFeature(final Class type) { From 99537e16208b09964dfa87702a4502e8c39aef7f Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 7 Aug 2023 16:25:34 +0200 Subject: [PATCH 08/84] Refactor OAUth and STS flow to allow reuse of interceptors. --- .../java/ch/cyberduck/core/s3/S3Session.java | 71 ++++--- ...entityTokenExpiredResponseInterceptor.java | 130 ------------ .../STSAssumeRoleAuthorizationService.java | 100 +++++++++ ...sumeRoleCredentialsRequestInterceptor.java | 110 ++++++++++ ...meRoleTokenExpiredResponseInterceptor.java | 79 +++++++ .../sts/STSCredentialsRequestInterceptor.java | 196 ------------------ .../java/ch/cyberduck/core/sts/STSTokens.java | 71 +++++++ .../core/sts/OidcAuthenticationTest.java | 4 +- 8 files changed, 399 insertions(+), 362 deletions(-) delete mode 100644 s3/src/main/java/ch/cyberduck/core/sts/S3WebIdentityTokenExpiredResponseInterceptor.java create mode 100644 s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java create mode 100644 s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java create mode 100644 s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java delete mode 100644 s3/src/main/java/ch/cyberduck/core/sts/STSCredentialsRequestInterceptor.java create mode 100644 s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 9009b863f1e..dff58009895 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -49,6 +49,7 @@ import ch.cyberduck.core.http.HttpSession; import ch.cyberduck.core.kms.KMSEncryptionFeature; import ch.cyberduck.core.oauth.OAuth2AuthorizationService; +import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; import ch.cyberduck.core.preferences.HostPreferences; import ch.cyberduck.core.preferences.PreferencesReader; import ch.cyberduck.core.proxy.Proxy; @@ -65,8 +66,10 @@ import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.sts.AWSProfileSTSCredentialsConfigurator; -import ch.cyberduck.core.sts.S3WebIdentityTokenExpiredResponseInterceptor; -import ch.cyberduck.core.sts.STSCredentialsRequestInterceptor; +import ch.cyberduck.core.sts.S3TokenExpiredResponseInterceptor; +import ch.cyberduck.core.sts.STSAssumeRoleTokenExpiredResponseInterceptor; +import ch.cyberduck.core.sts.STSAssumeRoleCredentialsRequestInterceptor; +import ch.cyberduck.core.sts.STSTokens; import ch.cyberduck.core.threading.BackgroundExceptionCallable; import ch.cyberduck.core.threading.CancelCallback; import ch.cyberduck.core.transfer.TransferStatus; @@ -120,7 +123,8 @@ public class S3Session extends HttpSession { private final PreferencesReader preferences = new HostPreferences(host); - private STSCredentialsRequestInterceptor authorizationService; + private OAuth2RequestInterceptor oauth; + private STSAssumeRoleCredentialsRequestInterceptor sts; private final S3AccessControlListFeature acl = new S3AccessControlListFeature(this); @@ -209,15 +213,13 @@ protected String getRestMetadataPrefix() { @Override protected RequestEntityRestStorageService connect(final Proxy proxy, final HostKeyCallback hostkey, final LoginCallback prompt, final CancelCallback cancel) { final HttpClientBuilder configuration = builder.build(proxy, this, prompt); - - if((host.getProtocol().isOAuthConfigurable())) { - authorizationService = new STSCredentialsRequestInterceptor(builder.build(ProxyFactory.get() - .find(host.getProtocol().getSTSEndpoint()), this, prompt).build(), host, trust, key, prompt, this) + if(host.getProtocol().isOAuthConfigurable()) { + configuration.addInterceptorLast(oauth = new OAuth2RequestInterceptor(builder.build(ProxyFactory.get() + .find(host.getProtocol().getOAuthAuthorizationUrl()), this, prompt).build(), host) .withRedirectUri(host.getProtocol().getOAuthRedirectUrl()) - .withFlowType(OAuth2AuthorizationService.FlowType.valueOf(host.getProtocol().getAuthorization())); - - configuration.addInterceptorLast(authorizationService); - configuration.setServiceUnavailableRetryStrategy(new S3WebIdentityTokenExpiredResponseInterceptor(this, prompt, authorizationService)); + .withFlowType(OAuth2AuthorizationService.FlowType.valueOf(host.getProtocol().getAuthorization()))); + configuration.addInterceptorLast(sts = new STSAssumeRoleCredentialsRequestInterceptor(oauth, this, trust, key)); + configuration.setServiceUnavailableRetryStrategy(new STSAssumeRoleTokenExpiredResponseInterceptor(this, oauth, sts, prompt)); } if(preferences.getBoolean("s3.upload.expect-continue")) { final String header = HTTP.EXPECT_DIRECTIVE; @@ -421,12 +423,9 @@ private Map getHeadersAsObject(final HttpRequest request) { @Override public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - if(host.getProtocol().isOAuthConfigurable()) { - authorizationService.authorize(host, prompt, cancel); - } - if(Scheme.isURL(host.getProtocol().getContext())) { try { + // Obtain credentials from instance metadata final Credentials temporary = new AWSSessionCredentialsRetriever(trust, key, this, host.getProtocol().getContext()).get(); client.setProviderCredentials(new AWSSessionCredentials(temporary.getUsername(), temporary.getPassword(), temporary.getToken())); @@ -438,28 +437,32 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal } } else { - final Credentials credentials; - // get temporary STS credentials with oAuth token if(host.getProtocol().isOAuthConfigurable()) { - credentials = authorizationService.assumeRoleWithWebIdentity(); - } - // Only for AWS - else if(isAwsHostname(host.getHostname())) { - // Try auto-configure - credentials = new AWSProfileSTSCredentialsConfigurator( - new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt).configure(host); + // Get temporary credentials from STS using Web Identity (OIDC) token + final STSTokens tokens = sts.refresh(oauth.authorize(host, prompt, cancel)); + client.setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKey(), + tokens.getSecretAccessKey(), tokens.getSessionToken())); } else { - credentials = host.getCredentials(); - } - if(StringUtils.isNotBlank(credentials.getToken())) { - client.setProviderCredentials(credentials.isAnonymousLogin() ? null : - new AWSSessionCredentials(credentials.getUsername(), credentials.getPassword(), - credentials.getToken())); - } - else { - client.setProviderCredentials(credentials.isAnonymousLogin() ? null : - new AWSCredentials(credentials.getUsername(), credentials.getPassword())); + final Credentials credentials; + // Only for AWS + if(isAwsHostname(host.getHostname())) { + // Try auto-configure + credentials = new AWSProfileSTSCredentialsConfigurator( + new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt).configure(host); + } + else { + credentials = host.getCredentials(); + } + if(StringUtils.isNotBlank(credentials.getToken())) { + client.setProviderCredentials(credentials.isAnonymousLogin() ? null : + new AWSSessionCredentials(credentials.getUsername(), credentials.getPassword(), + credentials.getToken())); + } + else { + client.setProviderCredentials(credentials.isAnonymousLogin() ? null : + new AWSCredentials(credentials.getUsername(), credentials.getPassword())); + } } } if(host.getCredentials().isPassed()) { diff --git a/s3/src/main/java/ch/cyberduck/core/sts/S3WebIdentityTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/S3WebIdentityTokenExpiredResponseInterceptor.java deleted file mode 100644 index 432d522c60e..00000000000 --- a/s3/src/main/java/ch/cyberduck/core/sts/S3WebIdentityTokenExpiredResponseInterceptor.java +++ /dev/null @@ -1,130 +0,0 @@ -package ch.cyberduck.core.sts; - -/* - * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. - * https://cyberduck.io/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -import ch.cyberduck.core.Credentials; -import ch.cyberduck.core.DisabledCancelCallback; -import ch.cyberduck.core.Host; -import ch.cyberduck.core.LoginCallback; -import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.ExpiredTokenException; -import ch.cyberduck.core.exception.InteroperabilityException; -import ch.cyberduck.core.exception.LoginFailureException; -import ch.cyberduck.core.http.DisabledServiceUnavailableRetryStrategy; -import ch.cyberduck.core.s3.S3ExceptionMappingService; -import ch.cyberduck.core.s3.S3Session; - -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.entity.BufferedHttpEntity; -import org.apache.http.protocol.HttpContext; -import org.apache.http.util.EntityUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.security.AWSSessionCredentials; - -import java.io.IOException; - -public class S3WebIdentityTokenExpiredResponseInterceptor extends DisabledServiceUnavailableRetryStrategy { - private static final Logger log = LogManager.getLogger(S3WebIdentityTokenExpiredResponseInterceptor.class); - - private static final int MAX_RETRIES = 1; - - private final S3Session session; - private final Host host; - private final STSCredentialsRequestInterceptor authorizationService; - private final LoginCallback prompt; - - public S3WebIdentityTokenExpiredResponseInterceptor(final S3Session session, final LoginCallback prompt, - STSCredentialsRequestInterceptor authorizationService) { - this.session = session; - this.host = session.getHost(); - this.authorizationService = authorizationService; - this.prompt = prompt; - } - - @Override - public boolean retryRequest(final HttpResponse response, final int executionCount, final HttpContext context) { - if(executionCount <= MAX_RETRIES) { - switch(response.getStatusLine().getStatusCode()) { - case HttpStatus.SC_BAD_REQUEST: - case HttpStatus.SC_FORBIDDEN: - try { - final S3ServiceException failure; - if(null != response.getEntity()) { - EntityUtils.updateEntity(response, new BufferedHttpEntity(response.getEntity())); - failure = new S3ServiceException(response.getStatusLine().getReasonPhrase(), - EntityUtils.toString(response.getEntity())); - } - // In case of a http HEAD request minio packs the error code and description in the response header - else { - failure = new S3ServiceException(response.getStatusLine().getReasonPhrase()); - if(response.containsHeader("x-minio-error-code")) { - failure.setErrorCode(response.getFirstHeader("x-minio-error-code").getValue()); - } - if(response.containsHeader("x-minio-error-desc")) { - failure.setErrorMessage(response.getFirstHeader("x-minio-error-desc").getValue()); - } - } - - failure.setResponseCode(response.getStatusLine().getStatusCode()); - BackgroundException s3exception = new S3ExceptionMappingService().map(failure); - - if(failure.getErrorCode().equals("InvalidAccessKeyId") || s3exception instanceof ExpiredTokenException) { - refreshOAuthAndSTS(); - } - return true; - } - catch(IOException e) { - log.warn(String.format("Failure parsing response entity from %s", response)); - } - } - } - - else { - if(log.isWarnEnabled()) { - log.warn(String.format("Skip retry for response %s after %d executions", response, executionCount)); - } - } - return false; - } - - private void refreshOAuthAndSTS() { - try { - try { - authorizationService.save(authorizationService.refresh()); - log.debug("OAuth refreshed. Refreshing STS token."); - } - catch(InteroperabilityException | LoginFailureException e3) { - log.warn(String.format("Failure %s refreshing OAuth tokens", e3)); - authorizationService.save(authorizationService.authorize(host, prompt, new DisabledCancelCallback())); - } - try { - Credentials credentials = authorizationService.assumeRoleWithWebIdentity(); - session.getClient().setProviderCredentials(credentials.isAnonymousLogin() ? null : - new AWSSessionCredentials(credentials.getUsername(), credentials.getPassword(), - credentials.getToken())); - } - catch(BackgroundException e) { - log.warn(String.format("Failure %s fetching temporary STS credentials with oAuth token", e)); - } - } - catch(BackgroundException e) { - log.warn(String.format("Failure %s refreshing OAuth tokens", e)); - } - } -} \ No newline at end of file diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java new file mode 100644 index 00000000000..810f68ad9ce --- /dev/null +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -0,0 +1,100 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.AsciiRandomStringService; +import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.Host; +import ch.cyberduck.core.OAuthTokens; +import ch.cyberduck.core.PreferencesUseragentProvider; +import ch.cyberduck.core.aws.CustomClientConfiguration; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.LoginFailureException; +import ch.cyberduck.core.preferences.HostPreferences; +import ch.cyberduck.core.ssl.ThreadLocalHostnameDelegatingTrustManager; +import ch.cyberduck.core.ssl.X509KeyManager; +import ch.cyberduck.core.ssl.X509TrustManager; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.internal.StaticCredentialsProvider; +import com.amazonaws.services.securitytoken.AWSSecurityTokenService; +import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; +import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException; +import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityRequest; +import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityResult; + +public class STSAssumeRoleAuthorizationService { + private static final Logger log = LogManager.getLogger(STSAssumeRoleAuthorizationService.class); + + private final AWSSecurityTokenService service; + + public STSAssumeRoleAuthorizationService(final Host bookmark, final X509TrustManager trust, final X509KeyManager key) { + this(AWSSecurityTokenServiceClientBuilder + .standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(bookmark.getProtocol().getSTSEndpoint(), null)) + .withCredentials(new StaticCredentialsProvider(new AnonymousAWSCredentials())) + .withClientConfiguration(new CustomClientConfiguration(bookmark, + new ThreadLocalHostnameDelegatingTrustManager(trust, bookmark.getProtocol().getSTSEndpoint()), key)) + .build()); + } + + public STSAssumeRoleAuthorizationService(final AWSSecurityTokenService service) { + this.service = service; + } + + public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws BackgroundException { + final AssumeRoleWithWebIdentityRequest request = new AssumeRoleWithWebIdentityRequest(); + request.withWebIdentityToken(StringUtils.isNotBlank(oauth.getIdToken()) ? oauth.getIdToken() : oauth.getAccessToken()); + if(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds") != 0) { + request.withDurationSeconds(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds")); + } + if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.policy"))) { + request.withPolicy(new HostPreferences(bookmark).getProperty("s3.assumerole.policy")); + } + if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.rolearn"))) { + request.withRoleArn(new HostPreferences(bookmark).getProperty("s3.assumerole.rolearn")); + } + if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.rolesessionname"))) { + request.withRoleSessionName(new HostPreferences(bookmark).getProperty("s3.assumerole.rolesessionname")); + } + else { + request.withRoleSessionName(new AsciiRandomStringService().random()); + } + try { + final AssumeRoleWithWebIdentityResult result = service.assumeRoleWithWebIdentity(request); + if(log.isDebugEnabled()) { + log.debug(String.format("Received assume role identity result %s", result)); + } + final Credentials credentials = bookmark.getCredentials(); + final STSTokens tokens = new STSTokens(result.getCredentials().getAccessKeyId(), + result.getCredentials().getSecretAccessKey(), + result.getCredentials().getSessionToken(), + result.getCredentials().getExpiration().getTime()); + credentials.setUsername(tokens.getAccessKey()); + credentials.setPassword(tokens.getSecretAccessKey()); + credentials.setToken(tokens.getSessionToken()); + return tokens; + } + catch(AWSSecurityTokenServiceException e) { + throw new LoginFailureException(e.getErrorMessage(), e); + } + } +} diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java new file mode 100644 index 00000000000..f9f63775c5c --- /dev/null +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -0,0 +1,110 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Host; +import ch.cyberduck.core.HostPasswordStore; +import ch.cyberduck.core.LoginOptions; +import ch.cyberduck.core.OAuthTokens; +import ch.cyberduck.core.PasswordStoreFactory; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.LocalAccessDeniedException; +import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; +import ch.cyberduck.core.s3.S3Session; +import ch.cyberduck.core.ssl.X509KeyManager; +import ch.cyberduck.core.ssl.X509TrustManager; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.protocol.HttpContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jets3t.service.security.AWSSessionCredentials; + +import java.io.IOException; + +import static com.amazonaws.services.s3.Headers.SECURITY_TOKEN; + +public class STSAssumeRoleCredentialsRequestInterceptor extends STSAssumeRoleAuthorizationService implements HttpRequestInterceptor { + private static final Logger log = LogManager.getLogger(STSAssumeRoleCredentialsRequestInterceptor.class); + + /** + * Currently valid tokens + */ + private STSTokens tokens = STSTokens.EMPTY; + + private final HostPasswordStore store = PasswordStoreFactory.get(); + private final OAuth2RequestInterceptor oauth; + private final S3Session session; + private final Host host; + + public STSAssumeRoleCredentialsRequestInterceptor(final OAuth2RequestInterceptor oauth, final S3Session session, final X509TrustManager trust, final X509KeyManager key) { + super(session.getHost(), trust, key); + this.oauth = oauth; + this.session = session; + this.host = session.getHost(); + } + + public STSTokens refresh() throws BackgroundException { + return this.refresh(oauth.refresh()); + } + + public STSTokens refresh(final OAuthTokens oauth) throws BackgroundException { + return this.tokens = this.authorize(host, oauth); + } + + /** + * Save updated tokens in keychain + * + * @return Same tokens saved + */ + public STSTokens save(final STSTokens tokens) throws LocalAccessDeniedException { + host.getCredentials() + .withUsername(tokens.getAccessKey()) + .withPassword(tokens.getSecretAccessKey()) + .withToken(tokens.getSessionToken()) + .withSaved(new LoginOptions().keychain); + if(log.isDebugEnabled()) { + log.debug(String.format("Save new tokens %s for %s", tokens, host)); + } + store.save(host); + return tokens; + } + + @Override + public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + if(tokens.isExpired()) { + try { + this.save(this.refresh()); + } + catch(BackgroundException e) { + log.warn(String.format("Failure %s refreshing STS tokens %s", e, tokens)); + // Follow-up error 401 handled in error interceptor + } + } + if(StringUtils.isNotBlank(tokens.getSessionToken())) { + if(log.isInfoEnabled()) { + log.info(String.format("Authorizing service request with STS tokens %s", tokens)); + } + request.setHeader(SECURITY_TOKEN, tokens.getSessionToken()); + + session.getClient().setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKey(), tokens.getSecretAccessKey(), + tokens.getSessionToken())); + } + } +} diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java new file mode 100644 index 00000000000..e4dfab801b8 --- /dev/null +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java @@ -0,0 +1,79 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.LoginCallback; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.oauth.OAuth2ErrorResponseInterceptor; +import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; +import ch.cyberduck.core.s3.S3Session; + +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.protocol.HttpContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jets3t.service.security.AWSSessionCredentials; + +public class STSAssumeRoleTokenExpiredResponseInterceptor extends OAuth2ErrorResponseInterceptor { + private static final Logger log = LogManager.getLogger(STSAssumeRoleTokenExpiredResponseInterceptor.class); + + private static final int MAX_RETRIES = 1; + + private final S3Session session; + private final STSAssumeRoleCredentialsRequestInterceptor service; + + public STSAssumeRoleTokenExpiredResponseInterceptor(final S3Session session, + final OAuth2RequestInterceptor oauth, + final STSAssumeRoleCredentialsRequestInterceptor sts, + final LoginCallback prompt) { + super(session.getHost(), oauth, prompt); + this.session = session; + this.service = sts; + } + + @Override + public boolean retryRequest(final HttpResponse response, final int executionCount, final HttpContext context) { + switch(response.getStatusLine().getStatusCode()) { + case HttpStatus.SC_UNAUTHORIZED: + if(!super.retryRequest(response, executionCount, context)) { + return false; + } + } + switch(response.getStatusLine().getStatusCode()) { + case HttpStatus.SC_UNAUTHORIZED: + case HttpStatus.SC_BAD_REQUEST: + if(executionCount <= MAX_RETRIES) { + try { + log.warn(String.format("Attempt to refresh STS token for failure %s", response)); + final STSTokens tokens = service.refresh(); + session.getClient().setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKey(), + tokens.getSecretAccessKey(), tokens.getSessionToken())); + // Try again + return true; + } + catch(BackgroundException e) { + log.warn(String.format("Failure %s refreshing STS token", e)); + } + } + else { + log.warn(String.format("Skip retry for response %s after %d executions", response, executionCount)); + } + break; + } + return false; + } +} \ No newline at end of file diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSCredentialsRequestInterceptor.java deleted file mode 100644 index 133fdf81edb..00000000000 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSCredentialsRequestInterceptor.java +++ /dev/null @@ -1,196 +0,0 @@ -package ch.cyberduck.core.sts; - -/* - * Copyright (c) 2002-2019 iterate GmbH. All rights reserved. - * https://cyberduck.io/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -import ch.cyberduck.core.Credentials; -import ch.cyberduck.core.DisabledCancelCallback; -import ch.cyberduck.core.Host; -import ch.cyberduck.core.LoginCallback; -import ch.cyberduck.core.aws.CustomClientConfiguration; -import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.InteroperabilityException; -import ch.cyberduck.core.exception.LoginFailureException; -import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; -import ch.cyberduck.core.preferences.HostPreferences; -import ch.cyberduck.core.s3.S3Session; -import ch.cyberduck.core.ssl.ThreadLocalHostnameDelegatingTrustManager; -import ch.cyberduck.core.ssl.X509KeyManager; -import ch.cyberduck.core.ssl.X509TrustManager; - -import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpException; -import org.apache.http.client.HttpClient; -import org.apache.http.protocol.HttpContext; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jets3t.service.security.AWSSessionCredentials; - -import java.io.IOException; - -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.AnonymousAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.services.securitytoken.AWSSecurityTokenService; -import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; -import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException; -import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityRequest; -import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityResult; -import com.google.api.client.auth.oauth2.Credential; - -public class STSCredentialsRequestInterceptor extends OAuth2RequestInterceptor { - private static final Logger log = LogManager.getLogger(STSCredentialsRequestInterceptor.class); - - private final X509TrustManager trust; - private final X509KeyManager key; - private final LoginCallback prompt; - private final S3Session session; - - private long stsExpiryInMilliseconds; - - public STSCredentialsRequestInterceptor(HttpClient client, Host host, final X509TrustManager trust, final X509KeyManager key, - LoginCallback prompt, S3Session session) { - super(client, host); - this.trust = trust; - this.key = key; - this.prompt = prompt; - this.session = session; - } - - @Override - public void process(final org.apache.http.HttpRequest request, final HttpContext context) throws HttpException, IOException { - if(System.currentTimeMillis() >= stsExpiryInMilliseconds) { - try { - if(tokens.isExpired()) { - try { - this.save(this.refresh(tokens)); - } - catch(InteroperabilityException | LoginFailureException e3) { - log.warn(String.format("Failure %s refreshing OAuth tokens", e3)); - try { - this.save(this.authorize(host, prompt, new DisabledCancelCallback())); - } - catch(BackgroundException e) { - log.warn(String.format("Failure %s OAuth authentication", e)); - } - } - } - try { - Credentials credentials = assumeRoleWithWebIdentity(); - session.getClient().setProviderCredentials(credentials.isAnonymousLogin() ? null : - new AWSSessionCredentials(credentials.getUsername(), credentials.getPassword(), - credentials.getToken())); - } - catch(BackgroundException e) { - log.warn(String.format("Failure %s to fetch temporary sts credentials", e)); - // Follow-up error 400 or 403 handled in web identity token expired interceptor - } - } - catch(BackgroundException e) { - log.warn(String.format("Failure %s getting web identity", e)); - } - } - } - - public Credentials assumeRoleWithWebIdentity() throws BackgroundException { - AWSSecurityTokenService service = this.getTokenService(host); - - AssumeRoleWithWebIdentityRequest webIdReq = new AssumeRoleWithWebIdentityRequest(); - if(StringUtils.isNotBlank(tokens.getIdToken())) { - webIdReq.withWebIdentityToken(tokens.getIdToken()); - } - else { - webIdReq.withWebIdentityToken(tokens.getAccessToken()); - } - - - if(new HostPreferences(host).getInteger("s3.assumerole.durationseconds") != 0) { - webIdReq.withDurationSeconds(new HostPreferences(host).getInteger("s3.assumerole.durationseconds")); - } - - if(StringUtils.isNotBlank(new HostPreferences(host).getProperty("s3.assumerole.policy"))) { - webIdReq.withPolicy(new HostPreferences(host).getProperty("s3.assumerole.policy")); - } - - if(StringUtils.isNotBlank(new HostPreferences(host).getProperty("s3.assumerole.rolearn"))) { - webIdReq.withRoleArn(new HostPreferences(host).getProperty("s3.assumerole.rolearn")); - } - - if(StringUtils.isNotBlank(new HostPreferences(host).getProperty("s3.assumerole.rolesessionname"))) { - webIdReq.withRoleSessionName(new HostPreferences(host).getProperty("s3.assumerole.rolesessionname")); - } - - - Credentials credentials = new Credentials(); - try { - AssumeRoleWithWebIdentityResult result = service.assumeRoleWithWebIdentity(webIdReq); - com.amazonaws.services.securitytoken.model.Credentials cred = result.getCredentials(); - - if(log.isDebugEnabled()) { - log.debug(cred.toString()); - } - - stsExpiryInMilliseconds = cred.getExpiration().getTime(); - - credentials.setUsername(cred.getAccessKeyId()); - credentials.setPassword(cred.getSecretAccessKey()); - credentials.setToken(cred.getSessionToken()); - } - catch(AWSSecurityTokenServiceException e) { - throw new LoginFailureException(e.getMessage(), e); - } - return credentials; - } - - private AWSSecurityTokenService getTokenService(final Host host) { - final ClientConfiguration configuration = new CustomClientConfiguration(host, - new ThreadLocalHostnameDelegatingTrustManager(trust, host.getProtocol().getSTSEndpoint()), key); - return AWSSecurityTokenServiceClientBuilder - .standard() - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(host.getProtocol().getSTSEndpoint(), null)) - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) - .withClientConfiguration(configuration) - .build(); - } - - @Override - public STSCredentialsRequestInterceptor withMethod(final Credential.AccessMethod method) { - super.withMethod(method); - return this; - } - - @Override - public STSCredentialsRequestInterceptor withRedirectUri(final String redirectUri) { - super.withRedirectUri(redirectUri); - return this; - } - - @Override - public STSCredentialsRequestInterceptor withFlowType(final FlowType flowType) { - super.withFlowType(flowType); - return this; - } - - @Override - public STSCredentialsRequestInterceptor withParameter(final String key, final String value) { - super.withParameter(key, value); - return this; - } - - public long getStsExpiryInMilliseconds() { - return stsExpiryInMilliseconds; - } -} diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java b/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java new file mode 100644 index 00000000000..4d63d138809 --- /dev/null +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java @@ -0,0 +1,71 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import org.apache.commons.lang3.StringUtils; + +public final class STSTokens { + + public static final STSTokens EMPTY + = new STSTokens(null, null, null, Long.MAX_VALUE); + + private final String accessKey; + private final String secretAccessKey; + private final Long expiryInMilliseconds; + private final String sessionToken; + + public STSTokens(final String accessKey, final String secretAccessKey, final String sessionToken, final Long expiryInMilliseconds) { + this.accessKey = accessKey; + this.secretAccessKey = secretAccessKey; + this.sessionToken = sessionToken; + this.expiryInMilliseconds = expiryInMilliseconds; + } + + public boolean validate() { + return StringUtils.isNotEmpty(sessionToken); + } + + public String getAccessKey() { + return accessKey; + } + + public String getSecretAccessKey() { + return secretAccessKey; + } + + public Long getExpiryInMilliseconds() { + return expiryInMilliseconds; + } + + public String getSessionToken() { + return sessionToken; + } + + public boolean isExpired() { + return System.currentTimeMillis() >= expiryInMilliseconds; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("OAuthTokens{"); + sb.append("accessKey='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(accessKey)))).append('\''); + sb.append(", secretAccessKey='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(secretAccessKey)))).append('\''); + sb.append(", sessionToken='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(secretAccessKey)))).append('\''); + sb.append(", expiryInMilliseconds=").append(expiryInMilliseconds); + sb.append('}'); + return sb.toString(); + } +} diff --git a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java index b6428906563..4886fa9a882 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java @@ -54,8 +54,8 @@ public void testSuccessfulLoginViaOidc() throws BackgroundException { Credentials creds = host.getCredentials(); assertNotEquals(StringUtils.EMPTY, creds.getUsername()); assertNotEquals(StringUtils.EMPTY, creds.getPassword()); - // credentials from STS are written to the S3Session's client object and not into the credential object from the Host. - assertTrue(creds.getToken().isEmpty()); + + assertFalse(creds.getToken().isEmpty()); assertNotNull(creds.getOauth().getIdToken()); assertNotNull(creds.getOauth().getRefreshToken()); assertNotEquals(Optional.of(Long.MAX_VALUE).get(), creds.getOauth().getExpiryInMilliseconds()); From 049e6b03eb83bf0755b0d91a02f206458afe3082 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 7 Aug 2023 16:27:43 +0200 Subject: [PATCH 09/84] Delete bundled key. --- s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile b/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile index 5790f70cb4d..1e6a5ba7615 100644 --- a/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile +++ b/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile @@ -20,9 +20,7 @@ Protocol s3 Vendor - s3-sts - Bundled - + s3-sts-minio-keycloak Description S3 STS AssumeRoleWithWebIdentity Default Hostname From 7490c15bed0b359013bf180feb1ca0c527c51366 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 7 Aug 2023 18:44:13 +0200 Subject: [PATCH 10/84] Must only handle specific 400 error codes. --- ...meRoleTokenExpiredResponseInterceptor.java | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java index e4dfab801b8..ed4c5ea9194 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java @@ -17,24 +17,32 @@ import ch.cyberduck.core.LoginCallback; import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.ExpiredTokenException; import ch.cyberduck.core.oauth.OAuth2ErrorResponseInterceptor; import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; +import ch.cyberduck.core.s3.S3ExceptionMappingService; import ch.cyberduck.core.s3.S3Session; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; +import org.apache.http.entity.BufferedHttpEntity; import org.apache.http.protocol.HttpContext; +import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jets3t.service.S3ServiceException; import org.jets3t.service.security.AWSSessionCredentials; +import java.io.IOException; + public class STSAssumeRoleTokenExpiredResponseInterceptor extends OAuth2ErrorResponseInterceptor { private static final Logger log = LogManager.getLogger(STSAssumeRoleTokenExpiredResponseInterceptor.class); private static final int MAX_RETRIES = 1; private final S3Session session; - private final STSAssumeRoleCredentialsRequestInterceptor service; + private final OAuth2RequestInterceptor oauth; + private final STSAssumeRoleCredentialsRequestInterceptor sts; public STSAssumeRoleTokenExpiredResponseInterceptor(final S3Session session, final OAuth2RequestInterceptor oauth, @@ -42,7 +50,8 @@ public STSAssumeRoleTokenExpiredResponseInterceptor(final S3Session session, final LoginCallback prompt) { super(session.getHost(), oauth, prompt); this.session = session; - this.service = sts; + this.oauth = oauth; + this.sts = sts; } @Override @@ -54,12 +63,33 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun } } switch(response.getStatusLine().getStatusCode()) { - case HttpStatus.SC_UNAUTHORIZED: case HttpStatus.SC_BAD_REQUEST: + final S3ServiceException failure; + try { + if(null != response.getEntity()) { + EntityUtils.updateEntity(response, new BufferedHttpEntity(response.getEntity())); + failure = new S3ServiceException(response.getStatusLine().getReasonPhrase(), + EntityUtils.toString(response.getEntity())); + if(new S3ExceptionMappingService().map(failure) instanceof ExpiredTokenException) { + if(log.isWarnEnabled()) { + log.warn(String.format("Handle failure %s", failure)); + } + } + else { + // Ignore other 400 failures + return false; + } + } + } + catch(IOException e) { + log.warn(String.format("Failure parsing response entity from %s", response)); + } + // Break through + case HttpStatus.SC_UNAUTHORIZED: if(executionCount <= MAX_RETRIES) { try { log.warn(String.format("Attempt to refresh STS token for failure %s", response)); - final STSTokens tokens = service.refresh(); + final STSTokens tokens = sts.refresh(oauth.getTokens()); session.getClient().setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKey(), tokens.getSecretAccessKey(), tokens.getSessionToken())); // Try again From d81acca44243f5f045146d119fb2caf157140ad8 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 7 Aug 2023 18:56:46 +0200 Subject: [PATCH 11/84] Logging. --- core/src/main/java/ch/cyberduck/core/OAuthTokens.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/ch/cyberduck/core/OAuthTokens.java b/core/src/main/java/ch/cyberduck/core/OAuthTokens.java index aee3c1badba..8ce1f8b6609 100644 --- a/core/src/main/java/ch/cyberduck/core/OAuthTokens.java +++ b/core/src/main/java/ch/cyberduck/core/OAuthTokens.java @@ -65,6 +65,7 @@ public String toString() { final StringBuilder sb = new StringBuilder("OAuthTokens{"); sb.append("accessToken='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(accessToken)))).append('\''); sb.append(", refreshToken='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(refreshToken)))).append('\''); + sb.append(", idToken='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(idToken)))).append('\''); sb.append(", expiryInMilliseconds=").append(expiryInMilliseconds); sb.append('}'); return sb.toString(); From 918b2a5ff8de8bade50f93e76727f5c7b14608cb Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 7 Aug 2023 19:04:38 +0200 Subject: [PATCH 12/84] Fix retrieving Id (OIDC) token. --- .../main/java/ch/cyberduck/core/DefaultHostPasswordStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/ch/cyberduck/core/DefaultHostPasswordStore.java b/core/src/main/java/ch/cyberduck/core/DefaultHostPasswordStore.java index 0c3e2470f70..d53258e390c 100644 --- a/core/src/main/java/ch/cyberduck/core/DefaultHostPasswordStore.java +++ b/core/src/main/java/ch/cyberduck/core/DefaultHostPasswordStore.java @@ -233,7 +233,7 @@ public void save(final Host bookmark) throws LocalAccessDeniedException { if(StringUtils.isNotBlank(credentials.getOauth().getIdToken())) { this.addPassword(bookmark.getProtocol().getScheme(), bookmark.getPort(), this.getOAuthHostname(bookmark), - String.format("%s OIDC Id Token", prefix), credentials.getOauth().getRefreshToken()); + String.format("%s OIDC Id Token", prefix), credentials.getOauth().getIdToken()); } } } From 033016582a0dc8d8da569bb6266d48f8ac600584 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 7 Aug 2023 20:51:07 +0200 Subject: [PATCH 13/84] Make sure to fetch tokens from instance metadata when expired when using "S3 (Credentials from EC2 Instance Metadata)" connection profile. --- .../auth/AWSSessionCredentialsRetriever.java | 34 +++++++++++++++++++ .../java/ch/cyberduck/core/s3/S3Session.java | 9 ++++- .../S3TokenExpiredResponseInterceptor.java | 20 +++++------ 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java b/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java index 30a3d9a18c9..729642bebde 100644 --- a/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java +++ b/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java @@ -16,6 +16,7 @@ */ import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.CredentialsConfigurator; import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.DisabledCancelCallback; import ch.cyberduck.core.DisabledConnectionCallback; @@ -37,6 +38,8 @@ import ch.cyberduck.core.transfer.TransferStatus; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.IOException; import java.io.InputStream; @@ -48,6 +51,7 @@ import com.google.gson.stream.MalformedJsonException; public class AWSSessionCredentialsRetriever { + private static final Logger log = LogManager.getLogger(AWSSessionCredentialsRetriever.class); private final TranscriptListener transcript; private final ProtocolFactory factory; @@ -67,7 +71,37 @@ public AWSSessionCredentialsRetriever(final X509TrustManager trust, final X509Ke this.url = url; } + public static class Configurator implements CredentialsConfigurator { + + private final AWSSessionCredentialsRetriever retriever; + private final String url; + + public Configurator(final X509TrustManager trust, final X509KeyManager key, final TranscriptListener transcript, final String url) { + this.url = url; + this.retriever = new AWSSessionCredentialsRetriever(trust, key, transcript, url); + } + + @Override + public Configurator reload() { + return this; + } + + @Override + public Credentials configure(final Host host) { + try { + return retriever.get(); + } + catch(BackgroundException e) { + log.warn(String.format("Ignore failure %s retrieving credentials from %s", e, url)); + return host.getCredentials(); + } + } + } + public Credentials get() throws BackgroundException { + if(log.isDebugEnabled()) { + log.debug(String.format("Configure credentials from %s", url)); + } final Host address = new HostParser(factory).get(url); final Path access = new Path(PathNormalizer.normalize(address.getDefaultPath()), EnumSet.of(Path.Type.file)); address.setDefaultPath(String.valueOf(Path.DELIMITER)); diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index dff58009895..c28e3ae25c7 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -21,6 +21,8 @@ import ch.cyberduck.core.AttributedList; import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.CredentialsConfigurator; +import ch.cyberduck.core.DisabledTranscriptListener; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostKeyCallback; import ch.cyberduck.core.ListProgressListener; @@ -284,7 +286,12 @@ public void process(final HttpRequest request, final HttpContext context) { } }); configuration.setServiceUnavailableRetryStrategy(new S3TokenExpiredResponseInterceptor(this, - new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt)); + S3Session.isAwsHostname(host.getHostname()) ? + Scheme.isURL(host.getProtocol().getContext()) ? + // Obtain credentials from instance metadata + new AWSSessionCredentialsRetriever.Configurator(trust, key, this, host.getProtocol().getContext()) : + new AWSProfileSTSCredentialsConfigurator(new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt) : CredentialsConfigurator.DISABLED + )); } configuration.addInterceptorLast(new HttpRequestInterceptor() { @Override diff --git a/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java index 36bd204b489..4c698b05364 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java @@ -17,13 +17,10 @@ import ch.cyberduck.core.Credentials; import ch.cyberduck.core.CredentialsConfigurator; -import ch.cyberduck.core.LoginCallback; import ch.cyberduck.core.exception.ExpiredTokenException; import ch.cyberduck.core.http.DisabledServiceUnavailableRetryStrategy; import ch.cyberduck.core.s3.S3ExceptionMappingService; import ch.cyberduck.core.s3.S3Session; -import ch.cyberduck.core.ssl.X509KeyManager; -import ch.cyberduck.core.ssl.X509TrustManager; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; @@ -33,6 +30,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jets3t.service.S3ServiceException; +import org.jets3t.service.security.AWSCredentials; import org.jets3t.service.security.AWSSessionCredentials; import java.io.IOException; @@ -45,11 +43,7 @@ public class S3TokenExpiredResponseInterceptor extends DisabledServiceUnavailabl private final S3Session session; private final CredentialsConfigurator configurator; - public S3TokenExpiredResponseInterceptor(final S3Session session, final X509TrustManager trust, final X509KeyManager key, final LoginCallback prompt) { - this(session, new AWSProfileSTSCredentialsConfigurator(trust, key, prompt)); - } - - public S3TokenExpiredResponseInterceptor(final S3Session session, final AWSProfileSTSCredentialsConfigurator configurator) { + public S3TokenExpiredResponseInterceptor(final S3Session session, final CredentialsConfigurator configurator) { this.session = session; this.configurator = configurator; } @@ -73,8 +67,14 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun if(log.isDebugEnabled()) { log.debug(String.format("Reconfigure client with credentials %s", credentials)); } - session.getClient().setProviderCredentials(new AWSSessionCredentials( - credentials.getUsername(), credentials.getPassword(), credentials.getToken())); + if(credentials.isTokenAuthentication()) { + session.getClient().setProviderCredentials(new AWSSessionCredentials( + credentials.getUsername(), credentials.getPassword(), credentials.getToken())); + } + else { + session.getClient().setProviderCredentials(new AWSCredentials( + credentials.getUsername(), credentials.getPassword())); + } return true; } } From 8dfa5526c112813adf311dae54dd5190567ba550 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 7 Aug 2023 22:24:27 +0200 Subject: [PATCH 14/84] Extract authentication handlers classes. --- .../s3/S3AWS2SignatureRequestInterceptor.java | 111 +++++++++++++ .../s3/S3AWS4SignatureRequestInterceptor.java | 143 ++++++++++++++++ .../java/ch/cyberduck/core/s3/S3Session.java | 152 +----------------- 3 files changed, 262 insertions(+), 144 deletions(-) create mode 100644 s3/src/main/java/ch/cyberduck/core/s3/S3AWS2SignatureRequestInterceptor.java create mode 100644 s3/src/main/java/ch/cyberduck/core/s3/S3AWS4SignatureRequestInterceptor.java diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3AWS2SignatureRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/s3/S3AWS2SignatureRequestInterceptor.java new file mode 100644 index 00000000000..470b587cc9a --- /dev/null +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3AWS2SignatureRequestInterceptor.java @@ -0,0 +1,111 @@ +package ch.cyberduck.core.s3; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.preferences.HostPreferences; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.protocol.HttpContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jets3t.service.security.ProviderCredentials; +import org.jets3t.service.utils.RestUtils; +import org.jets3t.service.utils.ServiceUtils; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public class S3AWS2SignatureRequestInterceptor implements HttpRequestInterceptor { + private static final Logger log = LogManager.getLogger(S3Session.class); + + private final S3Session session; + + public S3AWS2SignatureRequestInterceptor(final S3Session session) { + this.session = session; + } + + @Override + public void process(final HttpRequest request, final HttpContext context) throws IOException { + if(!session.getClient().isAuthenticatedConnection()) { + log.warn(String.format("Skip authentication request %s", request)); + return; + } + final ProviderCredentials credentials = session.getClient().getProviderCredentials(); + final String bucketName = context.getAttribute("bucket").toString(); + if(log.isDebugEnabled()) { + log.debug(String.format("Use bucket name %s from context", bucketName)); + } + final URI uri; + try { + uri = new URI(request.getRequestLine().getUri()); + } + catch(URISyntaxException e) { + throw new IOException(e); + } + String path = uri.getRawPath(); + // If bucket name is not already part of the full path, add it. + // This can be the case if the Host name has a bucket-name prefix, + // or if the Host name constitutes the bucket name for DNS-redirects. + if(!StringUtils.startsWith(path, bucketName)) { + path = String.format("/%s%s", bucketName, path); + } + final String queryString = uri.getRawQuery(); + if(StringUtils.isNotBlank(queryString)) { + path += String.format("?%s", queryString); + } + // Generate a canonical string representing the operation. + final String canonicalString = RestUtils.makeServiceCanonicalString( + request.getRequestLine().getMethod(), + path, + this.getHeaders(request), + null, + session.getRestHeaderPrefix(), + session.getClient().getResourceParameterNames()); + // Sign the canonical string. + final String signedCanonical = ServiceUtils.signWithHmacSha1( + credentials.getSecretKey(), canonicalString); + // Add encoded authorization to connection as HTTP Authorization header. + final String authorizationString = session.getSignatureIdentifier() + " " + + credentials.getAccessKey() + ":" + signedCanonical; + request.setHeader(HttpHeaders.AUTHORIZATION, authorizationString); + } + + final class HttpHeaderFilter implements Predicate
{ + @Override + public boolean test(final Header header) { + return !new HostPreferences(session.getHost()).getList("s3.signature.headers.exclude").stream() + .filter(s -> StringUtils.equalsIgnoreCase(s, header.getName())).findAny().isPresent(); + } + } + + private Map getHeaders(final HttpRequest request) { + final Map headers = new HashMap<>(); + for(Header header : Arrays.stream(request.getAllHeaders()).filter(new HttpHeaderFilter()).collect(Collectors.toList())) { + headers.put(StringUtils.lowerCase(StringUtils.trim(header.getName())), StringUtils.trim(header.getValue())); + } + return headers; + } +} diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3AWS4SignatureRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/s3/S3AWS4SignatureRequestInterceptor.java new file mode 100644 index 00000000000..2124eed5c46 --- /dev/null +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3AWS4SignatureRequestInterceptor.java @@ -0,0 +1,143 @@ +package ch.cyberduck.core.s3; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.preferences.HostPreferences; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpCoreContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jets3t.service.security.ProviderCredentials; +import org.jets3t.service.utils.ServiceUtils; +import org.jets3t.service.utils.SignatureUtils; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import com.amazonaws.auth.internal.SignerConstants; + +import static com.amazonaws.services.s3.Headers.S3_ALTERNATE_DATE; + +public class S3AWS4SignatureRequestInterceptor implements HttpRequestInterceptor { + private static final Logger log = LogManager.getLogger(S3Session.class); + + private final S3Session session; + + public S3AWS4SignatureRequestInterceptor(final S3Session session) { + this.session = session; + } + + @Override + public void process(final HttpRequest request, final HttpContext context) throws IOException { + if(!session.getClient().isAuthenticatedConnection()) { + log.warn(String.format("Skip authentication request %s", request)); + return; + } + final ProviderCredentials credentials = session.getClient().getProviderCredentials(); + final String bucketName = context.getAttribute("bucket").toString(); + if(log.isDebugEnabled()) { + log.debug(String.format("Use bucket name %s from context", bucketName)); + } + final URI uri; + try { + uri = new URI(request.getRequestLine().getUri()); + } + catch(URISyntaxException e) { + throw new IOException(e); + } + String region = session.getClient().getRegionEndpointCache().getRegionForBucketName(bucketName); + if(null == region) { + final HttpHost host = (HttpHost) context.getAttribute(HttpCoreContext.HTTP_TARGET_HOST); + if(host != null) { + try { + region = SignatureUtils.awsRegionForRequest(new URI(host.toURI())); + } + catch(URISyntaxException e) { + throw new IOException(e); + } + } + if(region != null) { + if(log.isDebugEnabled()) { + log.debug(String.format("Cache region %s for bucket %s", region, bucketName)); + } + session.getClient().getRegionEndpointCache().putRegionForBucketName(bucketName, region); + } + } + if(null == region) { + region = session.getHost().getRegion(); + } + if(null == region) { + region = new HostPreferences(session.getHost()).getProperty("s3.location"); + } + final HttpUriRequest message = (HttpUriRequest) request; + String requestPayloadHexSHA256Hash = + SignatureUtils.awsV4GetOrCalculatePayloadHash(message); + message.setHeader(SignerConstants.X_AMZ_CONTENT_SHA256, requestPayloadHexSHA256Hash); + // Generate AWS-flavoured ISO8601 timestamp string + final String timestampISO8601 = message.getFirstHeader(S3_ALTERNATE_DATE).getValue(); + // Canonical request string + final String canonicalRequestString = + SignatureUtils.awsV4BuildCanonicalRequestString(uri, + request.getRequestLine().getMethod(), this.getHeaders(request), requestPayloadHexSHA256Hash); + // String to sign + final String stringToSign = SignatureUtils.awsV4BuildStringToSign( + session.getSignatureVersion().toString(), canonicalRequestString, + timestampISO8601, region); + // Signing key + final byte[] signingKey = SignatureUtils.awsV4BuildSigningKey( + credentials.getSecretKey(), timestampISO8601, region); + // Request signature + final String signature = ServiceUtils.toHex(ServiceUtils.hmacSHA256( + signingKey, ServiceUtils.stringToBytes(stringToSign))); + // Authorization header value + final String authorizationHeaderValue = + SignatureUtils.awsV4BuildAuthorizationHeaderValue( + credentials.getAccessKey(), signature, + session.getSignatureVersion().toString(), canonicalRequestString, + timestampISO8601, region); + message.setHeader(HttpHeaders.AUTHORIZATION, authorizationHeaderValue); + } + + final class HttpHeaderFilter implements Predicate
{ + @Override + public boolean test(final Header header) { + return !new HostPreferences(session.getHost()).getList("s3.signature.headers.exclude").stream() + .filter(s -> StringUtils.equalsIgnoreCase(s, header.getName())).findAny().isPresent(); + } + } + + private Map getHeaders(final HttpRequest request) { + final Map headers = new HashMap<>(); + for(Header header : Arrays.stream(request.getAllHeaders()).filter(new HttpHeaderFilter()).collect(Collectors.toList())) { + headers.put(StringUtils.lowerCase(StringUtils.trim(header.getName())), StringUtils.trim(header.getValue())); + } + return headers; + } +} diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index c28e3ae25c7..88e4765c1c8 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -22,7 +22,6 @@ import ch.cyberduck.core.AttributedList; import ch.cyberduck.core.Credentials; import ch.cyberduck.core.CredentialsConfigurator; -import ch.cyberduck.core.DisabledTranscriptListener; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostKeyCallback; import ch.cyberduck.core.ListProgressListener; @@ -77,7 +76,6 @@ import ch.cyberduck.core.transfer.TransferStatus; import org.apache.commons.lang3.StringUtils; -import org.apache.http.Header; import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; @@ -85,7 +83,6 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.protocol.HTTP; import org.apache.http.protocol.HttpContext; @@ -99,23 +96,12 @@ import org.jets3t.service.security.AWSCredentials; import org.jets3t.service.security.AWSSessionCredentials; import org.jets3t.service.security.ProviderCredentials; -import org.jets3t.service.utils.RestUtils; -import org.jets3t.service.utils.ServiceUtils; import org.jets3t.service.utils.SignatureUtils; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import com.amazonaws.auth.internal.SignerConstants; import static com.amazonaws.services.s3.Headers.*; @@ -293,136 +279,14 @@ public void process(final HttpRequest request, final HttpContext context) { new AWSProfileSTSCredentialsConfigurator(new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt) : CredentialsConfigurator.DISABLED )); } - configuration.addInterceptorLast(new HttpRequestInterceptor() { - @Override - public void process(final HttpRequest request, final HttpContext context) throws IOException { - if(!client.isAuthenticatedConnection()) { - log.warn(String.format("Skip authentication request %s", request)); - return; - } - final ProviderCredentials credentials = client.getProviderCredentials(); - final String bucketName = context.getAttribute("bucket").toString(); - if(log.isDebugEnabled()) { - log.debug(String.format("Use bucket name %s from context for %s signature", bucketName, authenticationHeaderSignatureVersion)); - } - final URI uri; - try { - uri = new URI(request.getRequestLine().getUri()); - } - catch(URISyntaxException e) { - throw new IOException(e); - } - switch(authenticationHeaderSignatureVersion) { - case AWS2: { - String path = uri.getRawPath(); - // If bucket name is not already part of the full path, add it. - // This can be the case if the Host name has a bucket-name prefix, - // or if the Host name constitutes the bucket name for DNS-redirects. - if(!StringUtils.startsWith(path, bucketName)) { - path = String.format("/%s%s", bucketName, path); - } - final String queryString = uri.getRawQuery(); - if(StringUtils.isNotBlank(queryString)) { - path += String.format("?%s", queryString); - } - // Generate a canonical string representing the operation. - final String canonicalString = RestUtils.makeServiceCanonicalString( - request.getRequestLine().getMethod(), - path, - this.getHeadersAsObject(request), - null, - getRestHeaderPrefix(), - client.getResourceParameterNames()); - // Sign the canonical string. - final String signedCanonical = ServiceUtils.signWithHmacSha1( - credentials.getSecretKey(), canonicalString); - // Add encoded authorization to connection as HTTP Authorization header. - final String authorizationString = getSignatureIdentifier() + " " - + credentials.getAccessKey() + ":" + signedCanonical; - request.setHeader(HttpHeaders.AUTHORIZATION, authorizationString); - break; - } - case AWS4HMACSHA256: { - String region = regions.getRegionForBucketName(bucketName); - if(null == region) { - final HttpHost host = (HttpHost) context.getAttribute(HttpCoreContext.HTTP_TARGET_HOST); - if(host != null) { - try { - region = SignatureUtils.awsRegionForRequest(new URI(host.toURI())); - } - catch(URISyntaxException e) { - throw new IOException(e); - } - } - if(region != null) { - if(log.isDebugEnabled()) { - log.debug(String.format("Cache region %s for bucket %s", region, bucketName)); - } - regions.putRegionForBucketName(bucketName, region); - } - } - if(null == region) { - region = host.getRegion(); - } - if(null == region) { - region = new HostPreferences(host).getProperty("s3.location"); - } - final HttpUriRequest message = (HttpUriRequest) request; - String requestPayloadHexSHA256Hash = - SignatureUtils.awsV4GetOrCalculatePayloadHash(message); - message.setHeader(SignerConstants.X_AMZ_CONTENT_SHA256, requestPayloadHexSHA256Hash); - // Generate AWS-flavoured ISO8601 timestamp string - final String timestampISO8601 = message.getFirstHeader(S3_ALTERNATE_DATE).getValue(); - // Canonical request string - final String canonicalRequestString = - SignatureUtils.awsV4BuildCanonicalRequestString(uri, - request.getRequestLine().getMethod(), this.getHeadersAsString(request), requestPayloadHexSHA256Hash); - // String to sign - final String stringToSign = SignatureUtils.awsV4BuildStringToSign( - authenticationHeaderSignatureVersion.toString(), canonicalRequestString, - timestampISO8601, region); - // Signing key - final byte[] signingKey = SignatureUtils.awsV4BuildSigningKey( - credentials.getSecretKey(), timestampISO8601, region); - // Request signature - final String signature = ServiceUtils.toHex(ServiceUtils.hmacSHA256( - signingKey, ServiceUtils.stringToBytes(stringToSign))); - // Authorization header value - final String authorizationHeaderValue = - SignatureUtils.awsV4BuildAuthorizationHeaderValue( - credentials.getAccessKey(), signature, - authenticationHeaderSignatureVersion.toString(), canonicalRequestString, - timestampISO8601, region); - message.setHeader(HttpHeaders.AUTHORIZATION, authorizationHeaderValue); - break; - } - } - } - - final class HttpHeaderFilter implements Predicate
{ - @Override - public boolean test(final Header header) { - return !new HostPreferences(host).getList("s3.signature.headers.exclude").stream() - .filter(s -> StringUtils.equalsIgnoreCase(s, header.getName())).findAny().isPresent(); - } - } - - private Map getHeadersAsString(final HttpRequest request) { - final Map headers = new HashMap<>(); - for(Header header : Arrays.stream(request.getAllHeaders()).filter(new HttpHeaderFilter()).collect(Collectors.toList())) { - headers.put(StringUtils.lowerCase(StringUtils.trim(header.getName())), StringUtils.trim(header.getValue())); - } - return headers; - } - - private Map getHeadersAsObject(final HttpRequest request) { - final Map headers = new HashMap<>(); - for(Header header : Arrays.stream(request.getAllHeaders()).filter(new HttpHeaderFilter()).collect(Collectors.toList())) { - headers.put(StringUtils.lowerCase(StringUtils.trim(header.getName())), StringUtils.trim(header.getValue())); - } - return headers; - } - }); + switch(authenticationHeaderSignatureVersion) { + case AWS4HMACSHA256: + configuration.addInterceptorLast(new S3AWS4SignatureRequestInterceptor(this)); + break; + case AWS2: + configuration.addInterceptorLast(new S3AWS2SignatureRequestInterceptor(this)); + break; + } final RequestEntityRestStorageService client = new RequestEntityRestStorageService(this, configuration); client.setRegionEndpointCache(regions); return client; From 220916b2dcffc9b643753b39bcedf7c609d7ce14 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 8 Aug 2023 10:34:06 +0200 Subject: [PATCH 15/84] Skip x-amz-security-token header set in general handler. --- .../core/sts/STSAssumeRoleCredentialsRequestInterceptor.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java index f9f63775c5c..e540e483810 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -38,8 +38,6 @@ import java.io.IOException; -import static com.amazonaws.services.s3.Headers.SECURITY_TOKEN; - public class STSAssumeRoleCredentialsRequestInterceptor extends STSAssumeRoleAuthorizationService implements HttpRequestInterceptor { private static final Logger log = LogManager.getLogger(STSAssumeRoleCredentialsRequestInterceptor.class); @@ -101,8 +99,6 @@ public void process(final HttpRequest request, final HttpContext context) throws if(log.isInfoEnabled()) { log.info(String.format("Authorizing service request with STS tokens %s", tokens)); } - request.setHeader(SECURITY_TOKEN, tokens.getSessionToken()); - session.getClient().setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKey(), tokens.getSecretAccessKey(), tokens.getSessionToken())); } From e3c5167182c167b9018ac86b3f7091e4779c04ab Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 8 Aug 2023 10:34:22 +0200 Subject: [PATCH 16/84] Review registration of handlers depending on setup from AWS CLI. --- .../java/ch/cyberduck/core/s3/S3Session.java | 108 +++++++++--------- 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 88e4765c1c8..438574f1737 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -209,6 +209,20 @@ protected RequestEntityRestStorageService connect(final Proxy proxy, final HostK configuration.addInterceptorLast(sts = new STSAssumeRoleCredentialsRequestInterceptor(oauth, this, trust, key)); configuration.setServiceUnavailableRetryStrategy(new STSAssumeRoleTokenExpiredResponseInterceptor(this, oauth, sts, prompt)); } + else { + if(S3Session.isAwsHostname(host.getHostname())) { + // Try auto-configure + if(Scheme.isURL(host.getProtocol().getContext())) { + configuration.setServiceUnavailableRetryStrategy(new S3TokenExpiredResponseInterceptor(this, + new AWSSessionCredentialsRetriever.Configurator(trust, key, this, host.getProtocol().getContext()) + )); + } + else { + configuration.setServiceUnavailableRetryStrategy(new S3TokenExpiredResponseInterceptor(this, + new AWSProfileSTSCredentialsConfigurator(new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt))); + } + } + } if(preferences.getBoolean("s3.upload.expect-continue")) { final String header = HTTP.EXPECT_DIRECTIVE; if(log.isDebugEnabled()) { @@ -261,24 +275,15 @@ public void process(final HttpRequest request, final HttpContext context) { request.setHeader(S3_ALTERNATE_DATE, SignatureUtils.formatAwsFlavouredISO8601Date(client.getCurrentTimeWithOffset())); } }); - if(host.getProtocol().isTokenConfigurable()) { - configuration.addInterceptorLast(new HttpRequestInterceptor() { - @Override - public void process(final HttpRequest request, final HttpContext context) { - final ProviderCredentials credentials = client.getProviderCredentials(); - if(credentials instanceof AWSSessionCredentials) { - request.setHeader(SECURITY_TOKEN, ((AWSSessionCredentials) credentials).getSessionToken()); - } + configuration.addInterceptorLast(new HttpRequestInterceptor() { + @Override + public void process(final HttpRequest request, final HttpContext context) { + final ProviderCredentials credentials = client.getProviderCredentials(); + if(credentials instanceof AWSSessionCredentials) { + request.setHeader(SECURITY_TOKEN, ((AWSSessionCredentials) credentials).getSessionToken()); } - }); - configuration.setServiceUnavailableRetryStrategy(new S3TokenExpiredResponseInterceptor(this, - S3Session.isAwsHostname(host.getHostname()) ? - Scheme.isURL(host.getProtocol().getContext()) ? - // Obtain credentials from instance metadata - new AWSSessionCredentialsRetriever.Configurator(trust, key, this, host.getProtocol().getContext()) : - new AWSProfileSTSCredentialsConfigurator(new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt) : CredentialsConfigurator.DISABLED - )); - } + } + }); switch(authenticationHeaderSignatureVersion) { case AWS4HMACSHA256: configuration.addInterceptorLast(new S3AWS4SignatureRequestInterceptor(this)); @@ -294,47 +299,46 @@ public void process(final HttpRequest request, final HttpContext context) { @Override public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - if(Scheme.isURL(host.getProtocol().getContext())) { - try { - // Obtain credentials from instance metadata - final Credentials temporary = new AWSSessionCredentialsRetriever(trust, key, this, host.getProtocol().getContext()).get(); - client.setProviderCredentials(new AWSSessionCredentials(temporary.getUsername(), temporary.getPassword(), - temporary.getToken())); - } - catch(ConnectionTimeoutException | ConnectionRefusedException | ResolveFailedException | NotfoundException | - InteroperabilityException e) { - log.warn(String.format("Failure to retrieve session credentials from . %s", e.getMessage())); - throw new LoginFailureException(e.getDetail(false), e); - } + if(host.getProtocol().isOAuthConfigurable()) { + // Get temporary credentials from STS using Web Identity (OIDC) token + final STSTokens tokens = sts.refresh(oauth.authorize(host, prompt, cancel)); + client.setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKey(), + tokens.getSecretAccessKey(), tokens.getSessionToken())); } else { - if(host.getProtocol().isOAuthConfigurable()) { - // Get temporary credentials from STS using Web Identity (OIDC) token - final STSTokens tokens = sts.refresh(oauth.authorize(host, prompt, cancel)); - client.setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKey(), - tokens.getSecretAccessKey(), tokens.getSessionToken())); - } - else { - final Credentials credentials; - // Only for AWS - if(isAwsHostname(host.getHostname())) { - // Try auto-configure - credentials = new AWSProfileSTSCredentialsConfigurator( - new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt).configure(host); - } - else { - credentials = host.getCredentials(); - } - if(StringUtils.isNotBlank(credentials.getToken())) { - client.setProviderCredentials(credentials.isAnonymousLogin() ? null : - new AWSSessionCredentials(credentials.getUsername(), credentials.getPassword(), - credentials.getToken())); + final Credentials credentials; + // Only for AWS + if(isAwsHostname(host.getHostname())) { + // Try auto-configure + if(Scheme.isURL(host.getProtocol().getContext())) { + try { + // Obtain credentials from instance metadata + credentials = new AWSSessionCredentialsRetriever(trust, key, this, host.getProtocol().getContext()).get(); + } + catch(ConnectionTimeoutException | ConnectionRefusedException | ResolveFailedException | + NotfoundException | + InteroperabilityException e) { + log.warn(String.format("Failure to retrieve session credentials from . %s", e.getMessage())); + throw new LoginFailureException(e.getDetail(false), e); + } } else { - client.setProviderCredentials(credentials.isAnonymousLogin() ? null : - new AWSCredentials(credentials.getUsername(), credentials.getPassword())); + credentials = new AWSProfileSTSCredentialsConfigurator( + new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt).configure(host); } } + else { + credentials = host.getCredentials(); + } + if(StringUtils.isNotBlank(credentials.getToken())) { + client.setProviderCredentials(credentials.isAnonymousLogin() ? null : + new AWSSessionCredentials(credentials.getUsername(), credentials.getPassword(), + credentials.getToken())); + } + else { + client.setProviderCredentials(credentials.isAnonymousLogin() ? null : + new AWSCredentials(credentials.getUsername(), credentials.getPassword())); + } } if(host.getCredentials().isPassed()) { log.warn(String.format("Skip verifying credentials with previous successful authentication event for %s", this)); From 4f2c2c8ae65affed11fcd8db2c58cef595d8842b Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 8 Aug 2023 11:17:53 +0200 Subject: [PATCH 17/84] Allow failure when reloading credentials configuration. --- .../core/brick/BrickCredentialsConfigurator.java | 3 ++- .../java/ch/cyberduck/core/CredentialsConfigurator.java | 4 +++- .../cyberduck/core/CredentialsConfiguratorFactory.java | 9 ++++++++- .../ch/cyberduck/core/JumpHostConfiguratorFactory.java | 9 ++++++++- .../java/ch/cyberduck/core/JumphostConfigurator.java | 4 +++- .../core/WindowsIntegratedCredentialsConfigurator.java | 3 ++- .../cyberduck/core/auth/AWSCredentialsConfigurator.java | 3 ++- .../core/auth/AWSSessionCredentialsRetriever.java | 3 ++- .../core/sts/AWSProfileSTSCredentialsConfigurator.java | 2 +- .../sftp/openssh/OpenSSHCredentialsConfigurator.java | 3 ++- .../core/sftp/openssh/OpenSSHJumpHostConfigurator.java | 3 ++- 11 files changed, 35 insertions(+), 11 deletions(-) diff --git a/brick/src/main/java/ch/cyberduck/core/brick/BrickCredentialsConfigurator.java b/brick/src/main/java/ch/cyberduck/core/brick/BrickCredentialsConfigurator.java index f02cf5076d8..24d9137e73a 100644 --- a/brick/src/main/java/ch/cyberduck/core/brick/BrickCredentialsConfigurator.java +++ b/brick/src/main/java/ch/cyberduck/core/brick/BrickCredentialsConfigurator.java @@ -19,6 +19,7 @@ import ch.cyberduck.core.Credentials; import ch.cyberduck.core.CredentialsConfigurator; import ch.cyberduck.core.Host; +import ch.cyberduck.core.exception.LoginCanceledException; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -41,7 +42,7 @@ public Credentials configure(final Host host) { } @Override - public CredentialsConfigurator reload() { + public CredentialsConfigurator reload() throws LoginCanceledException { return this; } } diff --git a/core/src/main/java/ch/cyberduck/core/CredentialsConfigurator.java b/core/src/main/java/ch/cyberduck/core/CredentialsConfigurator.java index 8646b50f39b..9f47a84faf0 100644 --- a/core/src/main/java/ch/cyberduck/core/CredentialsConfigurator.java +++ b/core/src/main/java/ch/cyberduck/core/CredentialsConfigurator.java @@ -18,6 +18,8 @@ * dkocher@cyberduck.ch */ +import ch.cyberduck.core.exception.LoginCanceledException; + public interface CredentialsConfigurator { /** @@ -27,7 +29,7 @@ public interface CredentialsConfigurator { */ Credentials configure(Host host); - CredentialsConfigurator reload(); + CredentialsConfigurator reload() throws LoginCanceledException; CredentialsConfigurator DISABLED = new CredentialsConfigurator() { @Override diff --git a/core/src/main/java/ch/cyberduck/core/CredentialsConfiguratorFactory.java b/core/src/main/java/ch/cyberduck/core/CredentialsConfiguratorFactory.java index e1b379ea991..0a9c7439264 100644 --- a/core/src/main/java/ch/cyberduck/core/CredentialsConfiguratorFactory.java +++ b/core/src/main/java/ch/cyberduck/core/CredentialsConfiguratorFactory.java @@ -18,6 +18,8 @@ * dkocher@cyberduck.ch */ +import ch.cyberduck.core.exception.LoginCanceledException; + public final class CredentialsConfiguratorFactory { private CredentialsConfiguratorFactory() { @@ -29,6 +31,11 @@ private CredentialsConfiguratorFactory() { * @return Configurator for default settings */ public static CredentialsConfigurator get(final Protocol protocol) { - return protocol.getCredentialsFinder().reload(); + try { + return protocol.getCredentialsFinder().reload(); + } + catch(LoginCanceledException e) { + return CredentialsConfigurator.DISABLED; + } } } diff --git a/core/src/main/java/ch/cyberduck/core/JumpHostConfiguratorFactory.java b/core/src/main/java/ch/cyberduck/core/JumpHostConfiguratorFactory.java index 806d1c93959..80e6802c0df 100644 --- a/core/src/main/java/ch/cyberduck/core/JumpHostConfiguratorFactory.java +++ b/core/src/main/java/ch/cyberduck/core/JumpHostConfiguratorFactory.java @@ -18,6 +18,8 @@ * dkocher@cyberduck.ch */ +import ch.cyberduck.core.exception.LoginCanceledException; + public final class JumpHostConfiguratorFactory { private JumpHostConfiguratorFactory() { @@ -29,6 +31,11 @@ private JumpHostConfiguratorFactory() { * @return Configurator for default settings */ public static JumphostConfigurator get(final Protocol protocol) { - return protocol.getJumpHostFinder().reload(); + try { + return protocol.getJumpHostFinder().reload(); + } + catch(LoginCanceledException e) { + return JumphostConfigurator.DISABLED; + } } } diff --git a/core/src/main/java/ch/cyberduck/core/JumphostConfigurator.java b/core/src/main/java/ch/cyberduck/core/JumphostConfigurator.java index f6762af0bed..afc3a1aeb1a 100644 --- a/core/src/main/java/ch/cyberduck/core/JumphostConfigurator.java +++ b/core/src/main/java/ch/cyberduck/core/JumphostConfigurator.java @@ -18,11 +18,13 @@ * dkocher@cyberduck.ch */ +import ch.cyberduck.core.exception.LoginCanceledException; + public interface JumphostConfigurator { Host getJumphost(String alias); - JumphostConfigurator reload(); + JumphostConfigurator reload() throws LoginCanceledException; JumphostConfigurator DISABLED = new JumphostConfigurator() { @Override diff --git a/core/src/main/java/ch/cyberduck/core/WindowsIntegratedCredentialsConfigurator.java b/core/src/main/java/ch/cyberduck/core/WindowsIntegratedCredentialsConfigurator.java index 41e1e48970f..89ab3478e27 100644 --- a/core/src/main/java/ch/cyberduck/core/WindowsIntegratedCredentialsConfigurator.java +++ b/core/src/main/java/ch/cyberduck/core/WindowsIntegratedCredentialsConfigurator.java @@ -15,6 +15,7 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.preferences.HostPreferences; import org.apache.commons.lang3.StringUtils; @@ -51,7 +52,7 @@ public Credentials configure(final Host host) { } @Override - public CredentialsConfigurator reload() { + public CredentialsConfigurator reload() throws LoginCanceledException { return this; } } diff --git a/s3/src/main/java/ch/cyberduck/core/auth/AWSCredentialsConfigurator.java b/s3/src/main/java/ch/cyberduck/core/auth/AWSCredentialsConfigurator.java index 0fd4ae1b1b5..21ad09189e6 100644 --- a/s3/src/main/java/ch/cyberduck/core/auth/AWSCredentialsConfigurator.java +++ b/s3/src/main/java/ch/cyberduck/core/auth/AWSCredentialsConfigurator.java @@ -19,6 +19,7 @@ import ch.cyberduck.core.CredentialsConfigurator; import ch.cyberduck.core.Host; import ch.cyberduck.core.LoginOptions; +import ch.cyberduck.core.exception.LoginCanceledException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -66,7 +67,7 @@ public Credentials configure(final Host host) { } @Override - public CredentialsConfigurator reload() { + public CredentialsConfigurator reload() throws LoginCanceledException { for(AWSCredentialsProvider provider : providers) { provider.refresh(); } diff --git a/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java b/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java index 729642bebde..3b155d244ae 100644 --- a/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java +++ b/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java @@ -32,6 +32,7 @@ import ch.cyberduck.core.dav.DAVSession; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.InteroperabilityException; +import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.proxy.ProxyFactory; import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; @@ -82,7 +83,7 @@ public Configurator(final X509TrustManager trust, final X509KeyManager key, fina } @Override - public Configurator reload() { + public Configurator reload() throws LoginCanceledException { return this; } diff --git a/s3/src/main/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfigurator.java b/s3/src/main/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfigurator.java index 9948de43cbc..986269cafa9 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfigurator.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfigurator.java @@ -288,7 +288,7 @@ else if(StringUtils.isNotBlank(basicProfile.getAwsSessionToken())) { } @Override - public CredentialsConfigurator reload() { + public CredentialsConfigurator reload() throws LoginCanceledException { // See https://docs.aws.amazon.com/sdkref/latest/guide/creds-config-files.html for configuration behavior final Local configFile = LocalFactory.get(directory, "config"); final Local credentialsFile = LocalFactory.get(directory, "credentials"); diff --git a/ssh/src/main/java/ch/cyberduck/core/sftp/openssh/OpenSSHCredentialsConfigurator.java b/ssh/src/main/java/ch/cyberduck/core/sftp/openssh/OpenSSHCredentialsConfigurator.java index d23e4a6c039..f2c1e9f4cc5 100644 --- a/ssh/src/main/java/ch/cyberduck/core/sftp/openssh/OpenSSHCredentialsConfigurator.java +++ b/ssh/src/main/java/ch/cyberduck/core/sftp/openssh/OpenSSHCredentialsConfigurator.java @@ -21,6 +21,7 @@ import ch.cyberduck.core.Local; import ch.cyberduck.core.LocalFactory; import ch.cyberduck.core.LoginOptions; +import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.preferences.HostPreferences; import ch.cyberduck.core.sftp.openssh.config.transport.OpenSshConfig; @@ -91,7 +92,7 @@ public Credentials configure(final Host host) { } @Override - public CredentialsConfigurator reload() { + public CredentialsConfigurator reload() throws LoginCanceledException { configuration.refresh(); return this; } diff --git a/ssh/src/main/java/ch/cyberduck/core/sftp/openssh/OpenSSHJumpHostConfigurator.java b/ssh/src/main/java/ch/cyberduck/core/sftp/openssh/OpenSSHJumpHostConfigurator.java index 815f7966d99..da72a518f91 100644 --- a/ssh/src/main/java/ch/cyberduck/core/sftp/openssh/OpenSSHJumpHostConfigurator.java +++ b/ssh/src/main/java/ch/cyberduck/core/sftp/openssh/OpenSSHJumpHostConfigurator.java @@ -21,6 +21,7 @@ import ch.cyberduck.core.LocalFactory; import ch.cyberduck.core.ProtocolFactory; import ch.cyberduck.core.exception.HostParserException; +import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.sftp.SFTPProtocol; import ch.cyberduck.core.sftp.openssh.config.transport.OpenSshConfig; @@ -75,7 +76,7 @@ public Host getJumphost(final String alias) { } @Override - public JumphostConfigurator reload() { + public JumphostConfigurator reload() throws LoginCanceledException { hostname.reload(); credentials.reload(); return this; From ef567c1d5e3f5b9c793f3f8691fbb60721058cc5 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 8 Aug 2023 11:18:10 +0200 Subject: [PATCH 18/84] Reuse interceptor to provide login credentials from AWS CLI config. --- .../java/ch/cyberduck/core/s3/S3Session.java | 41 ++++++------ .../S3TokenExpiredResponseInterceptor.java | 5 ++ .../ch/cyberduck/core/s3/S3SessionTest.java | 5 +- .../cyberduck/core/s3/S3UrlProviderTest.java | 65 ++++++++++++------- .../core/spectra/SpectraSession.java | 4 +- 5 files changed, 73 insertions(+), 47 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 438574f1737..9b923368f19 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -21,7 +21,6 @@ import ch.cyberduck.core.AttributedList; import ch.cyberduck.core.Credentials; -import ch.cyberduck.core.CredentialsConfigurator; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostKeyCallback; import ch.cyberduck.core.ListProgressListener; @@ -68,8 +67,8 @@ import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.sts.AWSProfileSTSCredentialsConfigurator; import ch.cyberduck.core.sts.S3TokenExpiredResponseInterceptor; -import ch.cyberduck.core.sts.STSAssumeRoleTokenExpiredResponseInterceptor; import ch.cyberduck.core.sts.STSAssumeRoleCredentialsRequestInterceptor; +import ch.cyberduck.core.sts.STSAssumeRoleTokenExpiredResponseInterceptor; import ch.cyberduck.core.sts.STSTokens; import ch.cyberduck.core.threading.BackgroundExceptionCallable; import ch.cyberduck.core.threading.CancelCallback; @@ -111,8 +110,18 @@ public class S3Session extends HttpSession { private final PreferencesReader preferences = new HostPreferences(host); + /** + * Handle authentication with OpenID connect retrieving token for STS + */ private OAuth2RequestInterceptor oauth; + /** + * Swap OIDC Id token with temporary security credentials + */ private STSAssumeRoleCredentialsRequestInterceptor sts; + /** + * Fetch latest temporary session token from AWS CLI configuration or instance metadata + */ + private S3TokenExpiredResponseInterceptor token; private final S3AccessControlListFeature acl = new S3AccessControlListFeature(this); @@ -199,7 +208,7 @@ protected String getRestMetadataPrefix() { } @Override - protected RequestEntityRestStorageService connect(final Proxy proxy, final HostKeyCallback hostkey, final LoginCallback prompt, final CancelCallback cancel) { + protected RequestEntityRestStorageService connect(final Proxy proxy, final HostKeyCallback hostkey, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { final HttpClientBuilder configuration = builder.build(proxy, this, prompt); if(host.getProtocol().isOAuthConfigurable()) { configuration.addInterceptorLast(oauth = new OAuth2RequestInterceptor(builder.build(ProxyFactory.get() @@ -213,12 +222,12 @@ protected RequestEntityRestStorageService connect(final Proxy proxy, final HostK if(S3Session.isAwsHostname(host.getHostname())) { // Try auto-configure if(Scheme.isURL(host.getProtocol().getContext())) { - configuration.setServiceUnavailableRetryStrategy(new S3TokenExpiredResponseInterceptor(this, + configuration.setServiceUnavailableRetryStrategy(token = new S3TokenExpiredResponseInterceptor(this, new AWSSessionCredentialsRetriever.Configurator(trust, key, this, host.getProtocol().getContext()) )); } else { - configuration.setServiceUnavailableRetryStrategy(new S3TokenExpiredResponseInterceptor(this, + configuration.setServiceUnavailableRetryStrategy(token = new S3TokenExpiredResponseInterceptor(this, new AWSProfileSTSCredentialsConfigurator(new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt))); } } @@ -309,22 +318,14 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal final Credentials credentials; // Only for AWS if(isAwsHostname(host.getHostname())) { - // Try auto-configure - if(Scheme.isURL(host.getProtocol().getContext())) { - try { - // Obtain credentials from instance metadata - credentials = new AWSSessionCredentialsRetriever(trust, key, this, host.getProtocol().getContext()).get(); - } - catch(ConnectionTimeoutException | ConnectionRefusedException | ResolveFailedException | - NotfoundException | - InteroperabilityException e) { - log.warn(String.format("Failure to retrieve session credentials from . %s", e.getMessage())); - throw new LoginFailureException(e.getDetail(false), e); - } + try { + // Try auto-configure + credentials = token.refresh(); } - else { - credentials = new AWSProfileSTSCredentialsConfigurator( - new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt).configure(host); + catch(ConnectionTimeoutException | ConnectionRefusedException | ResolveFailedException | + NotfoundException | InteroperabilityException e) { + log.warn(String.format("Failure to retrieve session credentials from . %s", e.getMessage())); + throw new LoginFailureException(e.getDetail(false), e); } } else { diff --git a/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java index 4c698b05364..7c69bce75bd 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java @@ -17,6 +17,7 @@ import ch.cyberduck.core.Credentials; import ch.cyberduck.core.CredentialsConfigurator; +import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ExpiredTokenException; import ch.cyberduck.core.http.DisabledServiceUnavailableRetryStrategy; import ch.cyberduck.core.s3.S3ExceptionMappingService; @@ -91,4 +92,8 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun } return false; } + + public Credentials refresh() throws BackgroundException { + return configurator.reload().configure(session.getHost()); + } } diff --git a/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java b/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java index 77c80efc12d..35e9a4dd412 100644 --- a/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java +++ b/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java @@ -225,9 +225,10 @@ public void testFeatures() { } @Test - public void testBucketVirtualHostStyleCustomHost() { + public void testBucketVirtualHostStyleCustomHost() throws Exception { final Host host = new Host(new S3Protocol(), "test-eu-central-1-cyberduck"); - assertFalse(new S3Session(host).connect(new DisabledProxyFinder().find(host.getHostname()), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()) + assertFalse(new S3Session(host).connect + (new DisabledProxyFinder().find(host.getHostname()), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()) .getDisableDnsBuckets()); } diff --git a/s3/src/test/java/ch/cyberduck/core/s3/S3UrlProviderTest.java b/s3/src/test/java/ch/cyberduck/core/s3/S3UrlProviderTest.java index 879b471dc91..4fae1d38bc5 100644 --- a/s3/src/test/java/ch/cyberduck/core/s3/S3UrlProviderTest.java +++ b/s3/src/test/java/ch/cyberduck/core/s3/S3UrlProviderTest.java @@ -9,6 +9,7 @@ import ch.cyberduck.core.DisabledPasswordStore; import ch.cyberduck.core.Host; import ch.cyberduck.core.Path; +import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.proxy.Proxy; import ch.cyberduck.test.IntegrationTest; @@ -25,7 +26,7 @@ public class S3UrlProviderTest extends AbstractS3Test { @Test - public void testToHttpURL() { + public void testToHttpURL() throws Exception { final S3Session session = new S3Session(new Host(new S3Protocol() { @Override public String getAuthorization() { @@ -34,7 +35,13 @@ public String getAuthorization() { }, new S3Protocol().getDefaultHostname())) { @Override public RequestEntityRestStorageService getClient() { - return this.connect(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + try { + return this.connect(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + } + catch(BackgroundException e) { + fail(); + throw new RuntimeException(e); + } } }; Path p = new Path("/bucket/f/key f", EnumSet.of(Path.Type.file)); @@ -50,7 +57,7 @@ public String findLoginPassword(final Host bookmark) { public void testWebUrl() { { final DescriptiveUrlBag list = new S3UrlProvider(session, Collections.emptyMap()).toUrl(new Path("/test-eu-west-1-cyberduck/key", - EnumSet.of(Path.Type.file))).filter(DescriptiveUrl.Type.http); + EnumSet.of(Path.Type.file))).filter(DescriptiveUrl.Type.http); assertEquals(2, list.size()); final Iterator provider = list.iterator(); assertEquals("https://test-eu-west-1-cyberduck.s3.amazonaws.com/key", provider.next().getUrl()); @@ -59,7 +66,7 @@ public void testWebUrl() { session.getHost().setWebURL("https://cdn.cyberduck.io/"); { final DescriptiveUrlBag list = new S3UrlProvider(session, Collections.emptyMap()).toUrl(new Path("/test-eu-west-1-cyberduck/key", - EnumSet.of(Path.Type.file))).filter(DescriptiveUrl.Type.http); + EnumSet.of(Path.Type.file))).filter(DescriptiveUrl.Type.http); assertEquals(3, list.size()); final Iterator provider = list.iterator(); assertEquals("https://test-eu-west-1-cyberduck.s3.amazonaws.com/key", provider.next().getUrl()); @@ -71,21 +78,21 @@ public void testWebUrl() { @Test public void testProviderUriWithKey() { final Iterator provider = new S3UrlProvider(session, Collections.emptyMap()).toUrl(new Path("/test-eu-west-1-cyberduck/key", - EnumSet.of(Path.Type.file))).filter(DescriptiveUrl.Type.provider).iterator(); + EnumSet.of(Path.Type.file))).filter(DescriptiveUrl.Type.provider).iterator(); assertEquals("s3://test-eu-west-1-cyberduck/key", provider.next().getUrl()); } @Test public void testProviderUriRoot() { final Iterator provider = new S3UrlProvider(session, Collections.emptyMap()).toUrl(new Path("/test-eu-west-1-cyberduck", - EnumSet.of(Path.Type.directory))).filter(DescriptiveUrl.Type.provider).iterator(); + EnumSet.of(Path.Type.directory))).filter(DescriptiveUrl.Type.provider).iterator(); assertEquals("s3://test-eu-west-1-cyberduck/", provider.next().getUrl()); } @Test public void testHttpUri() { final Iterator http = new S3UrlProvider(session, Collections.emptyMap()).toUrl(new Path("/test-eu-west-1-cyberduck/key", - EnumSet.of(Path.Type.file))).filter(DescriptiveUrl.Type.http).iterator(); + EnumSet.of(Path.Type.file))).filter(DescriptiveUrl.Type.http).iterator(); assertEquals("https://test-eu-west-1-cyberduck.s3.amazonaws.com/key", http.next().getUrl()); assertEquals("http://test-eu-west-1-cyberduck.s3.amazonaws.com/key", http.next().getUrl()); } @@ -94,37 +101,49 @@ public void testHttpUri() { public void testHttpUriCustomPort() { session.getHost().setPort(8443); final Iterator http = new S3UrlProvider(session, Collections.emptyMap()).toUrl(new Path("/test-eu-west-1-cyberduck/key", - EnumSet.of(Path.Type.file))).filter(DescriptiveUrl.Type.http).iterator(); + EnumSet.of(Path.Type.file))).filter(DescriptiveUrl.Type.http).iterator(); assertEquals("https://test-eu-west-1-cyberduck.s3.amazonaws.com:8443/key", http.next().getUrl()); assertEquals("http://test-eu-west-1-cyberduck.s3.amazonaws.com/key", http.next().getUrl()); } @Test - public void testToSignedUrlAnonymous() { + public void testToSignedUrlAnonymous() throws Exception { final S3Session session = new S3Session(new Host(new S3Protocol(), new S3Protocol().getDefaultHostname(), - new Credentials("anonymous", null))) { + new Credentials("anonymous", null))) { @Override public RequestEntityRestStorageService getClient() { - return this.connect(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + try { + return this.connect(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + } + catch(BackgroundException e) { + fail(); + throw new RuntimeException(e); + } } }; assertEquals(DescriptiveUrl.EMPTY, - new S3UrlProvider(session, Collections.emptyMap(), new DisabledPasswordStore() { - @Override - public String findLoginPassword(final Host bookmark) { - return "k"; - } - }).toUrl(new Path("/test-eu-west-1-cyberduck/test f", EnumSet.of(Path.Type.file))).find(DescriptiveUrl.Type.signed) + new S3UrlProvider(session, Collections.emptyMap(), new DisabledPasswordStore() { + @Override + public String findLoginPassword(final Host bookmark) { + return "k"; + } + }).toUrl(new Path("/test-eu-west-1-cyberduck/test f", EnumSet.of(Path.Type.file))).find(DescriptiveUrl.Type.signed) ); } @Test - public void testToSignedUrlThirdparty() { + public void testToSignedUrlThirdparty() throws Exception { final S3Session session = new S3Session(new Host(new S3Protocol(), "s.greenqloud.com", - new Credentials("k", "s"))) { + new Credentials("k", "s"))) { @Override public RequestEntityRestStorageService getClient() { - return this.connect(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + try { + return this.connect(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + } + catch(BackgroundException e) { + fail(); + throw new RuntimeException(e); + } } }; final S3UrlProvider provider = new S3UrlProvider(session, Collections.emptyMap(), new DisabledPasswordStore() { @@ -134,7 +153,7 @@ public String findLoginPassword(final Host bookmark) { } }); assertNotNull( - provider.toUrl(new Path("/test-eu-west-1-cyberduck/test", EnumSet.of(Path.Type.file))).find(DescriptiveUrl.Type.signed) + provider.toUrl(new Path("/test-eu-west-1-cyberduck/test", EnumSet.of(Path.Type.file))).find(DescriptiveUrl.Type.signed) ); } @@ -147,12 +166,12 @@ public String findLoginPassword(final Host bookmark) { } }); assertTrue(provider.toSignedUrl(new Path("/test-eu-west-1-cyberduck/test", EnumSet.of(Path.Type.file)), 30).getUrl().startsWith( - "https://test-eu-west-1-cyberduck.s3.amazonaws.com/test?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=")); + "https://test-eu-west-1-cyberduck.s3.amazonaws.com/test?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=")); } @Test public void testPlaceholder() { assertTrue( - new S3UrlProvider(session, Collections.emptyMap()).toUrl(new Path("/test-eu-west-1-cyberduck/test", EnumSet.of(Path.Type.directory))).filter(DescriptiveUrl.Type.signed).isEmpty()); + new S3UrlProvider(session, Collections.emptyMap()).toUrl(new Path("/test-eu-west-1-cyberduck/test", EnumSet.of(Path.Type.directory))).filter(DescriptiveUrl.Type.signed).isEmpty()); } } diff --git a/spectra/src/main/java/ch/cyberduck/core/spectra/SpectraSession.java b/spectra/src/main/java/ch/cyberduck/core/spectra/SpectraSession.java index 9df40a60e40..b434d4421b6 100644 --- a/spectra/src/main/java/ch/cyberduck/core/spectra/SpectraSession.java +++ b/spectra/src/main/java/ch/cyberduck/core/spectra/SpectraSession.java @@ -21,8 +21,8 @@ import ch.cyberduck.core.LoginCallback; import ch.cyberduck.core.UrlProvider; import ch.cyberduck.core.cdn.DistributionConfiguration; +import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.*; -import ch.cyberduck.core.preferences.HostPreferences; import ch.cyberduck.core.proxy.Proxy; import ch.cyberduck.core.s3.RequestEntityRestStorageService; import ch.cyberduck.core.s3.S3Session; @@ -41,7 +41,7 @@ public SpectraSession(final Host host, final X509TrustManager trust, final X509K } @Override - protected RequestEntityRestStorageService connect(final Proxy proxy, final HostKeyCallback hostkey, final LoginCallback prompt, final CancelCallback cancel) { + protected RequestEntityRestStorageService connect(final Proxy proxy, final HostKeyCallback hostkey, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { final RequestEntityRestStorageService client = super.connect(proxy, hostkey, prompt, cancel); final Jets3tProperties configuration = client.getConfiguration(); configuration.setProperty("s3service.disable-dns-buckets", String.valueOf(true)); From 6c9a54ec4b4cc2496bffa92d837093f10723bcb6 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 8 Aug 2023 14:23:20 +0200 Subject: [PATCH 19/84] Use configured directory. --- .../core/sts/AWSProfileSTSCredentialsConfigurator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfigurator.java b/s3/src/main/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfigurator.java index 986269cafa9..d7162993ea3 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfigurator.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfigurator.java @@ -341,7 +341,6 @@ public CredentialsConfigurator reload() throws LoginCanceledException { * @return Null on error reading from file or expired SSO credentials in cache */ private CachedCredential fetchSsoCredentials(final Map properties) { - final Local awsDirectory = LocalFactory.get(LocalFactory.get(), ".aws"); // See https://github.com/boto/botocore/blob/412aeb96c9a6ebc72aa1bdf33e58ddd48c7b048d/botocore/credentials.py#L2078-L2098 try { final ObjectMapper mapper = JsonMapper.builder() @@ -364,7 +363,7 @@ private CachedCredential fetchSsoCredentials(final Map propertie final String hash = BaseEncoding.base16().lowerCase().encode(hashCode.asBytes()); final String cachedCredentialsJson = String.format("%s.json", hash); final Local cachedCredentialsFile = - LocalFactory.get(LocalFactory.get(LocalFactory.get(awsDirectory, "cli"), "cache"), cachedCredentialsJson); + LocalFactory.get(LocalFactory.get(LocalFactory.get(directory, "cli"), "cache"), cachedCredentialsJson); if(log.isDebugEnabled()) { log.debug(String.format("Attempting to read SSO credentials from %s", cachedCredentialsFile.getAbsolute())); } From 23d02f3d8d3a436e7ef96a147fd09a1996973d29 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 8 Aug 2023 16:27:28 +0200 Subject: [PATCH 20/84] Fix #10917. --- .../cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java | 4 ++-- .../sts/STSAssumeRoleTokenExpiredResponseInterceptor.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java index 7c69bce75bd..d887f33de78 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java @@ -54,12 +54,12 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun if(executionCount <= MAX_RETRIES) { switch(response.getStatusLine().getStatusCode()) { case HttpStatus.SC_BAD_REQUEST: - final S3ServiceException failure; try { if(null != response.getEntity()) { EntityUtils.updateEntity(response, new BufferedHttpEntity(response.getEntity())); - failure = new S3ServiceException(response.getStatusLine().getReasonPhrase(), + final S3ServiceException failure = new S3ServiceException(response.getStatusLine().getReasonPhrase(), EntityUtils.toString(response.getEntity())); + failure.setResponseCode(response.getStatusLine().getStatusCode()); if(new S3ExceptionMappingService().map(failure) instanceof ExpiredTokenException) { if(log.isWarnEnabled()) { log.warn(String.format("Handle failure %s", failure)); diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java index ed4c5ea9194..d12bb4d45cc 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java @@ -64,12 +64,12 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun } switch(response.getStatusLine().getStatusCode()) { case HttpStatus.SC_BAD_REQUEST: - final S3ServiceException failure; try { if(null != response.getEntity()) { EntityUtils.updateEntity(response, new BufferedHttpEntity(response.getEntity())); - failure = new S3ServiceException(response.getStatusLine().getReasonPhrase(), + final S3ServiceException failure = new S3ServiceException(response.getStatusLine().getReasonPhrase(), EntityUtils.toString(response.getEntity())); + failure.setResponseCode(response.getStatusLine().getStatusCode()); if(new S3ExceptionMappingService().map(failure) instanceof ExpiredTokenException) { if(log.isWarnEnabled()) { log.warn(String.format("Handle failure %s", failure)); From 2b1669f9f1330e4fbfffe38bd6f3c09c0b8435e2 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 8 Aug 2023 22:43:39 +0200 Subject: [PATCH 21/84] Set subject from JWT claims as role session name by default. --- s3/pom.xml | 12 ++++++++++- .../STSAssumeRoleAuthorizationService.java | 21 ++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/s3/pom.xml b/s3/pom.xml index 3c89a68fc43..f5094d1dcf5 100644 --- a/s3/pom.xml +++ b/s3/pom.xml @@ -13,7 +13,8 @@ ~ GNU General Public License for more details. --> - + 4.0.0 ch.cyberduck @@ -23,6 +24,10 @@ s3 jar + + 4.2.1 + + ch.cyberduck @@ -65,6 +70,11 @@ com.amazonaws aws-java-sdk-s3 + + com.auth0 + java-jwt + ${jwt.version} + ch.cyberduck cryptomator diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 810f68ad9ce..dda9543b46e 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -19,7 +19,6 @@ import ch.cyberduck.core.Credentials; import ch.cyberduck.core.Host; import ch.cyberduck.core.OAuthTokens; -import ch.cyberduck.core.PreferencesUseragentProvider; import ch.cyberduck.core.aws.CustomClientConfiguration; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LoginFailureException; @@ -40,6 +39,8 @@ import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException; import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityRequest; import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityResult; +import com.auth0.jwt.JWT; +import com.auth0.jwt.exceptions.JWTDecodeException; public class STSAssumeRoleAuthorizationService { private static final Logger log = LogManager.getLogger(STSAssumeRoleAuthorizationService.class); @@ -62,7 +63,8 @@ public STSAssumeRoleAuthorizationService(final AWSSecurityTokenService service) public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws BackgroundException { final AssumeRoleWithWebIdentityRequest request = new AssumeRoleWithWebIdentityRequest(); - request.withWebIdentityToken(StringUtils.isNotBlank(oauth.getIdToken()) ? oauth.getIdToken() : oauth.getAccessToken()); + final String token = StringUtils.isNotBlank(oauth.getIdToken()) ? oauth.getIdToken() : oauth.getAccessToken(); + request.withWebIdentityToken(token); if(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds") != 0) { request.withDurationSeconds(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds")); } @@ -76,7 +78,20 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws request.withRoleSessionName(new HostPreferences(bookmark).getProperty("s3.assumerole.rolesessionname")); } else { - request.withRoleSessionName(new AsciiRandomStringService().random()); + try { + final String sub = JWT.decode(token).getSubject(); + if(StringUtils.isNotBlank(sub)) { + request.withRoleSessionName(sub); + } + else { + log.warn(String.format("Missing subject in decoding JWT %s", token)); + request.withRoleSessionName(new AsciiRandomStringService().random()); + } + } + catch(JWTDecodeException e) { + log.warn(String.format("Failure decoding JWT %s", token)); + request.withRoleSessionName(new AsciiRandomStringService().random()); + } } try { final AssumeRoleWithWebIdentityResult result = service.assumeRoleWithWebIdentity(request); From 0b78be938512e845ca0e290fc35f3956e07199d1 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 8 Aug 2023 22:51:25 +0200 Subject: [PATCH 22/84] Move class. --- s3/src/main/java/ch/cyberduck/core/s3/S3Session.java | 1 - .../core/{sts => s3}/S3TokenExpiredResponseInterceptor.java | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) rename s3/src/main/java/ch/cyberduck/core/{sts => s3}/S3TokenExpiredResponseInterceptor.java (97%) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 9b923368f19..dff30de3369 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -66,7 +66,6 @@ import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.sts.AWSProfileSTSCredentialsConfigurator; -import ch.cyberduck.core.sts.S3TokenExpiredResponseInterceptor; import ch.cyberduck.core.sts.STSAssumeRoleCredentialsRequestInterceptor; import ch.cyberduck.core.sts.STSAssumeRoleTokenExpiredResponseInterceptor; import ch.cyberduck.core.sts.STSTokens; diff --git a/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java similarity index 97% rename from s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java rename to s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java index d887f33de78..40596c4351c 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/S3TokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java @@ -1,4 +1,4 @@ -package ch.cyberduck.core.sts; +package ch.cyberduck.core.s3; /* * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. @@ -20,8 +20,6 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ExpiredTokenException; import ch.cyberduck.core.http.DisabledServiceUnavailableRetryStrategy; -import ch.cyberduck.core.s3.S3ExceptionMappingService; -import ch.cyberduck.core.s3.S3Session; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; From b55ec15f033bb1d2f35db0caf5a0b7b9f20fb5a4 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 8 Aug 2023 22:53:07 +0200 Subject: [PATCH 23/84] Move and rename class. --- .../S3CredentialsConfigurator.java} | 12 ++++++------ s3/src/main/java/ch/cyberduck/core/s3/S3Session.java | 3 +-- .../S3CredentialsConfiguratorTest.java} | 12 ++++++------ 3 files changed, 13 insertions(+), 14 deletions(-) rename s3/src/main/java/ch/cyberduck/core/{sts/AWSProfileSTSCredentialsConfigurator.java => s3/S3CredentialsConfigurator.java} (97%) rename s3/src/test/java/ch/cyberduck/core/{sts/AWSProfileSTSCredentialsConfiguratorTest.java => s3/S3CredentialsConfiguratorTest.java} (77%) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfigurator.java b/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java similarity index 97% rename from s3/src/main/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfigurator.java rename to s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java index d7162993ea3..b8db23d57e8 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfigurator.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java @@ -1,7 +1,7 @@ -package ch.cyberduck.core.sts; +package ch.cyberduck.core.s3; /* - * Copyright (c) 2002-2018 iterate GmbH. All rights reserved. + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify @@ -77,8 +77,8 @@ /** * Configure credentials from AWS CLI configuration and SSO cache */ -public class AWSProfileSTSCredentialsConfigurator implements CredentialsConfigurator { - private static final Logger log = LogManager.getLogger(AWSProfileSTSCredentialsConfigurator.class); +public class S3CredentialsConfigurator implements CredentialsConfigurator { + private static final Logger log = LogManager.getLogger(S3CredentialsConfigurator.class); private final Local directory; private final X509TrustManager trust; @@ -86,11 +86,11 @@ public class AWSProfileSTSCredentialsConfigurator implements CredentialsConfigur private final PasswordCallback prompt; private final Map profiles = new LinkedHashMap<>(); - public AWSProfileSTSCredentialsConfigurator(final X509TrustManager trust, final X509KeyManager key, final PasswordCallback prompt) { + public S3CredentialsConfigurator(final X509TrustManager trust, final X509KeyManager key, final PasswordCallback prompt) { this(LocalFactory.get(LocalFactory.get(), ".aws"), trust, key, prompt); } - public AWSProfileSTSCredentialsConfigurator(final Local directory, final X509TrustManager trust, final X509KeyManager key, final PasswordCallback prompt) { + public S3CredentialsConfigurator(final Local directory, final X509TrustManager trust, final X509KeyManager key, final PasswordCallback prompt) { this.directory = directory; this.trust = trust; this.key = key; diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index dff30de3369..461d68388a2 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -65,7 +65,6 @@ import ch.cyberduck.core.ssl.ThreadLocalHostnameDelegatingTrustManager; import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; -import ch.cyberduck.core.sts.AWSProfileSTSCredentialsConfigurator; import ch.cyberduck.core.sts.STSAssumeRoleCredentialsRequestInterceptor; import ch.cyberduck.core.sts.STSAssumeRoleTokenExpiredResponseInterceptor; import ch.cyberduck.core.sts.STSTokens; @@ -227,7 +226,7 @@ protected RequestEntityRestStorageService connect(final Proxy proxy, final HostK } else { configuration.setServiceUnavailableRetryStrategy(token = new S3TokenExpiredResponseInterceptor(this, - new AWSProfileSTSCredentialsConfigurator(new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt))); + new S3CredentialsConfigurator(new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt))); } } } diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfiguratorTest.java b/s3/src/test/java/ch/cyberduck/core/s3/S3CredentialsConfiguratorTest.java similarity index 77% rename from s3/src/test/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfiguratorTest.java rename to s3/src/test/java/ch/cyberduck/core/s3/S3CredentialsConfiguratorTest.java index 6a0f29efbc3..e7c022eb055 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AWSProfileSTSCredentialsConfiguratorTest.java +++ b/s3/src/test/java/ch/cyberduck/core/s3/S3CredentialsConfiguratorTest.java @@ -1,7 +1,7 @@ -package ch.cyberduck.core.sts; +package ch.cyberduck.core.s3; /* - * Copyright (c) 2002-2018 iterate GmbH. All rights reserved. + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify @@ -30,18 +30,18 @@ import static org.junit.Assert.assertEquals; -public class AWSProfileSTSCredentialsConfiguratorTest { +public class S3CredentialsConfiguratorTest { @Test public void testConfigure() throws Exception { - new AWSProfileSTSCredentialsConfigurator(new DisabledX509TrustManager(), new DefaultX509KeyManager(), new DisabledPasswordCallback()) + new S3CredentialsConfigurator(new DisabledX509TrustManager(), new DefaultX509KeyManager(), new DisabledPasswordCallback()) .reload().configure(new Host(new TestProtocol())); } @Test public void readFailureForInvalidAWSCredentialsProfileEntry() throws Exception { final Credentials credentials = new Credentials("test_s3_profile"); - final Credentials verify = new AWSProfileSTSCredentialsConfigurator(LocalFactory.get(new File("src/test/resources/invalid/.aws").getAbsolutePath()), + final Credentials verify = new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/invalid/.aws").getAbsolutePath()), new DisabledX509TrustManager(), new DefaultX509KeyManager(), new DisabledPasswordCallback()) .reload().configure(new Host(new TestProtocol(), StringUtils.EMPTY, credentials)); assertEquals(credentials, verify); @@ -49,7 +49,7 @@ public void readFailureForInvalidAWSCredentialsProfileEntry() throws Exception { @Test public void readSuccessForValidAWSCredentialsProfileEntry() throws Exception { - final Credentials verify = new AWSProfileSTSCredentialsConfigurator(LocalFactory.get(new File("src/test/resources/valid/.aws").getAbsolutePath()) + final Credentials verify = new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/valid/.aws").getAbsolutePath()) , new DisabledX509TrustManager(), new DefaultX509KeyManager(), new DisabledPasswordCallback()) .reload().configure(new Host(new TestProtocol(), StringUtils.EMPTY, new Credentials("test_s3_profile"))); assertEquals("EXAMPLEKEYID", verify.getUsername()); From 85891c6953cd5d084294117deaf3eac7cf885971 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 9 Aug 2023 00:23:34 +0200 Subject: [PATCH 24/84] Review. --- .../sts/STSAssumeRoleAuthorizationService.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index dda9543b46e..0a740f7ba03 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -64,33 +64,33 @@ public STSAssumeRoleAuthorizationService(final AWSSecurityTokenService service) public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws BackgroundException { final AssumeRoleWithWebIdentityRequest request = new AssumeRoleWithWebIdentityRequest(); final String token = StringUtils.isNotBlank(oauth.getIdToken()) ? oauth.getIdToken() : oauth.getAccessToken(); - request.withWebIdentityToken(token); + request.setWebIdentityToken(token); if(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds") != 0) { - request.withDurationSeconds(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds")); + request.setDurationSeconds(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds")); } if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.policy"))) { - request.withPolicy(new HostPreferences(bookmark).getProperty("s3.assumerole.policy")); + request.setPolicy(new HostPreferences(bookmark).getProperty("s3.assumerole.policy")); } if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.rolearn"))) { - request.withRoleArn(new HostPreferences(bookmark).getProperty("s3.assumerole.rolearn")); + request.setRoleArn(new HostPreferences(bookmark).getProperty("s3.assumerole.rolearn")); } if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.rolesessionname"))) { - request.withRoleSessionName(new HostPreferences(bookmark).getProperty("s3.assumerole.rolesessionname")); + request.setRoleSessionName(new HostPreferences(bookmark).getProperty("s3.assumerole.rolesessionname")); } else { try { final String sub = JWT.decode(token).getSubject(); if(StringUtils.isNotBlank(sub)) { - request.withRoleSessionName(sub); + request.setRoleSessionName(sub); } else { log.warn(String.format("Missing subject in decoding JWT %s", token)); - request.withRoleSessionName(new AsciiRandomStringService().random()); + request.setRoleSessionName(new AsciiRandomStringService().random()); } } catch(JWTDecodeException e) { log.warn(String.format("Failure decoding JWT %s", token)); - request.withRoleSessionName(new AsciiRandomStringService().random()); + request.setRoleSessionName(new AsciiRandomStringService().random()); } } try { From 1cd86317a91d7dc7c10d8a929e28349dde963fa1 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 9 Aug 2023 00:24:00 +0200 Subject: [PATCH 25/84] Add support for AssumeRoleWithSAML. --- .../STSAssumeRoleAuthorizationService.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 0a740f7ba03..3b678f182da 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -37,6 +37,8 @@ import com.amazonaws.services.securitytoken.AWSSecurityTokenService; import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException; +import com.amazonaws.services.securitytoken.model.AssumeRoleWithSAMLRequest; +import com.amazonaws.services.securitytoken.model.AssumeRoleWithSAMLResult; import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityRequest; import com.amazonaws.services.securitytoken.model.AssumeRoleWithWebIdentityResult; import com.auth0.jwt.JWT; @@ -61,6 +63,37 @@ public STSAssumeRoleAuthorizationService(final AWSSecurityTokenService service) this.service = service; } + public STSTokens authorize(final Host bookmark, final String sAMLAssertion) throws BackgroundException { + final AssumeRoleWithSAMLRequest request = new AssumeRoleWithSAMLRequest().withSAMLAssertion(sAMLAssertion); + if(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds") != 0) { + request.setDurationSeconds(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds")); + } + if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.policy"))) { + request.setPolicy(new HostPreferences(bookmark).getProperty("s3.assumerole.policy")); + } + if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.rolearn"))) { + request.setRoleArn(new HostPreferences(bookmark).getProperty("s3.assumerole.rolearn")); + } + try { + final AssumeRoleWithSAMLResult result = service.assumeRoleWithSAML(request); + if(log.isDebugEnabled()) { + log.debug(String.format("Received assume role identity result %s", result)); + } + final Credentials credentials = bookmark.getCredentials(); + final STSTokens tokens = new STSTokens(result.getCredentials().getAccessKeyId(), + result.getCredentials().getSecretAccessKey(), + result.getCredentials().getSessionToken(), + result.getCredentials().getExpiration().getTime()); + credentials.setUsername(tokens.getAccessKey()); + credentials.setPassword(tokens.getSecretAccessKey()); + credentials.setToken(tokens.getSessionToken()); + return tokens; + } + catch(AWSSecurityTokenServiceException e) { + throw new LoginFailureException(e.getErrorMessage(), e); + } + } + public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws BackgroundException { final AssumeRoleWithWebIdentityRequest request = new AssumeRoleWithWebIdentityRequest(); final String token = StringUtils.isNotBlank(oauth.getIdToken()) ? oauth.getIdToken() : oauth.getAccessToken(); From 3ca574ee5b61f652b5c6103e7bcf4dd64efb8f55 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 9 Aug 2023 00:24:36 +0200 Subject: [PATCH 26/84] Fix sts:GetSessionToken to include MFA token. --- .../core/s3/S3CredentialsConfigurator.java | 139 +++++++++--------- 1 file changed, 69 insertions(+), 70 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java b/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java index b8db23d57e8..dafd5d20452 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java @@ -126,6 +126,34 @@ else if(StringUtils.equals(awsAccessIdKey, profile)) { if(optional.isPresent()) { final Map.Entry entry = optional.get(); final BasicProfile basicProfile = entry.getValue(); + final String tokenCode; + if(basicProfile.getProperties().containsKey("mfa_serial")) { + try { + tokenCode = prompt.prompt( + host, LocaleFactory.localizedString("Provide additional login credentials", "Credentials"), + String.format("%s %s", LocaleFactory.localizedString("Multi-Factor Authentication", "S3"), + basicProfile.getPropertyValue("mfa_serial")), + new LoginOptions(host.getProtocol()) + .password(true) + .passwordPlaceholder(LocaleFactory.localizedString("MFA Authentication Code", "S3")) + .keychain(false) + ).getPassword(); + } + catch(LoginCanceledException e) { + log.warn(String.format("Canceled MFA prompt for profile %s", basicProfile)); + return credentials; + } + } + else { + tokenCode = null; + } + final Integer durationSeconds; + if(basicProfile.getProperties().containsKey("duration_seconds")) { + durationSeconds = Integer.valueOf(basicProfile.getPropertyValue("duration_seconds")); + } + else { + durationSeconds = null; + } if(basicProfile.isRoleBasedProfile()) { if(log.isDebugEnabled()) { log.debug(String.format("Configure credentials from role based profile %s", basicProfile.getProfileName())); @@ -157,36 +185,8 @@ else if(!profiles.containsKey(basicProfile.getRoleSourceProfile())) { sourceProfile.getAwsSecretAccessKey(), sourceProfile.getAwsSessionToken()); } - final String tokenCode; - if(basicProfile.getProperties().containsKey("mfa_serial")) { - try { - tokenCode = prompt.prompt( - host, LocaleFactory.localizedString("Provide additional login credentials", "Credentials"), - String.format("%s %s", LocaleFactory.localizedString("Multi-Factor Authentication", "S3"), - basicProfile.getPropertyValue("mfa_serial")), - new LoginOptions(host.getProtocol()) - .password(true) - .passwordPlaceholder(LocaleFactory.localizedString("MFA Authentication Code", "S3")) - .keychain(false) - ).getPassword(); - } - catch(LoginCanceledException e) { - log.warn(String.format("Canceled MFA prompt for profile %s", basicProfile)); - return credentials; - } - } - else { - tokenCode = null; - } - final Integer durationSeconds; - if(basicProfile.getProperties().containsKey("duration_seconds")) { - durationSeconds = Integer.valueOf(basicProfile.getPropertyValue("duration_seconds")); - } - else { - durationSeconds = null; - } // Starts a new session by sending a request to the AWS Security Token Service (STS) to assume a - // Role using the long-lived AWS credentials + // role using the long-lived AWS credentials final AssumeRoleRequest assumeRoleRequest = new AssumeRoleRequest() .withExternalId(basicProfile.getRoleExternalId()) .withRoleArn(basicProfile.getRoleArn()) @@ -219,6 +219,7 @@ else if(!profiles.containsKey(basicProfile.getRoleSourceProfile())) { } catch(AWSSecurityTokenServiceException e) { log.warn(e.getErrorMessage(), e); + return credentials; } } } @@ -233,55 +234,53 @@ else if(!profiles.containsKey(basicProfile.getRoleSourceProfile())) { if(null == cached) { return credentials; } - credentials.setUsername(cached.accessKey); - credentials.setPassword(cached.secretKey); - credentials.setToken(cached.sessionToken); + return credentials + .withUsername(cached.accessKey) + .withPassword(cached.secretKey) + .withToken(cached.sessionToken); } - else if(StringUtils.isNotBlank(basicProfile.getAwsSessionToken())) { - // No need to obtain session token if preconfigured in profile + if(tokenCode != null) { + // Obtain session token if(log.isDebugEnabled()) { - log.debug(String.format("Set session token credentials from profile %s", profile)); + log.debug(String.format("Get session token from credentials in profile %s", basicProfile.getProfileName())); } - credentials.setUsername(basicProfile.getAwsAccessIdKey()); - credentials.setPassword(basicProfile.getAwsSecretAccessKey()); - credentials.setToken(basicProfile.getAwsSessionToken()); - } - else { - if(host.getProtocol().isTokenConfigurable()) { - // Obtain session token - if(log.isDebugEnabled()) { - log.debug(String.format("Get session token from credentials in profile %s", basicProfile.getProfileName())); - } - final AWSSecurityTokenService service = this.getTokenService(host, - host.getRegion(), - basicProfile.getAwsAccessIdKey(), - basicProfile.getAwsSecretAccessKey(), - basicProfile.getAwsSessionToken()); - final GetSessionTokenRequest sessionTokenRequest = new GetSessionTokenRequest(); - if(log.isDebugEnabled()) { - log.debug(String.format("Request %s from %s", sessionTokenRequest, service)); - } - try { - final GetSessionTokenResult sessionTokenResult = service.getSessionToken(sessionTokenRequest); - if(log.isDebugEnabled()) { - log.debug(String.format("Set credentials from %s", sessionTokenResult)); - } - credentials.setUsername(sessionTokenResult.getCredentials().getAccessKeyId()); - credentials.setPassword(sessionTokenResult.getCredentials().getSecretAccessKey()); - credentials.setToken(sessionTokenResult.getCredentials().getSessionToken()); - } - catch(AWSSecurityTokenServiceException e) { - log.warn(e.getErrorMessage(), e); - } + final AWSSecurityTokenService service = this.getTokenService(host, + host.getRegion(), + basicProfile.getAwsAccessIdKey(), + basicProfile.getAwsSecretAccessKey(), + basicProfile.getAwsSessionToken()); + // The purpose of the sts:GetSessionToken operation is to authenticate the user using MFA. + final GetSessionTokenRequest sessionTokenRequest = new GetSessionTokenRequest() + // The value provided by the MFA device, if MFA is required + .withTokenCode(tokenCode) + // Specify this value if the IAM user has a policy that requires MFA authentication + .withSerialNumber(basicProfile.getPropertyValue("mfa_serial")) + .withDurationSeconds(durationSeconds); + if(log.isDebugEnabled()) { + log.debug(String.format("Request %s from %s", sessionTokenRequest, service)); } - else { + try { + final GetSessionTokenResult sessionTokenResult = service.getSessionToken(sessionTokenRequest); if(log.isDebugEnabled()) { - log.debug(String.format("Set static credentials from profile %s", basicProfile.getProfileName())); + log.debug(String.format("Set credentials from %s", sessionTokenResult)); } - credentials.setUsername(basicProfile.getAwsAccessIdKey()); - credentials.setPassword(basicProfile.getAwsSecretAccessKey()); + return credentials + .withUsername(sessionTokenResult.getCredentials().getAccessKeyId()) + .withPassword(sessionTokenResult.getCredentials().getSecretAccessKey()) + .withToken(sessionTokenResult.getCredentials().getSessionToken()); } + catch(AWSSecurityTokenServiceException e) { + log.warn(e.getErrorMessage(), e); + return credentials; + } + } + if(log.isDebugEnabled()) { + log.debug(String.format("Set credentials from profile %s", basicProfile.getProfileName())); } + return credentials + .withUsername(basicProfile.getAwsAccessIdKey()) + .withPassword(basicProfile.getAwsSecretAccessKey()) + .withToken(basicProfile.getAwsSessionToken()); } } return credentials; From fb59e7aef13cb0e2755a93b3998fad83bf8e4fbb Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 9 Aug 2023 16:59:11 +0200 Subject: [PATCH 27/84] Delete unused profile. --- .../S3-OIDC-Google-Testing.cyberduckprofile | 67 ------------------- 1 file changed, 67 deletions(-) delete mode 100644 s3/src/test/resources/S3-OIDC-Google-Testing.cyberduckprofile diff --git a/s3/src/test/resources/S3-OIDC-Google-Testing.cyberduckprofile b/s3/src/test/resources/S3-OIDC-Google-Testing.cyberduckprofile deleted file mode 100644 index 08ef3974852..00000000000 --- a/s3/src/test/resources/S3-OIDC-Google-Testing.cyberduckprofile +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - - - Protocol - s3 - Vendor - s3-sts - Bundled - - Description - S3 STS AssumeRoleWithWebIdentity - Default Hostname - testminiogoogle.duckdns.org - Default Nickname - MinIO-Google-Webbased-Test - Scheme - https - Authorization - AuthorizationCode - OAuth Authorization Url - https://accounts.google.com/o/oauth2/v2/auth - OAuth Token Url - https://oauth2.googleapis.com/token - OAuth Client ID - Your-Google-Client-ID - OAuth Client Secret - Your-Google-CLient-Secret - OAuth Redirect Url - https://cyberduck.io/oauth/ - STS Endpoint - https://testminiogoogle.duckdns.org - Scopes - - openid - profile - - Password Configurable - - Username Configurable - - Path Configurable - - Properties - - s3.bucket.virtualhost.disable=true - s3.assumerole.rolearn=arn:minio:iam:::role/Your-Configured-Role-ID - - - From 4249129de40751ec78e6c23303123c5974fc0c1f Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 9 Aug 2023 17:10:16 +0200 Subject: [PATCH 28/84] Add and review tests. --- .../java/ch/cyberduck/core/sts/AbstractOidcTest.java | 2 +- .../ch/cyberduck/core/sts/OidcAuthenticationTest.java | 8 ++++---- .../ch/cyberduck/core/sts/OidcAuthorizationTest.java | 9 +++------ .../java/ch/cyberduck/core/sts/S3OidcProfileTest.java | 7 ++++++- s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile | 1 + .../docker-compose.yml | 0 .../keycloak/Dockerfile | 0 .../keycloak/keycloak-realm.json | 0 .../minio/Dockerfile | 0 .../testcontainers.properties | 0 .../{oidcTestcontainer => testcontainer}/testfile.txt | 0 11 files changed, 15 insertions(+), 12 deletions(-) rename s3/src/test/resources/{oidcTestcontainer => testcontainer}/docker-compose.yml (100%) rename s3/src/test/resources/{oidcTestcontainer => testcontainer}/keycloak/Dockerfile (100%) rename s3/src/test/resources/{oidcTestcontainer => testcontainer}/keycloak/keycloak-realm.json (100%) rename s3/src/test/resources/{oidcTestcontainer => testcontainer}/minio/Dockerfile (100%) rename s3/src/test/resources/{oidcTestcontainer => testcontainer}/testcontainers.properties (100%) rename s3/src/test/resources/{oidcTestcontainer => testcontainer}/testfile.txt (100%) diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AbstractOidcTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AbstractOidcTest.java index 02eb3d73f23..7dde6f71bdc 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AbstractOidcTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AbstractOidcTest.java @@ -48,7 +48,7 @@ public abstract class AbstractOidcTest { static { compose = new DockerComposeContainer<>( - new File("src/test/resources/oidcTestcontainer/docker-compose.yml")) + new File(AbstractOidcTest.class.getResource("/testcontainer/docker-compose.yml").getFile())) .withPull(false) .withLocalCompose(true) .withOptions("--compatibility") diff --git a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java index 4886fa9a882..cb554f46b4a 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java @@ -62,21 +62,21 @@ public void testSuccessfulLoginViaOidc() throws BackgroundException { session.close(); } - @Test(expected = LoginFailureException.class) + @Test public void testInvalidUserName() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("WrongUsername", "rouser")); final S3Session session = new S3Session(host); session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + assertThrows(LoginFailureException.class, () -> session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback())); session.close(); } - @Test(expected = LoginFailureException.class) + @Test public void testInvalidPassword() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "invalidPassword")); final S3Session session = new S3Session(host); session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + assertThrows(LoginFailureException.class, () -> session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback())); session.close(); } diff --git a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthorizationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthorizationTest.java index 2237bb3fe8a..63f05dcdb27 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthorizationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthorizationTest.java @@ -42,8 +42,7 @@ import java.util.Collections; import java.util.EnumSet; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; @Category(TestcontainerTest.class) public class OidcAuthorizationTest extends AbstractOidcTest { @@ -86,7 +85,7 @@ public void testAuthorizationWritePermissionOnBucket() throws BackgroundExceptio session.close(); } - @Test(expected = AccessDeniedException.class) + @Test public void testAuthorizationNoWritePermissionOnBucket() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); final S3Session session = new S3Session(host); @@ -94,9 +93,7 @@ public void testAuthorizationNoWritePermissionOnBucket() throws BackgroundExcept session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); final Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); final Path test = new Path(container, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)); - new S3TouchFeature(session, new S3AccessControlListFeature(session)).touch(test, new TransferStatus()); - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(test)); - new S3DefaultDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); + assertThrows(AccessDeniedException.class, () -> new S3TouchFeature(session, new S3AccessControlListFeature(session)).touch(test, new TransferStatus())); assertFalse(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(test)); session.close(); } diff --git a/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java b/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java index 598d1cf049d..868c6e4e0e8 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java @@ -15,8 +15,10 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.Host; import ch.cyberduck.core.Profile; import ch.cyberduck.core.ProtocolFactory; +import ch.cyberduck.core.preferences.HostPreferences; import ch.cyberduck.core.s3.S3Protocol; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; @@ -37,7 +39,10 @@ public void testDefaultProfile() throws Exception { assertEquals("minio", profile.getOAuthClientId()); assertEquals("password", profile.getOAuthClientSecret()); assertNotNull(profile.getOAuthAuthorizationUrl()); - assertNotNull(profile.getOAuthTokenUrl()); + assertEquals("http://localhost:8080/realms/cyberduckrealm/protocol/openid-connect/token", profile.getOAuthTokenUrl()); + assertEquals("http://localhost:9000", profile.getSTSEndpoint()); assertFalse(profile.getOAuthScopes().isEmpty()); + assertTrue(profile.getOAuthScopes().contains("openid")); + assertEquals("", new HostPreferences(new Host(profile)).getProperty("s3.assumerole.rolearn")); } } diff --git a/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile b/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile index 1e6a5ba7615..e1c610598d0 100644 --- a/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile +++ b/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile @@ -59,6 +59,7 @@ Properties s3.bucket.virtualhost.disable=true + s3.assumerole.rolearn= diff --git a/s3/src/test/resources/oidcTestcontainer/docker-compose.yml b/s3/src/test/resources/testcontainer/docker-compose.yml similarity index 100% rename from s3/src/test/resources/oidcTestcontainer/docker-compose.yml rename to s3/src/test/resources/testcontainer/docker-compose.yml diff --git a/s3/src/test/resources/oidcTestcontainer/keycloak/Dockerfile b/s3/src/test/resources/testcontainer/keycloak/Dockerfile similarity index 100% rename from s3/src/test/resources/oidcTestcontainer/keycloak/Dockerfile rename to s3/src/test/resources/testcontainer/keycloak/Dockerfile diff --git a/s3/src/test/resources/oidcTestcontainer/keycloak/keycloak-realm.json b/s3/src/test/resources/testcontainer/keycloak/keycloak-realm.json similarity index 100% rename from s3/src/test/resources/oidcTestcontainer/keycloak/keycloak-realm.json rename to s3/src/test/resources/testcontainer/keycloak/keycloak-realm.json diff --git a/s3/src/test/resources/oidcTestcontainer/minio/Dockerfile b/s3/src/test/resources/testcontainer/minio/Dockerfile similarity index 100% rename from s3/src/test/resources/oidcTestcontainer/minio/Dockerfile rename to s3/src/test/resources/testcontainer/minio/Dockerfile diff --git a/s3/src/test/resources/oidcTestcontainer/testcontainers.properties b/s3/src/test/resources/testcontainer/testcontainers.properties similarity index 100% rename from s3/src/test/resources/oidcTestcontainer/testcontainers.properties rename to s3/src/test/resources/testcontainer/testcontainers.properties diff --git a/s3/src/test/resources/oidcTestcontainer/testfile.txt b/s3/src/test/resources/testcontainer/testfile.txt similarity index 100% rename from s3/src/test/resources/oidcTestcontainer/testfile.txt rename to s3/src/test/resources/testcontainer/testfile.txt From 50e3e1da975028b49839eff0cfe778c429fee476 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 9 Aug 2023 17:13:01 +0200 Subject: [PATCH 29/84] Logging. --- .../sts/STSAssumeRoleAuthorizationService.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 3b678f182da..3730ac31cff 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -96,7 +96,19 @@ public STSTokens authorize(final Host bookmark, final String sAMLAssertion) thro public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws BackgroundException { final AssumeRoleWithWebIdentityRequest request = new AssumeRoleWithWebIdentityRequest(); - final String token = StringUtils.isNotBlank(oauth.getIdToken()) ? oauth.getIdToken() : oauth.getAccessToken(); + final String token; + if(StringUtils.isNotBlank(oauth.getIdToken())) { + if(log.isDebugEnabled()) { + log.debug(String.format("Assume role with OIDC Id token for %s", bookmark)); + } + token = oauth.getIdToken(); + } + else { + if(log.isDebugEnabled()) { + log.debug(String.format("Assume role with OAuth access token for %s", bookmark)); + } + token = oauth.getAccessToken(); + } request.setWebIdentityToken(token); if(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds") != 0) { request.setDurationSeconds(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds")); From 162135016ef0f89087a77b00603953dfecc25868 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 9 Aug 2023 17:36:06 +0200 Subject: [PATCH 30/84] Extract variable. --- .../STSAssumeRoleAuthorizationService.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 3730ac31cff..4fb0f6afc7f 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -65,14 +65,15 @@ public STSAssumeRoleAuthorizationService(final AWSSecurityTokenService service) public STSTokens authorize(final Host bookmark, final String sAMLAssertion) throws BackgroundException { final AssumeRoleWithSAMLRequest request = new AssumeRoleWithSAMLRequest().withSAMLAssertion(sAMLAssertion); - if(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds") != 0) { - request.setDurationSeconds(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds")); + final HostPreferences preferences = new HostPreferences(bookmark); + if(preferences.getInteger("s3.assumerole.durationseconds") != 0) { + request.setDurationSeconds(preferences.getInteger("s3.assumerole.durationseconds")); } - if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.policy"))) { - request.setPolicy(new HostPreferences(bookmark).getProperty("s3.assumerole.policy")); + if(StringUtils.isNotBlank(preferences.getProperty("s3.assumerole.policy"))) { + request.setPolicy(preferences.getProperty("s3.assumerole.policy")); } - if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.rolearn"))) { - request.setRoleArn(new HostPreferences(bookmark).getProperty("s3.assumerole.rolearn")); + if(StringUtils.isNotBlank(preferences.getProperty("s3.assumerole.rolearn"))) { + request.setRoleArn(preferences.getProperty("s3.assumerole.rolearn")); } try { final AssumeRoleWithSAMLResult result = service.assumeRoleWithSAML(request); @@ -110,17 +111,18 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws token = oauth.getAccessToken(); } request.setWebIdentityToken(token); - if(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds") != 0) { - request.setDurationSeconds(new HostPreferences(bookmark).getInteger("s3.assumerole.durationseconds")); + final HostPreferences preferences = new HostPreferences(bookmark); + if(preferences.getInteger("s3.assumerole.durationseconds") != 0) { + request.setDurationSeconds(preferences.getInteger("s3.assumerole.durationseconds")); } - if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.policy"))) { - request.setPolicy(new HostPreferences(bookmark).getProperty("s3.assumerole.policy")); + if(StringUtils.isNotBlank(preferences.getProperty("s3.assumerole.policy"))) { + request.setPolicy(preferences.getProperty("s3.assumerole.policy")); } - if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.rolearn"))) { - request.setRoleArn(new HostPreferences(bookmark).getProperty("s3.assumerole.rolearn")); + if(StringUtils.isNotBlank(preferences.getProperty("s3.assumerole.rolearn"))) { + request.setRoleArn(preferences.getProperty("s3.assumerole.rolearn")); } - if(StringUtils.isNotBlank(new HostPreferences(bookmark).getProperty("s3.assumerole.rolesessionname"))) { - request.setRoleSessionName(new HostPreferences(bookmark).getProperty("s3.assumerole.rolesessionname")); + if(StringUtils.isNotBlank(preferences.getProperty("s3.assumerole.rolesessionname"))) { + request.setRoleSessionName(preferences.getProperty("s3.assumerole.rolesessionname")); } else { try { From 174a632b571ef5a18ef7fba0df2494a23fd04602 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 9 Aug 2023 17:37:57 +0200 Subject: [PATCH 31/84] Rename field. --- .../main/java/ch/cyberduck/core/s3/S3Session.java | 2 +- .../sts/STSAssumeRoleAuthorizationService.java | 4 ++-- ...STSAssumeRoleCredentialsRequestInterceptor.java | 4 ++-- ...SAssumeRoleTokenExpiredResponseInterceptor.java | 2 +- .../main/java/ch/cyberduck/core/sts/STSTokens.java | 14 +++++++------- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 461d68388a2..93f5bef2efc 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -309,7 +309,7 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal if(host.getProtocol().isOAuthConfigurable()) { // Get temporary credentials from STS using Web Identity (OIDC) token final STSTokens tokens = sts.refresh(oauth.authorize(host, prompt, cancel)); - client.setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKey(), + client.setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKeyId(), tokens.getSecretAccessKey(), tokens.getSessionToken())); } else { diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 4fb0f6afc7f..7b74209a2f6 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -85,7 +85,7 @@ public STSTokens authorize(final Host bookmark, final String sAMLAssertion) thro result.getCredentials().getSecretAccessKey(), result.getCredentials().getSessionToken(), result.getCredentials().getExpiration().getTime()); - credentials.setUsername(tokens.getAccessKey()); + credentials.setUsername(tokens.getAccessKeyId()); credentials.setPassword(tokens.getSecretAccessKey()); credentials.setToken(tokens.getSessionToken()); return tokens; @@ -150,7 +150,7 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws result.getCredentials().getSecretAccessKey(), result.getCredentials().getSessionToken(), result.getCredentials().getExpiration().getTime()); - credentials.setUsername(tokens.getAccessKey()); + credentials.setUsername(tokens.getAccessKeyId()); credentials.setPassword(tokens.getSecretAccessKey()); credentials.setToken(tokens.getSessionToken()); return tokens; diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java index e540e483810..b2306c9e745 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -73,7 +73,7 @@ public STSTokens refresh(final OAuthTokens oauth) throws BackgroundException { */ public STSTokens save(final STSTokens tokens) throws LocalAccessDeniedException { host.getCredentials() - .withUsername(tokens.getAccessKey()) + .withUsername(tokens.getAccessKeyId()) .withPassword(tokens.getSecretAccessKey()) .withToken(tokens.getSessionToken()) .withSaved(new LoginOptions().keychain); @@ -99,7 +99,7 @@ public void process(final HttpRequest request, final HttpContext context) throws if(log.isInfoEnabled()) { log.info(String.format("Authorizing service request with STS tokens %s", tokens)); } - session.getClient().setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKey(), tokens.getSecretAccessKey(), + session.getClient().setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKeyId(), tokens.getSecretAccessKey(), tokens.getSessionToken())); } } diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java index d12bb4d45cc..db9ad18a867 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java @@ -90,7 +90,7 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun try { log.warn(String.format("Attempt to refresh STS token for failure %s", response)); final STSTokens tokens = sts.refresh(oauth.getTokens()); - session.getClient().setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKey(), + session.getClient().setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKeyId(), tokens.getSecretAccessKey(), tokens.getSessionToken())); // Try again return true; diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java b/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java index 4d63d138809..6a688031a8e 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java @@ -22,13 +22,13 @@ public final class STSTokens { public static final STSTokens EMPTY = new STSTokens(null, null, null, Long.MAX_VALUE); - private final String accessKey; + private final String accessKeyId; private final String secretAccessKey; - private final Long expiryInMilliseconds; private final String sessionToken; + private final Long expiryInMilliseconds; - public STSTokens(final String accessKey, final String secretAccessKey, final String sessionToken, final Long expiryInMilliseconds) { - this.accessKey = accessKey; + public STSTokens(final String accessKeyId, final String secretAccessKey, final String sessionToken, final Long expiryInMilliseconds) { + this.accessKeyId = accessKeyId; this.secretAccessKey = secretAccessKey; this.sessionToken = sessionToken; this.expiryInMilliseconds = expiryInMilliseconds; @@ -38,8 +38,8 @@ public boolean validate() { return StringUtils.isNotEmpty(sessionToken); } - public String getAccessKey() { - return accessKey; + public String getAccessKeyId() { + return accessKeyId; } public String getSecretAccessKey() { @@ -61,7 +61,7 @@ public boolean isExpired() { @Override public String toString() { final StringBuilder sb = new StringBuilder("OAuthTokens{"); - sb.append("accessKey='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(accessKey)))).append('\''); + sb.append("accessKey='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(accessKeyId)))).append('\''); sb.append(", secretAccessKey='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(secretAccessKey)))).append('\''); sb.append(", sessionToken='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(secretAccessKey)))).append('\''); sb.append(", expiryInMilliseconds=").append(expiryInMilliseconds); From c452ca3e985a2dfd9ea24efb894ae89687c7ba41 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Wed, 9 Aug 2023 17:39:23 +0200 Subject: [PATCH 32/84] Fix deprecated usage. --- .../cyberduck/core/sts/STSAssumeRoleAuthorizationService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 7b74209a2f6..aead05e263d 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -31,6 +31,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.AnonymousAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.internal.StaticCredentialsProvider; @@ -53,7 +54,7 @@ public STSAssumeRoleAuthorizationService(final Host bookmark, final X509TrustMan this(AWSSecurityTokenServiceClientBuilder .standard() .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(bookmark.getProtocol().getSTSEndpoint(), null)) - .withCredentials(new StaticCredentialsProvider(new AnonymousAWSCredentials())) + .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) .withClientConfiguration(new CustomClientConfiguration(bookmark, new ThreadLocalHostnameDelegatingTrustManager(trust, bookmark.getProtocol().getSTSEndpoint()), key)) .build()); From 03b18a75a29447df7f286b66b301b2c5e1f4e8c9 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 10 Aug 2023 09:53:13 +0200 Subject: [PATCH 33/84] Use "Try Again" as default button title on alert to retry after OAuth login failure. --- core/src/main/java/ch/cyberduck/core/KeychainLoginService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/ch/cyberduck/core/KeychainLoginService.java b/core/src/main/java/ch/cyberduck/core/KeychainLoginService.java index 04e0ef27df3..60e8deffce7 100644 --- a/core/src/main/java/ch/cyberduck/core/KeychainLoginService.java +++ b/core/src/main/java/ch/cyberduck/core/KeychainLoginService.java @@ -160,7 +160,7 @@ public boolean prompt(final Host bookmark, final String message, final LoginCall } if(options.oauth) { prompt.warn(bookmark, LocaleFactory.localizedString("Login failed", "Credentials"), message, - LocaleFactory.localizedString("Continue", "Credentials"), + LocaleFactory.localizedString("Try Again", "Alert"), LocaleFactory.localizedString("Cancel", "Localizable"), null); log.warn(String.format("Reset OAuth tokens for %s", bookmark)); credentials.setOauth(OAuthTokens.EMPTY); From 0051e17774543f854f1c052225694aa14430c047 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 10 Aug 2023 10:07:27 +0200 Subject: [PATCH 34/84] Add mapping for `TokenRefreshRequired`. --- .../java/ch/cyberduck/core/s3/S3ExceptionMappingService.java | 1 + .../core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3ExceptionMappingService.java b/s3/src/main/java/ch/cyberduck/core/s3/S3ExceptionMappingService.java index 5338d855c5f..abfbfe767e4 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3ExceptionMappingService.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3ExceptionMappingService.java @@ -73,6 +73,7 @@ public BackgroundException map(final ServiceException e) { return new ConnectionTimeoutException(buffer.toString(), e); case "ExpiredToken": case "InvalidToken": + case "TokenRefreshRequired": return new ExpiredTokenException(buffer.toString(), e); } } diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java index db9ad18a867..10f491eeeab 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java @@ -71,6 +71,9 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun EntityUtils.toString(response.getEntity())); failure.setResponseCode(response.getStatusLine().getStatusCode()); if(new S3ExceptionMappingService().map(failure) instanceof ExpiredTokenException) { + // 400 Bad Request (ExpiredToken) The provided token has expired + // 400 Bad Request (InvalidToken) The provided token is malformed or otherwise not valid + // 400 Bad Request (TokenRefreshRequired) The provided token must be refreshed. if(log.isWarnEnabled()) { log.warn(String.format("Handle failure %s", failure)); } From 16e6cdccc192775b68a741cf91b3e410073e143f Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 10 Aug 2023 11:21:31 +0200 Subject: [PATCH 35/84] Review. --- .../core/oauth/OAuth2AuthorizationService.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java index e255fd7001f..b53697768b1 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java @@ -144,21 +144,18 @@ public OAuthTokens authorize(final Host bookmark, final LoginCallback prompt, fi switch(flowType) { case AuthorizationCode: response = this.authorizeWithCode(bookmark, prompt); - return credentials.withOauth(new OAuthTokens( - response.getAccessToken(), response.getRefreshToken(), - null == response.getExpiresInSeconds() ? System.currentTimeMillis() : - System.currentTimeMillis() + response.getExpiresInSeconds() * 1000, response.getIdToken())) - .withSaved(new LoginOptions().keychain).getOauth(); + break; case PasswordGrant: response = this.authorizeWithPassword(credentials); - return credentials.withOauth(new OAuthTokens( - response.getAccessToken(), response.getRefreshToken(), - null == response.getExpiresInSeconds() ? Long.MAX_VALUE : - System.currentTimeMillis() + response.getExpiresInSeconds() * 1000, response.getIdToken())) - .withSaved(new LoginOptions().keychain).getOauth(); + break; default: throw new LoginCanceledException(); } + return credentials.withOauth(new OAuthTokens( + response.getAccessToken(), response.getRefreshToken(), + null == response.getExpiresInSeconds() ? Long.MAX_VALUE : + System.currentTimeMillis() + response.getExpiresInSeconds() * 1000, response.getIdToken())) + .withSaved(new LoginOptions().keychain).getOauth(); } private IdTokenResponse authorizeWithCode(final Host bookmark, final LoginCallback prompt) throws BackgroundException { From d4bc80ba9148fb3720e3c1859f1caecd4b8714df Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 10 Aug 2023 14:38:49 +0200 Subject: [PATCH 36/84] Extract method to allow refresh from subclass on different error code. --- .../oauth/OAuth2ErrorResponseInterceptor.java | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java index 4e4c63dbe4f..3ea06285a6d 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java @@ -18,6 +18,7 @@ import ch.cyberduck.core.DisabledCancelCallback; import ch.cyberduck.core.Host; import ch.cyberduck.core.LoginCallback; +import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.InteroperabilityException; import ch.cyberduck.core.exception.LoginFailureException; @@ -52,14 +53,7 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun case HttpStatus.SC_UNAUTHORIZED: if(executionCount <= MAX_RETRIES) { try { - try { - log.warn(String.format("Attempt to refresh OAuth tokens for failure %s", response)); - service.save(service.refresh()); - } - catch(InteroperabilityException | LoginFailureException e) { - log.warn(String.format("Failure %s refreshing OAuth tokens", e)); - service.save(service.authorize(bookmark, prompt, new DisabledCancelCallback())); - } + this.refresh(response); // Try again return true; } @@ -74,4 +68,17 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun } return false; } + + protected OAuthTokens refresh(final HttpResponse response) throws BackgroundException { + OAuthTokens tokens; + try { + log.warn(String.format("Attempt to refresh OAuth tokens for failure %s", response)); + tokens = service.refresh(); + } + catch(InteroperabilityException | LoginFailureException e) { + log.warn(String.format("Failure %s refreshing OAuth tokens", e)); + tokens = service.authorize(bookmark, prompt, new DisabledCancelCallback()); + } + return service.save(tokens); + } } From b35f63309b2a8b6b0f56e26d0cd38f76230ea91a Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 10 Aug 2023 15:58:24 +0200 Subject: [PATCH 37/84] Only reset provider credentials after refresh. --- ...STSAssumeRoleCredentialsRequestInterceptor.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java index b2306c9e745..eb2445c7801 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -88,19 +88,17 @@ public STSTokens save(final STSTokens tokens) throws LocalAccessDeniedException public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { if(tokens.isExpired()) { try { - this.save(this.refresh()); + this.save(this.refresh(oauth.getTokens())); + if(log.isInfoEnabled()) { + log.info(String.format("Authorizing service request with STS tokens %s", tokens)); + } + session.getClient().setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKeyId(), tokens.getSecretAccessKey(), + tokens.getSessionToken())); } catch(BackgroundException e) { log.warn(String.format("Failure %s refreshing STS tokens %s", e, tokens)); // Follow-up error 401 handled in error interceptor } } - if(StringUtils.isNotBlank(tokens.getSessionToken())) { - if(log.isInfoEnabled()) { - log.info(String.format("Authorizing service request with STS tokens %s", tokens)); - } - session.getClient().setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKeyId(), tokens.getSecretAccessKey(), - tokens.getSessionToken())); - } } } From a315db1cdb419277ed1f636d87f7b3ba8df9831d Mon Sep 17 00:00:00 2001 From: David Kocher Date: Thu, 10 Aug 2023 16:15:07 +0200 Subject: [PATCH 38/84] Review implementation. --- ...meRoleTokenExpiredResponseInterceptor.java | 76 +++++++++++-------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java index 10f491eeeab..a62b8be8f3c 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java @@ -16,8 +16,10 @@ */ import ch.cyberduck.core.LoginCallback; +import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ExpiredTokenException; +import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.oauth.OAuth2ErrorResponseInterceptor; import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; import ch.cyberduck.core.s3.S3ExceptionMappingService; @@ -58,54 +60,68 @@ public STSAssumeRoleTokenExpiredResponseInterceptor(final S3Session session, public boolean retryRequest(final HttpResponse response, final int executionCount, final HttpContext context) { switch(response.getStatusLine().getStatusCode()) { case HttpStatus.SC_UNAUTHORIZED: - if(!super.retryRequest(response, executionCount, context)) { + case HttpStatus.SC_FORBIDDEN: + case HttpStatus.SC_BAD_REQUEST: + if(executionCount > MAX_RETRIES) { + log.warn(String.format("Skip retry for response %s after %d executions", response, executionCount)); return false; } - } - switch(response.getStatusLine().getStatusCode()) { - case HttpStatus.SC_BAD_REQUEST: - try { - if(null != response.getEntity()) { + if(null != response.getEntity()) { + try { EntityUtils.updateEntity(response, new BufferedHttpEntity(response.getEntity())); final S3ServiceException failure = new S3ServiceException(response.getStatusLine().getReasonPhrase(), EntityUtils.toString(response.getEntity())); failure.setResponseCode(response.getStatusLine().getStatusCode()); - if(new S3ExceptionMappingService().map(failure) instanceof ExpiredTokenException) { + final BackgroundException type = new S3ExceptionMappingService().map(failure); + final OAuthTokens oAuthTokens; + if(type instanceof ExpiredTokenException) { // 400 Bad Request (ExpiredToken) The provided token has expired // 400 Bad Request (InvalidToken) The provided token is malformed or otherwise not valid // 400 Bad Request (TokenRefreshRequired) The provided token must be refreshed. - if(log.isWarnEnabled()) { - log.warn(String.format("Handle failure %s", failure)); + // No refresh of OAuth tokens + oAuthTokens = oauth.getTokens(); + } + else if(type instanceof LoginFailureException) { + // 401 (InvalidAccessKeyId) The AWS access key ID that you provided does not exist in our records. + // 401 (InvalidSecurity) The provided security credentials are not valid. + // 403 (Forbidden) Access Denied + try { + // Refresh OAuth tokens + oAuthTokens = super.refresh(response); + } + catch(BackgroundException e) { + log.warn(String.format("Failure %s refreshing OAuth tokens", e)); + return false; } } else { // Ignore other 400 failures + if(log.isDebugEnabled()) { + log.debug(String.format("Ignore failure %s", type)); + } + return false; + } + if(log.isWarnEnabled()) { + log.warn(String.format("Handle failure %s", failure)); + } + try { + log.warn(String.format("Attempt to refresh STS token for failure %s", response)); + final STSTokens stsTokens = sts.refresh(oAuthTokens); + session.getClient().setProviderCredentials(new AWSSessionCredentials(stsTokens.getAccessKeyId(), + stsTokens.getSecretAccessKey(), stsTokens.getSessionToken())); + // Try again + return true; + } + catch(BackgroundException e) { + log.warn(String.format("Failure %s refreshing STS token", e)); return false; } } - } - catch(IOException e) { - log.warn(String.format("Failure parsing response entity from %s", response)); - } - // Break through - case HttpStatus.SC_UNAUTHORIZED: - if(executionCount <= MAX_RETRIES) { - try { - log.warn(String.format("Attempt to refresh STS token for failure %s", response)); - final STSTokens tokens = sts.refresh(oauth.getTokens()); - session.getClient().setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKeyId(), - tokens.getSecretAccessKey(), tokens.getSessionToken())); - // Try again - return true; - } - catch(BackgroundException e) { - log.warn(String.format("Failure %s refreshing STS token", e)); + catch(IOException e) { + log.warn(String.format("Failure parsing response entity from %s", response)); + return false; } } - else { - log.warn(String.format("Skip retry for response %s after %d executions", response, executionCount)); - } - break; } return false; } From ef14ea13ecd82eda12cb5148ee86a4e132daa927 Mon Sep 17 00:00:00 2001 From: chenkins Date: Fri, 11 Aug 2023 08:29:29 +0200 Subject: [PATCH 39/84] Slimmed keycloak-realm.json. --- .../keycloak/keycloak-realm.json | 2467 ++--------------- 1 file changed, 219 insertions(+), 2248 deletions(-) diff --git a/s3/src/test/resources/testcontainer/keycloak/keycloak-realm.json b/s3/src/test/resources/testcontainer/keycloak/keycloak-realm.json index bc77ca959ff..8434c40622c 100644 --- a/s3/src/test/resources/testcontainer/keycloak/keycloak-realm.json +++ b/s3/src/test/resources/testcontainer/keycloak/keycloak-realm.json @@ -1,2260 +1,231 @@ { - "id": "e1afe7e1-b8f5-4032-b161-bac1d4c6455c", - "realm": "cyberduckrealm", - "notBefore": 0, - "defaultSignatureAlgorithm": "RS256", - "revokeRefreshToken": false, - "refreshTokenMaxReuse": 0, - "accessTokenLifespan": 300, - "accessTokenLifespanForImplicitFlow": 900, - "ssoSessionIdleTimeout": 180, - "ssoSessionMaxLifespan": 300, - "ssoSessionIdleTimeoutRememberMe": 0, - "ssoSessionMaxLifespanRememberMe": 0, - "offlineSessionIdleTimeout": 2592000, - "offlineSessionMaxLifespanEnabled": false, - "offlineSessionMaxLifespan": 5184000, - "clientSessionIdleTimeout": 0, - "clientSessionMaxLifespan": 0, - "clientOfflineSessionIdleTimeout": 0, - "clientOfflineSessionMaxLifespan": 0, - "accessCodeLifespan": 60, - "accessCodeLifespanUserAction": 300, - "accessCodeLifespanLogin": 1800, - "actionTokenGeneratedByAdminLifespan": 43200, - "actionTokenGeneratedByUserLifespan": 300, - "oauth2DeviceCodeLifespan": 600, - "oauth2DevicePollingInterval": 5, - "enabled": true, - "sslRequired": "external", - "registrationAllowed": false, - "registrationEmailAsUsername": false, - "rememberMe": false, - "verifyEmail": false, - "loginWithEmailAllowed": true, - "duplicateEmailsAllowed": false, - "resetPasswordAllowed": false, - "editUsernameAllowed": false, - "bruteForceProtected": false, - "permanentLockout": false, - "maxFailureWaitSeconds": 900, - "minimumQuickLoginWaitSeconds": 60, - "waitIncrementSeconds": 60, - "quickLoginCheckMilliSeconds": 1000, - "maxDeltaTimeSeconds": 43200, - "failureFactor": 30, - "roles": { - "realm": [ - { - "id": "8194a695-2e25-4f0f-b327-72ff146af1d3", - "name": "default-roles-cyberduckrealm", - "description": "${role_default-roles}", - "composite": true, - "composites": { - "realm": [ - "offline_access", - "uma_authorization" - ], - "client": { - "account": [ - "manage-account", - "view-profile" - ] - } - }, - "clientRole": false, - "containerId": "e1afe7e1-b8f5-4032-b161-bac1d4c6455c", - "attributes": {} - }, - { - "id": "f482863a-6db4-4474-8f74-113fbd1e1936", - "name": "offline_access", - "description": "${role_offline-access}", - "composite": false, - "clientRole": false, - "containerId": "e1afe7e1-b8f5-4032-b161-bac1d4c6455c", - "attributes": {} - }, - { - "id": "68b4bf91-1719-47d5-96e7-d61c23861ff3", - "name": "uma_authorization", - "description": "${role_uma_authorization}", - "composite": false, - "clientRole": false, - "containerId": "e1afe7e1-b8f5-4032-b161-bac1d4c6455c", - "attributes": {} - } - ], - "client": { - "realm-management": [ - { - "id": "0b92f964-d7ac-406e-9b1b-686a4c9294f6", - "name": "query-clients", - "description": "${role_query-clients}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "265eafec-47c8-4c3d-9943-5ba90cbb89b5", - "name": "view-authorization", - "description": "${role_view-authorization}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "052b711b-98cc-4d5e-b82a-4ae771376872", - "name": "view-realm", - "description": "${role_view-realm}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "0e793a88-ec20-4f49-9618-7d15b209991c", - "name": "realm-admin", - "description": "${role_realm-admin}", - "composite": true, - "composites": { - "client": { - "realm-management": [ - "query-clients", - "view-authorization", - "view-realm", - "create-client", - "manage-users", - "manage-events", - "manage-authorization", - "query-realms", - "view-identity-providers", - "query-users", - "view-clients", - "manage-realm", - "view-users", - "query-groups", - "impersonation", - "manage-identity-providers", - "manage-clients", - "view-events" - ] - } - }, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "49fb3abc-a339-454a-aa7f-0645c8d63fa9", - "name": "create-client", - "description": "${role_create-client}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "6f0a4e65-02ac-4dbf-961a-a038ee4a80a1", - "name": "manage-users", - "description": "${role_manage-users}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "6d5f8beb-2257-4112-b0a9-9dadd8a5bb85", - "name": "manage-events", - "description": "${role_manage-events}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "2dc59d62-411f-4d5a-94af-a65924e0a1c1", - "name": "manage-authorization", - "description": "${role_manage-authorization}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "e4d3c86a-6d45-406c-b9e7-530b427ff4a1", - "name": "query-realms", - "description": "${role_query-realms}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "fe3c4b94-25c4-4ee4-947e-d6ebae4a2432", - "name": "view-identity-providers", - "description": "${role_view-identity-providers}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "7555375d-6bb7-41de-8acf-c43fa059a1ba", - "name": "query-users", - "description": "${role_query-users}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "f915e51e-9887-4896-822f-6b73c1033de3", - "name": "view-clients", - "description": "${role_view-clients}", - "composite": true, - "composites": { - "client": { - "realm-management": [ - "query-clients" - ] - } - }, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "be4ad746-7105-4f30-8c8a-e3a6eafb6f0a", - "name": "manage-realm", - "description": "${role_manage-realm}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "0b361b2e-f79c-4c1f-8f75-4b077b50eb07", - "name": "view-users", - "description": "${role_view-users}", - "composite": true, - "composites": { - "client": { - "realm-management": [ - "query-users", - "query-groups" - ] - } - }, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "8180252b-b94f-4f0a-8755-6ca517bbdcb1", - "name": "impersonation", - "description": "${role_impersonation}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "09f264ef-8c4f-4e51-8051-c1364adf8bec", - "name": "query-groups", - "description": "${role_query-groups}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "06b213ed-9bca-4071-9c9f-c41ccecc81d9", - "name": "manage-identity-providers", - "description": "${role_manage-identity-providers}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "2f6f693a-ddcd-45a4-bac8-b4ab83573399", - "name": "manage-clients", - "description": "${role_manage-clients}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - }, - { - "id": "e44f8384-814b-4fdf-90e2-246472751829", - "name": "view-events", - "description": "${role_view-events}", - "composite": false, - "clientRole": true, - "containerId": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "attributes": {} - } - ], - "security-admin-console": [], - "admin-cli": [], - "minio": [], - "account-console": [], - "broker": [ - { - "id": "c6d3cbe9-7852-44d6-b956-03208cdf590b", - "name": "read-token", - "description": "${role_read-token}", - "composite": false, - "clientRole": true, - "containerId": "40e2c9c5-ca68-43e6-bdca-1bec70485771", - "attributes": {} - } - ], - "account": [ - { - "id": "2010e2cc-7804-45f4-9ffe-8ce50460366d", - "name": "view-consent", - "description": "${role_view-consent}", - "composite": false, - "clientRole": true, - "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", - "attributes": {} - }, - { - "id": "7cc96e86-6f91-44d8-b3d2-6a92b26e23b9", - "name": "view-applications", - "description": "${role_view-applications}", - "composite": false, - "clientRole": true, - "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", - "attributes": {} - }, - { - "id": "68c10534-629e-403b-bba2-09bd9de3f56d", - "name": "manage-consent", - "description": "${role_manage-consent}", - "composite": true, - "composites": { - "client": { - "account": [ - "view-consent" - ] + "id": "cyberduckrealm", + "realm": "cyberduckrealm", + "displayName": "Cyberduck", + "enabled": true, + "sslRequired": "external", + "defaultRole": { + "name": "user", + "description": "User" + }, + "roles": { + "realm": [ + { + "name": "user", + "description": "User", + "composite": false + }, + { + "name": "admin", + "description": "Administrator", + "composite": true, + "composites": { + "realm": [ + "user" + ], + "client": { + "realm-management": [ + "realm-admin" + ] + } + } + }, + { + "name": "syncer", + "description": "Syncer", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "view-users", + "manage-users" + ] + } + } } - }, - "clientRole": true, - "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", - "attributes": {} - }, - { - "id": "413c38d0-50bf-4733-8320-6b3c004e3163", - "name": "manage-account", - "description": "${role_manage-account}", - "composite": true, - "composites": { - "client": { - "account": [ - "manage-account-links" - ] - } - }, - "clientRole": true, - "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", - "attributes": {} - }, - { - "id": "0439c628-4012-4d0a-a51c-522d51b204e6", - "name": "view-profile", - "description": "${role_view-profile}", - "composite": false, - "clientRole": true, - "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", - "attributes": {} - }, - { - "id": "a697a2f8-0517-4658-a5aa-89e79705ce2b", - "name": "manage-account-links", - "description": "${role_manage-account-links}", - "composite": false, - "clientRole": true, - "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", - "attributes": {} - }, - { - "id": "f7376ccf-182c-4227-bb8c-16f7a101d913", - "name": "delete-account", - "description": "${role_delete-account}", - "composite": false, - "clientRole": true, - "containerId": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", - "attributes": {} - } - ] - } - }, - "groups": [], - "defaultRole": { - "id": "8194a695-2e25-4f0f-b327-72ff146af1d3", - "name": "default-roles-cyberduckrealm", - "description": "${role_default-roles}", - "composite": true, - "clientRole": false, - "containerId": "e1afe7e1-b8f5-4032-b161-bac1d4c6455c" - }, - "requiredCredentials": [ - "password" - ], - "otpPolicyType": "totp", - "otpPolicyAlgorithm": "HmacSHA1", - "otpPolicyInitialCounter": 0, - "otpPolicyDigits": 6, - "otpPolicyLookAheadWindow": 1, - "otpPolicyPeriod": 30, - "otpSupportedApplications": [ - "FreeOTP", - "Google Authenticator" - ], - "webAuthnPolicyRpEntityName": "keycloak", - "webAuthnPolicySignatureAlgorithms": [ - "ES256" - ], - "webAuthnPolicyRpId": "", - "webAuthnPolicyAttestationConveyancePreference": "not specified", - "webAuthnPolicyAuthenticatorAttachment": "not specified", - "webAuthnPolicyRequireResidentKey": "not specified", - "webAuthnPolicyUserVerificationRequirement": "not specified", - "webAuthnPolicyCreateTimeout": 0, - "webAuthnPolicyAvoidSameAuthenticatorRegister": false, - "webAuthnPolicyAcceptableAaguids": [], - "webAuthnPolicyPasswordlessRpEntityName": "keycloak", - "webAuthnPolicyPasswordlessSignatureAlgorithms": [ - "ES256" - ], - "webAuthnPolicyPasswordlessRpId": "", - "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified", - "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified", - "webAuthnPolicyPasswordlessRequireResidentKey": "not specified", - "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified", - "webAuthnPolicyPasswordlessCreateTimeout": 0, - "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false, - "webAuthnPolicyPasswordlessAcceptableAaguids": [], - "scopeMappings": [ - { - "clientScope": "offline_access", - "roles": [ - "offline_access" - ] - } - ], - "clientScopeMappings": { - "account": [ - { - "client": "account-console", - "roles": [ - "manage-account" ] - } - ] - }, - "clients": [ - { - "id": "c8bdfa3b-e7d0-4ea7-af06-885539c6218e", - "clientId": "account", - "name": "${client_account}", - "rootUrl": "${authBaseUrl}", - "baseUrl": "/realms/cyberduckrealm/account/", - "surrogateAuthRequired": false, - "enabled": true, - "alwaysDisplayInConsole": false, - "clientAuthenticatorType": "client-secret", - "redirectUris": [ - "/realms/cyberduckrealm/account/*" - ], - "webOrigins": ["*"], - "notBefore": 0, - "bearerOnly": false, - "consentRequired": false, - "standardFlowEnabled": true, - "implicitFlowEnabled": false, - "directAccessGrantsEnabled": false, - "serviceAccountsEnabled": false, - "publicClient": true, - "frontchannelLogout": false, - "protocol": "openid-connect", - "attributes": {}, - "authenticationFlowBindingOverrides": {}, - "fullScopeAllowed": false, - "nodeReRegistrationTimeout": 0, - "defaultClientScopes": [ - "web-origins", - "roles", - "profile", - "email" - ], - "optionalClientScopes": [ - "address", - "phone", - "offline_access", - "microprofile-jwt" - ] - }, - { - "id": "dd489003-815a-4c63-acf4-0a7f8b41bd79", - "clientId": "account-console", - "name": "${client_account-console}", - "rootUrl": "${authBaseUrl}", - "baseUrl": "/realms/cyberduckrealm/account/", - "surrogateAuthRequired": false, - "enabled": true, - "alwaysDisplayInConsole": false, - "clientAuthenticatorType": "client-secret", - "redirectUris": [ - "/realms/cyberduckrealm/account/*" - ], - "webOrigins": [], - "notBefore": 0, - "bearerOnly": false, - "consentRequired": false, - "standardFlowEnabled": true, - "implicitFlowEnabled": false, - "directAccessGrantsEnabled": false, - "serviceAccountsEnabled": false, - "publicClient": true, - "frontchannelLogout": false, - "protocol": "openid-connect", - "attributes": { - "pkce.code.challenge.method": "S256" - }, - "authenticationFlowBindingOverrides": {}, - "fullScopeAllowed": false, - "nodeReRegistrationTimeout": 0, - "protocolMappers": [ - { - "id": "daf73eea-eebe-4823-ae33-e1276b9a6ddc", - "name": "audience resolve", - "protocol": "openid-connect", - "protocolMapper": "oidc-audience-resolve-mapper", - "consentRequired": false, - "config": {} - } - ], - "defaultClientScopes": [ - "web-origins", - "roles", - "profile", - "email" - ], - "optionalClientScopes": [ - "address", - "phone", - "offline_access", - "microprofile-jwt" - ] - }, - { - "id": "c902977d-6c6a-4bf0-8949-565ab2bfb297", - "clientId": "admin-cli", - "name": "${client_admin-cli}", - "surrogateAuthRequired": false, - "enabled": true, - "alwaysDisplayInConsole": false, - "clientAuthenticatorType": "client-secret", - "redirectUris": [], - "webOrigins": [], - "notBefore": 0, - "bearerOnly": false, - "consentRequired": false, - "standardFlowEnabled": false, - "implicitFlowEnabled": false, - "directAccessGrantsEnabled": true, - "serviceAccountsEnabled": false, - "publicClient": true, - "frontchannelLogout": false, - "protocol": "openid-connect", - "attributes": {}, - "authenticationFlowBindingOverrides": {}, - "fullScopeAllowed": false, - "nodeReRegistrationTimeout": 0, - "defaultClientScopes": [ - "web-origins", - "roles", - "profile", - "email" - ], - "optionalClientScopes": [ - "address", - "phone", - "offline_access", - "microprofile-jwt" - ] - }, - { - "id": "40e2c9c5-ca68-43e6-bdca-1bec70485771", - "clientId": "broker", - "name": "${client_broker}", - "surrogateAuthRequired": false, - "enabled": true, - "alwaysDisplayInConsole": false, - "clientAuthenticatorType": "client-secret", - "redirectUris": [], - "webOrigins": [], - "notBefore": 0, - "bearerOnly": true, - "consentRequired": false, - "standardFlowEnabled": true, - "implicitFlowEnabled": false, - "directAccessGrantsEnabled": false, - "serviceAccountsEnabled": false, - "publicClient": false, - "frontchannelLogout": false, - "protocol": "openid-connect", - "attributes": {}, - "authenticationFlowBindingOverrides": {}, - "fullScopeAllowed": false, - "nodeReRegistrationTimeout": 0, - "defaultClientScopes": [ - "web-origins", - "roles", - "profile", - "email" - ], - "optionalClientScopes": [ - "address", - "phone", - "offline_access", - "microprofile-jwt" - ] - }, - { - "id": "8af8c8a6-7778-4a39-bd1f-9a2879fa00c1", - "clientId": "minio", - "rootUrl": "${authBaseUrl}", - "baseUrl": "/realms/cyberduckrealm/account/", - "surrogateAuthRequired": false, - "enabled": true, - "alwaysDisplayInConsole": true, - "clientAuthenticatorType": "client-secret", - "secret": "password", - "redirectUris": [ - "*" - ], - "webOrigins": [ - "http://minio:9000" - ], - "notBefore": 0, - "bearerOnly": false, - "consentRequired": false, - "standardFlowEnabled": true, - "implicitFlowEnabled": false, - "directAccessGrantsEnabled": true, - "serviceAccountsEnabled": false, - "publicClient": false, - "frontchannelLogout": false, - "protocol": "openid-connect", - "attributes": { - "access.token.lifespan": "30", - "saml.force.post.binding": "false", - "saml.multivalued.roles": "false", - "oauth2.device.authorization.grant.enabled": "false", - "use.jwks.url": "true", - "backchannel.logout.revoke.offline.tokens": "false", - "s3IdentityProviderEndpoint": "http://minio:9000", - "s3IdentityProviderSecretKey": "cyberduckSecretKey", - "saml.server.signature.keyinfo.ext": "false", - "use.refresh.tokens": "true", - "oidc.ciba.grant.enabled": "false", - "use.jwks.string": "false", - "backchannel.logout.session.required": "false", - "client_credentials.use_refresh_token": "false", - "require.pushed.authorization.requests": "false", - "saml.client.signature": "false", - "s3IdentityProviderAccessKey": "cyberduckAccessKey", - "id.token.as.detached.signature": "false", - "saml.assertion.signature": "false", - "saml.encrypt": "false", - "saml.server.signature": "false", - "exclude.session.state.from.auth.response": "false", - "saml.artifact.binding": "false", - "saml_force_name_id_format": "false", - "tls.client.certificate.bound.access.tokens": "false", - "saml.authnstatement": "false", - "display.on.consent.screen": "false", - "s3IdentityProviderEnabled": "true", - "saml.onetimeuse.condition": "false" - }, - "authenticationFlowBindingOverrides": {}, - "fullScopeAllowed": true, - "nodeReRegistrationTimeout": -1, - "defaultClientScopes": [ - "web-origins", - "roles", - "profile", - "minio-authorization", - "email" - ], - "optionalClientScopes": [ - "address", - "phone", - "offline_access", - "microprofile-jwt" - ] - }, - { - "id": "82124274-9a44-4ae4-8a26-f9f938cbcadc", - "clientId": "realm-management", - "name": "${client_realm-management}", - "surrogateAuthRequired": false, - "enabled": true, - "alwaysDisplayInConsole": false, - "clientAuthenticatorType": "client-secret", - "redirectUris": [], - "webOrigins": [], - "notBefore": 0, - "bearerOnly": true, - "consentRequired": false, - "standardFlowEnabled": true, - "implicitFlowEnabled": false, - "directAccessGrantsEnabled": false, - "serviceAccountsEnabled": false, - "publicClient": false, - "frontchannelLogout": false, - "protocol": "openid-connect", - "attributes": {}, - "authenticationFlowBindingOverrides": {}, - "fullScopeAllowed": false, - "nodeReRegistrationTimeout": 0, - "defaultClientScopes": [ - "web-origins", - "roles", - "profile", - "email" - ], - "optionalClientScopes": [ - "address", - "phone", - "offline_access", - "microprofile-jwt" - ] }, - { - "id": "70c5b895-1dbd-4b32-964a-db47c5f5afa9", - "clientId": "security-admin-console", - "name": "${client_security-admin-console}", - "rootUrl": "${authAdminUrl}", - "baseUrl": "/admin/cyberduckrealm/console/", - "surrogateAuthRequired": false, - "enabled": true, - "alwaysDisplayInConsole": false, - "clientAuthenticatorType": "client-secret", - "redirectUris": [ - "/admin/cyberduckrealm/console/*" - ], - "webOrigins": [ - "+" - ], - "notBefore": 0, - "bearerOnly": false, - "consentRequired": false, - "standardFlowEnabled": true, - "implicitFlowEnabled": false, - "directAccessGrantsEnabled": false, - "serviceAccountsEnabled": false, - "publicClient": true, - "frontchannelLogout": false, - "protocol": "openid-connect", - "attributes": { - "pkce.code.challenge.method": "S256" - }, - "authenticationFlowBindingOverrides": {}, - "fullScopeAllowed": false, - "nodeReRegistrationTimeout": 0, - "protocolMappers": [ - { - "id": "a4779ad7-c6ac-4050-924e-5e851c6580fd", - "name": "locale", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "locale", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "locale", - "jsonType.label": "String" - } - } - ], - "defaultClientScopes": [ - "web-origins", - "roles", - "profile", - "email" - ], - "optionalClientScopes": [ - "address", - "phone", - "offline_access", - "microprofile-jwt" - ] - } - ], - "clientScopes": [ - { - "id": "0941c887-88c7-455a-9dae-b0982c4a8ba0", - "name": "phone", - "description": "OpenID Connect built-in scope: phone", - "protocol": "openid-connect", - "attributes": { - "include.in.token.scope": "true", - "display.on.consent.screen": "true", - "consent.screen.text": "${phoneScopeConsentText}" - }, - "protocolMappers": [ - { - "id": "cb825f68-9f26-4feb-a111-5cf2d214f77c", - "name": "phone number verified", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "phoneNumberVerified", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "phone_number_verified", - "jsonType.label": "boolean" - } - }, - { - "id": "becf72e9-7bbf-4737-a937-64ea66603f4b", - "name": "phone number", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "phoneNumber", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "phone_number", - "jsonType.label": "String" - } - } - ] - }, - { - "id": "23647c33-d481-4b6a-a822-85d93b2950be", - "name": "web-origins", - "description": "OpenID Connect scope for add allowed web origins to the access token", - "protocol": "openid-connect", - "attributes": { - "include.in.token.scope": "false", - "display.on.consent.screen": "false", - "consent.screen.text": "" - }, - "protocolMappers": [ - { - "id": "868104af-32f4-4f67-8652-89f7bc097044", - "name": "allowed web origins", - "protocol": "openid-connect", - "protocolMapper": "oidc-allowed-origins-mapper", - "consentRequired": false, - "config": {} - } - ] - }, - { - "id": "52399e9d-b715-478f-b3d5-3c4c41a19492", - "name": "roles", - "description": "OpenID Connect scope for add user roles to the access token", - "protocol": "openid-connect", - "attributes": { - "include.in.token.scope": "false", - "display.on.consent.screen": "true", - "consent.screen.text": "${rolesScopeConsentText}" - }, - "protocolMappers": [ - { - "id": "27648e0c-ff42-4fe8-85a1-3a13b3fc4187", - "name": "client roles", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-client-role-mapper", - "consentRequired": false, - "config": { - "user.attribute": "foo", - "access.token.claim": "true", - "claim.name": "resource_access.${client_id}.roles", - "jsonType.label": "String", - "multivalued": "true" - } - }, - { - "id": "9b824cc3-1685-49b3-8bc2-a3ffaecb1b71", - "name": "audience resolve", - "protocol": "openid-connect", - "protocolMapper": "oidc-audience-resolve-mapper", - "consentRequired": false, - "config": {} - }, - { - "id": "0bbebb2f-d53a-41d8-9f81-fe85b7dbfef1", - "name": "realm roles", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-realm-role-mapper", - "consentRequired": false, - "config": { - "user.attribute": "foo", - "access.token.claim": "true", - "claim.name": "realm_access.roles", - "jsonType.label": "String", - "multivalued": "true" - } - } - ] - }, - { - "id": "adf7c33f-01cb-4d63-9186-646ba28daabd", - "name": "address", - "description": "OpenID Connect built-in scope: address", - "protocol": "openid-connect", - "attributes": { - "include.in.token.scope": "true", - "display.on.consent.screen": "true", - "consent.screen.text": "${addressScopeConsentText}" - }, - "protocolMappers": [ - { - "id": "ff1c2e7f-6170-4915-9a15-a2f796665429", - "name": "address", - "protocol": "openid-connect", - "protocolMapper": "oidc-address-mapper", - "consentRequired": false, - "config": { - "user.attribute.formatted": "formatted", - "user.attribute.country": "country", - "user.attribute.postal_code": "postal_code", - "userinfo.token.claim": "true", - "user.attribute.street": "street", - "id.token.claim": "true", - "user.attribute.region": "region", - "access.token.claim": "true", - "user.attribute.locality": "locality" - } - } - ] - }, - { - "id": "c1326a37-f3b3-4d45-9c0e-d92e3d21b00f", - "name": "role_list", - "description": "SAML role list", - "protocol": "saml", - "attributes": { - "consent.screen.text": "${samlRoleListScopeConsentText}", - "display.on.consent.screen": "true" - }, - "protocolMappers": [ - { - "id": "a0cbecae-fa99-493e-b840-109e8daeadb5", - "name": "role list", - "protocol": "saml", - "protocolMapper": "saml-role-list-mapper", - "consentRequired": false, - "config": { - "single": "false", - "attribute.nameformat": "Basic", - "attribute.name": "Role" - } - } - ] - }, - { - "id": "736f4ffd-516a-4608-bc7c-dead9baf4d4c", - "name": "microprofile-jwt", - "description": "Microprofile - JWT built-in scope", - "protocol": "openid-connect", - "attributes": { - "include.in.token.scope": "true", - "display.on.consent.screen": "false" - }, - "protocolMappers": [ - { - "id": "d45c912b-c4df-49c9-afd9-7b80451f52fd", - "name": "upn", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-property-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "username", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "upn", - "jsonType.label": "String" - } - }, - { - "id": "303703a4-adbb-4f7a-a16b-41bdacad23fa", - "name": "groups", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-realm-role-mapper", - "consentRequired": false, - "config": { - "multivalued": "true", - "user.attribute": "foo", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "groups", - "jsonType.label": "String" - } - } - ] - }, - { - "id": "7169650f-742e-4f08-9e47-ea57342bea74", - "name": "email", - "description": "OpenID Connect built-in scope: email", - "protocol": "openid-connect", - "attributes": { - "include.in.token.scope": "true", - "display.on.consent.screen": "true", - "consent.screen.text": "${emailScopeConsentText}" - }, - "protocolMappers": [ - { - "id": "dce18712-0535-475c-97b7-d35b9254a2f1", - "name": "email", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-property-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "email", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "email", - "jsonType.label": "String" - } - }, - { - "id": "bd1489c6-68d7-4311-9e95-875d18376c15", - "name": "email verified", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-property-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "emailVerified", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "email_verified", - "jsonType.label": "boolean" - } - } - ] - }, - { - "id": "26b0e632-2359-4570-95b7-573fa84aa01a", - "name": "offline_access", - "description": "OpenID Connect built-in scope: offline_access", - "protocol": "openid-connect", - "attributes": { - "consent.screen.text": "${offlineAccessScopeConsentText}", - "display.on.consent.screen": "true" - } - }, - { - "id": "5df385b5-e928-4c70-ad48-817680493427", - "name": "profile", - "description": "OpenID Connect built-in scope: profile", - "protocol": "openid-connect", - "attributes": { - "include.in.token.scope": "true", - "display.on.consent.screen": "true", - "consent.screen.text": "${profileScopeConsentText}" - }, - "protocolMappers": [ - { - "id": "341891af-ee4c-4c39-b5f1-e774a8665835", - "name": "profile", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "profile", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "profile", - "jsonType.label": "String" - } - }, - { - "id": "d9141fff-4bb4-414c-a98e-773e195d45fd", - "name": "locale", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "locale", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "locale", - "jsonType.label": "String" - } - }, - { - "id": "02381dc5-699e-4763-abee-156c77d3bb10", - "name": "nickname", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "nickname", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "nickname", - "jsonType.label": "String" - } - }, - { - "id": "897f4d28-b959-4154-8cc8-45331d3a8702", - "name": "zoneinfo", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "zoneinfo", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "zoneinfo", - "jsonType.label": "String" - } - }, - { - "id": "e0fe20ed-dbe1-4c87-8ecb-0e986461036d", - "name": "full name", - "protocol": "openid-connect", - "protocolMapper": "oidc-full-name-mapper", - "consentRequired": false, - "config": { - "id.token.claim": "true", - "access.token.claim": "true", - "userinfo.token.claim": "true" - } - }, - { - "id": "ed3bdec1-29ff-4c46-90fb-8dfaa602b5c0", - "name": "birthdate", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "birthdate", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "birthdate", - "jsonType.label": "String" - } - }, - { - "id": "423ab9b6-a48a-436c-aad1-43875b1ab9f5", - "name": "picture", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "picture", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "picture", - "jsonType.label": "String" - } - }, - { - "id": "4e8de822-abfd-4194-a30e-c6c50b337abd", - "name": "updated at", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "updatedAt", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "updated_at", - "jsonType.label": "String" - } - }, - { - "id": "7b937773-ec5c-4137-be86-f85d252aea2d", - "name": "middle name", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "middleName", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "middle_name", - "jsonType.label": "String" - } - }, - { - "id": "c9ce2856-dc8a-4fa6-a3eb-2b0376aabfab", - "name": "given name", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-property-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "firstName", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "given_name", - "jsonType.label": "String" - } - }, - { - "id": "1ce2cd9a-174c-4f0e-b1e3-14b9172c65d5", - "name": "family name", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-property-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "lastName", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "family_name", - "jsonType.label": "String" - } - }, - { - "id": "c29dccad-6e2a-4952-9c54-0693640acf3b", - "name": "gender", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "gender", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "gender", - "jsonType.label": "String" - } - }, - { - "id": "adbe68c1-ae8f-46e3-a06e-6efb18e8878c", - "name": "username", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-property-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "username", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "preferred_username", - "jsonType.label": "String" - } + "users": [ + { + "username": "rawuser", + "enabled": true, + "email": "readandwrite@test.com", + "attributes": { + "policy": "readwrite" + }, + "credentials": [ + { + "type": "password", + "value": "rawuser" + } + ], + "realmRoles": [ + "user" + ], + "clientRoles": { + "account": [ + "view-profile", + "manage-account" + ] + } }, { - "id": "f8254933-a164-4407-9a7c-079f88e47f57", - "name": "website", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "userinfo.token.claim": "true", - "user.attribute": "website", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "website", - "jsonType.label": "String" - } - } - ] - }, - { - "id": "00e3796f-ae81-403d-a6e3-757a4481e050", - "name": "minio-authorization", - "protocol": "openid-connect", - "attributes": { - "include.in.token.scope": "true", - "display.on.consent.screen": "true" - }, - "protocolMappers": [ - { - "id": "bd14ce8f-e019-4094-92e1-0b2ea1ba84e1", - "name": "minio-policy-mapper", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-attribute-mapper", - "consentRequired": false, - "config": { - "aggregate.attrs": "true", - "multivalued": "true", - "userinfo.token.claim": "true", - "user.attribute": "policy", - "id.token.claim": "true", - "access.token.claim": "true", - "claim.name": "policy", - "jsonType.label": "String" - } - } - ] - } - ], - "defaultDefaultClientScopes": [ - "web-origins", - "roles", - "profile", - "email", - "role_list" - ], - "defaultOptionalClientScopes": [ - "phone", - "offline_access", - "microprofile-jwt", - "address" - ], - "browserSecurityHeaders": { - "contentSecurityPolicyReportOnly": "", - "xContentTypeOptions": "nosniff", - "xRobotsTag": "none", - "xFrameOptions": "SAMEORIGIN", - "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';", - "xXSSProtection": "1; mode=block", - "strictTransportSecurity": "max-age=31536000; includeSubDomains" - }, - "smtpServer": {}, - "eventsEnabled": false, - "eventsListeners": [ - "jboss-logging" - ], - "enabledEventTypes": [], - "adminEventsEnabled": false, - "adminEventsDetailsEnabled": false, - "identityProviders": [], - "identityProviderMappers": [], - "components": { - "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [ - { - "id": "cf127fb5-bcf4-461e-849a-4caf70fdb100", - "name": "Max Clients Limit", - "providerId": "max-clients", - "subType": "anonymous", - "subComponents": {}, - "config": { - "max-clients": [ - "200" - ] - } - }, - { - "id": "ad413384-9c48-4bfa-b71f-312893f50689", - "name": "Allowed Protocol Mapper Types", - "providerId": "allowed-protocol-mappers", - "subType": "anonymous", - "subComponents": {}, - "config": { - "allowed-protocol-mapper-types": [ - "oidc-address-mapper", - "oidc-full-name-mapper", - "saml-user-attribute-mapper", - "saml-role-list-mapper", - "oidc-usermodel-property-mapper", - "saml-user-property-mapper", - "oidc-sha256-pairwise-sub-mapper", - "oidc-usermodel-attribute-mapper" - ] - } - }, - { - "id": "18a947be-6679-4bcf-98be-afed9b0cf80e", - "name": "Full Scope Disabled", - "providerId": "scope", - "subType": "anonymous", - "subComponents": {}, - "config": {} - }, - { - "id": "fb98c161-3ea2-4d75-b4ac-bbf8e4f95060", - "name": "Consent Required", - "providerId": "consent-required", - "subType": "anonymous", - "subComponents": {}, - "config": {} - }, - { - "id": "3e76db0d-ba83-4fd9-b46f-60c76869791f", - "name": "Allowed Protocol Mapper Types", - "providerId": "allowed-protocol-mappers", - "subType": "authenticated", - "subComponents": {}, - "config": { - "allowed-protocol-mapper-types": [ - "saml-user-attribute-mapper", - "oidc-address-mapper", - "oidc-usermodel-property-mapper", - "saml-role-list-mapper", - "oidc-usermodel-attribute-mapper", - "saml-user-property-mapper", - "oidc-full-name-mapper", - "oidc-sha256-pairwise-sub-mapper" - ] - } - }, - { - "id": "5777b57a-f847-4fa9-9ea0-3776bb98ba81", - "name": "Trusted Hosts", - "providerId": "trusted-hosts", - "subType": "anonymous", - "subComponents": {}, - "config": { - "host-sending-registration-request-must-match": [ - "true" - ], - "client-uris-must-match": [ - "true" - ] - } - }, - { - "id": "57ffa16e-0cc8-4144-96f9-811bdcc68095", - "name": "Allowed Client Scopes", - "providerId": "allowed-client-templates", - "subType": "anonymous", - "subComponents": {}, - "config": { - "allow-default-scopes": [ - "true" - ] - } - }, - { - "id": "7e5883b7-051f-4aaf-aca4-1b3f90dc5d1f", - "name": "Allowed Client Scopes", - "providerId": "allowed-client-templates", - "subType": "authenticated", - "subComponents": {}, - "config": { - "allow-default-scopes": [ - "true" - ] + "username": "rouser", + "enabled": true, + "email": "readonly@test.com", + "attributes": { + "policy": "readonly" + }, + "credentials": [ + { + "type": "password", + "value": "rouser" + } + ], + "realmRoles": [ + "user" + ], + "clientRoles": { + "account": [ + "view-profile", + "manage-account" + ] + } } - } ], - "org.keycloak.userprofile.UserProfileProvider": [ - { - "id": "5cf69331-9fe2-4b0c-8632-98fd2deb393b", - "providerId": "declarative-user-profile", - "subComponents": {}, - "config": {} - } + "scopeMappings": [ ], - "org.keycloak.keys.KeyProvider": [ - { - "id": "f678f09a-8dcf-44f2-bb92-6e6a8a215d81", - "name": "hmac-generated", - "providerId": "hmac-generated", - "subComponents": {}, - "config": { - "priority": [ - "100" - ], - "algorithm": [ - "HS256" - ] - } - }, - { - "id": "81086736-cd47-4b8e-8f68-8d7e27814537", - "name": "rsa-generated", - "providerId": "rsa-generated", - "subComponents": {}, - "config": { - "priority": [ - "100" - ] - } - }, - { - "id": "028d8122-2150-40c9-9770-144dc7516792", - "name": "rsa-enc-generated", - "providerId": "rsa-enc-generated", - "subComponents": {}, - "config": { - "priority": [ - "100" - ], - "algorithm": [ - "RSA-OAEP" - ] - } - }, - { - "id": "d607e956-c8d0-424f-ae9b-476caac12f3a", - "name": "aes-generated", - "providerId": "aes-generated", - "subComponents": {}, - "config": { - "priority": [ - "100" - ] - } - } - ] - }, - "internationalizationEnabled": false, - "supportedLocales": [], - "authenticationFlows": [ - { - "id": "8f395abd-02f0-4fca-9142-73120a6af711", - "alias": "Account verification options", - "description": "Method with which to verity the existing account", - "providerId": "basic-flow", - "topLevel": false, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "idp-email-verification", - "authenticatorFlow": false, - "requirement": "ALTERNATIVE", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticatorFlow": true, - "requirement": "ALTERNATIVE", - "priority": 20, - "flowAlias": "Verify Existing Account by Re-authentication", - "userSetupAllowed": false, - "autheticatorFlow": true - } - ] - }, - { - "id": "7a7f2eff-b041-484b-b93d-1df4cd916496", - "alias": "Authentication Options", - "description": "Authentication options.", - "providerId": "basic-flow", - "topLevel": false, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "basic-auth", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "basic-auth-otp", - "authenticatorFlow": false, - "requirement": "DISABLED", - "priority": 20, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "auth-spnego", - "authenticatorFlow": false, - "requirement": "DISABLED", - "priority": 30, - "userSetupAllowed": false, - "autheticatorFlow": false - } - ] - }, - { - "id": "98f32dcd-203e-4179-8000-b3d6fc42dd35", - "alias": "Browser - Conditional OTP", - "description": "Flow to determine if the OTP is required for the authentication", - "providerId": "basic-flow", - "topLevel": false, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "conditional-user-configured", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "auth-otp-form", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 20, - "userSetupAllowed": false, - "autheticatorFlow": false - } - ] - }, - { - "id": "95b7e6df-f963-4930-950e-46c6eff90480", - "alias": "Direct Grant - Conditional OTP", - "description": "Flow to determine if the OTP is required for the authentication", - "providerId": "basic-flow", - "topLevel": false, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "conditional-user-configured", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "direct-grant-validate-otp", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 20, - "userSetupAllowed": false, - "autheticatorFlow": false - } - ] - }, - { - "id": "9e55e789-b7a3-4c32-a843-60a2ecf0518b", - "alias": "First broker login - Conditional OTP", - "description": "Flow to determine if the OTP is required for the authentication", - "providerId": "basic-flow", - "topLevel": false, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "conditional-user-configured", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "auth-otp-form", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 20, - "userSetupAllowed": false, - "autheticatorFlow": false - } - ] - }, - { - "id": "c2c7436f-b39a-4af4-a596-b1c0c343dd18", - "alias": "Handle Existing Account", - "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider", - "providerId": "basic-flow", - "topLevel": false, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "idp-confirm-link", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticatorFlow": true, - "requirement": "REQUIRED", - "priority": 20, - "flowAlias": "Account verification options", - "userSetupAllowed": false, - "autheticatorFlow": true - } - ] - }, - { - "id": "5a707131-58e2-4cc3-8d62-ddfaa12a2ee8", - "alias": "Reset - Conditional OTP", - "description": "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", - "providerId": "basic-flow", - "topLevel": false, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "conditional-user-configured", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "reset-otp", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 20, - "userSetupAllowed": false, - "autheticatorFlow": false - } - ] - }, - { - "id": "4612185d-8e99-4fb9-b772-4251c96af818", - "alias": "User creation or linking", - "description": "Flow for the existing/non-existing user alternatives", - "providerId": "basic-flow", - "topLevel": false, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticatorConfig": "create unique user config", - "authenticator": "idp-create-user-if-unique", - "authenticatorFlow": false, - "requirement": "ALTERNATIVE", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticatorFlow": true, - "requirement": "ALTERNATIVE", - "priority": 20, - "flowAlias": "Handle Existing Account", - "userSetupAllowed": false, - "autheticatorFlow": true - } - ] - }, - { - "id": "6d129dc6-2fd3-4cf6-8bcd-5e207cf832d6", - "alias": "Verify Existing Account by Re-authentication", - "description": "Reauthentication of existing account", - "providerId": "basic-flow", - "topLevel": false, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "idp-username-password-form", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticatorFlow": true, - "requirement": "CONDITIONAL", - "priority": 20, - "flowAlias": "First broker login - Conditional OTP", - "userSetupAllowed": false, - "autheticatorFlow": true - } - ] - }, - { - "id": "259142b1-b8ad-4fb8-a464-2d53b999a0cd", - "alias": "browser", - "description": "browser based authentication", - "providerId": "basic-flow", - "topLevel": true, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "auth-cookie", - "authenticatorFlow": false, - "requirement": "ALTERNATIVE", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "auth-spnego", - "authenticatorFlow": false, - "requirement": "DISABLED", - "priority": 20, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "identity-provider-redirector", - "authenticatorFlow": false, - "requirement": "ALTERNATIVE", - "priority": 25, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticatorFlow": true, - "requirement": "ALTERNATIVE", - "priority": 30, - "flowAlias": "forms", - "userSetupAllowed": false, - "autheticatorFlow": true - } - ] - }, - { - "id": "45be8796-9ea8-4b47-91d0-b3ed012b1030", - "alias": "clients", - "description": "Base authentication for clients", - "providerId": "client-flow", - "topLevel": true, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "client-secret", - "authenticatorFlow": false, - "requirement": "ALTERNATIVE", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "client-jwt", - "authenticatorFlow": false, - "requirement": "ALTERNATIVE", - "priority": 20, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "client-secret-jwt", - "authenticatorFlow": false, - "requirement": "ALTERNATIVE", - "priority": 30, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "client-x509", - "authenticatorFlow": false, - "requirement": "ALTERNATIVE", - "priority": 40, - "userSetupAllowed": false, - "autheticatorFlow": false - } - ] - }, - { - "id": "728be498-3ab4-4e6f-9791-79b720abc8ad", - "alias": "direct grant", - "description": "OpenID Connect Resource Owner Grant", - "providerId": "basic-flow", - "topLevel": true, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "direct-grant-validate-username", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "direct-grant-validate-password", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 20, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticatorFlow": true, - "requirement": "CONDITIONAL", - "priority": 30, - "flowAlias": "Direct Grant - Conditional OTP", - "userSetupAllowed": false, - "autheticatorFlow": true - } - ] - }, - { - "id": "b14135ca-f14d-46ac-9421-fca779668d0d", - "alias": "docker auth", - "description": "Used by Docker clients to authenticate against the IDP", - "providerId": "basic-flow", - "topLevel": true, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "docker-http-basic-authenticator", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - } - ] - }, - { - "id": "551ea373-9762-4327-affb-41fc55860b9b", - "alias": "first broker login", - "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", - "providerId": "basic-flow", - "topLevel": true, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticatorConfig": "review profile config", - "authenticator": "idp-review-profile", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticatorFlow": true, - "requirement": "REQUIRED", - "priority": 20, - "flowAlias": "User creation or linking", - "userSetupAllowed": false, - "autheticatorFlow": true - } - ] - }, - { - "id": "01570967-c01b-4631-a0d4-99b10c4382b3", - "alias": "forms", - "description": "Username, password, otp and other auth forms.", - "providerId": "basic-flow", - "topLevel": false, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "auth-username-password-form", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticatorFlow": true, - "requirement": "CONDITIONAL", - "priority": 20, - "flowAlias": "Browser - Conditional OTP", - "userSetupAllowed": false, - "autheticatorFlow": true - } - ] - }, - { - "id": "6a63fd0b-ca42-4658-9d8f-eb7b72341343", - "alias": "http challenge", - "description": "An authentication flow based on challenge-response HTTP Authentication Schemes", - "providerId": "basic-flow", - "topLevel": true, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "no-cookie-redirect", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticatorFlow": true, - "requirement": "REQUIRED", - "priority": 20, - "flowAlias": "Authentication Options", - "userSetupAllowed": false, - "autheticatorFlow": true - } - ] - }, - { - "id": "c05fa3bf-5b0d-4fc0-b251-bdef09cad34a", - "alias": "registration", - "description": "registration flow", - "providerId": "basic-flow", - "topLevel": true, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "registration-page-form", - "authenticatorFlow": true, - "requirement": "REQUIRED", - "priority": 10, - "flowAlias": "registration form", - "userSetupAllowed": false, - "autheticatorFlow": true - } - ] - }, - { - "id": "8b021678-bb93-4514-8a86-3288d5ec0c63", - "alias": "registration form", - "description": "registration form", - "providerId": "form-flow", - "topLevel": false, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "registration-user-creation", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 20, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "registration-profile-action", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 40, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "registration-password-action", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 50, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "registration-recaptcha-action", - "authenticatorFlow": false, - "requirement": "DISABLED", - "priority": 60, - "userSetupAllowed": false, - "autheticatorFlow": false - } - ] - }, - { - "id": "70720740-95ba-4cce-868a-9ba5d4ed9a8a", - "alias": "reset credentials", - "description": "Reset credentials for a user if they forgot their password or something", - "providerId": "basic-flow", - "topLevel": true, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "reset-credentials-choose-user", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "reset-credential-email", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 20, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticator": "reset-password", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 30, - "userSetupAllowed": false, - "autheticatorFlow": false - }, - { - "authenticatorFlow": true, - "requirement": "CONDITIONAL", - "priority": 40, - "flowAlias": "Reset - Conditional OTP", - "userSetupAllowed": false, - "autheticatorFlow": true - } - ] - }, - { - "id": "c42ef4b2-d9ea-4e76-876c-71c05131ea27", - "alias": "saml ecp", - "description": "SAML ECP Profile Authentication Flow", - "providerId": "basic-flow", - "topLevel": true, - "builtIn": true, - "authenticationExecutions": [ - { - "authenticator": "http-basic-authenticator", - "authenticatorFlow": false, - "requirement": "REQUIRED", - "priority": 10, - "userSetupAllowed": false, - "autheticatorFlow": false - } - ] - } - ], - "authenticatorConfig": [ - { - "id": "ff127d3c-27e0-49bc-8500-7241eae7f232", - "alias": "create unique user config", - "config": { - "require.password.update.after.registration": "false" - } - }, - { - "id": "76ffa39a-94d4-480f-8cbb-8ac375865c4b", - "alias": "review profile config", - "config": { - "update.profile.on.first.login": "missing" - } - } - ], - "requiredActions": [ - { - "alias": "CONFIGURE_TOTP", - "name": "Configure OTP", - "providerId": "CONFIGURE_TOTP", - "enabled": true, - "defaultAction": false, - "priority": 10, - "config": {} - }, - { - "alias": "terms_and_conditions", - "name": "Terms and Conditions", - "providerId": "terms_and_conditions", - "enabled": false, - "defaultAction": false, - "priority": 20, - "config": {} - }, - { - "alias": "UPDATE_PASSWORD", - "name": "Update Password", - "providerId": "UPDATE_PASSWORD", - "enabled": true, - "defaultAction": false, - "priority": 30, - "config": {} - }, - { - "alias": "UPDATE_PROFILE", - "name": "Update Profile", - "providerId": "UPDATE_PROFILE", - "enabled": true, - "defaultAction": false, - "priority": 40, - "config": {} - }, - { - "alias": "VERIFY_EMAIL", - "name": "Verify Email", - "providerId": "VERIFY_EMAIL", - "enabled": true, - "defaultAction": false, - "priority": 50, - "config": {} - }, - { - "alias": "delete_account", - "name": "Delete Account", - "providerId": "delete_account", - "enabled": false, - "defaultAction": false, - "priority": 60, - "config": {} - }, - { - "alias": "update_user_locale", - "name": "Update User Locale", - "providerId": "update_user_locale", - "enabled": true, - "defaultAction": false, - "priority": 1000, - "config": {} - } - ], - "browserFlow": "browser", - "registrationFlow": "registration", - "directGrantFlow": "direct grant", - "resetCredentialsFlow": "reset credentials", - "clientAuthenticationFlow": "clients", - "dockerAuthenticationFlow": "docker auth", - "attributes": { - "cibaBackchannelTokenDeliveryMode": "poll", - "cibaExpiresIn": "120", - "cibaAuthRequestedUserHint": "login_hint", - "oauth2DeviceCodeLifespan": "600", - "oauth2DevicePollingInterval": "5", - "clientOfflineSessionMaxLifespan": "0", - "clientSessionIdleTimeout": "0", - "userProfileEnabled": "false", - "parRequestUriLifespan": "60", - "clientSessionMaxLifespan": "0", - "clientOfflineSessionIdleTimeout": "0", - "cibaInterval": "5" - }, - "keycloakVersion": "16.1.0", - "userManagedAccessAllowed": false, - "clientProfiles": { - "profiles": [] - }, - "clientPolicies": { - "policies": [] - }, - "users": [ - { - "username": "rawuser", - "enabled": true, - "email": "readandwrite@test.com", - "attributes": - { - "policy":"readwrite" - }, - "credentials": [ - { - "type": "password", - "value": "rawuser" + "clients": [ + { + "clientId": "minio", + "rootUrl": "${authBaseUrl}", + "baseUrl": "/realms/cyberduckrealm/account/", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": true, + "clientAuthenticatorType": "client-secret", + "secret": "password", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "http://minio:9000" + ], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "access.token.lifespan": "30", + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "oauth2.device.authorization.grant.enabled": "false", + "use.jwks.url": "true", + "backchannel.logout.revoke.offline.tokens": "false", + "s3IdentityProviderEndpoint": "http://minio:9000", + "s3IdentityProviderSecretKey": "cyberduckSecretKey", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "oidc.ciba.grant.enabled": "false", + "use.jwks.string": "false", + "backchannel.logout.session.required": "false", + "client_credentials.use_refresh_token": "false", + "require.pushed.authorization.requests": "false", + "saml.client.signature": "false", + "s3IdentityProviderAccessKey": "cyberduckAccessKey", + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "saml.encrypt": "false", + "saml.server.signature": "false", + "exclude.session.state.from.auth.response": "false", + "saml.artifact.binding": "false", + "saml_force_name_id_format": "false", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "false", + "s3IdentityProviderEnabled": "true", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "roles", + "profile", + "minio-authorization", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ], + "protocolMappers": [ + { + "name": "realm roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "consentRequired": false, + "config": { + "access.token.claim": "true", + "claim.name": "realm_access.roles", + "jsonType.label": "String", + "multivalued": "true" + } + }, + { + "name": "client roles", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-client-role-mapper", + "consentRequired": false, + "config": { + "access.token.claim": "true", + "claim.name": "resource_access.${client_id}.roles", + "jsonType.label": "String", + "multivalued": "true", + "usermodel.clientRoleMapping.clientId": "cryptomatorhub" + } + } + ] } - ], - "realmRoles": [ - "user" - ], - "clientRoles": { - "account": [ - "view-profile", - "manage-account" - ] - } - }, - { - "username": "rouser", - "enabled": true, - "email": "readonly@test.com", - "attributes": - { - "policy":"readonly" - }, - "credentials": [ - { - "type": "password", - "value": "rouser" + ], + "clientScopes": [ + { + "name": "minio-authorization", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "true" + }, + "protocolMappers": [ + { + "name": "minio-policy-mapper", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-attribute-mapper", + "consentRequired": false, + "config": { + "aggregate.attrs": "true", + "multivalued": "true", + "userinfo.token.claim": "true", + "user.attribute": "policy", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "policy", + "jsonType.label": "String" + } + } + ] } - ], - "realmRoles": [ - "user" - ], - "clientRoles": { - "account": [ - "view-profile", - "manage-account" - ] - } + ], + "browserSecurityHeaders": { + "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self' http://localhost:*; object-src 'none';" } - ] -} +} \ No newline at end of file From d3bd6e1eee1137ff131926de2c84966f9130f5b2 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 12:06:13 +0200 Subject: [PATCH 40/84] Throw failure on missing or invalid JWT in Id token. --- .../STSAssumeRoleAuthorizationService.java | 38 ++++++++----------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index aead05e263d..881834b03e6 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -98,18 +98,9 @@ public STSTokens authorize(final Host bookmark, final String sAMLAssertion) thro public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws BackgroundException { final AssumeRoleWithWebIdentityRequest request = new AssumeRoleWithWebIdentityRequest(); - final String token; - if(StringUtils.isNotBlank(oauth.getIdToken())) { - if(log.isDebugEnabled()) { - log.debug(String.format("Assume role with OIDC Id token for %s", bookmark)); - } - token = oauth.getIdToken(); - } - else { - if(log.isDebugEnabled()) { - log.debug(String.format("Assume role with OAuth access token for %s", bookmark)); - } - token = oauth.getAccessToken(); + final String token = oauth.getIdToken(); + if(log.isDebugEnabled()) { + log.debug(String.format("Assume role with OIDC Id token for %s", bookmark)); } request.setWebIdentityToken(token); final HostPreferences preferences = new HostPreferences(bookmark); @@ -122,22 +113,23 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws if(StringUtils.isNotBlank(preferences.getProperty("s3.assumerole.rolearn"))) { request.setRoleArn(preferences.getProperty("s3.assumerole.rolearn")); } + final String sub; + try { + sub = JWT.decode(token).getSubject(); + } + catch(JWTDecodeException e) { + log.warn(String.format("Failure %s decoding JWT %s", e, token)); + throw new LoginFailureException("Invalid JWT or JSON format in authentication token", e); + } if(StringUtils.isNotBlank(preferences.getProperty("s3.assumerole.rolesessionname"))) { request.setRoleSessionName(preferences.getProperty("s3.assumerole.rolesessionname")); } else { - try { - final String sub = JWT.decode(token).getSubject(); - if(StringUtils.isNotBlank(sub)) { - request.setRoleSessionName(sub); - } - else { - log.warn(String.format("Missing subject in decoding JWT %s", token)); - request.setRoleSessionName(new AsciiRandomStringService().random()); - } + if(StringUtils.isNotBlank(sub)) { + request.setRoleSessionName(sub); } - catch(JWTDecodeException e) { - log.warn(String.format("Failure decoding JWT %s", token)); + else { + log.warn(String.format("Missing subject in decoding JWT %s", token)); request.setRoleSessionName(new AsciiRandomStringService().random()); } } From 0db18150d344356f1a2a571d898a6dea3b1be799 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 12:17:54 +0200 Subject: [PATCH 41/84] Fix typo. --- s3/src/main/java/ch/cyberduck/core/s3/S3Session.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 93f5bef2efc..90ab89d7407 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -113,7 +113,7 @@ public class S3Session extends HttpSession { */ private OAuth2RequestInterceptor oauth; /** - * Swap OIDC Id token with temporary security credentials + * Swap OIDC Id token for temporary security credentials */ private STSAssumeRoleCredentialsRequestInterceptor sts; /** From 62b0c2282ac3f740afcd560d93ce0c905af5d61a Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 12:19:00 +0200 Subject: [PATCH 42/84] Remove Role ARN configuration not required. --- s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile b/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile index e1c610598d0..aa86f4f0987 100644 --- a/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile +++ b/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile @@ -59,7 +59,7 @@ Properties s3.bucket.virtualhost.disable=true - s3.assumerole.rolearn= + From f6c68a21112895009c064bedc6666424d2b19589 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 13:41:07 +0200 Subject: [PATCH 43/84] Delete obsolete environment passed as parameter below. --- s3/src/test/resources/testcontainer/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/s3/src/test/resources/testcontainer/docker-compose.yml b/s3/src/test/resources/testcontainer/docker-compose.yml index 6a44e2c183c..536ead01410 100644 --- a/s3/src/test/resources/testcontainer/docker-compose.yml +++ b/s3/src/test/resources/testcontainer/docker-compose.yml @@ -10,7 +10,6 @@ services: environment: KEYCLOAK_ADMIN: admin KEYCLOAK_ADMIN_PASSWORD: admin - KEYCLOAK_IMPORT: /tmp/keycloak-realm.json PROXY_ADDRESS_FORWARDING: "true" KEYCLOAK_LOGLEVEL: DEBUG DB_VENDOR: h2 From 99fa10eedc1b848890ea960ba157c51ec0db7f6e Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 13:43:47 +0200 Subject: [PATCH 44/84] Add missing login prompt parameter. --- s3/src/main/java/ch/cyberduck/core/s3/S3Session.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 90ab89d7407..55bf3d9e472 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -210,7 +210,7 @@ protected RequestEntityRestStorageService connect(final Proxy proxy, final HostK final HttpClientBuilder configuration = builder.build(proxy, this, prompt); if(host.getProtocol().isOAuthConfigurable()) { configuration.addInterceptorLast(oauth = new OAuth2RequestInterceptor(builder.build(ProxyFactory.get() - .find(host.getProtocol().getOAuthAuthorizationUrl()), this, prompt).build(), host) + .find(host.getProtocol().getOAuthAuthorizationUrl()), this, prompt).build(), host, prompt) .withRedirectUri(host.getProtocol().getOAuthRedirectUrl()) .withFlowType(OAuth2AuthorizationService.FlowType.valueOf(host.getProtocol().getAuthorization()))); configuration.addInterceptorLast(sts = new STSAssumeRoleCredentialsRequestInterceptor(oauth, this, trust, key)); From cb0d603556ed9c9e6c7e11314be8112e68e2a1a6 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 13:47:47 +0200 Subject: [PATCH 45/84] Pull out case for anonymous credentials. --- .../main/java/ch/cyberduck/core/s3/S3Session.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 55bf3d9e472..17fe313542e 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -329,14 +329,17 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal else { credentials = host.getCredentials(); } - if(StringUtils.isNotBlank(credentials.getToken())) { - client.setProviderCredentials(credentials.isAnonymousLogin() ? null : - new AWSSessionCredentials(credentials.getUsername(), credentials.getPassword(), - credentials.getToken())); + if(credentials.isAnonymousLogin()) { + client.setProviderCredentials(null); } else { - client.setProviderCredentials(credentials.isAnonymousLogin() ? null : - new AWSCredentials(credentials.getUsername(), credentials.getPassword())); + if(StringUtils.isNotBlank(credentials.getToken())) { + client.setProviderCredentials(new AWSSessionCredentials( + credentials.getUsername(), credentials.getPassword(), credentials.getToken())); + } + else { + client.setProviderCredentials(new AWSCredentials(credentials.getUsername(), credentials.getPassword())); + } } } if(host.getCredentials().isPassed()) { From 79aba79ea8360c1c34503273b3bd8d8644910ecd Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 13:49:47 +0200 Subject: [PATCH 46/84] Log credentials format. --- s3/src/main/java/ch/cyberduck/core/s3/S3Session.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 17fe313542e..99c6bbd2455 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -330,14 +330,23 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal credentials = host.getCredentials(); } if(credentials.isAnonymousLogin()) { + if(log.isDebugEnabled()) { + log.debug(String.format("Connect with no credentials to %s", host)); + } client.setProviderCredentials(null); } else { if(StringUtils.isNotBlank(credentials.getToken())) { + if(log.isDebugEnabled()) { + log.debug(String.format("Connect with session credentials to %s", host)); + } client.setProviderCredentials(new AWSSessionCredentials( credentials.getUsername(), credentials.getPassword(), credentials.getToken())); } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Connect with basic credentials to %s", host)); + } client.setProviderCredentials(new AWSCredentials(credentials.getUsername(), credentials.getPassword())); } } From 0d5bdbdfbf2ddaf6b98b0835941ec14c4e79618a Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 13:51:43 +0200 Subject: [PATCH 47/84] Logging. --- .../cyberduck/core/auth/AWSCredentialsConfigurator.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/s3/src/main/java/ch/cyberduck/core/auth/AWSCredentialsConfigurator.java b/s3/src/main/java/ch/cyberduck/core/auth/AWSCredentialsConfigurator.java index 21ad09189e6..b9e51c0a136 100644 --- a/s3/src/main/java/ch/cyberduck/core/auth/AWSCredentialsConfigurator.java +++ b/s3/src/main/java/ch/cyberduck/core/auth/AWSCredentialsConfigurator.java @@ -25,6 +25,8 @@ import org.apache.logging.log4j.Logger; import org.jets3t.service.security.ProviderCredentials; +import java.util.Arrays; + import com.amazonaws.SdkClientException; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; @@ -49,6 +51,9 @@ public Credentials configure(final Host host) { for(AWSCredentialsProvider provider : providers) { try { final AWSCredentials c = provider.getCredentials(); + if(log.isDebugEnabled()) { + log.debug(String.format("Configure %s with %s", host, c)); + } credentials.setUsername(c.getAWSAccessKeyId()); credentials.setPassword(c.getAWSSecretKey()); if(c instanceof AWSSessionCredentials) { @@ -68,6 +73,9 @@ public Credentials configure(final Host host) { @Override public CredentialsConfigurator reload() throws LoginCanceledException { + if(log.isDebugEnabled()) { + log.debug(String.format("Reload from %s", Arrays.toString(providers))); + } for(AWSCredentialsProvider provider : providers) { provider.refresh(); } From d93d5a10e1415db2ba5c7e584c330e1929b92c97 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 14:42:31 +0200 Subject: [PATCH 48/84] Logging. --- .../cyberduck/core/sts/STSAssumeRoleAuthorizationService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 881834b03e6..c64bc3df609 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -134,6 +134,9 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws } } try { + if(log.isDebugEnabled()) { + log.debug(String.format("Use request %s", request)); + } final AssumeRoleWithWebIdentityResult result = service.assumeRoleWithWebIdentity(request); if(log.isDebugEnabled()) { log.debug(String.format("Received assume role identity result %s", result)); From 5f387958565cde247ffe4934bdf164a94b84165f Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 14:54:01 +0200 Subject: [PATCH 49/84] Prompt for Role ARN when defined as empty value in connection profile. --- .../java/ch/cyberduck/core/s3/S3Session.java | 2 +- .../STSAssumeRoleAuthorizationService.java | 28 ++++++++++++++++--- ...sumeRoleCredentialsRequestInterceptor.java | 7 +++-- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 99c6bbd2455..1396098cb13 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -213,7 +213,7 @@ protected RequestEntityRestStorageService connect(final Proxy proxy, final HostK .find(host.getProtocol().getOAuthAuthorizationUrl()), this, prompt).build(), host, prompt) .withRedirectUri(host.getProtocol().getOAuthRedirectUrl()) .withFlowType(OAuth2AuthorizationService.FlowType.valueOf(host.getProtocol().getAuthorization()))); - configuration.addInterceptorLast(sts = new STSAssumeRoleCredentialsRequestInterceptor(oauth, this, trust, key)); + configuration.addInterceptorLast(sts = new STSAssumeRoleCredentialsRequestInterceptor(oauth, this, trust, key, prompt)); configuration.setServiceUnavailableRetryStrategy(new STSAssumeRoleTokenExpiredResponseInterceptor(this, oauth, sts, prompt)); } else { diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index c64bc3df609..9333a9093b3 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -18,6 +18,9 @@ import ch.cyberduck.core.AsciiRandomStringService; import ch.cyberduck.core.Credentials; import ch.cyberduck.core.Host; +import ch.cyberduck.core.LocaleFactory; +import ch.cyberduck.core.LoginCallback; +import ch.cyberduck.core.LoginOptions; import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.aws.CustomClientConfiguration; import ch.cyberduck.core.exception.BackgroundException; @@ -34,7 +37,6 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.AnonymousAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.securitytoken.AWSSecurityTokenService; import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException; @@ -49,19 +51,21 @@ public class STSAssumeRoleAuthorizationService { private static final Logger log = LogManager.getLogger(STSAssumeRoleAuthorizationService.class); private final AWSSecurityTokenService service; + private final LoginCallback prompt; - public STSAssumeRoleAuthorizationService(final Host bookmark, final X509TrustManager trust, final X509KeyManager key) { + public STSAssumeRoleAuthorizationService(final Host bookmark, final X509TrustManager trust, final X509KeyManager key, final LoginCallback prompt) { this(AWSSecurityTokenServiceClientBuilder .standard() .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(bookmark.getProtocol().getSTSEndpoint(), null)) .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) .withClientConfiguration(new CustomClientConfiguration(bookmark, new ThreadLocalHostnameDelegatingTrustManager(trust, bookmark.getProtocol().getSTSEndpoint()), key)) - .build()); + .build(), prompt); } - public STSAssumeRoleAuthorizationService(final AWSSecurityTokenService service) { + public STSAssumeRoleAuthorizationService(final AWSSecurityTokenService service, final LoginCallback prompt) { this.service = service; + this.prompt = prompt; } public STSTokens authorize(final Host bookmark, final String sAMLAssertion) throws BackgroundException { @@ -113,6 +117,22 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws if(StringUtils.isNotBlank(preferences.getProperty("s3.assumerole.rolearn"))) { request.setRoleArn(preferences.getProperty("s3.assumerole.rolearn")); } + else { + if(StringUtils.EMPTY.equals(preferences.getProperty("s3.assumerole.rolearn"))) { + // When defined in connection profile but with empty value + if(log.isDebugEnabled()) { + log.debug("Prompt for Role ARN"); + } + final Credentials input = prompt.prompt(bookmark, + LocaleFactory.localizedString("Role Amazon Resource Name (ARN)", "Credentials"), + LocaleFactory.localizedString("Provide additional login credentials", "Credentials"), + new LoginOptions().icon(bookmark.getProtocol().disk())); + if(input.isSaved()) { + bookmark.setProperty("s3.assumerole.rolearn", input.getPassword()); + } + request.setRoleArn(input.getPassword()); + } + } final String sub; try { sub = JWT.decode(token).getSubject(); diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java index eb2445c7801..9b2b0f28baf 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -17,6 +17,7 @@ import ch.cyberduck.core.Host; import ch.cyberduck.core.HostPasswordStore; +import ch.cyberduck.core.LoginCallback; import ch.cyberduck.core.LoginOptions; import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.PasswordStoreFactory; @@ -27,7 +28,6 @@ import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; -import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpException; import org.apache.http.HttpRequest; import org.apache.http.HttpRequestInterceptor; @@ -51,8 +51,9 @@ public class STSAssumeRoleCredentialsRequestInterceptor extends STSAssumeRoleAut private final S3Session session; private final Host host; - public STSAssumeRoleCredentialsRequestInterceptor(final OAuth2RequestInterceptor oauth, final S3Session session, final X509TrustManager trust, final X509KeyManager key) { - super(session.getHost(), trust, key); + public STSAssumeRoleCredentialsRequestInterceptor(final OAuth2RequestInterceptor oauth, final S3Session session, + final X509TrustManager trust, final X509KeyManager key, final LoginCallback prompt) { + super(session.getHost(), trust, key, prompt); this.oauth = oauth; this.session = session; this.host = session.getHost(); From f09329729e241a1c9661c2ad688129e42f8cb6e3 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 14:54:20 +0200 Subject: [PATCH 50/84] Review tests. --- ...oleWithWebIdentityAuthenticationTest.java} | 57 ++++++++++--------- ...RoleWithWebIdentityAuthorizationTest.java} | 21 +++---- 2 files changed, 42 insertions(+), 36 deletions(-) rename s3/src/test/java/ch/cyberduck/core/sts/{OidcAuthenticationTest.java => AssumeRoleWithWebIdentityAuthenticationTest.java} (62%) rename s3/src/test/java/ch/cyberduck/core/sts/{OidcAuthorizationTest.java => AssumeRoleWithWebIdentityAuthorizationTest.java} (75%) diff --git a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java similarity index 62% rename from s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java rename to s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index cb554f46b4a..52515984709 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -18,15 +18,17 @@ import ch.cyberduck.core.Credentials; import ch.cyberduck.core.DisabledCancelCallback; import ch.cyberduck.core.DisabledHostKeyCallback; +import ch.cyberduck.core.DisabledListProgressListener; import ch.cyberduck.core.DisabledLoginCallback; import ch.cyberduck.core.Host; +import ch.cyberduck.core.HostUrlProvider; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LoginFailureException; -import ch.cyberduck.core.preferences.HostPreferences; -import ch.cyberduck.core.proxy.Proxy; +import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.s3.S3AccessControlListFeature; import ch.cyberduck.core.s3.S3FindFeature; +import ch.cyberduck.core.s3.S3ObjectListService; import ch.cyberduck.core.s3.S3Session; import ch.cyberduck.test.TestcontainerTest; @@ -42,14 +44,14 @@ import static org.junit.Assert.*; @Category(TestcontainerTest.class) -public class OidcAuthenticationTest extends AbstractOidcTest { +public class AssumeRoleWithWebIdentityAuthenticationTest extends AbstractOidcTest { @Test public void testSuccessfulLoginViaOidc() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); Credentials creds = host.getCredentials(); assertNotEquals(StringUtils.EMPTY, creds.getUsername()); @@ -66,8 +68,8 @@ public void testSuccessfulLoginViaOidc() throws BackgroundException { public void testInvalidUserName() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("WrongUsername", "rouser")); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - assertThrows(LoginFailureException.class, () -> session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback())); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + assertThrows(LoginFailureException.class, () -> session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback())); session.close(); } @@ -75,8 +77,8 @@ public void testInvalidUserName() throws BackgroundException { public void testInvalidPassword() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "invalidPassword")); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - assertThrows(LoginFailureException.class, () -> session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback())); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + assertThrows(LoginFailureException.class, () -> session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback())); session.close(); } @@ -84,8 +86,8 @@ public void testInvalidPassword() throws BackgroundException { public void testTokenRefresh() throws BackgroundException, InterruptedException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); String firstAccessToken = host.getCredentials().getOauth().getIdToken(); String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); @@ -102,42 +104,45 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException session.close(); } - /** only use with the below specified changes in the keycloak config json file and run as separate test + /** + * only use with the below specified changes in the keycloak config json file and run as separate test * set config keycloak-realm.json: - * "access.token.lifespan": "930" - * "ssoSessionMaxLifespan": 1100, + * "access.token.lifespan": "930" + * "ssoSessionMaxLifespan": 1100, */ @Test @Ignore - public void testSTSCredentialsExpiredValidOAuthToken() throws BackgroundException, InterruptedException { + public void testSTSCredentialsExpiredOAuthToken() throws BackgroundException, InterruptedException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); - host.setProperty("s3.assumerole.durationseconds", "900"); - assertEquals(new HostPreferences(host).getInteger("s3.assumerole.durationseconds"), 900); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); String firstAccessToken = host.getCredentials().getOauth().getAccessToken(); String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); + assertTrue(session.getClient().getProviderCredentials() instanceof AWSSessionCredentials); + + String firstSessionToken = ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken(); Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + assertFalse(new S3ObjectListService(session, new S3AccessControlListFeature(session)).list(container, new DisabledListProgressListener()).isEmpty()); Thread.sleep(1000 * 910); - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + assertFalse(new S3ObjectListService(session, new S3AccessControlListFeature(session)).list(container, new DisabledListProgressListener()).isEmpty()); assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); + assertNotEquals(firstSessionToken, ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken()); assertEquals(firstAccessToken, host.getCredentials().getOauth().getAccessToken()); } /** - * This test fails if the x-minio Headers are not read because of InvalidAccessKeyId error code which has no response body. - * Adjust the sleep time according to the network latency + * This test fails if the x-minio Headers are not read because of InvalidAccessKeyId error code which has no response body. + * Adjust the sleep time according to the network latency */ @Test @Ignore public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws BackgroundException, InterruptedException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); String firstAccessToken = host.getCredentials().getOauth().getIdToken(); String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); @@ -145,7 +150,7 @@ public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws Bac // Time of latency may vary and so the time needs to be adjusted accordingly Thread.sleep(28820); Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + assertFalse(new S3ObjectListService(session, new S3AccessControlListFeature(session)).list(container, new DisabledListProgressListener()).isEmpty()); assertNotEquals(firstAccessToken, host.getCredentials().getOauth().getIdToken()); assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); diff --git a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthorizationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java similarity index 75% rename from s3/src/test/java/ch/cyberduck/core/sts/OidcAuthorizationTest.java rename to s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java index 63f05dcdb27..09363a05512 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/OidcAuthorizationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java @@ -22,11 +22,12 @@ import ch.cyberduck.core.DisabledHostKeyCallback; import ch.cyberduck.core.DisabledLoginCallback; import ch.cyberduck.core.Host; +import ch.cyberduck.core.HostUrlProvider; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.AccessDeniedException; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Delete; -import ch.cyberduck.core.proxy.Proxy; +import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.s3.S3AccessControlListFeature; import ch.cyberduck.core.s3.S3DefaultDeleteFeature; import ch.cyberduck.core.s3.S3FindFeature; @@ -45,14 +46,14 @@ import static org.junit.Assert.*; @Category(TestcontainerTest.class) -public class OidcAuthorizationTest extends AbstractOidcTest { +public class AssumeRoleWithWebIdentityAuthorizationTest extends AbstractOidcTest { @Test public void testAuthorizationFindBucket() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); final Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); session.close(); @@ -62,8 +63,8 @@ public void testAuthorizationFindBucket() throws BackgroundException { public void testAuthorizationUserReadAccessOnBucket() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); final TransferStatus status = new TransferStatus(); final Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); new S3ReadFeature(session).read(new Path(container, "testfile.txt", EnumSet.of(Path.Type.file)), status, new DisabledConnectionCallback()); @@ -74,8 +75,8 @@ public void testAuthorizationUserReadAccessOnBucket() throws BackgroundException public void testAuthorizationWritePermissionOnBucket() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); final Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); final Path test = new Path(container, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)); new S3TouchFeature(session, new S3AccessControlListFeature(session)).touch(test, new TransferStatus()); @@ -89,8 +90,8 @@ public void testAuthorizationWritePermissionOnBucket() throws BackgroundExceptio public void testAuthorizationNoWritePermissionOnBucket() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); final Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); final Path test = new Path(container, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)); assertThrows(AccessDeniedException.class, () -> new S3TouchFeature(session, new S3AccessControlListFeature(session)).touch(test, new TransferStatus())); From 0794fab9cd00c9416f753435b147954034667b4b Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 15:57:18 +0200 Subject: [PATCH 51/84] Allow explicit empty definition of client secret in profile to disable prompt. --- .../ch/cyberduck/core/oauth/OAuth2AuthorizationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java index b53697768b1..7bb92a02395 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java @@ -174,7 +174,7 @@ private IdTokenResponse authorizeWithCode(final Host bookmark, final LoginCallba method, transport, json, new GenericUrl(tokenServerUrl), - new ClientParametersAuthentication(clientid, clientsecret), + new ClientParametersAuthentication(clientid, StringUtils.isNotBlank(clientsecret) ? clientsecret : null), clientid, authorizationServerUrl) .setScopes(scopes) From 74c170d102ad60e9d1320f66684c5cbd3f4699c0 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 17:31:31 +0200 Subject: [PATCH 52/84] Rename profile. --- s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java | 2 +- ...OIDC-Testing.cyberduckprofile => S3 (OIDC).cyberduckprofile} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename s3/src/test/resources/{S3-OIDC-Testing.cyberduckprofile => S3 (OIDC).cyberduckprofile} (100%) diff --git a/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java b/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java index 868c6e4e0e8..f67f05d832b 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java @@ -35,7 +35,7 @@ public class S3OidcProfileTest { public void testDefaultProfile() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/S3-OIDC-Testing.cyberduckprofile")); + this.getClass().getResourceAsStream("/S3 (OIDC).cyberduckprofile")); assertEquals("minio", profile.getOAuthClientId()); assertEquals("password", profile.getOAuthClientSecret()); assertNotNull(profile.getOAuthAuthorizationUrl()); diff --git a/s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile b/s3/src/test/resources/S3 (OIDC).cyberduckprofile similarity index 100% rename from s3/src/test/resources/S3-OIDC-Testing.cyberduckprofile rename to s3/src/test/resources/S3 (OIDC).cyberduckprofile From 165de986d67ef7d718ab6fff739fe3b55abb278e Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 17:31:40 +0200 Subject: [PATCH 53/84] Rename base test class. --- ...st.java => AbstractAssumeRoleWithWebIdentityTest.java} | 8 ++++---- .../sts/AssumeRoleWithWebIdentityAuthenticationTest.java | 2 +- .../sts/AssumeRoleWithWebIdentityAuthorizationTest.java | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename s3/src/test/java/ch/cyberduck/core/sts/{AbstractOidcTest.java => AbstractAssumeRoleWithWebIdentityTest.java} (89%) diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AbstractOidcTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java similarity index 89% rename from s3/src/test/java/ch/cyberduck/core/sts/AbstractOidcTest.java rename to s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java index 7dde6f71bdc..5720405fe32 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AbstractOidcTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java @@ -39,16 +39,16 @@ import java.util.HashSet; @Category(TestcontainerTest.class) -public abstract class AbstractOidcTest { +public abstract class AbstractAssumeRoleWithWebIdentityTest { - protected static final Logger log = LogManager.getLogger(AbstractOidcTest.class); + protected static final Logger log = LogManager.getLogger(AbstractAssumeRoleWithWebIdentityTest.class); protected static Profile profile = null; private static Network network; private static final DockerComposeContainer compose; static { compose = new DockerComposeContainer<>( - new File(AbstractOidcTest.class.getResource("/testcontainer/docker-compose.yml").getFile())) + new File(AbstractAssumeRoleWithWebIdentityTest.class.getResource("/testcontainer/docker-compose.yml").getFile())) .withPull(false) .withLocalCompose(true) .withOptions("--compatibility") @@ -69,7 +69,7 @@ public void setup() throws BackgroundException { private Profile readProfile() throws AccessDeniedException { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); return new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/S3-OIDC-Testing.cyberduckprofile")); + this.getClass().getResourceAsStream("/S3 (OIDC).cyberduckprofile")); } @AfterClass diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index 52515984709..8aa90929b52 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -44,7 +44,7 @@ import static org.junit.Assert.*; @Category(TestcontainerTest.class) -public class AssumeRoleWithWebIdentityAuthenticationTest extends AbstractOidcTest { +public class AssumeRoleWithWebIdentityAuthenticationTest extends AbstractAssumeRoleWithWebIdentityTest { @Test public void testSuccessfulLoginViaOidc() throws BackgroundException { diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java index 09363a05512..5505a444983 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java @@ -46,7 +46,7 @@ import static org.junit.Assert.*; @Category(TestcontainerTest.class) -public class AssumeRoleWithWebIdentityAuthorizationTest extends AbstractOidcTest { +public class AssumeRoleWithWebIdentityAuthorizationTest extends AbstractAssumeRoleWithWebIdentityTest { @Test public void testAuthorizationFindBucket() throws BackgroundException { From 1cd797d8bd03096f9e40b760004dc608aa734f56 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 22:25:47 +0200 Subject: [PATCH 54/84] Add handling for response headers from Minio for HEAD requests with no error response body. --- .../core/s3/S3ExceptionMappingService.java | 27 ++++++ .../s3/S3TokenExpiredResponseInterceptor.java | 38 ++++---- ...meRoleTokenExpiredResponseInterceptor.java | 89 +++++++++---------- ...RoleWithWebIdentityAuthenticationTest.java | 6 +- 4 files changed, 87 insertions(+), 73 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3ExceptionMappingService.java b/s3/src/main/java/ch/cyberduck/core/s3/S3ExceptionMappingService.java index abfbfe767e4..132495f76cf 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3ExceptionMappingService.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3ExceptionMappingService.java @@ -29,8 +29,12 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; +import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.HttpResponseException; +import org.apache.http.entity.BufferedHttpEntity; +import org.apache.http.util.EntityUtils; +import org.jets3t.service.S3ServiceException; import org.jets3t.service.ServiceException; import org.xml.sax.SAXException; @@ -38,6 +42,29 @@ public class S3ExceptionMappingService extends AbstractExceptionMappingService { + private static final String MINIO_ERROR_CODE = "x-minio-error-code"; + private static final String MINIO_ERROR_DESCRIPTION = "x-minio-error-desc"; + + public BackgroundException map(HttpResponse response) throws IOException { + final S3ServiceException failure; + if(null == response.getEntity()) { + failure = new S3ServiceException(response.getStatusLine().getReasonPhrase()); + } + else { + EntityUtils.updateEntity(response, new BufferedHttpEntity(response.getEntity())); + failure = new S3ServiceException(response.getStatusLine().getReasonPhrase(), + EntityUtils.toString(response.getEntity())); + } + failure.setResponseCode(response.getStatusLine().getStatusCode()); + if(response.containsHeader(MINIO_ERROR_CODE)) { + failure.setErrorCode(response.getFirstHeader(MINIO_ERROR_CODE).getValue()); + } + if(response.containsHeader(MINIO_ERROR_DESCRIPTION)) { + failure.setErrorMessage(response.getFirstHeader(MINIO_ERROR_DESCRIPTION).getValue()); + } + return this.map(failure); + } + @Override public BackgroundException map(final ServiceException e) { if(e.getCause() instanceof ServiceException) { diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java index 40596c4351c..c47ab43b247 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java @@ -53,29 +53,23 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun switch(response.getStatusLine().getStatusCode()) { case HttpStatus.SC_BAD_REQUEST: try { - if(null != response.getEntity()) { - EntityUtils.updateEntity(response, new BufferedHttpEntity(response.getEntity())); - final S3ServiceException failure = new S3ServiceException(response.getStatusLine().getReasonPhrase(), - EntityUtils.toString(response.getEntity())); - failure.setResponseCode(response.getStatusLine().getStatusCode()); - if(new S3ExceptionMappingService().map(failure) instanceof ExpiredTokenException) { - if(log.isWarnEnabled()) { - log.warn(String.format("Handle failure %s", failure)); - } - final Credentials credentials = configurator.configure(session.getHost()); - if(log.isDebugEnabled()) { - log.debug(String.format("Reconfigure client with credentials %s", credentials)); - } - if(credentials.isTokenAuthentication()) { - session.getClient().setProviderCredentials(new AWSSessionCredentials( - credentials.getUsername(), credentials.getPassword(), credentials.getToken())); - } - else { - session.getClient().setProviderCredentials(new AWSCredentials( - credentials.getUsername(), credentials.getPassword())); - } - return true; + if(new S3ExceptionMappingService().map(response) instanceof ExpiredTokenException) { + if(log.isWarnEnabled()) { + log.warn(String.format("Handle failure %s", response)); } + final Credentials credentials = configurator.configure(session.getHost()); + if(log.isDebugEnabled()) { + log.debug(String.format("Reconfigure client with credentials %s", credentials)); + } + if(credentials.isTokenAuthentication()) { + session.getClient().setProviderCredentials(new AWSSessionCredentials( + credentials.getUsername(), credentials.getPassword(), credentials.getToken())); + } + else { + session.getClient().setProviderCredentials(new AWSCredentials( + credentials.getUsername(), credentials.getPassword())); + } + return true; } } catch(IOException e) { diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java index a62b8be8f3c..bb8d37684ad 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java @@ -32,7 +32,6 @@ import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jets3t.service.S3ServiceException; import org.jets3t.service.security.AWSSessionCredentials; import java.io.IOException; @@ -66,61 +65,55 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun log.warn(String.format("Skip retry for response %s after %d executions", response, executionCount)); return false; } - if(null != response.getEntity()) { - try { - EntityUtils.updateEntity(response, new BufferedHttpEntity(response.getEntity())); - final S3ServiceException failure = new S3ServiceException(response.getStatusLine().getReasonPhrase(), - EntityUtils.toString(response.getEntity())); - failure.setResponseCode(response.getStatusLine().getStatusCode()); - final BackgroundException type = new S3ExceptionMappingService().map(failure); - final OAuthTokens oAuthTokens; - if(type instanceof ExpiredTokenException) { - // 400 Bad Request (ExpiredToken) The provided token has expired - // 400 Bad Request (InvalidToken) The provided token is malformed or otherwise not valid - // 400 Bad Request (TokenRefreshRequired) The provided token must be refreshed. - // No refresh of OAuth tokens - oAuthTokens = oauth.getTokens(); - } - else if(type instanceof LoginFailureException) { - // 401 (InvalidAccessKeyId) The AWS access key ID that you provided does not exist in our records. - // 401 (InvalidSecurity) The provided security credentials are not valid. - // 403 (Forbidden) Access Denied - try { - // Refresh OAuth tokens - oAuthTokens = super.refresh(response); - } - catch(BackgroundException e) { - log.warn(String.format("Failure %s refreshing OAuth tokens", e)); - return false; - } - } - else { - // Ignore other 400 failures - if(log.isDebugEnabled()) { - log.debug(String.format("Ignore failure %s", type)); - } - return false; - } - if(log.isWarnEnabled()) { - log.warn(String.format("Handle failure %s", failure)); - } + try { + final BackgroundException type = new S3ExceptionMappingService().map(response); + final OAuthTokens oAuthTokens; + if(type instanceof ExpiredTokenException) { + // 400 Bad Request (ExpiredToken) The provided token has expired + // 400 Bad Request (InvalidToken) The provided token is malformed or otherwise not valid + // 400 Bad Request (TokenRefreshRequired) The provided token must be refreshed. + // No refresh of OAuth tokens + oAuthTokens = oauth.getTokens(); + } + else if(type instanceof LoginFailureException) { + // 401 (InvalidAccessKeyId) The AWS access key ID that you provided does not exist in our records. + // 401 (InvalidSecurity) The provided security credentials are not valid. + // 403 (Forbidden) Access Denied try { - log.warn(String.format("Attempt to refresh STS token for failure %s", response)); - final STSTokens stsTokens = sts.refresh(oAuthTokens); - session.getClient().setProviderCredentials(new AWSSessionCredentials(stsTokens.getAccessKeyId(), - stsTokens.getSecretAccessKey(), stsTokens.getSessionToken())); - // Try again - return true; + // Refresh OAuth tokens + oAuthTokens = super.refresh(response); } catch(BackgroundException e) { - log.warn(String.format("Failure %s refreshing STS token", e)); + log.warn(String.format("Failure %s refreshing OAuth tokens", e)); return false; } } - catch(IOException e) { - log.warn(String.format("Failure parsing response entity from %s", response)); + else { + // Ignore other 400 failures + if(log.isDebugEnabled()) { + log.debug(String.format("Ignore failure %s", type)); + } return false; } + if(log.isWarnEnabled()) { + log.warn(String.format("Handle failure %s", response)); + } + try { + log.warn(String.format("Attempt to refresh STS token for failure %s", response)); + final STSTokens stsTokens = sts.refresh(oAuthTokens); + session.getClient().setProviderCredentials(new AWSSessionCredentials(stsTokens.getAccessKeyId(), + stsTokens.getSecretAccessKey(), stsTokens.getSessionToken())); + // Try again + return true; + } + catch(BackgroundException e) { + log.warn(String.format("Failure %s refreshing STS token", e)); + return false; + } + } + catch(IOException e) { + log.warn(String.format("Failure parsing response entity from %s", response)); + return false; } } return false; diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index 8aa90929b52..7d9b6fd408f 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -124,9 +124,9 @@ public void testSTSCredentialsExpiredOAuthToken() throws BackgroundException, In String firstSessionToken = ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken(); Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); - assertFalse(new S3ObjectListService(session, new S3AccessControlListFeature(session)).list(container, new DisabledListProgressListener()).isEmpty()); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); Thread.sleep(1000 * 910); - assertFalse(new S3ObjectListService(session, new S3AccessControlListFeature(session)).list(container, new DisabledListProgressListener()).isEmpty()); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); assertNotEquals(firstSessionToken, ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken()); assertEquals(firstAccessToken, host.getCredentials().getOauth().getAccessToken()); @@ -150,7 +150,7 @@ public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws Bac // Time of latency may vary and so the time needs to be adjusted accordingly Thread.sleep(28820); Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); - assertFalse(new S3ObjectListService(session, new S3AccessControlListFeature(session)).list(container, new DisabledListProgressListener()).isEmpty()); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); assertNotEquals(firstAccessToken, host.getCredentials().getOauth().getIdToken()); assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); From 8df18143780b7664767e64e48cc542ac3f062dcc Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 22:46:43 +0200 Subject: [PATCH 55/84] Missing int in preferences defaults to -1. --- .../cyberduck/core/sts/STSAssumeRoleAuthorizationService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 9333a9093b3..2b19dff03f8 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -71,7 +71,7 @@ public STSAssumeRoleAuthorizationService(final AWSSecurityTokenService service, public STSTokens authorize(final Host bookmark, final String sAMLAssertion) throws BackgroundException { final AssumeRoleWithSAMLRequest request = new AssumeRoleWithSAMLRequest().withSAMLAssertion(sAMLAssertion); final HostPreferences preferences = new HostPreferences(bookmark); - if(preferences.getInteger("s3.assumerole.durationseconds") != 0) { + if(preferences.getInteger("s3.assumerole.durationseconds") != -1) { request.setDurationSeconds(preferences.getInteger("s3.assumerole.durationseconds")); } if(StringUtils.isNotBlank(preferences.getProperty("s3.assumerole.policy"))) { @@ -108,7 +108,7 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws } request.setWebIdentityToken(token); final HostPreferences preferences = new HostPreferences(bookmark); - if(preferences.getInteger("s3.assumerole.durationseconds") != 0) { + if(preferences.getInteger("s3.assumerole.durationseconds") != -1) { request.setDurationSeconds(preferences.getInteger("s3.assumerole.durationseconds")); } if(StringUtils.isNotBlank(preferences.getProperty("s3.assumerole.policy"))) { From f4655a7f194beb46e3d5cb877ad2fd51a3a699f8 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 11 Aug 2023 22:47:00 +0200 Subject: [PATCH 56/84] Do not set empty values in defaults. --- defaults/src/main/resources/default.properties | 10 +++++----- .../java/ch/cyberduck/core/sts/S3OidcProfileTest.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/defaults/src/main/resources/default.properties b/defaults/src/main/resources/default.properties index dd6a6178c56..272f51d7536 100644 --- a/defaults/src/main/resources/default.properties +++ b/defaults/src/main/resources/default.properties @@ -297,11 +297,10 @@ s3.endpoint.format.ipv6=s3.dualstack.%s.amazonaws.com s3.acl.default=private # STS Assume Role request parameters -s3.assumerole.durationseconds=0 -s3.assumerole.policy= -s3.assumerole.rolearn= -s3.assumerole.rolesessionname= - +#s3.assumerole.durationseconds= +#s3.assumerole.policy= +#s3.assumerole.rolearn= +#s3.assumerole.rolesessionname= # Default redundancy level s3.storage.class=STANDARD @@ -485,6 +484,7 @@ storegate.login.hint= # Mobile ctera.attach.devicetype=DriveConnect +oauth.application.identifier=io.cyberduck oauth.browser.open.warn=false brick.pairing.nickname.configure=false diff --git a/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java b/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java index f67f05d832b..7d5bbb6b60c 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java @@ -43,6 +43,6 @@ public void testDefaultProfile() throws Exception { assertEquals("http://localhost:9000", profile.getSTSEndpoint()); assertFalse(profile.getOAuthScopes().isEmpty()); assertTrue(profile.getOAuthScopes().contains("openid")); - assertEquals("", new HostPreferences(new Host(profile)).getProperty("s3.assumerole.rolearn")); + assertNull(new HostPreferences(new Host(profile)).getProperty("s3.assumerole.rolearn")); } } From 54413616b7cf0d051c4cff8bd3ab29311e85a855 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sat, 12 Aug 2023 11:00:39 +0200 Subject: [PATCH 57/84] Mark as token configurable to ensure AWS Session Token is saved and retrieved from keychain. Try to authenticate with retrieved AWS Session token from keychain. --- s3/src/main/java/ch/cyberduck/core/s3/S3Session.java | 2 +- s3/src/test/resources/S3 (OIDC).cyberduckprofile | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 1396098cb13..f376981698a 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -308,7 +308,7 @@ public void process(final HttpRequest request, final HttpContext context) { public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { if(host.getProtocol().isOAuthConfigurable()) { // Get temporary credentials from STS using Web Identity (OIDC) token - final STSTokens tokens = sts.refresh(oauth.authorize(host, prompt, cancel)); + final STSTokens tokens = sts.authorize(host, oauth.authorize(host, prompt, cancel)); client.setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKeyId(), tokens.getSecretAccessKey(), tokens.getSessionToken())); } diff --git a/s3/src/test/resources/S3 (OIDC).cyberduckprofile b/s3/src/test/resources/S3 (OIDC).cyberduckprofile index aa86f4f0987..da33f61b927 100644 --- a/s3/src/test/resources/S3 (OIDC).cyberduckprofile +++ b/s3/src/test/resources/S3 (OIDC).cyberduckprofile @@ -54,8 +54,10 @@ Username Configurable - Path Configurable + Token Configurable + Token Placeholder + AWS Session Token Properties s3.bucket.virtualhost.disable=true From c5dccbaa6ef8a5bcba9e670c35d5fb397dd384d5 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sat, 12 Aug 2023 11:06:50 +0200 Subject: [PATCH 58/84] Add mapper for STS API failures. Map to `ExpiredTokenException` when web identity token is no longer valid. --- .../STSAssumeRoleAuthorizationService.java | 4 +- .../core/sts/STSExceptionMappingService.java | 42 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 s3/src/main/java/ch/cyberduck/core/sts/STSExceptionMappingService.java diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 2b19dff03f8..8ccf2b2b14a 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -96,7 +96,7 @@ public STSTokens authorize(final Host bookmark, final String sAMLAssertion) thro return tokens; } catch(AWSSecurityTokenServiceException e) { - throw new LoginFailureException(e.getErrorMessage(), e); + throw new STSExceptionMappingService().map(e); } } @@ -172,7 +172,7 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws return tokens; } catch(AWSSecurityTokenServiceException e) { - throw new LoginFailureException(e.getErrorMessage(), e); + throw new STSExceptionMappingService().map(e); } } } diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSExceptionMappingService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSExceptionMappingService.java new file mode 100644 index 00000000000..a1515b8644d --- /dev/null +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSExceptionMappingService.java @@ -0,0 +1,42 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.ExceptionMappingService; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.ExpiredTokenException; +import ch.cyberduck.core.exception.LoginFailureException; + +import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException; +import com.amazonaws.services.securitytoken.model.InvalidIdentityTokenException; + +public class STSExceptionMappingService implements ExceptionMappingService { + + @Override + public BackgroundException map(final AWSSecurityTokenServiceException e) { + if(e instanceof com.amazonaws.services.securitytoken.model.ExpiredTokenException) { + // The web identity token that was passed is expired or is not valid. Get a new identity token from the identity + // provider and then retry the request. + return new ExpiredTokenException(e.getErrorMessage(), e); + } + if(e instanceof InvalidIdentityTokenException) { + // The web identity token that was passed could not be validated by Amazon Web Services. Get a new identity token from + // the identity provider and then retry the request. + return new ExpiredTokenException(e.getErrorMessage(), e); + } + return new LoginFailureException(e.getErrorMessage(), e); + } +} From 9d9cb54347a599196cda9e5f2a9dfa806bbf45a5 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sat, 12 Aug 2023 11:38:05 +0200 Subject: [PATCH 59/84] Explicitly skip validation for OAuth. --- core/src/main/java/ch/cyberduck/core/AbstractProtocol.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/java/ch/cyberduck/core/AbstractProtocol.java b/core/src/main/java/ch/cyberduck/core/AbstractProtocol.java index f27eee6ecbb..92e8e9c246a 100644 --- a/core/src/main/java/ch/cyberduck/core/AbstractProtocol.java +++ b/core/src/main/java/ch/cyberduck/core/AbstractProtocol.java @@ -321,6 +321,10 @@ public boolean validate(final Credentials credentials, final LoginOptions option return StringUtils.isNotBlank(credentials.getPassword()); } } + if(options.oauth) { + // Always refresh tokens in login + return true; + } if(options.token) { return StringUtils.isNotBlank(credentials.getToken()); } From de95cc54beb5beeb9b19ba3ce22818a03362eaec Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sat, 12 Aug 2023 11:47:52 +0200 Subject: [PATCH 60/84] Logging. --- s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java b/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java index 6a688031a8e..fbf4b41f0e8 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java @@ -60,10 +60,10 @@ public boolean isExpired() { @Override public String toString() { - final StringBuilder sb = new StringBuilder("OAuthTokens{"); - sb.append("accessKey='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(accessKeyId)))).append('\''); + final StringBuilder sb = new StringBuilder("STSTokens{"); + sb.append("accessKeyId='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(accessKeyId)))).append('\''); sb.append(", secretAccessKey='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(secretAccessKey)))).append('\''); - sb.append(", sessionToken='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(secretAccessKey)))).append('\''); + sb.append(", sessionToken='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(sessionToken)))).append('\''); sb.append(", expiryInMilliseconds=").append(expiryInMilliseconds); sb.append('}'); return sb.toString(); From fee11df846f14f97e31f048a50bb9b2fcb0a6067 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sat, 12 Aug 2023 18:03:37 +0200 Subject: [PATCH 61/84] Explict field for STS tokens in credentials used by configurator and returned from STS assume role requests. --- .../java/ch/cyberduck/core/Credentials.java | 26 +++++++++++++-- .../java/ch/cyberduck/core}/STSTokens.java | 9 ++++- .../auth/AWSSessionCredentialsRetriever.java | 21 ++++++++---- .../core/s3/S3CredentialsConfigurator.java | 33 ++++++++++--------- .../java/ch/cyberduck/core/s3/S3Session.java | 8 ++--- .../s3/S3TokenExpiredResponseInterceptor.java | 8 ++--- .../STSAssumeRoleAuthorizationService.java | 13 +++----- ...sumeRoleCredentialsRequestInterceptor.java | 6 ++-- ...meRoleTokenExpiredResponseInterceptor.java | 1 + .../AWSSessionCredentialsRetrieverTest.java | 7 ++-- .../s3/S3CredentialsConfiguratorTest.java | 6 ++-- ...RoleWithWebIdentityAuthenticationTest.java | 4 ++- 12 files changed, 88 insertions(+), 54 deletions(-) rename {s3/src/main/java/ch/cyberduck/core/sts => core/src/main/java/ch/cyberduck/core}/STSTokens.java (93%) diff --git a/core/src/main/java/ch/cyberduck/core/Credentials.java b/core/src/main/java/ch/cyberduck/core/Credentials.java index 128fda68c0e..479d38b5687 100644 --- a/core/src/main/java/ch/cyberduck/core/Credentials.java +++ b/core/src/main/java/ch/cyberduck/core/Credentials.java @@ -39,6 +39,7 @@ public class Credentials implements Comparable { */ private String password = StringUtils.EMPTY; private String token = StringUtils.EMPTY; + private STSTokens tokens = STSTokens.EMPTY; private OAuthTokens oauth = OAuthTokens.EMPTY; /** @@ -73,6 +74,7 @@ public Credentials(final Credentials copy) { this.user = copy.user; this.password = copy.password; this.token = copy.token; + this.tokens = copy.tokens; this.oauth = copy.oauth; this.identity = copy.identity; this.identityPassphrase = copy.identityPassphrase; @@ -156,6 +158,21 @@ public Credentials withToken(final String token) { return this; } + public STSTokens getTokens() { + return tokens; + } + + public void setTokens(final STSTokens tokens) { + this.tokens = tokens; + this.passed = false; + } + + public Credentials withTokens(final STSTokens tokens) { + this.tokens = tokens; + this.passed = false; + return this; + } + public OAuthTokens getOauth() { return oauth; } @@ -300,6 +317,7 @@ public boolean validate(final Protocol protocol, final LoginOptions options) { public void reset() { this.setPassword(StringUtils.EMPTY); this.setToken(StringUtils.EMPTY); + this.setTokens(STSTokens.EMPTY); this.setOauth(OAuthTokens.EMPTY); this.setIdentityPassphrase(StringUtils.EMPTY); } @@ -329,14 +347,15 @@ public boolean equals(final Object o) { final Credentials that = (Credentials) o; return Objects.equals(user, that.user) && Objects.equals(password, that.password) && - Objects.equals(token, that.token) && + Objects.equals(token, that.token) && + Objects.equals(tokens, that.tokens) && Objects.equals(identity, that.identity) && Objects.equals(certificate, that.certificate); } @Override public int hashCode() { - return Objects.hash(user, password, token, identity, certificate); + return Objects.hash(user, password, tokens, identity, certificate); } @Override @@ -344,8 +363,9 @@ public String toString() { final StringBuilder sb = new StringBuilder("Credentials{"); sb.append("user='").append(user).append('\''); sb.append(", password='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(password)))).append('\''); - sb.append(", oauth='").append(oauth).append('\''); sb.append(", token='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(token)))).append('\''); + sb.append(", tokens='").append(tokens).append('\''); + sb.append(", oauth='").append(oauth).append('\''); sb.append(", identity=").append(identity); sb.append('}'); return sb.toString(); diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java b/core/src/main/java/ch/cyberduck/core/STSTokens.java similarity index 93% rename from s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java rename to core/src/main/java/ch/cyberduck/core/STSTokens.java index fbf4b41f0e8..8245c238cc0 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSTokens.java +++ b/core/src/main/java/ch/cyberduck/core/STSTokens.java @@ -1,4 +1,4 @@ -package ch.cyberduck.core.sts; +package ch.cyberduck.core; /* * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. @@ -17,6 +17,9 @@ import org.apache.commons.lang3.StringUtils; +/** + * Temporary access credentials + */ public final class STSTokens { public static final STSTokens EMPTY @@ -27,6 +30,10 @@ public final class STSTokens { private final String sessionToken; private final Long expiryInMilliseconds; + public STSTokens(final String sessionToken) { + this(null, null, sessionToken, -1L); + } + public STSTokens(final String accessKeyId, final String secretAccessKey, final String sessionToken, final Long expiryInMilliseconds) { this.accessKeyId = accessKeyId; this.secretAccessKey = secretAccessKey; diff --git a/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java b/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java index 3b155d244ae..7b86aaac3f7 100644 --- a/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java +++ b/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java @@ -27,7 +27,10 @@ import ch.cyberduck.core.Path; import ch.cyberduck.core.PathNormalizer; import ch.cyberduck.core.ProtocolFactory; +import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.TranscriptListener; +import ch.cyberduck.core.date.ISO8601DateParser; +import ch.cyberduck.core.date.InvalidDateException; import ch.cyberduck.core.dav.DAVReadFeature; import ch.cyberduck.core.dav.DAVSession; import ch.cyberduck.core.exception.BackgroundException; @@ -38,7 +41,6 @@ import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.transfer.TransferStatus; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -46,6 +48,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.Date; import java.util.EnumSet; import com.google.gson.stream.JsonReader; @@ -73,7 +76,6 @@ public AWSSessionCredentialsRetriever(final X509TrustManager trust, final X509Ke } public static class Configurator implements CredentialsConfigurator { - private final AWSSessionCredentialsRetriever retriever; private final String url; @@ -126,6 +128,7 @@ protected Credentials parse(final InputStream in) throws BackgroundException { String key = null; String secret = null; String token = null; + Date expiration = null; while(reader.hasNext()) { final String name = reader.nextName(); final String value = reader.nextString(); @@ -139,14 +142,18 @@ protected Credentials parse(final InputStream in) throws BackgroundException { case "Token": token = value; break; + case "Expiration": + try { + expiration = new ISO8601DateParser().parse(value); + } + catch(InvalidDateException e) { + log.warn(String.format("Failure %s parsing %s", e, value)); + } + break; } } reader.endObject(); - final Credentials credentials = new Credentials(key, secret); - if(StringUtils.isNotBlank(token)) { - credentials.setToken(token); - } - return credentials; + return new Credentials().withTokens(new STSTokens(key, secret, token, expiration != null ? expiration.getTime() : -1L)); } catch(MalformedJsonException e) { throw new InteroperabilityException("Invalid JSON response", e); diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java b/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java index dafd5d20452..8ff754af9fb 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java @@ -30,6 +30,7 @@ import ch.cyberduck.core.ssl.ThreadLocalHostnameDelegatingTrustManager; import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; +import ch.cyberduck.core.STSTokens; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -213,9 +214,11 @@ else if(!profiles.containsKey(basicProfile.getRoleSourceProfile())) { if(log.isDebugEnabled()) { log.debug(String.format("Set credentials from %s", assumeRoleResult)); } - credentials.setUsername(assumeRoleResult.getCredentials().getAccessKeyId()); - credentials.setPassword(assumeRoleResult.getCredentials().getSecretAccessKey()); - credentials.setToken(assumeRoleResult.getCredentials().getSessionToken()); + credentials.setTokens(new STSTokens( + assumeRoleResult.getCredentials().getAccessKeyId(), + assumeRoleResult.getCredentials().getSecretAccessKey(), + assumeRoleResult.getCredentials().getSessionToken(), + assumeRoleResult.getCredentials().getExpiration().getTime())); } catch(AWSSecurityTokenServiceException e) { log.warn(e.getErrorMessage(), e); @@ -234,10 +237,8 @@ else if(!profiles.containsKey(basicProfile.getRoleSourceProfile())) { if(null == cached) { return credentials; } - return credentials - .withUsername(cached.accessKey) - .withPassword(cached.secretKey) - .withToken(cached.sessionToken); + return credentials.withTokens(new STSTokens( + cached.accessKey, cached.secretKey, cached.sessionToken, Long.valueOf(cached.expiration))); } if(tokenCode != null) { // Obtain session token @@ -264,10 +265,11 @@ else if(!profiles.containsKey(basicProfile.getRoleSourceProfile())) { if(log.isDebugEnabled()) { log.debug(String.format("Set credentials from %s", sessionTokenResult)); } - return credentials - .withUsername(sessionTokenResult.getCredentials().getAccessKeyId()) - .withPassword(sessionTokenResult.getCredentials().getSecretAccessKey()) - .withToken(sessionTokenResult.getCredentials().getSessionToken()); + return credentials.withTokens(new STSTokens( + sessionTokenResult.getCredentials().getAccessKeyId(), + sessionTokenResult.getCredentials().getSecretAccessKey(), + sessionTokenResult.getCredentials().getSessionToken(), + sessionTokenResult.getCredentials().getExpiration().getTime())); } catch(AWSSecurityTokenServiceException e) { log.warn(e.getErrorMessage(), e); @@ -277,10 +279,11 @@ else if(!profiles.containsKey(basicProfile.getRoleSourceProfile())) { if(log.isDebugEnabled()) { log.debug(String.format("Set credentials from profile %s", basicProfile.getProfileName())); } - return credentials - .withUsername(basicProfile.getAwsAccessIdKey()) - .withPassword(basicProfile.getAwsSecretAccessKey()) - .withToken(basicProfile.getAwsSessionToken()); + return credentials.withTokens(new STSTokens( + basicProfile.getAwsAccessIdKey(), + basicProfile.getAwsSecretAccessKey(), + basicProfile.getAwsSessionToken(), + -1L)); } } return credentials; diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index f376981698a..2686b083745 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -29,6 +29,7 @@ import ch.cyberduck.core.Path; import ch.cyberduck.core.PathAttributes; import ch.cyberduck.core.PathContainerService; +import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.Scheme; import ch.cyberduck.core.UrlProvider; import ch.cyberduck.core.auth.AWSSessionCredentialsRetriever; @@ -67,12 +68,10 @@ import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.sts.STSAssumeRoleCredentialsRequestInterceptor; import ch.cyberduck.core.sts.STSAssumeRoleTokenExpiredResponseInterceptor; -import ch.cyberduck.core.sts.STSTokens; import ch.cyberduck.core.threading.BackgroundExceptionCallable; import ch.cyberduck.core.threading.CancelCallback; import ch.cyberduck.core.transfer.TransferStatus; -import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; @@ -336,12 +335,13 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal client.setProviderCredentials(null); } else { - if(StringUtils.isNotBlank(credentials.getToken())) { + if(credentials.getTokens().validate()) { if(log.isDebugEnabled()) { log.debug(String.format("Connect with session credentials to %s", host)); } client.setProviderCredentials(new AWSSessionCredentials( - credentials.getUsername(), credentials.getPassword(), credentials.getToken())); + credentials.getTokens().getAccessKeyId(), credentials.getTokens().getSecretAccessKey(), + credentials.getTokens().getSessionToken())); } else { if(log.isDebugEnabled()) { diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java index c47ab43b247..80458b18311 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java @@ -23,12 +23,9 @@ import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; -import org.apache.http.entity.BufferedHttpEntity; import org.apache.http.protocol.HttpContext; -import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jets3t.service.S3ServiceException; import org.jets3t.service.security.AWSCredentials; import org.jets3t.service.security.AWSSessionCredentials; @@ -61,9 +58,10 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun if(log.isDebugEnabled()) { log.debug(String.format("Reconfigure client with credentials %s", credentials)); } - if(credentials.isTokenAuthentication()) { + if(credentials.getTokens().validate()) { session.getClient().setProviderCredentials(new AWSSessionCredentials( - credentials.getUsername(), credentials.getPassword(), credentials.getToken())); + credentials.getTokens().getAccessKeyId(), credentials.getTokens().getSecretAccessKey(), + credentials.getTokens().getSessionToken())); } else { session.getClient().setProviderCredentials(new AWSCredentials( diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 8ccf2b2b14a..d614dd3375a 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -22,6 +22,7 @@ import ch.cyberduck.core.LoginCallback; import ch.cyberduck.core.LoginOptions; import ch.cyberduck.core.OAuthTokens; +import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.aws.CustomClientConfiguration; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LoginFailureException; @@ -90,9 +91,7 @@ public STSTokens authorize(final Host bookmark, final String sAMLAssertion) thro result.getCredentials().getSecretAccessKey(), result.getCredentials().getSessionToken(), result.getCredentials().getExpiration().getTime()); - credentials.setUsername(tokens.getAccessKeyId()); - credentials.setPassword(tokens.getSecretAccessKey()); - credentials.setToken(tokens.getSessionToken()); + credentials.setTokens(tokens); return tokens; } catch(AWSSecurityTokenServiceException e) { @@ -162,14 +161,10 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws log.debug(String.format("Received assume role identity result %s", result)); } final Credentials credentials = bookmark.getCredentials(); - final STSTokens tokens = new STSTokens(result.getCredentials().getAccessKeyId(), + return credentials.withTokens(new STSTokens(result.getCredentials().getAccessKeyId(), result.getCredentials().getSecretAccessKey(), result.getCredentials().getSessionToken(), - result.getCredentials().getExpiration().getTime()); - credentials.setUsername(tokens.getAccessKeyId()); - credentials.setPassword(tokens.getSecretAccessKey()); - credentials.setToken(tokens.getSessionToken()); - return tokens; + result.getCredentials().getExpiration().getTime())).getTokens(); } catch(AWSSecurityTokenServiceException e) { throw new STSExceptionMappingService().map(e); diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java index 9b2b0f28baf..48051c8d79d 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -21,7 +21,9 @@ import ch.cyberduck.core.LoginOptions; import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.PasswordStoreFactory; +import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.ExpiredTokenException; import ch.cyberduck.core.exception.LocalAccessDeniedException; import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; import ch.cyberduck.core.s3.S3Session; @@ -74,9 +76,7 @@ public STSTokens refresh(final OAuthTokens oauth) throws BackgroundException { */ public STSTokens save(final STSTokens tokens) throws LocalAccessDeniedException { host.getCredentials() - .withUsername(tokens.getAccessKeyId()) - .withPassword(tokens.getSecretAccessKey()) - .withToken(tokens.getSessionToken()) + .withTokens(tokens) .withSaved(new LoginOptions().keychain); if(log.isDebugEnabled()) { log.debug(String.format("Save new tokens %s for %s", tokens, host)); diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java index bb8d37684ad..48157876d80 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java @@ -17,6 +17,7 @@ import ch.cyberduck.core.LoginCallback; import ch.cyberduck.core.OAuthTokens; +import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ExpiredTokenException; import ch.cyberduck.core.exception.LoginFailureException; diff --git a/s3/src/test/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetrieverTest.java b/s3/src/test/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetrieverTest.java index ff9b617d1e1..53b266929dd 100644 --- a/s3/src/test/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetrieverTest.java +++ b/s3/src/test/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetrieverTest.java @@ -47,9 +47,10 @@ public void testParse() throws Exception { " \"Token\" : \"token\",\n" + " \"Expiration\" : \"2012-04-27T22:39:16Z\"\n" + "}", Charset.defaultCharset())); - assertEquals("AKIAIOSFODNN7EXAMPLE", c.getUsername()); - assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", c.getPassword()); - assertEquals("token", c.getToken()); + assertEquals("AKIAIOSFODNN7EXAMPLE", c.getTokens().getAccessKeyId()); + assertEquals("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", c.getTokens().getSecretAccessKey()); + assertEquals("token", c.getTokens().getSessionToken()); + assertEquals(1335566356000L, c.getTokens().getExpiryInMilliseconds(), 0L); } @Test(expected = ConnectionTimeoutException.class) diff --git a/s3/src/test/java/ch/cyberduck/core/s3/S3CredentialsConfiguratorTest.java b/s3/src/test/java/ch/cyberduck/core/s3/S3CredentialsConfiguratorTest.java index e7c022eb055..75e1b5ba60a 100644 --- a/s3/src/test/java/ch/cyberduck/core/s3/S3CredentialsConfiguratorTest.java +++ b/s3/src/test/java/ch/cyberduck/core/s3/S3CredentialsConfiguratorTest.java @@ -52,8 +52,8 @@ public void readSuccessForValidAWSCredentialsProfileEntry() throws Exception { final Credentials verify = new S3CredentialsConfigurator(LocalFactory.get(new File("src/test/resources/valid/.aws").getAbsolutePath()) , new DisabledX509TrustManager(), new DefaultX509KeyManager(), new DisabledPasswordCallback()) .reload().configure(new Host(new TestProtocol(), StringUtils.EMPTY, new Credentials("test_s3_profile"))); - assertEquals("EXAMPLEKEYID", verify.getUsername()); - assertEquals("EXAMPLESECRETKEY", verify.getPassword()); - assertEquals("EXAMPLETOKEN", verify.getToken()); + assertEquals("EXAMPLEKEYID", verify.getTokens().getAccessKeyId()); + assertEquals("EXAMPLESECRETKEY", verify.getTokens().getSecretAccessKey()); + assertEquals("EXAMPLETOKEN", verify.getTokens().getSessionToken()); } } diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index 7d9b6fd408f..b1e551cde96 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -57,7 +57,9 @@ public void testSuccessfulLoginViaOidc() throws BackgroundException { assertNotEquals(StringUtils.EMPTY, creds.getUsername()); assertNotEquals(StringUtils.EMPTY, creds.getPassword()); - assertFalse(creds.getToken().isEmpty()); + assertNotNull(creds.getTokens().getAccessKeyId()); + assertNotNull(creds.getTokens().getSecretAccessKey()); + assertNotNull(creds.getTokens().getSessionToken()); assertNotNull(creds.getOauth().getIdToken()); assertNotNull(creds.getOauth().getRefreshToken()); assertNotEquals(Optional.of(Long.MAX_VALUE).get(), creds.getOauth().getExpiryInMilliseconds()); From 57ea0770aeceaa4da8906392698f3f4a6f1531a5 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sat, 12 Aug 2023 18:04:36 +0200 Subject: [PATCH 62/84] Simplify retry handling. Refresh OAuth token when STS request fails because of invalid web identity token. --- ...sumeRoleCredentialsRequestInterceptor.java | 14 +++++-- ...meRoleTokenExpiredResponseInterceptor.java | 42 ++++++------------- 2 files changed, 23 insertions(+), 33 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java index 48051c8d79d..913ee2e5cf8 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -62,11 +62,19 @@ public STSAssumeRoleCredentialsRequestInterceptor(final OAuth2RequestInterceptor } public STSTokens refresh() throws BackgroundException { - return this.refresh(oauth.refresh()); + return this.tokens = this.authorize(host, oauth.refresh()); } - public STSTokens refresh(final OAuthTokens oauth) throws BackgroundException { - return this.tokens = this.authorize(host, oauth); + public STSTokens refresh(final OAuthTokens oauthTokens) throws BackgroundException { + try { + return this.tokens = this.authorize(host, oauthTokens); + } + catch(ExpiredTokenException e) { + if(log.isWarnEnabled()) { + log.warn(String.format("Failure %s authorizing. Retry with refreshed OAuth tokens", e)); + } + return this.tokens = this.authorize(host, oauth.refresh()); + } } /** diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java index 48157876d80..505e18c0fac 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java @@ -19,7 +19,6 @@ import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.ExpiredTokenException; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.oauth.OAuth2ErrorResponseInterceptor; import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; @@ -28,9 +27,7 @@ import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; -import org.apache.http.entity.BufferedHttpEntity; import org.apache.http.protocol.HttpContext; -import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jets3t.service.security.AWSSessionCredentials; @@ -59,7 +56,6 @@ public STSAssumeRoleTokenExpiredResponseInterceptor(final S3Session session, @Override public boolean retryRequest(final HttpResponse response, final int executionCount, final HttpContext context) { switch(response.getStatusLine().getStatusCode()) { - case HttpStatus.SC_UNAUTHORIZED: case HttpStatus.SC_FORBIDDEN: case HttpStatus.SC_BAD_REQUEST: if(executionCount > MAX_RETRIES) { @@ -69,23 +65,24 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun try { final BackgroundException type = new S3ExceptionMappingService().map(response); final OAuthTokens oAuthTokens; - if(type instanceof ExpiredTokenException) { + if(type instanceof LoginFailureException) { + // 403 Forbidden (InvalidAccessKeyId) The provided token has expired // 400 Bad Request (ExpiredToken) The provided token has expired // 400 Bad Request (InvalidToken) The provided token is malformed or otherwise not valid // 400 Bad Request (TokenRefreshRequired) The provided token must be refreshed. - // No refresh of OAuth tokens - oAuthTokens = oauth.getTokens(); - } - else if(type instanceof LoginFailureException) { - // 401 (InvalidAccessKeyId) The AWS access key ID that you provided does not exist in our records. - // 401 (InvalidSecurity) The provided security credentials are not valid. - // 403 (Forbidden) Access Denied + if(log.isWarnEnabled()) { + log.warn(String.format("Handle failure %s", response)); + } try { - // Refresh OAuth tokens - oAuthTokens = super.refresh(response); + log.warn(String.format("Attempt to refresh STS token for failure %s", response)); + final STSTokens stsTokens = sts.refresh(oauth.getTokens()); + session.getClient().setProviderCredentials(new AWSSessionCredentials(stsTokens.getAccessKeyId(), + stsTokens.getSecretAccessKey(), stsTokens.getSessionToken())); + // Try again + return true; } catch(BackgroundException e) { - log.warn(String.format("Failure %s refreshing OAuth tokens", e)); + log.warn(String.format("Failure %s refreshing STS token", e)); return false; } } @@ -96,21 +93,6 @@ else if(type instanceof LoginFailureException) { } return false; } - if(log.isWarnEnabled()) { - log.warn(String.format("Handle failure %s", response)); - } - try { - log.warn(String.format("Attempt to refresh STS token for failure %s", response)); - final STSTokens stsTokens = sts.refresh(oAuthTokens); - session.getClient().setProviderCredentials(new AWSSessionCredentials(stsTokens.getAccessKeyId(), - stsTokens.getSecretAccessKey(), stsTokens.getSessionToken())); - // Try again - return true; - } - catch(BackgroundException e) { - log.warn(String.format("Failure %s refreshing STS token", e)); - return false; - } } catch(IOException e) { log.warn(String.format("Failure parsing response entity from %s", response)); From 293741e240452b6ab56142daa6f410f7f1a553f3 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sat, 12 Aug 2023 18:11:56 +0200 Subject: [PATCH 63/84] Set subject from JWT token as username to make sure OAuth tokens can be identified in keychain. --- .../core/sts/STSAssumeRoleAuthorizationService.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index d614dd3375a..062aa648130 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -161,10 +161,12 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws log.debug(String.format("Received assume role identity result %s", result)); } final Credentials credentials = bookmark.getCredentials(); - return credentials.withTokens(new STSTokens(result.getCredentials().getAccessKeyId(), - result.getCredentials().getSecretAccessKey(), - result.getCredentials().getSessionToken(), - result.getCredentials().getExpiration().getTime())).getTokens(); + return credentials + .withUsername(sub) + .withTokens(new STSTokens(result.getCredentials().getAccessKeyId(), + result.getCredentials().getSecretAccessKey(), + result.getCredentials().getSessionToken(), + result.getCredentials().getExpiration().getTime())).getTokens(); } catch(AWSSecurityTokenServiceException e) { throw new STSExceptionMappingService().map(e); From 11b79b04e24ffc0f9d076ddfb4ee8e78df02833f Mon Sep 17 00:00:00 2001 From: chenkins Date: Sat, 12 Aug 2023 12:12:12 +0200 Subject: [PATCH 64/84] Inject OAuth TTL into TestContainer tests. --- ...AbstractAssumeRoleWithWebIdentityTest.java | 48 ++----- ...RoleWithWebIdentityAuthenticationTest.java | 57 +-------- ...eTokenExpiryFailsBecauseOfLatencyTest.java | 84 +++++++++++++ ...CredentialsExpiredValidOAuthTokenTest.java | 103 +++++++++++++++ .../core/sts/STSOAuthExpiredValidSTSTest.java | 73 +++++++++++ .../ch/cyberduck/core/sts/STSTestSetup.java | 117 ++++++++++++++++++ .../testcontainer/docker-compose.yml | 11 +- .../testcontainer/keycloak/Dockerfile | 1 - 8 files changed, 395 insertions(+), 99 deletions(-) create mode 100644 s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java create mode 100644 s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java create mode 100644 s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java create mode 100644 s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java delete mode 100644 s3/src/test/resources/testcontainer/keycloak/Dockerfile diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java index 5720405fe32..2312055aab4 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java @@ -17,65 +17,39 @@ import ch.cyberduck.core.Profile; -import ch.cyberduck.core.ProtocolFactory; -import ch.cyberduck.core.exception.AccessDeniedException; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.s3.S3Protocol; -import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.test.TestcontainerTest; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.junit.AfterClass; import org.junit.Before; -import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.experimental.categories.Category; import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.Network; -import org.testcontainers.containers.wait.strategy.Wait; -import java.io.File; -import java.util.Collections; -import java.util.HashSet; +import static ch.cyberduck.core.sts.STSTestSetup.*; @Category(TestcontainerTest.class) public abstract class AbstractAssumeRoleWithWebIdentityTest { + protected static final int MILLIS = 1000; + + // lag to wait after token expiry + protected static final int LAG = 10 * MILLIS; protected static final Logger log = LogManager.getLogger(AbstractAssumeRoleWithWebIdentityTest.class); + protected static int OAUTH_TTL_SECS = 30; + protected static Profile profile = null; - private static Network network; - private static final DockerComposeContainer compose; - static { - compose = new DockerComposeContainer<>( - new File(AbstractAssumeRoleWithWebIdentityTest.class.getResource("/testcontainer/docker-compose.yml").getFile())) - .withPull(false) - .withLocalCompose(true) - .withOptions("--compatibility") - .withExposedService("keycloak_1", 8080, Wait.forListeningPort()) - .withExposedService("minio_1", 9000, Wait.forListeningPort()); - } - @BeforeClass - public static void beforeAll() { - compose.start(); - } + @ClassRule + public static DockerComposeContainer compose = prepareDockerComposeContainer(getKeyCloakFile()); + @Before public void setup() throws BackgroundException { profile = readProfile(); } - private Profile readProfile() throws AccessDeniedException { - final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); - return new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/S3 (OIDC).cyberduckprofile")); - } - @AfterClass - public static void disconnect() { - if(compose == null && network != null) { - network.close(); - } - } } \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index b1e551cde96..6791958feb4 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -18,7 +18,6 @@ import ch.cyberduck.core.Credentials; import ch.cyberduck.core.DisabledCancelCallback; import ch.cyberduck.core.DisabledHostKeyCallback; -import ch.cyberduck.core.DisabledListProgressListener; import ch.cyberduck.core.DisabledLoginCallback; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostUrlProvider; @@ -28,12 +27,10 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.s3.S3AccessControlListFeature; import ch.cyberduck.core.s3.S3FindFeature; -import ch.cyberduck.core.s3.S3ObjectListService; import ch.cyberduck.core.s3.S3Session; import ch.cyberduck.test.TestcontainerTest; import org.jets3t.service.security.AWSSessionCredentials; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.testcontainers.shaded.org.apache.commons.lang3.StringUtils; @@ -97,7 +94,8 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - Thread.sleep(1100 * 30); + + Thread.sleep(MILLIS * OAUTH_TTL_SECS + LAG); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); assertNotEquals(firstAccessToken, host.getCredentials().getOauth().getIdToken()); @@ -106,56 +104,5 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException session.close(); } - /** - * only use with the below specified changes in the keycloak config json file and run as separate test - * set config keycloak-realm.json: - * "access.token.lifespan": "930" - * "ssoSessionMaxLifespan": 1100, - */ - @Test - @Ignore - public void testSTSCredentialsExpiredOAuthToken() throws BackgroundException, InterruptedException { - final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); - final S3Session session = new S3Session(host); - session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); - - String firstAccessToken = host.getCredentials().getOauth().getAccessToken(); - String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); - assertTrue(session.getClient().getProviderCredentials() instanceof AWSSessionCredentials); - - String firstSessionToken = ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken(); - Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - Thread.sleep(1000 * 910); - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); - assertNotEquals(firstSessionToken, ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken()); - assertEquals(firstAccessToken, host.getCredentials().getOauth().getAccessToken()); - } - - /** - * This test fails if the x-minio Headers are not read because of InvalidAccessKeyId error code which has no response body. - * Adjust the sleep time according to the network latency - */ - @Test - @Ignore - public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws BackgroundException, InterruptedException { - final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); - final S3Session session = new S3Session(host); - session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); - - String firstAccessToken = host.getCredentials().getOauth().getIdToken(); - String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); - - // Time of latency may vary and so the time needs to be adjusted accordingly - Thread.sleep(28820); - Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - assertNotEquals(firstAccessToken, host.getCredentials().getOauth().getIdToken()); - assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); - session.close(); - } } \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java b/s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java new file mode 100644 index 00000000000..90ca49ae9d7 --- /dev/null +++ b/s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java @@ -0,0 +1,84 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.DisabledCancelCallback; +import ch.cyberduck.core.DisabledHostKeyCallback; +import ch.cyberduck.core.DisabledLoginCallback; +import ch.cyberduck.core.Host; +import ch.cyberduck.core.HostUrlProvider; +import ch.cyberduck.core.Path; +import ch.cyberduck.core.Profile; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.proxy.DisabledProxyFinder; +import ch.cyberduck.core.s3.S3AccessControlListFeature; +import ch.cyberduck.core.s3.S3FindFeature; +import ch.cyberduck.core.s3.S3Session; +import ch.cyberduck.test.TestcontainerTest; + +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.EnumSet; + +import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.MILLIS; +import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.OAUTH_TTL_SECS; +import static ch.cyberduck.core.sts.STSTestSetup.readProfile; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + + +@Category(TestcontainerTest.class) +public class STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest { + + + /** + * If an HTTP HEAD request to MinIO fails, the error code and error description are written to MinIO-specific error headers, since HTTP HEAD requests do not use a body. + * For example, a HEAD request is used to check whether an S3 bucket or folder is discoverable. + * When a HEAD request uses expired STS credentials. Because of preemtively refresh tokens this case is only possible it the credentials are still valid but a few milliseconds before expire. Because of the latency in the network the request will be invalid when reaching the MinIO Service. + * But the sleep time needs to be ajusted according to the network latency. + * Adjust the sleep time according to the network latency. + * Overall the test may be removed and the general question is how to handle the MinIO-specific HTTP-Headers when a HEAD-Request is failing. + * This test fails if the x-minio Headers are not read because of InvalidAccessKeyId error code which has no response body. + */ + @Test + @Ignore("Time of network latency may vary and so the time needs to be adjusted manually") // TODO should we remove this test or keep for documentation? + public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws BackgroundException, InterruptedException { + final Profile profile = readProfile(); + + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); + final S3Session session = new S3Session(host); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); + + String firstAccessToken = host.getCredentials().getOauth().getIdToken(); + String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); + + // Time of latency may vary and so the time needs to be adjusted accordingly + final int NETWORK_LATENCY = 1180; + final int wait = OAUTH_TTL_SECS * MILLIS - NETWORK_LATENCY; + Thread.sleep(wait); + + Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + + assertNotEquals(firstAccessToken, host.getCredentials().getOauth().getIdToken()); + assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); + session.close(); + } +} \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java b/s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java new file mode 100644 index 00000000000..7b03a241caa --- /dev/null +++ b/s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java @@ -0,0 +1,103 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.DisabledCancelCallback; +import ch.cyberduck.core.DisabledHostKeyCallback; +import ch.cyberduck.core.DisabledLoginCallback; +import ch.cyberduck.core.Host; +import ch.cyberduck.core.Path; +import ch.cyberduck.core.Profile; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.preferences.HostPreferences; +import ch.cyberduck.core.proxy.Proxy; +import ch.cyberduck.core.s3.S3AccessControlListFeature; +import ch.cyberduck.core.s3.S3FindFeature; +import ch.cyberduck.core.s3.S3Session; +import ch.cyberduck.test.TestcontainerTest; + +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.testcontainers.containers.DockerComposeContainer; + +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; + +import com.amazonaws.auth.AWSSessionCredentials; + +import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.LAG; +import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.MILLIS; +import static ch.cyberduck.core.sts.STSTestSetup.*; +import static org.junit.Assert.*; + +@Category(TestcontainerTest.class) +public class STSCredentialsExpiredValidOAuthTokenTest { + + + /** + * Adjust OAuth token TTL in Keycloak: + * "access.token.lifespan": "930" + * "ssoSessionMaxLifespan": 1100, + */ + private static Map overrideKeycloakDefaults() { + Map m = new HashMap<>(); + m.put("access.token.lifespan", Integer.toString(930)); + m.put("ssoSessionMaxLifespan", Integer.toString(1100)); + return m; + } + + @ClassRule + public static DockerComposeContainer compose = prepareDockerComposeContainer( + getKeyCloakFile(overrideKeycloakDefaults()) + ); + + + @Test + @Ignore("Takes 15 minutes, skip by default") // TODO does this test run through? + public void testSTSCredentialsExpiredValidOAuthToken() throws BackgroundException, InterruptedException { + final Profile profile = readProfile(); + // 900 secs = 15 min is mininmum value: https://min.io/docs/minio/linux/developers/security-token-service/AssumeRoleWithWebIdentity.html + final int assumeRoleDurationSeconds = 900; + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); + host.setProperty("s3.assumerole.durationseconds", Integer.toString(assumeRoleDurationSeconds)); + + assertEquals(new HostPreferences(host).getInteger("s3.assumerole.durationseconds"), assumeRoleDurationSeconds); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + + String firstAccessToken = host.getCredentials().getOauth().getAccessToken(); + String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); + assertTrue(session.getClient().getProviderCredentials() instanceof AWSSessionCredentials); + + String firstSessionToken = ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken(); + Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + + Thread.sleep(MILLIS * assumeRoleDurationSeconds + LAG); + + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); + assertNotEquals(firstSessionToken, ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken()); + assertEquals(firstAccessToken, host.getCredentials().getOauth().getAccessToken()); + } + + +} \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java b/s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java new file mode 100644 index 00000000000..21f3069c50f --- /dev/null +++ b/s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java @@ -0,0 +1,73 @@ +package ch.cyberduck.core.sts; + +/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.DisabledCancelCallback; +import ch.cyberduck.core.DisabledHostKeyCallback; +import ch.cyberduck.core.DisabledLoginCallback; +import ch.cyberduck.core.Host; +import ch.cyberduck.core.Profile; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.proxy.Proxy; +import ch.cyberduck.core.s3.S3Session; +import ch.cyberduck.test.TestcontainerTest; + +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.testcontainers.containers.DockerComposeContainer; + +import java.util.HashMap; +import java.util.Map; + +import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.LAG; +import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.MILLIS; +import static ch.cyberduck.core.sts.STSTestSetup.*; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@Category(TestcontainerTest.class) +public class STSOAuthExpiredValidSTSTest { + + private static final int OAUTH_TTL_SECS = 5; + + private static Map overrideKeycloakDefaults() { + Map m = new HashMap<>(); + m.put("access.token.lifespan", Integer.toString(OAUTH_TTL_SECS)); + return m; + } + + @ClassRule + public static DockerComposeContainer compose = prepareDockerComposeContainer(getKeyCloakFile(overrideKeycloakDefaults())); + + @Test + public void testOAuthExpiry() throws BackgroundException, InterruptedException { + // TODO STS refresh in this case of early OAuth expiry... + final Profile profile = readProfile(); + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); + final S3Session session = new S3Session(host); + session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + + assertFalse(host.getCredentials().getOauth().isExpired()); + Thread.sleep(MILLIS * OAUTH_TTL_SECS + LAG); + assertTrue(host.getCredentials().getOauth().isExpired()); + session.close(); + } + + +} \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java b/s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java new file mode 100644 index 00000000000..3c029a52b39 --- /dev/null +++ b/s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java @@ -0,0 +1,117 @@ +package ch.cyberduck.core.sts;/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Profile; +import ch.cyberduck.core.ProtocolFactory; +import ch.cyberduck.core.exception.AccessDeniedException; +import ch.cyberduck.core.s3.S3Protocol; +import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +public class STSTestSetup { + private static final Logger log = LogManager.getLogger(STSTestSetup.class); + + public static DockerComposeContainer prepareDockerComposeContainer(String tempFile) { + log.info("Preparing docker compose container..."); + System.out.println(tempFile); + return new DockerComposeContainer<>( + new File(STSTestSetup.class.getResource("/testcontainer/docker-compose.yml").getFile())) + .withEnv("KEYCLOAK_REALM_JSON", tempFile) + .withPull(false) + .withLocalCompose(true) + .withOptions("--compatibility") + .withExposedService("keycloak_1", 8080, Wait.forListeningPort()) + .withExposedService("minio_1", 9000, Wait.forListeningPort()); + } + + public static String getKeyCloakFile(Map replacements) { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + JsonElement je = new Gson().fromJson(new InputStreamReader(STSTestSetup.class.getResourceAsStream("/testcontainer/keycloak/keycloak-realm.json")), JsonElement.class); + JsonObject jo = je.getAsJsonObject(); + + for(Map.Entry replacement : replacements.entrySet()) { + updateJsonValues(jo,replacement.getKey(), replacement.getValue()); + } + + String content = gson.toJson(jo); + try { + final Path tempFile = Files.createTempFile(null, null); + Files.write(tempFile, content.getBytes(StandardCharsets.UTF_8)); + return tempFile.toAbsolutePath().toString(); + + } + catch(IOException e) { + throw new RuntimeException(e); + } + } + + private static void updateJsonValues(JsonObject jsonObj, String key, String newVal) { + for (Map.Entry entry : jsonObj.entrySet()) { + JsonElement element = entry.getValue(); + if (element.isJsonArray()) { + updateJsonValues(element.getAsJsonArray(), key, newVal); + } else if (element.isJsonObject()) { + updateJsonValues(element.getAsJsonObject(), key,newVal); + } else if (entry.getKey().equals(key)) { + jsonObj.remove(key); + jsonObj.addProperty(key, newVal); + break; + } + } + } + + private static void updateJsonValues(JsonArray asJsonArray, String key, String newVal) { + for (int index = 0; index < asJsonArray.size(); index++) { + JsonElement element = asJsonArray.get(index); + if (element.isJsonArray()) { + updateJsonValues(element.getAsJsonArray(), key, newVal); + } else if (element.isJsonObject()) { + updateJsonValues(element.getAsJsonObject(), key,newVal); + } + + } + } + + public static String getKeyCloakFile() { + return getKeyCloakFile(Collections.emptyMap()); + } + + public static Profile readProfile() throws AccessDeniedException { + final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); + return new ProfilePlistReader(factory).read( + STSTestSetup.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); + } + +} diff --git a/s3/src/test/resources/testcontainer/docker-compose.yml b/s3/src/test/resources/testcontainer/docker-compose.yml index 536ead01410..d31002e7c76 100644 --- a/s3/src/test/resources/testcontainer/docker-compose.yml +++ b/s3/src/test/resources/testcontainer/docker-compose.yml @@ -4,32 +4,31 @@ services: keycloak: hostname: keycloak - build: keycloak + image: quay.io/keycloak/keycloak:21.1.1 ports: - "8080:8080" environment: KEYCLOAK_ADMIN: admin KEYCLOAK_ADMIN_PASSWORD: admin - PROXY_ADDRESS_FORWARDING: "true" KEYCLOAK_LOGLEVEL: DEBUG DB_VENDOR: h2 KC_HEALTH_ENABLED: "true" KC_METRICS_ENABLED: "true" + volumes: - - ./keycloak/keycloak-realm.json:/opt/keycloak/data/import/keycloak-realm.json + - ${KEYCLOAK_REALM_JSON:-./keycloak/keycloak-realm.json}:/opt/keycloak/data/import/keycloak-realm.json command: start-dev --import-realm --db=dev-mem --health-enabled=true networks: - testContainerNetwork - healthcheck: hostname: healthcheck image: busybox depends_on: keycloak: condition: service_started - command: sh -c "until wget -q -O- http://keycloak:8080/realms/cyberduckrealm/.well-known/openid-configuration >/dev/null 2>&1; do sleep 1; done" + command: sh -c "until wget -q -O- http://keycloak:8080/realms/cyberduckrealm/.well-known/openid-configuration >/dev/null 2>&1; do sleep 1; done" healthcheck: - test: ["CMD-SHELL", "exit 0"] + test: [ "CMD-SHELL", "exit 0" ] interval: 1s timeout: 1s retries: 5 diff --git a/s3/src/test/resources/testcontainer/keycloak/Dockerfile b/s3/src/test/resources/testcontainer/keycloak/Dockerfile deleted file mode 100644 index d5cbb0ca507..00000000000 --- a/s3/src/test/resources/testcontainer/keycloak/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM quay.io/keycloak/keycloak:21.1.1 \ No newline at end of file From 1479ace25cae21273e14b431a7ed8fdd3440dfba Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sat, 12 Aug 2023 21:36:22 +0200 Subject: [PATCH 65/84] Implement equals. --- .../java/ch/cyberduck/core/OAuthTokens.java | 31 +++++++++++++++++++ .../java/ch/cyberduck/core/STSTokens.java | 31 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/core/src/main/java/ch/cyberduck/core/OAuthTokens.java b/core/src/main/java/ch/cyberduck/core/OAuthTokens.java index 8ce1f8b6609..3d652bc0383 100644 --- a/core/src/main/java/ch/cyberduck/core/OAuthTokens.java +++ b/core/src/main/java/ch/cyberduck/core/OAuthTokens.java @@ -17,6 +17,8 @@ import org.apache.commons.lang3.StringUtils; +import java.util.Objects; + public final class OAuthTokens { public static final OAuthTokens EMPTY = new OAuthTokens(null, null, Long.MAX_VALUE, null); @@ -60,6 +62,35 @@ public boolean isExpired() { return System.currentTimeMillis() >= expiryInMilliseconds; } + @Override + public boolean equals(final Object o) { + if(this == o) { + return true; + } + if(o == null || getClass() != o.getClass()) { + return false; + } + final OAuthTokens that = (OAuthTokens) o; + if(!Objects.equals(accessToken, that.accessToken)) { + return false; + } + if(!Objects.equals(refreshToken, that.refreshToken)) { + return false; + } + if(!Objects.equals(idToken, that.idToken)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + int result = accessToken != null ? accessToken.hashCode() : 0; + result = 31 * result + (refreshToken != null ? refreshToken.hashCode() : 0); + result = 31 * result + (idToken != null ? idToken.hashCode() : 0); + return result; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("OAuthTokens{"); diff --git a/core/src/main/java/ch/cyberduck/core/STSTokens.java b/core/src/main/java/ch/cyberduck/core/STSTokens.java index 8245c238cc0..4757845dc29 100644 --- a/core/src/main/java/ch/cyberduck/core/STSTokens.java +++ b/core/src/main/java/ch/cyberduck/core/STSTokens.java @@ -17,6 +17,8 @@ import org.apache.commons.lang3.StringUtils; +import java.util.Objects; + /** * Temporary access credentials */ @@ -65,6 +67,35 @@ public boolean isExpired() { return System.currentTimeMillis() >= expiryInMilliseconds; } + @Override + public boolean equals(final Object o) { + if(this == o) { + return true; + } + if(o == null || getClass() != o.getClass()) { + return false; + } + final STSTokens stsTokens = (STSTokens) o; + if(!Objects.equals(accessKeyId, stsTokens.accessKeyId)) { + return false; + } + if(!Objects.equals(secretAccessKey, stsTokens.secretAccessKey)) { + return false; + } + if(!Objects.equals(sessionToken, stsTokens.sessionToken)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + int result = accessKeyId != null ? accessKeyId.hashCode() : 0; + result = 31 * result + (secretAccessKey != null ? secretAccessKey.hashCode() : 0); + result = 31 * result + (sessionToken != null ? sessionToken.hashCode() : 0); + return result; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("STSTokens{"); From 65168005b9f30dab37f0a66aec5d6f41f58309f8 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sat, 12 Aug 2023 22:13:23 +0200 Subject: [PATCH 66/84] Review tests. --- ...AbstractAssumeRoleWithWebIdentityTest.java | 8 +--- ...RoleWithWebIdentityAuthenticationTest.java | 36 ++++++++--------- ...eTokenExpiryFailsBecauseOfLatencyTest.java | 32 +++++++-------- ...CredentialsExpiredValidOAuthTokenTest.java | 30 ++++++-------- .../core/sts/STSOAuthExpiredValidSTSTest.java | 40 ++++++++++++++----- .../ch/cyberduck/core/sts/STSTestSetup.java | 32 +++++++-------- 6 files changed, 95 insertions(+), 83 deletions(-) diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java index 2312055aab4..f558afd6cf4 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java @@ -15,7 +15,6 @@ * GNU General Public License for more details. */ - import ch.cyberduck.core.Profile; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.test.TestcontainerTest; @@ -31,25 +30,22 @@ @Category(TestcontainerTest.class) public abstract class AbstractAssumeRoleWithWebIdentityTest { + protected static final Logger log = LogManager.getLogger(AbstractAssumeRoleWithWebIdentityTest.class); + protected static final int MILLIS = 1000; // lag to wait after token expiry protected static final int LAG = 10 * MILLIS; - protected static final Logger log = LogManager.getLogger(AbstractAssumeRoleWithWebIdentityTest.class); protected static int OAUTH_TTL_SECS = 30; protected static Profile profile = null; - @ClassRule public static DockerComposeContainer compose = prepareDockerComposeContainer(getKeyCloakFile()); - @Before public void setup() throws BackgroundException { profile = readProfile(); } - - } \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index 6791958feb4..4a052353e75 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -21,7 +21,9 @@ import ch.cyberduck.core.DisabledLoginCallback; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostUrlProvider; +import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.Path; +import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.proxy.DisabledProxyFinder; @@ -44,22 +46,22 @@ public class AssumeRoleWithWebIdentityAuthenticationTest extends AbstractAssumeRoleWithWebIdentityTest { @Test - public void testSuccessfulLoginViaOidc() throws BackgroundException { + public void testSuccessfulLogin() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); final S3Session session = new S3Session(host); session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); - Credentials creds = host.getCredentials(); - assertNotEquals(StringUtils.EMPTY, creds.getUsername()); - assertNotEquals(StringUtils.EMPTY, creds.getPassword()); + final Credentials credentials = host.getCredentials(); + assertEquals("95555b63-6798-45a4-9a65-8fb38ad49a97", credentials.getUsername()); + assertNotEquals(StringUtils.EMPTY, credentials.getPassword()); - assertNotNull(creds.getTokens().getAccessKeyId()); - assertNotNull(creds.getTokens().getSecretAccessKey()); - assertNotNull(creds.getTokens().getSessionToken()); - assertNotNull(creds.getOauth().getIdToken()); - assertNotNull(creds.getOauth().getRefreshToken()); - assertNotEquals(Optional.of(Long.MAX_VALUE).get(), creds.getOauth().getExpiryInMilliseconds()); + assertNotNull(credentials.getTokens().getAccessKeyId()); + assertNotNull(credentials.getTokens().getSecretAccessKey()); + assertNotNull(credentials.getTokens().getSessionToken()); + assertNotNull(credentials.getOauth().getIdToken()); + assertNotNull(credentials.getOauth().getRefreshToken()); + assertNotEquals(Optional.of(Long.MAX_VALUE).get(), credentials.getOauth().getExpiryInMilliseconds()); session.close(); } @@ -88,9 +90,10 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); - String firstAccessToken = host.getCredentials().getOauth().getIdToken(); - String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); - String firstSessionToken = ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken(); + final OAuthTokens oauth = host.getCredentials().getOauth(); + assertTrue(oauth.validate()); + final STSTokens tokens = host.getCredentials().getTokens(); + assertTrue(tokens.validate()); Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); @@ -98,11 +101,8 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException Thread.sleep(MILLIS * OAUTH_TTL_SECS + LAG); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - assertNotEquals(firstAccessToken, host.getCredentials().getOauth().getIdToken()); - assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); - assertNotEquals(firstSessionToken, ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken()); + assertNotEquals(oauth, host.getCredentials().getOauth()); + assertNotEquals(tokens, host.getCredentials().getTokens()); session.close(); } - - } \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java b/s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java index 90ca49ae9d7..7ab7d13e252 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java @@ -21,8 +21,10 @@ import ch.cyberduck.core.DisabledLoginCallback; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostUrlProvider; +import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.Path; import ch.cyberduck.core.Profile; +import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.s3.S3AccessControlListFeature; @@ -30,34 +32,30 @@ import ch.cyberduck.core.s3.S3Session; import ch.cyberduck.test.TestcontainerTest; +import org.junit.ClassRule; import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.testcontainers.containers.DockerComposeContainer; import java.util.EnumSet; import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.MILLIS; import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.OAUTH_TTL_SECS; -import static ch.cyberduck.core.sts.STSTestSetup.readProfile; +import static ch.cyberduck.core.sts.STSTestSetup.*; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; - @Category(TestcontainerTest.class) public class STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest { + @ClassRule + public static DockerComposeContainer compose = prepareDockerComposeContainer( + getKeyCloakFile() + ); - /** - * If an HTTP HEAD request to MinIO fails, the error code and error description are written to MinIO-specific error headers, since HTTP HEAD requests do not use a body. - * For example, a HEAD request is used to check whether an S3 bucket or folder is discoverable. - * When a HEAD request uses expired STS credentials. Because of preemtively refresh tokens this case is only possible it the credentials are still valid but a few milliseconds before expire. Because of the latency in the network the request will be invalid when reaching the MinIO Service. - * But the sleep time needs to be ajusted according to the network latency. - * Adjust the sleep time according to the network latency. - * Overall the test may be removed and the general question is how to handle the MinIO-specific HTTP-Headers when a HEAD-Request is failing. - * This test fails if the x-minio Headers are not read because of InvalidAccessKeyId error code which has no response body. - */ @Test - @Ignore("Time of network latency may vary and so the time needs to be adjusted manually") // TODO should we remove this test or keep for documentation? + @Ignore("Time of network latency may vary and so the time needs to be adjusted manually") public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws BackgroundException, InterruptedException { final Profile profile = readProfile(); @@ -66,8 +64,10 @@ public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws Bac session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); - String firstAccessToken = host.getCredentials().getOauth().getIdToken(); - String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); + final OAuthTokens oauth = host.getCredentials().getOauth(); + assertTrue(oauth.validate()); + final STSTokens tokens = host.getCredentials().getTokens(); + assertTrue(tokens.validate()); // Time of latency may vary and so the time needs to be adjusted accordingly final int NETWORK_LATENCY = 1180; @@ -77,8 +77,8 @@ public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws Bac Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - assertNotEquals(firstAccessToken, host.getCredentials().getOauth().getIdToken()); - assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); + assertNotEquals(oauth, host.getCredentials().getOauth()); + assertNotEquals(tokens, host.getCredentials().getTokens()); session.close(); } } \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java b/s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java index 7b03a241caa..cca31f93c9c 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java @@ -20,11 +20,14 @@ import ch.cyberduck.core.DisabledHostKeyCallback; import ch.cyberduck.core.DisabledLoginCallback; import ch.cyberduck.core.Host; +import ch.cyberduck.core.HostUrlProvider; +import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.Path; import ch.cyberduck.core.Profile; +import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.preferences.HostPreferences; -import ch.cyberduck.core.proxy.Proxy; +import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.s3.S3AccessControlListFeature; import ch.cyberduck.core.s3.S3FindFeature; import ch.cyberduck.core.s3.S3Session; @@ -40,8 +43,6 @@ import java.util.HashMap; import java.util.Map; -import com.amazonaws.auth.AWSSessionCredentials; - import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.LAG; import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.MILLIS; import static ch.cyberduck.core.sts.STSTestSetup.*; @@ -50,7 +51,6 @@ @Category(TestcontainerTest.class) public class STSCredentialsExpiredValidOAuthTokenTest { - /** * Adjust OAuth token TTL in Keycloak: * "access.token.lifespan": "930" @@ -68,9 +68,8 @@ private static Map overrideKeycloakDefaults() { getKeyCloakFile(overrideKeycloakDefaults()) ); - @Test - @Ignore("Takes 15 minutes, skip by default") // TODO does this test run through? + @Ignore("Takes 15 minutes, skip by default") public void testSTSCredentialsExpiredValidOAuthToken() throws BackgroundException, InterruptedException { final Profile profile = readProfile(); // 900 secs = 15 min is mininmum value: https://min.io/docs/minio/linux/developers/security-token-service/AssumeRoleWithWebIdentity.html @@ -80,24 +79,21 @@ public void testSTSCredentialsExpiredValidOAuthToken() throws BackgroundExceptio assertEquals(new HostPreferences(host).getInteger("s3.assumerole.durationseconds"), assumeRoleDurationSeconds); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); - String firstAccessToken = host.getCredentials().getOauth().getAccessToken(); - String firstAccessKey = session.getClient().getProviderCredentials().getAccessKey(); - assertTrue(session.getClient().getProviderCredentials() instanceof AWSSessionCredentials); + final OAuthTokens oauth = host.getCredentials().getOauth(); + assertTrue(oauth.validate()); + final STSTokens tokens = host.getCredentials().getTokens(); + assertTrue(tokens.validate()); - String firstSessionToken = ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken(); Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); Thread.sleep(MILLIS * assumeRoleDurationSeconds + LAG); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - assertNotEquals(firstAccessKey, session.getClient().getProviderCredentials().getAccessKey()); - assertNotEquals(firstSessionToken, ((AWSSessionCredentials) session.getClient().getProviderCredentials()).getSessionToken()); - assertEquals(firstAccessToken, host.getCredentials().getOauth().getAccessToken()); + assertEquals(oauth, host.getCredentials().getOauth()); + assertNotEquals(tokens, host.getCredentials().getTokens()); } - - } \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java b/s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java index 21f3069c50f..ae25559eec1 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java @@ -20,9 +20,15 @@ import ch.cyberduck.core.DisabledHostKeyCallback; import ch.cyberduck.core.DisabledLoginCallback; import ch.cyberduck.core.Host; +import ch.cyberduck.core.HostUrlProvider; +import ch.cyberduck.core.OAuthTokens; +import ch.cyberduck.core.Path; import ch.cyberduck.core.Profile; +import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.proxy.Proxy; +import ch.cyberduck.core.proxy.DisabledProxyFinder; +import ch.cyberduck.core.s3.S3AccessControlListFeature; +import ch.cyberduck.core.s3.S3FindFeature; import ch.cyberduck.core.s3.S3Session; import ch.cyberduck.test.TestcontainerTest; @@ -31,14 +37,15 @@ import org.junit.experimental.categories.Category; import org.testcontainers.containers.DockerComposeContainer; +import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.LAG; import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.MILLIS; import static ch.cyberduck.core.sts.STSTestSetup.*; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; +import static org.junit.Assert.assertNotEquals; @Category(TestcontainerTest.class) public class STSOAuthExpiredValidSTSTest { @@ -52,22 +59,35 @@ private static Map overrideKeycloakDefaults() { } @ClassRule - public static DockerComposeContainer compose = prepareDockerComposeContainer(getKeyCloakFile(overrideKeycloakDefaults())); + public static DockerComposeContainer compose = prepareDockerComposeContainer( + getKeyCloakFile(overrideKeycloakDefaults()) + ); @Test public void testOAuthExpiry() throws BackgroundException, InterruptedException { - // TODO STS refresh in this case of early OAuth expiry... final Profile profile = readProfile(); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); final S3Session session = new S3Session(host); - session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback()); + session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); + + final Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + + final OAuthTokens oauth = host.getCredentials().getOauth(); + assertTrue(oauth.validate()); + assertFalse(oauth.isExpired()); + final STSTokens tokens = host.getCredentials().getTokens(); + assertTrue(tokens.validate()); - assertFalse(host.getCredentials().getOauth().isExpired()); Thread.sleep(MILLIS * OAUTH_TTL_SECS + LAG); + assertTrue(host.getCredentials().getOauth().isExpired()); - session.close(); - } + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); + assertNotEquals(oauth, host.getCredentials().getOauth()); + assertNotEquals(tokens, host.getCredentials().getTokens()); + session.close(); + } } \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java b/s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java index 3c029a52b39..c87653ef928 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java @@ -43,12 +43,11 @@ public class STSTestSetup { private static final Logger log = LogManager.getLogger(STSTestSetup.class); - public static DockerComposeContainer prepareDockerComposeContainer(String tempFile) { + public static DockerComposeContainer prepareDockerComposeContainer(final String keyCloakRealmTempFile) { log.info("Preparing docker compose container..."); - System.out.println(tempFile); return new DockerComposeContainer<>( new File(STSTestSetup.class.getResource("/testcontainer/docker-compose.yml").getFile())) - .withEnv("KEYCLOAK_REALM_JSON", tempFile) + .withEnv("KEYCLOAK_REALM_JSON", keyCloakRealmTempFile) .withPull(false) .withLocalCompose(true) .withOptions("--compatibility") @@ -56,13 +55,13 @@ public static DockerComposeContainer prepareDockerComposeContainer(String tempFi .withExposedService("minio_1", 9000, Wait.forListeningPort()); } - public static String getKeyCloakFile(Map replacements) { + public static String getKeyCloakFile(Map replacements) { Gson gson = new GsonBuilder().setPrettyPrinting().create(); JsonElement je = new Gson().fromJson(new InputStreamReader(STSTestSetup.class.getResourceAsStream("/testcontainer/keycloak/keycloak-realm.json")), JsonElement.class); JsonObject jo = je.getAsJsonObject(); for(Map.Entry replacement : replacements.entrySet()) { - updateJsonValues(jo,replacement.getKey(), replacement.getValue()); + updateJsonValues(jo, replacement.getKey(), replacement.getValue()); } String content = gson.toJson(jo); @@ -78,13 +77,15 @@ public static String getKeyCloakFile(Map replacements) { } private static void updateJsonValues(JsonObject jsonObj, String key, String newVal) { - for (Map.Entry entry : jsonObj.entrySet()) { + for(Map.Entry entry : jsonObj.entrySet()) { JsonElement element = entry.getValue(); - if (element.isJsonArray()) { + if(element.isJsonArray()) { updateJsonValues(element.getAsJsonArray(), key, newVal); - } else if (element.isJsonObject()) { - updateJsonValues(element.getAsJsonObject(), key,newVal); - } else if (entry.getKey().equals(key)) { + } + else if(element.isJsonObject()) { + updateJsonValues(element.getAsJsonObject(), key, newVal); + } + else if(entry.getKey().equals(key)) { jsonObj.remove(key); jsonObj.addProperty(key, newVal); break; @@ -93,14 +94,14 @@ private static void updateJsonValues(JsonObject jsonObj, String key, String newV } private static void updateJsonValues(JsonArray asJsonArray, String key, String newVal) { - for (int index = 0; index < asJsonArray.size(); index++) { + for(int index = 0; index < asJsonArray.size(); index++) { JsonElement element = asJsonArray.get(index); - if (element.isJsonArray()) { + if(element.isJsonArray()) { updateJsonValues(element.getAsJsonArray(), key, newVal); - } else if (element.isJsonObject()) { - updateJsonValues(element.getAsJsonObject(), key,newVal); } - + else if(element.isJsonObject()) { + updateJsonValues(element.getAsJsonObject(), key, newVal); + } } } @@ -113,5 +114,4 @@ public static Profile readProfile() throws AccessDeniedException { return new ProfilePlistReader(factory).read( STSTestSetup.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); } - } From eb35e006cc630928294028cad4439c3f5a919423 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sat, 12 Aug 2023 22:25:56 +0200 Subject: [PATCH 67/84] Temporary credentials are not saved. --- .../STSAssumeRoleCredentialsRequestInterceptor.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java index 913ee2e5cf8..3ce394e7c63 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -77,19 +77,7 @@ public STSTokens refresh(final OAuthTokens oauthTokens) throws BackgroundExcepti } } - /** - * Save updated tokens in keychain - * - * @return Same tokens saved - */ public STSTokens save(final STSTokens tokens) throws LocalAccessDeniedException { - host.getCredentials() - .withTokens(tokens) - .withSaved(new LoginOptions().keychain); - if(log.isDebugEnabled()) { - log.debug(String.format("Save new tokens %s for %s", tokens, host)); - } - store.save(host); return tokens; } From e2baf3947157dd82865f8f5fa6b5b301e4061966 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sat, 12 Aug 2023 23:15:55 +0200 Subject: [PATCH 68/84] Add abstraction for authentication strategy. --- .../auth/AWSSessionCredentialsRetriever.java | 14 ++- .../core/s3/S3CredentialsStrategy.java | 25 +++++ .../java/ch/cyberduck/core/s3/S3Session.java | 101 +++++++----------- .../s3/S3TokenExpiredResponseInterceptor.java | 24 +++-- ...sumeRoleCredentialsRequestInterceptor.java | 25 ++++- 5 files changed, 112 insertions(+), 77 deletions(-) create mode 100644 s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsStrategy.java diff --git a/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java b/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java index 7b86aaac3f7..6cefc315bd0 100644 --- a/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java +++ b/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java @@ -34,9 +34,15 @@ import ch.cyberduck.core.dav.DAVReadFeature; import ch.cyberduck.core.dav.DAVSession; import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.ConnectionRefusedException; +import ch.cyberduck.core.exception.ConnectionTimeoutException; import ch.cyberduck.core.exception.InteroperabilityException; import ch.cyberduck.core.exception.LoginCanceledException; +import ch.cyberduck.core.exception.LoginFailureException; +import ch.cyberduck.core.exception.NotfoundException; +import ch.cyberduck.core.exception.ResolveFailedException; import ch.cyberduck.core.proxy.ProxyFactory; +import ch.cyberduck.core.s3.S3CredentialsStrategy; import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.transfer.TransferStatus; @@ -54,7 +60,7 @@ import com.google.gson.stream.JsonReader; import com.google.gson.stream.MalformedJsonException; -public class AWSSessionCredentialsRetriever { +public class AWSSessionCredentialsRetriever implements S3CredentialsStrategy { private static final Logger log = LogManager.getLogger(AWSSessionCredentialsRetriever.class); private final TranscriptListener transcript; @@ -101,6 +107,7 @@ public Credentials configure(final Host host) { } } + @Override public Credentials get() throws BackgroundException { if(log.isDebugEnabled()) { log.debug(String.format("Configure credentials from %s", url)); @@ -116,6 +123,11 @@ public Credentials get() throws BackgroundException { connection.close(); return credentials; } + catch(ConnectionTimeoutException | ConnectionRefusedException | ResolveFailedException | NotfoundException | + InteroperabilityException e) { + log.warn(String.format("Failure %s to retrieve session credentials", e)); + throw new LoginFailureException(e.getDetail(false), e); + } finally { connection.removeListener(transcript); } diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsStrategy.java b/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsStrategy.java new file mode 100644 index 00000000000..8b2e02e4673 --- /dev/null +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsStrategy.java @@ -0,0 +1,25 @@ +package ch.cyberduck.core.s3;/* + * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.Credentials; +import ch.cyberduck.core.exception.BackgroundException; + +public interface S3CredentialsStrategy { + + /** + * @return Retrieve credentials to sign requests + */ + Credentials get() throws BackgroundException; +} diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 2686b083745..11e7ec0c157 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -29,7 +29,6 @@ import ch.cyberduck.core.Path; import ch.cyberduck.core.PathAttributes; import ch.cyberduck.core.PathContainerService; -import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.Scheme; import ch.cyberduck.core.UrlProvider; import ch.cyberduck.core.auth.AWSSessionCredentialsRetriever; @@ -40,12 +39,7 @@ import ch.cyberduck.core.date.RFC822DateFormatter; import ch.cyberduck.core.exception.AccessDeniedException; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.ConnectionRefusedException; -import ch.cyberduck.core.exception.ConnectionTimeoutException; import ch.cyberduck.core.exception.InteroperabilityException; -import ch.cyberduck.core.exception.LoginFailureException; -import ch.cyberduck.core.exception.NotfoundException; -import ch.cyberduck.core.exception.ResolveFailedException; import ch.cyberduck.core.features.*; import ch.cyberduck.core.http.HttpSession; import ch.cyberduck.core.kms.KMSEncryptionFeature; @@ -107,19 +101,6 @@ public class S3Session extends HttpSession { private final PreferencesReader preferences = new HostPreferences(host); - /** - * Handle authentication with OpenID connect retrieving token for STS - */ - private OAuth2RequestInterceptor oauth; - /** - * Swap OIDC Id token for temporary security credentials - */ - private STSAssumeRoleCredentialsRequestInterceptor sts; - /** - * Fetch latest temporary session token from AWS CLI configuration or instance metadata - */ - private S3TokenExpiredResponseInterceptor token; - private final S3AccessControlListFeature acl = new S3AccessControlListFeature(this); private final Versioning versioning = preferences.getBoolean("s3.versioning.enable") @@ -150,6 +131,7 @@ public Distribution read(final Path container, final Distribution.Method method, private final Map> distributions = new ConcurrentHashMap<>(); + private S3CredentialsStrategy authentication; private S3Protocol.AuthenticationHeaderSignatureVersion authenticationHeaderSignatureVersion = S3Protocol.AuthenticationHeaderSignatureVersion.getDefault(host.getProtocol()); @@ -208,25 +190,39 @@ protected String getRestMetadataPrefix() { protected RequestEntityRestStorageService connect(final Proxy proxy, final HostKeyCallback hostkey, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { final HttpClientBuilder configuration = builder.build(proxy, this, prompt); if(host.getProtocol().isOAuthConfigurable()) { - configuration.addInterceptorLast(oauth = new OAuth2RequestInterceptor(builder.build(ProxyFactory.get() + final OAuth2RequestInterceptor oauth = new OAuth2RequestInterceptor(builder.build(ProxyFactory.get() .find(host.getProtocol().getOAuthAuthorizationUrl()), this, prompt).build(), host, prompt) .withRedirectUri(host.getProtocol().getOAuthRedirectUrl()) - .withFlowType(OAuth2AuthorizationService.FlowType.valueOf(host.getProtocol().getAuthorization()))); - configuration.addInterceptorLast(sts = new STSAssumeRoleCredentialsRequestInterceptor(oauth, this, trust, key, prompt)); + .withFlowType(OAuth2AuthorizationService.FlowType.valueOf(host.getProtocol().getAuthorization())); + configuration.addInterceptorLast(oauth); + final STSAssumeRoleCredentialsRequestInterceptor sts = new STSAssumeRoleCredentialsRequestInterceptor(oauth, this, trust, key, prompt, cancel); + configuration.addInterceptorLast(sts); configuration.setServiceUnavailableRetryStrategy(new STSAssumeRoleTokenExpiredResponseInterceptor(this, oauth, sts, prompt)); + authentication = sts; } else { if(S3Session.isAwsHostname(host.getHostname())) { + final S3TokenExpiredResponseInterceptor interceptor; // Try auto-configure if(Scheme.isURL(host.getProtocol().getContext())) { - configuration.setServiceUnavailableRetryStrategy(token = new S3TokenExpiredResponseInterceptor(this, - new AWSSessionCredentialsRetriever.Configurator(trust, key, this, host.getProtocol().getContext()) - )); + interceptor = new S3TokenExpiredResponseInterceptor(this, + new AWSSessionCredentialsRetriever(trust, key, this, host.getProtocol().getContext()) + ); } else { - configuration.setServiceUnavailableRetryStrategy(token = new S3TokenExpiredResponseInterceptor(this, - new S3CredentialsConfigurator(new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt))); + interceptor = new S3TokenExpiredResponseInterceptor(this, new S3CredentialsStrategy() { + @Override + public Credentials get() { + return new S3CredentialsConfigurator( + new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key, prompt).configure(host); + } + }); } + configuration.setServiceUnavailableRetryStrategy(interceptor); + authentication = interceptor; + } + else { + authentication = host::getCredentials; } } if(preferences.getBoolean("s3.upload.expect-continue")) { @@ -305,50 +301,27 @@ public void process(final HttpRequest request, final HttpContext context) { @Override public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - if(host.getProtocol().isOAuthConfigurable()) { - // Get temporary credentials from STS using Web Identity (OIDC) token - final STSTokens tokens = sts.authorize(host, oauth.authorize(host, prompt, cancel)); - client.setProviderCredentials(new AWSSessionCredentials(tokens.getAccessKeyId(), - tokens.getSecretAccessKey(), tokens.getSessionToken())); + final Credentials credentials = authentication.get(); + if(credentials.isAnonymousLogin()) { + if(log.isDebugEnabled()) { + log.debug(String.format("Connect with no credentials to %s", host)); + } + client.setProviderCredentials(null); } else { - final Credentials credentials; - // Only for AWS - if(isAwsHostname(host.getHostname())) { - try { - // Try auto-configure - credentials = token.refresh(); - } - catch(ConnectionTimeoutException | ConnectionRefusedException | ResolveFailedException | - NotfoundException | InteroperabilityException e) { - log.warn(String.format("Failure to retrieve session credentials from . %s", e.getMessage())); - throw new LoginFailureException(e.getDetail(false), e); - } - } - else { - credentials = host.getCredentials(); - } - if(credentials.isAnonymousLogin()) { + if(credentials.getTokens().validate()) { if(log.isDebugEnabled()) { - log.debug(String.format("Connect with no credentials to %s", host)); + log.debug(String.format("Connect with session credentials to %s", host)); } - client.setProviderCredentials(null); + client.setProviderCredentials(new AWSSessionCredentials( + credentials.getTokens().getAccessKeyId(), credentials.getTokens().getSecretAccessKey(), + credentials.getTokens().getSessionToken())); } else { - if(credentials.getTokens().validate()) { - if(log.isDebugEnabled()) { - log.debug(String.format("Connect with session credentials to %s", host)); - } - client.setProviderCredentials(new AWSSessionCredentials( - credentials.getTokens().getAccessKeyId(), credentials.getTokens().getSecretAccessKey(), - credentials.getTokens().getSessionToken())); - } - else { - if(log.isDebugEnabled()) { - log.debug(String.format("Connect with basic credentials to %s", host)); - } - client.setProviderCredentials(new AWSCredentials(credentials.getUsername(), credentials.getPassword())); + if(log.isDebugEnabled()) { + log.debug(String.format("Connect with basic credentials to %s", host)); } + client.setProviderCredentials(new AWSCredentials(credentials.getUsername(), credentials.getPassword())); } } if(host.getCredentials().isPassed()) { diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java index 80458b18311..409297cd83d 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java @@ -16,7 +16,6 @@ */ import ch.cyberduck.core.Credentials; -import ch.cyberduck.core.CredentialsConfigurator; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ExpiredTokenException; import ch.cyberduck.core.http.DisabledServiceUnavailableRetryStrategy; @@ -31,19 +30,27 @@ import java.io.IOException; -public class S3TokenExpiredResponseInterceptor extends DisabledServiceUnavailableRetryStrategy { +/** + * Fetch latest temporary session token from AWS CLI configuration or instance metadata + */ +public class S3TokenExpiredResponseInterceptor extends DisabledServiceUnavailableRetryStrategy implements S3CredentialsStrategy { private static final Logger log = LogManager.getLogger(S3TokenExpiredResponseInterceptor.class); private static final int MAX_RETRIES = 1; private final S3Session session; - private final CredentialsConfigurator configurator; + private final S3CredentialsStrategy configurator; - public S3TokenExpiredResponseInterceptor(final S3Session session, final CredentialsConfigurator configurator) { + public S3TokenExpiredResponseInterceptor(final S3Session session, final S3CredentialsStrategy configurator) { this.session = session; this.configurator = configurator; } + @Override + public Credentials get() throws BackgroundException { + return configurator.get(); + } + @Override public boolean retryRequest(final HttpResponse response, final int executionCount, final HttpContext context) { if(executionCount <= MAX_RETRIES) { @@ -54,7 +61,7 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun if(log.isWarnEnabled()) { log.warn(String.format("Handle failure %s", response)); } - final Credentials credentials = configurator.configure(session.getHost()); + final Credentials credentials = configurator.get(); if(log.isDebugEnabled()) { log.debug(String.format("Reconfigure client with credentials %s", credentials)); } @@ -73,6 +80,9 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun catch(IOException e) { log.warn(String.format("Failure parsing response entity from %s", response)); } + catch(BackgroundException e) { + log.warn(String.format("Failure %s retrieving credentials", e)); + } } } else { @@ -82,8 +92,4 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun } return false; } - - public Credentials refresh() throws BackgroundException { - return configurator.reload().configure(session.getHost()); - } } diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java index 3ce394e7c63..2f3da0ce4c1 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -15,10 +15,10 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.Credentials; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostPasswordStore; import ch.cyberduck.core.LoginCallback; -import ch.cyberduck.core.LoginOptions; import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.PasswordStoreFactory; import ch.cyberduck.core.STSTokens; @@ -26,9 +26,11 @@ import ch.cyberduck.core.exception.ExpiredTokenException; import ch.cyberduck.core.exception.LocalAccessDeniedException; import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; +import ch.cyberduck.core.s3.S3CredentialsStrategy; import ch.cyberduck.core.s3.S3Session; import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; +import ch.cyberduck.core.threading.CancelCallback; import org.apache.http.HttpException; import org.apache.http.HttpRequest; @@ -40,7 +42,10 @@ import java.io.IOException; -public class STSAssumeRoleCredentialsRequestInterceptor extends STSAssumeRoleAuthorizationService implements HttpRequestInterceptor { +/** + * Swap OIDC Id token for temporary security credentials + */ +public class STSAssumeRoleCredentialsRequestInterceptor extends STSAssumeRoleAuthorizationService implements S3CredentialsStrategy, HttpRequestInterceptor { private static final Logger log = LogManager.getLogger(STSAssumeRoleCredentialsRequestInterceptor.class); /** @@ -49,16 +54,24 @@ public class STSAssumeRoleCredentialsRequestInterceptor extends STSAssumeRoleAut private STSTokens tokens = STSTokens.EMPTY; private final HostPasswordStore store = PasswordStoreFactory.get(); + /** + * Handle authentication with OpenID connect retrieving token for STS + */ private final OAuth2RequestInterceptor oauth; private final S3Session session; private final Host host; + private final LoginCallback prompt; + private final CancelCallback cancel; public STSAssumeRoleCredentialsRequestInterceptor(final OAuth2RequestInterceptor oauth, final S3Session session, - final X509TrustManager trust, final X509KeyManager key, final LoginCallback prompt) { + final X509TrustManager trust, final X509KeyManager key, + final LoginCallback prompt, final CancelCallback cancel) { super(session.getHost(), trust, key, prompt); this.oauth = oauth; this.session = session; this.host = session.getHost(); + this.prompt = prompt; + this.cancel = cancel; } public STSTokens refresh() throws BackgroundException { @@ -98,4 +111,10 @@ public void process(final HttpRequest request, final HttpContext context) throws } } } + + @Override + public Credentials get() throws BackgroundException { + // Get temporary credentials from STS using Web Identity (OIDC) token + return host.getCredentials().withTokens(this.authorize(host, oauth.authorize(host, prompt, cancel))); + } } From c157b42b23f0a9217b63eadd54b70672854c46e3 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sun, 13 Aug 2023 13:22:00 +0200 Subject: [PATCH 69/84] Review. --- ...SAssumeRoleCredentialsRequestInterceptor.java | 16 +++------------- ...ssumeRoleTokenExpiredResponseInterceptor.java | 4 +--- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java index 2f3da0ce4c1..929f6c6904d 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -19,12 +19,10 @@ import ch.cyberduck.core.Host; import ch.cyberduck.core.HostPasswordStore; import ch.cyberduck.core.LoginCallback; -import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.PasswordStoreFactory; import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ExpiredTokenException; -import ch.cyberduck.core.exception.LocalAccessDeniedException; import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; import ch.cyberduck.core.s3.S3CredentialsStrategy; import ch.cyberduck.core.s3.S3Session; @@ -75,12 +73,8 @@ public STSAssumeRoleCredentialsRequestInterceptor(final OAuth2RequestInterceptor } public STSTokens refresh() throws BackgroundException { - return this.tokens = this.authorize(host, oauth.refresh()); - } - - public STSTokens refresh(final OAuthTokens oauthTokens) throws BackgroundException { try { - return this.tokens = this.authorize(host, oauthTokens); + return this.tokens = this.authorize(host, oauth.authorize(host, prompt, cancel)); } catch(ExpiredTokenException e) { if(log.isWarnEnabled()) { @@ -90,15 +84,11 @@ public STSTokens refresh(final OAuthTokens oauthTokens) throws BackgroundExcepti } } - public STSTokens save(final STSTokens tokens) throws LocalAccessDeniedException { - return tokens; - } - @Override public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { if(tokens.isExpired()) { try { - this.save(this.refresh(oauth.getTokens())); + this.refresh(); if(log.isInfoEnabled()) { log.info(String.format("Authorizing service request with STS tokens %s", tokens)); } @@ -115,6 +105,6 @@ public void process(final HttpRequest request, final HttpContext context) throws @Override public Credentials get() throws BackgroundException { // Get temporary credentials from STS using Web Identity (OIDC) token - return host.getCredentials().withTokens(this.authorize(host, oauth.authorize(host, prompt, cancel))); + return host.getCredentials().withTokens(this.refresh()); } } diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java index 505e18c0fac..634d7c09068 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java @@ -40,7 +40,6 @@ public class STSAssumeRoleTokenExpiredResponseInterceptor extends OAuth2ErrorRes private static final int MAX_RETRIES = 1; private final S3Session session; - private final OAuth2RequestInterceptor oauth; private final STSAssumeRoleCredentialsRequestInterceptor sts; public STSAssumeRoleTokenExpiredResponseInterceptor(final S3Session session, @@ -49,7 +48,6 @@ public STSAssumeRoleTokenExpiredResponseInterceptor(final S3Session session, final LoginCallback prompt) { super(session.getHost(), oauth, prompt); this.session = session; - this.oauth = oauth; this.sts = sts; } @@ -75,7 +73,7 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun } try { log.warn(String.format("Attempt to refresh STS token for failure %s", response)); - final STSTokens stsTokens = sts.refresh(oauth.getTokens()); + final STSTokens stsTokens = sts.refresh(); session.getClient().setProviderCredentials(new AWSSessionCredentials(stsTokens.getAccessKeyId(), stsTokens.getSecretAccessKey(), stsTokens.getSessionToken())); // Try again From 81a8f3bdb3a88d53533cc4956e804045f99bbcf7 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sun, 13 Aug 2023 13:26:35 +0200 Subject: [PATCH 70/84] Review. --- ...STSAssumeRoleTokenExpiredResponseInterceptor.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java index 634d7c09068..5992fc98abb 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java @@ -16,7 +16,6 @@ */ import ch.cyberduck.core.LoginCallback; -import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LoginFailureException; @@ -61,9 +60,7 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun return false; } try { - final BackgroundException type = new S3ExceptionMappingService().map(response); - final OAuthTokens oAuthTokens; - if(type instanceof LoginFailureException) { + if(new S3ExceptionMappingService().map(response) instanceof LoginFailureException) { // 403 Forbidden (InvalidAccessKeyId) The provided token has expired // 400 Bad Request (ExpiredToken) The provided token has expired // 400 Bad Request (InvalidToken) The provided token is malformed or otherwise not valid @@ -84,13 +81,6 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun return false; } } - else { - // Ignore other 400 failures - if(log.isDebugEnabled()) { - log.debug(String.format("Ignore failure %s", type)); - } - return false; - } } catch(IOException e) { log.warn(String.format("Failure parsing response entity from %s", response)); From 25a75766beb2e1c87fd6c9654023d17fdfe27268 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sun, 13 Aug 2023 13:40:50 +0200 Subject: [PATCH 71/84] Fix test. --- .../core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index 4a052353e75..d187b8fae9a 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -32,7 +32,6 @@ import ch.cyberduck.core.s3.S3Session; import ch.cyberduck.test.TestcontainerTest; -import org.jets3t.service.security.AWSSessionCredentials; import org.junit.Test; import org.junit.experimental.categories.Category; import org.testcontainers.shaded.org.apache.commons.lang3.StringUtils; @@ -53,7 +52,7 @@ public void testSuccessfulLogin() throws BackgroundException { session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); final Credentials credentials = host.getCredentials(); - assertEquals("95555b63-6798-45a4-9a65-8fb38ad49a97", credentials.getUsername()); + assertNotEquals("rouser", credentials.getUsername()); assertNotEquals(StringUtils.EMPTY, credentials.getPassword()); assertNotNull(credentials.getTokens().getAccessKeyId()); From a2eec98719cf972349af089c8ff6a630250b1c03 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Sun, 13 Aug 2023 13:41:15 +0200 Subject: [PATCH 72/84] Merge implementations retrieving latest credentials on error response. --- ... S3AuthenticationResponseInterceptor.java} | 25 +++-- .../java/ch/cyberduck/core/s3/S3Session.java | 11 ++- ...meRoleTokenExpiredResponseInterceptor.java | 92 ------------------- 3 files changed, 21 insertions(+), 107 deletions(-) rename s3/src/main/java/ch/cyberduck/core/s3/{S3TokenExpiredResponseInterceptor.java => S3AuthenticationResponseInterceptor.java} (74%) delete mode 100644 s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/s3/S3AuthenticationResponseInterceptor.java similarity index 74% rename from s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java rename to s3/src/main/java/ch/cyberduck/core/s3/S3AuthenticationResponseInterceptor.java index 409297cd83d..60ca968c88a 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3TokenExpiredResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3AuthenticationResponseInterceptor.java @@ -17,7 +17,7 @@ import ch.cyberduck.core.Credentials; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.ExpiredTokenException; +import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.http.DisabledServiceUnavailableRetryStrategy; import org.apache.http.HttpResponse; @@ -31,37 +31,42 @@ import java.io.IOException; /** - * Fetch latest temporary session token from AWS CLI configuration or instance metadata + * Update credentials on authentication failure */ -public class S3TokenExpiredResponseInterceptor extends DisabledServiceUnavailableRetryStrategy implements S3CredentialsStrategy { - private static final Logger log = LogManager.getLogger(S3TokenExpiredResponseInterceptor.class); +public class S3AuthenticationResponseInterceptor extends DisabledServiceUnavailableRetryStrategy implements S3CredentialsStrategy { + private static final Logger log = LogManager.getLogger(S3AuthenticationResponseInterceptor.class); private static final int MAX_RETRIES = 1; private final S3Session session; - private final S3CredentialsStrategy configurator; + private final S3CredentialsStrategy authenticator; - public S3TokenExpiredResponseInterceptor(final S3Session session, final S3CredentialsStrategy configurator) { + public S3AuthenticationResponseInterceptor(final S3Session session, final S3CredentialsStrategy authenticator) { this.session = session; - this.configurator = configurator; + this.authenticator = authenticator; } @Override public Credentials get() throws BackgroundException { - return configurator.get(); + return authenticator.get(); } @Override public boolean retryRequest(final HttpResponse response, final int executionCount, final HttpContext context) { if(executionCount <= MAX_RETRIES) { switch(response.getStatusLine().getStatusCode()) { + case HttpStatus.SC_FORBIDDEN: case HttpStatus.SC_BAD_REQUEST: try { - if(new S3ExceptionMappingService().map(response) instanceof ExpiredTokenException) { + if(new S3ExceptionMappingService().map(response) instanceof LoginFailureException) { + // 403 Forbidden (InvalidAccessKeyId) The provided token has expired + // 400 Bad Request (ExpiredToken) The provided token has expired + // 400 Bad Request (InvalidToken) The provided token is malformed or otherwise not valid + // 400 Bad Request (TokenRefreshRequired) The provided token must be refreshed. if(log.isWarnEnabled()) { log.warn(String.format("Handle failure %s", response)); } - final Credentials credentials = configurator.get(); + final Credentials credentials = authenticator.get(); if(log.isDebugEnabled()) { log.debug(String.format("Reconfigure client with credentials %s", credentials)); } diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java index 11e7ec0c157..a18cae77b00 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3Session.java @@ -61,7 +61,6 @@ import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.sts.STSAssumeRoleCredentialsRequestInterceptor; -import ch.cyberduck.core.sts.STSAssumeRoleTokenExpiredResponseInterceptor; import ch.cyberduck.core.threading.BackgroundExceptionCallable; import ch.cyberduck.core.threading.CancelCallback; import ch.cyberduck.core.transfer.TransferStatus; @@ -197,20 +196,22 @@ protected RequestEntityRestStorageService connect(final Proxy proxy, final HostK configuration.addInterceptorLast(oauth); final STSAssumeRoleCredentialsRequestInterceptor sts = new STSAssumeRoleCredentialsRequestInterceptor(oauth, this, trust, key, prompt, cancel); configuration.addInterceptorLast(sts); - configuration.setServiceUnavailableRetryStrategy(new STSAssumeRoleTokenExpiredResponseInterceptor(this, oauth, sts, prompt)); + configuration.setServiceUnavailableRetryStrategy(new S3AuthenticationResponseInterceptor(this, sts)); authentication = sts; } else { if(S3Session.isAwsHostname(host.getHostname())) { - final S3TokenExpiredResponseInterceptor interceptor; + final S3AuthenticationResponseInterceptor interceptor; // Try auto-configure if(Scheme.isURL(host.getProtocol().getContext())) { - interceptor = new S3TokenExpiredResponseInterceptor(this, + // Fetch temporary session token from instance metadata + interceptor = new S3AuthenticationResponseInterceptor(this, new AWSSessionCredentialsRetriever(trust, key, this, host.getProtocol().getContext()) ); } else { - interceptor = new S3TokenExpiredResponseInterceptor(this, new S3CredentialsStrategy() { + // Fetch temporary session token from AWS CLI configuration + interceptor = new S3AuthenticationResponseInterceptor(this, new S3CredentialsStrategy() { @Override public Credentials get() { return new S3CredentialsConfigurator( diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java deleted file mode 100644 index 5992fc98abb..00000000000 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleTokenExpiredResponseInterceptor.java +++ /dev/null @@ -1,92 +0,0 @@ -package ch.cyberduck.core.sts; - -/* - * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. - * https://cyberduck.io/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -import ch.cyberduck.core.LoginCallback; -import ch.cyberduck.core.STSTokens; -import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.LoginFailureException; -import ch.cyberduck.core.oauth.OAuth2ErrorResponseInterceptor; -import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; -import ch.cyberduck.core.s3.S3ExceptionMappingService; -import ch.cyberduck.core.s3.S3Session; - -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.protocol.HttpContext; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jets3t.service.security.AWSSessionCredentials; - -import java.io.IOException; - -public class STSAssumeRoleTokenExpiredResponseInterceptor extends OAuth2ErrorResponseInterceptor { - private static final Logger log = LogManager.getLogger(STSAssumeRoleTokenExpiredResponseInterceptor.class); - - private static final int MAX_RETRIES = 1; - - private final S3Session session; - private final STSAssumeRoleCredentialsRequestInterceptor sts; - - public STSAssumeRoleTokenExpiredResponseInterceptor(final S3Session session, - final OAuth2RequestInterceptor oauth, - final STSAssumeRoleCredentialsRequestInterceptor sts, - final LoginCallback prompt) { - super(session.getHost(), oauth, prompt); - this.session = session; - this.sts = sts; - } - - @Override - public boolean retryRequest(final HttpResponse response, final int executionCount, final HttpContext context) { - switch(response.getStatusLine().getStatusCode()) { - case HttpStatus.SC_FORBIDDEN: - case HttpStatus.SC_BAD_REQUEST: - if(executionCount > MAX_RETRIES) { - log.warn(String.format("Skip retry for response %s after %d executions", response, executionCount)); - return false; - } - try { - if(new S3ExceptionMappingService().map(response) instanceof LoginFailureException) { - // 403 Forbidden (InvalidAccessKeyId) The provided token has expired - // 400 Bad Request (ExpiredToken) The provided token has expired - // 400 Bad Request (InvalidToken) The provided token is malformed or otherwise not valid - // 400 Bad Request (TokenRefreshRequired) The provided token must be refreshed. - if(log.isWarnEnabled()) { - log.warn(String.format("Handle failure %s", response)); - } - try { - log.warn(String.format("Attempt to refresh STS token for failure %s", response)); - final STSTokens stsTokens = sts.refresh(); - session.getClient().setProviderCredentials(new AWSSessionCredentials(stsTokens.getAccessKeyId(), - stsTokens.getSecretAccessKey(), stsTokens.getSessionToken())); - // Try again - return true; - } - catch(BackgroundException e) { - log.warn(String.format("Failure %s refreshing STS token", e)); - return false; - } - } - } - catch(IOException e) { - log.warn(String.format("Failure parsing response entity from %s", response)); - return false; - } - } - return false; - } -} \ No newline at end of file From 3f7f77d0d36e2b5ea3cfc799ee0d96dee2583db0 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 14 Aug 2023 09:18:53 +0200 Subject: [PATCH 73/84] Fix test. --- .../ch/cyberduck/core/s3/S3SessionTest.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java b/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java index 35e9a4dd412..9970e72be43 100644 --- a/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java +++ b/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java @@ -95,23 +95,21 @@ public X509Certificate[] getAcceptedIssuers() { assertFalse(session.isConnected()); } - @Test(expected = ExpiredTokenException.class) + @Test public void testConnectSessionTokenStatic() throws Exception { - final S3Protocol protocol = new S3Protocol() { - @Override - public boolean isTokenConfigurable() { - return true; - } - }; - final Host host = new Host(protocol, protocol.getDefaultHostname(), new Credentials( - "ASIA5RMYTHDIR37CTCXI", "TsnhChH4FlBt7hql2KnzrwNizmktJnO8YzDQwFqx", - "FQoDYXdzEN3//////////wEaDLAz85HLZTQ7zu6/OSKrAfwLewUMHKaswh5sXv50BgMwbeKfCoMATjagvM+KV9++z0I6rItmMectuYoEGCOcnWHKZxtvpZAGcjlvgEDPw1KRYu16riUnd2Yo3doskqAoH0dlL2nH0eoj0d81H5e6IjdlGCm1E3K3zQPFLfMbvn1tdDQR1HV8o9eslmxo54hWMY2M14EpZhcXQMlns0mfYLYHLEVvgpz/8xYjR0yKDxJlXSATEpXtowHtqSi8tL7aBQ==" - )); + final S3Protocol protocol = new S3Protocol(); + final Host host = new Host(protocol, protocol.getDefaultHostname(), new Credentials() + .withTokens(new STSTokens( + "ASIA5RMYTHDIR37CTCXI", + "TsnhChH4FlBt7hql2KnzrwNizmktJnO8YzDQwFqx", + "FQoDYXdzEN3//////////wEaDLAz85HLZTQ7zu6/OSKrAfwLewUMHKaswh5sXv50BgMwbeKfCoMATjagvM+KV9++z0I6rItmMectuYoEGCOcnWHKZxtvpZAGcjlvgEDPw1KRYu16riUnd2Yo3doskqAoH0dlL2nH0eoj0d81H5e6IjdlGCm1E3K3zQPFLfMbvn1tdDQR1HV8o9eslmxo54hWMY2M14EpZhcXQMlns0mfYLYHLEVvgpz/8xYjR0yKDxJlXSATEpXtowHtqSi8tL7aBQ==", + -1L + ))); final S3Session session = new S3Session(host); assertNotNull(session.open(new DisabledProxyFinder().find(host.getHostname()), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback())); assertTrue(session.isConnected()); assertNotNull(session.getClient()); - session.login(new DisabledProxyFinder().find(host.getHostname()), new DisabledLoginCallback(), new DisabledCancelCallback()); + assertThrows(ExpiredTokenException.class, () -> session.login(new DisabledProxyFinder().find(host.getHostname()), new DisabledLoginCallback(), new DisabledCancelCallback())); } @Test From 1913156c9ca583e32173bf0794f3312156aa7dbe Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 14 Aug 2023 09:42:16 +0200 Subject: [PATCH 74/84] Logging. --- .../core/s3/S3AuthenticationResponseInterceptor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3AuthenticationResponseInterceptor.java b/s3/src/main/java/ch/cyberduck/core/s3/S3AuthenticationResponseInterceptor.java index 60ca968c88a..868fcf1123f 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3AuthenticationResponseInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3AuthenticationResponseInterceptor.java @@ -58,13 +58,14 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun case HttpStatus.SC_FORBIDDEN: case HttpStatus.SC_BAD_REQUEST: try { - if(new S3ExceptionMappingService().map(response) instanceof LoginFailureException) { + final BackgroundException failure = new S3ExceptionMappingService().map(response); + if(failure instanceof LoginFailureException) { // 403 Forbidden (InvalidAccessKeyId) The provided token has expired // 400 Bad Request (ExpiredToken) The provided token has expired // 400 Bad Request (InvalidToken) The provided token is malformed or otherwise not valid // 400 Bad Request (TokenRefreshRequired) The provided token must be refreshed. if(log.isWarnEnabled()) { - log.warn(String.format("Handle failure %s", response)); + log.warn(String.format("Handle failure %s from response %s", failure, response)); } final Credentials credentials = authenticator.get(); if(log.isDebugEnabled()) { From 03d9bca9cecf4e2e9b26b7e8b378e1f53d89011d Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 14 Aug 2023 09:53:31 +0200 Subject: [PATCH 75/84] Review error mapping to handle error codes. --- .../core/oauth/OAuth2AuthorizationService.java | 3 +-- .../core/oauth/OAuth2ErrorResponseInterceptor.java | 2 +- .../core/oauth/OAuthExceptionMappingService.java | 12 ++++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java index 7bb92a02395..c57aeae6c5d 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java @@ -27,7 +27,6 @@ import ch.cyberduck.core.Profile; import ch.cyberduck.core.StringAppender; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.InteroperabilityException; import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.http.DefaultHttpResponseExceptionMappingService; @@ -123,7 +122,7 @@ public OAuthTokens authorize(final Host bookmark, final LoginCallback prompt, fi try { return credentials.withOauth(this.refresh(saved)).withSaved(new LoginOptions().keychain).getOauth(); } - catch(LoginFailureException | InteroperabilityException e) { + catch(LoginFailureException e) { log.warn(String.format("Failure refreshing tokens from %s for %s", saved, bookmark)); // Continue with new OAuth 2 flow } diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java index 3ea06285a6d..80377502028 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java @@ -75,7 +75,7 @@ protected OAuthTokens refresh(final HttpResponse response) throws BackgroundExce log.warn(String.format("Attempt to refresh OAuth tokens for failure %s", response)); tokens = service.refresh(); } - catch(InteroperabilityException | LoginFailureException e) { + catch(LoginFailureException e) { log.warn(String.format("Failure %s refreshing OAuth tokens", e)); tokens = service.authorize(bookmark, prompt, new DisabledCancelCallback()); } diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuthExceptionMappingService.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuthExceptionMappingService.java index 58b03cd59e6..2c27e22160a 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuthExceptionMappingService.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuthExceptionMappingService.java @@ -17,6 +17,8 @@ import ch.cyberduck.core.AbstractExceptionMappingService; import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.ExpiredTokenException; +import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.http.DefaultHttpResponseExceptionMappingService; import org.apache.http.client.HttpResponseException; @@ -32,6 +34,16 @@ public BackgroundException map(final TokenResponseException failure) { final TokenErrorResponse details = failure.getDetails(); if(null != details) { this.append(buffer, details.getErrorDescription()); + switch(details.getError()) { + // Error code "invalid_request", "invalid_client", "invalid_grant", "unauthorized_client", "unsupported_grant_type", "invalid_scope" + case "invalid_client": + case "unauthorized_client": + case "unsupported_grant_type": + case "invalid_scope": + return new LoginFailureException(buffer.toString(), failure); + case "invalid_grant": + return new ExpiredTokenException(buffer.toString(), failure); + } } return new DefaultHttpResponseExceptionMappingService().map(new HttpResponseException(failure.getStatusCode(), buffer.toString())); } From 44f7d7f4c6d4e298313a410431e3fc426a107383 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 14 Aug 2023 11:08:10 +0200 Subject: [PATCH 76/84] Add tests. --- ...RoleWithWebIdentityAuthenticationTest.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index d187b8fae9a..ca7ae9d2c71 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -18,25 +18,36 @@ import ch.cyberduck.core.Credentials; import ch.cyberduck.core.DisabledCancelCallback; import ch.cyberduck.core.DisabledHostKeyCallback; +import ch.cyberduck.core.DisabledListProgressListener; import ch.cyberduck.core.DisabledLoginCallback; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostUrlProvider; import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.Path; +import ch.cyberduck.core.Profile; +import ch.cyberduck.core.ProtocolFactory; import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.exception.ExpiredTokenException; +import ch.cyberduck.core.exception.InteroperabilityException; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.s3.S3AccessControlListFeature; +import ch.cyberduck.core.s3.S3BucketListService; import ch.cyberduck.core.s3.S3FindFeature; +import ch.cyberduck.core.s3.S3Protocol; import ch.cyberduck.core.s3.S3Session; +import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.test.TestcontainerTest; +import org.jets3t.service.security.AWSSessionCredentials; import org.junit.Test; import org.junit.experimental.categories.Category; import org.testcontainers.shaded.org.apache.commons.lang3.StringUtils; +import java.util.Collections; import java.util.EnumSet; +import java.util.HashSet; import java.util.Optional; import static org.junit.Assert.*; @@ -104,4 +115,58 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException assertNotEquals(tokens, host.getCredentials().getTokens()); session.close(); } + + /** + * Test flow in S3AuthenticationResponseInterceptor> with invalid OAuth tokens found. Assuming role + * initially fails with InvalidParameterValue + * Fetch OpenID Connect Id token initially fails because of invalid refresh token. Must re-run OAuth flow. + */ + @Test + public void testLoginInvalidOAuthTokensLogin() throws Exception { + final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); + final Credentials credentials = new Credentials("rouser", "rouser") + .withOauth(new OAuthTokens( + "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJDQmpaYUNSeU5USmZqV0VmMU1fZXZLRVliMEdGLXU0QzhjZ3RZYnBtZUlFIn0.eyJleHAiOjE2OTE5OTc3MzUsImlhdCI6MTY5MTk5NzcwNSwianRpIjoiNDA1MGUxMGYtNzZjNC00MjYwLTk1YTctZTMyMTE2YTA3N2NlIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL3JlYWxtcy9jeWJlcmR1Y2tyZWFsbSIsInN1YiI6IjMzNGRiZWIwLTE5NWQtNDJhMS1hMWQ2LTEyODFmMDBiZmIxZCIsInR5cCI6IkJlYXJlciIsImF6cCI6Im1pbmlvIiwic2Vzc2lvbl9zdGF0ZSI6IjNkZDY0MDVlLTNkMGMtNDVjOS05MTZkLTllYTNkNWY1ODVkYiIsInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIiwidXNlciJdfSwic2NvcGUiOiJvcGVuaWQgbWluaW8tYXV0aG9yaXphdGlvbiIsInNpZCI6IjNkZDY0MDVlLTNkMGMtNDVjOS05MTZkLTllYTNkNWY1ODVkYiIsInBvbGljeSI6WyJyZWFkb25seSJdfQ.uKxLmSW6j2EQEo86j0WZOKWgavhS8Ub7TjrnynUi4m1ls0SchvgCilVpzIzNdFL9Y7khiqxl7si5BezbTLPgwyh4GDgrHcJwBk5D6aOcaH6hYcAtcbOiu1KEyfj1O_lwvDCHb-J07TIEeuvquOs2nD7FxqafHjLe-3pL6JuTtBtlx8WKloO9PY-Dn-ntuyqikr7ysLcDBfFJda487cmeTADxiMQ_MmoidW3uGXn0Ps6vhRgteUQO5JTKMa7MT1PKMTY8iNnSdNVuhKkBodnkXMSo5JEt4veqR9Yh-WPT_XL8caUiGInYvHty-n6-yhGhNckrlvtmJc0dJsts4hi1Mw", + "eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIxNzA0N2Q0NS0wMTVhLTQwYWItYjc5NS03Y2Y1ZDE2ZmFhMmQifQ.eyJleHAiOjE2OTE5OTk1MDUsImlhdCI6MTY5MTk5NzcwNSwianRpIjoiY2U4OGVlMjMtOTQ1Yi00YzlmLWExMjAtZjU2ODk0NzIwZDk0IiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL3JlYWxtcy9jeWJlcmR1Y2tyZWFsbSIsImF1ZCI6Imh0dHA6Ly9sb2NhbGhvc3Q6ODA4MC9yZWFsbXMvY3liZXJkdWNrcmVhbG0iLCJzdWIiOiIzMzRkYmViMC0xOTVkLTQyYTEtYTFkNi0xMjgxZjAwYmZiMWQiLCJ0eXAiOiJSZWZyZXNoIiwiYXpwIjoibWluaW8iLCJzZXNzaW9uX3N0YXRlIjoiM2RkNjQwNWUtM2QwYy00NWM5LTkxNmQtOWVhM2Q1ZjU4NWRiIiwic2NvcGUiOiJvcGVuaWQgbWluaW8tYXV0aG9yaXphdGlvbiIsInNpZCI6IjNkZDY0MDVlLTNkMGMtNDVjOS05MTZkLTllYTNkNWY1ODVkYiJ9.iRFLFjU-Uyv81flgieBht2K2BSlM-67fe5unvqI9PXA", + Long.MAX_VALUE, + "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJDQmpaYUNSeU5USmZqV0VmMU1fZXZLRVliMEdGLXU0QzhjZ3RZYnBtZUlFIn0.eyJleHAiOjE2OTE5OTc3MzUsImlhdCI6MTY5MTk5NzcwNSwiYXV0aF90aW1lIjowLCJqdGkiOiJlYWZiNWE5NS1lYmY3LTQ0OTEtODAwYy0yZjU1NTk2MjQ0YzIiLCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjgwODAvcmVhbG1zL2N5YmVyZHVja3JlYWxtIiwiYXVkIjoibWluaW8iLCJzdWIiOiIzMzRkYmViMC0xOTVkLTQyYTEtYTFkNi0xMjgxZjAwYmZiMWQiLCJ0eXAiOiJJRCIsImF6cCI6Im1pbmlvIiwic2Vzc2lvbl9zdGF0ZSI6IjNkZDY0MDVlLTNkMGMtNDVjOS05MTZkLTllYTNkNWY1ODVkYiIsImF0X2hhc2giOiJWX1lIZTVpc0UzY0IyOGF4cXQzRGpnIiwic2lkIjoiM2RkNjQwNWUtM2QwYy00NWM5LTkxNmQtOWVhM2Q1ZjU4NWRiIiwicG9saWN5IjpbInJlYWRvbmx5Il19.bXjcBJY7H79O9rtYr3b_EpKuclaRRsWGIVm5SEesqMM3aIkGq6ikWNmoL4Ffy48Frx1E3UnvG5PQfd8C2-XgNg_9EnWyR1MkgxJ67xQOAT10E77wZ0YbFWYIcdOojR98rmh4_TGVeTaGwDMMQZzRMr0nQwfZP3TQ8ciRhor8svnkFkk3FBzT1rSJA0bJv181HyerQl0f_TnTEnr3UjmmFmDrNASxHoXbwqiE4L-qZBnNiz97jLxGULfyVn4CZUub53x0ka0KGnLeicFHDh1asiHMW18o9-BUh8cGp-Ywm7Xu_f_c8XokNjG8ls56Xp7g8rQ4-d3J0F0-TAgnn7xO1g")) + .withTokens(STSTokens.EMPTY); + final Host host = new Host(profile, profile.getDefaultHostname(), credentials); + final S3Session session = new S3Session(host); + assertNotNull(session.open(new DisabledProxyFinder().find(host.getHostname()), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback())); + assertTrue(session.isConnected()); + assertNotNull(session.getClient()); + session.login(new DisabledProxyFinder().find(host.getHostname()), new DisabledLoginCallback(), new DisabledCancelCallback()); + assertNotEquals(OAuthTokens.EMPTY, credentials.getOauth()); + assertNotEquals(STSTokens.EMPTY, credentials.getTokens()); + } + + /** + * Test flow in S3AuthenticationResponseInterceptor> with no OAuth tokens found. + * Fetch OpenID Connect Id token and swap for temporary access credentials assuming role + */ + @Test + public void testBucketListInvalidOAuthTokensList() throws Exception { + final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); + final Credentials credentials = new Credentials("rouser", "rouser") + .withOauth(OAuthTokens.EMPTY) + .withTokens(new STSTokens( + "5K1AVE34L4U1SQ7QTMWM", + "LfkexzCDPojZpdIoNLNvHxrUi1KI5yP3Yken+DGI", + "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiI1SzFBVkUzNEw0VTFTUTdRVE1XTSIsImF0X2hhc2giOiJWX1lIZTVpc0UzY0IyOGF4cXQzRGpnIiwiYXVkIjoibWluaW8iLCJhdXRoX3RpbWUiOjAsImF6cCI6Im1pbmlvIiwiZXhwIjoxNjkxOTk3NzM1LCJpYXQiOjE2OTE5OTc3MDUsImlzcyI6Imh0dHA6Ly9sb2NhbGhvc3Q6ODA4MC9yZWFsbXMvY3liZXJkdWNrcmVhbG0iLCJqdGkiOiJlYWZiNWE5NS1lYmY3LTQ0OTEtODAwYy0yZjU1NTk2MjQ0YzIiLCJwb2xpY3kiOiJyZWFkb25seSIsInNlc3Npb25fc3RhdGUiOiIzZGQ2NDA1ZS0zZDBjLTQ1YzktOTE2ZC05ZWEzZDVmNTg1ZGIiLCJzaWQiOiIzZGQ2NDA1ZS0zZDBjLTQ1YzktOTE2ZC05ZWEzZDVmNTg1ZGIiLCJzdWIiOiIzMzRkYmViMC0xOTVkLTQyYTEtYTFkNi0xMjgxZjAwYmZiMWQiLCJ0eXAiOiJJRCJ9.HmyC7XuJw9XnsNUd2ZuGSVIPjnGHPpgbXX1HSbNJuhis1kUjhcrYY2HnQZ-uScoX57o_C3fF1eEv_t1kW2U6Rw", + -1L + )); + final Host host = new Host(profile, profile.getDefaultHostname(), credentials); + final S3Session session = new S3Session(host); + assertNotNull(session.open(new DisabledProxyFinder().find(host.getHostname()), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback())); + assertTrue(session.isConnected()); + assertNotNull(session.getClient()); + session.getClient().setProviderCredentials(new AWSSessionCredentials( + credentials.getTokens().getAccessKeyId(), credentials.getTokens().getSecretAccessKey(), + credentials.getTokens().getSessionToken())); + new S3BucketListService(session).list( + new Path(String.valueOf(Path.DELIMITER), EnumSet.of(Path.Type.volume, Path.Type.directory)), new DisabledListProgressListener()); + assertNotEquals(OAuthTokens.EMPTY, credentials.getOauth()); + assertNotEquals(STSTokens.EMPTY, credentials.getTokens()); + } } \ No newline at end of file From 5ccac0531d5e57990f47d7ef648a9171ac8eb5d9 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 15 Aug 2023 10:37:28 +0200 Subject: [PATCH 77/84] Refactor flows. --- .../ch/cyberduck/core/box/BoxSession.java | 5 +- ...PreconditionFailedResponseInterceptor.java | 10 +--- .../ch/cyberduck/core/sds/SDSSession.java | 8 ++- .../core/dropbox/DropboxSession.java | 5 +- .../ch/cyberduck/core/eue/EueSession.java | 17 +----- .../core/googledrive/DriveSession.java | 6 +- .../googlestorage/GoogleStorageSession.java | 13 ++--- .../ch/cyberduck/core/hubic/HubicSession.java | 10 ++-- .../oauth/OAuth2AuthorizationService.java | 56 ++++++++++++------- .../oauth/OAuth2ErrorResponseInterceptor.java | 31 ++-------- .../core/oauth/OAuth2RequestInterceptor.java | 23 ++++++-- .../cyberduck/core/onedrive/GraphSession.java | 6 +- .../core/owncloud/OwncloudSession.java | 5 +- .../STSAssumeRoleAuthorizationService.java | 30 +++++----- ...sumeRoleCredentialsRequestInterceptor.java | 34 ++++++++--- .../ch/cyberduck/core/s3/S3SessionTest.java | 40 +++---------- ...RoleWithWebIdentityAuthenticationTest.java | 3 +- .../core/storegate/StoregateSession.java | 6 +- 18 files changed, 140 insertions(+), 168 deletions(-) diff --git a/box/src/main/java/ch/cyberduck/core/box/BoxSession.java b/box/src/main/java/ch/cyberduck/core/box/BoxSession.java index d37ece339ac..8b7416fb070 100644 --- a/box/src/main/java/ch/cyberduck/core/box/BoxSession.java +++ b/box/src/main/java/ch/cyberduck/core/box/BoxSession.java @@ -71,15 +71,14 @@ public CloseableHttpClient connect(final Proxy proxy, final HostKeyCallback key, authorizationService = new OAuth2RequestInterceptor(configuration.build(), host, prompt) .withRedirectUri(host.getProtocol().getOAuthRedirectUrl()); configuration.addInterceptorLast(authorizationService); - configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService, prompt)); + configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService)); return configuration.build(); } @Override public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - authorizationService.authorize(host, prompt, cancel); try { - final Credentials credentials = host.getCredentials(); + final Credentials credentials = authorizationService.validate(); credentials.setUsername(new UsersApi(new BoxApiClient(client)).getUsersMe(Collections.emptyList()).getLogin()); } catch(ApiException e) { diff --git a/dracoon/src/main/java/ch/cyberduck/core/sds/PreconditionFailedResponseInterceptor.java b/dracoon/src/main/java/ch/cyberduck/core/sds/PreconditionFailedResponseInterceptor.java index 853d2275c19..785a9145edd 100644 --- a/dracoon/src/main/java/ch/cyberduck/core/sds/PreconditionFailedResponseInterceptor.java +++ b/dracoon/src/main/java/ch/cyberduck/core/sds/PreconditionFailedResponseInterceptor.java @@ -15,7 +15,6 @@ * GNU General Public License for more details. */ -import ch.cyberduck.core.DisabledCancelCallback; import ch.cyberduck.core.Host; import ch.cyberduck.core.LoginCallback; import ch.cyberduck.core.exception.BackgroundException; @@ -30,24 +29,19 @@ import org.apache.logging.log4j.Logger; public class PreconditionFailedResponseInterceptor extends OAuth2ErrorResponseInterceptor { - private static final Logger log = LogManager.getLogger(PreconditionFailedResponseInterceptor.class); private static final int MAX_RETRIES = 1; - private final Host bookmark; private final OAuth2RequestInterceptor service; - private final LoginCallback prompt; private final ServiceUnavailableRetryStrategy next; public PreconditionFailedResponseInterceptor(final Host bookmark, final OAuth2RequestInterceptor service, final LoginCallback prompt, final ServiceUnavailableRetryStrategy next) { - super(bookmark, service, prompt); - this.bookmark = bookmark; + super(bookmark, service); this.service = service; - this.prompt = prompt; this.next = next; } @@ -58,7 +52,7 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun if(executionCount <= MAX_RETRIES) { try { log.warn(String.format("Invalidate OAuth tokens due to failed precondition %s", response)); - service.save(service.authorize(bookmark, prompt, new DisabledCancelCallback())); + service.save(service.authorize()); // Try again return true; } diff --git a/dracoon/src/main/java/ch/cyberduck/core/sds/SDSSession.java b/dracoon/src/main/java/ch/cyberduck/core/sds/SDSSession.java index ddcf26545f7..3d1d9a2d056 100644 --- a/dracoon/src/main/java/ch/cyberduck/core/sds/SDSSession.java +++ b/dracoon/src/main/java/ch/cyberduck/core/sds/SDSSession.java @@ -182,7 +182,7 @@ public void process(final HttpRequest request, final HttpContext context) throws throw new DefaultIOExceptionMappingService().map(e); } configuration.setServiceUnavailableRetryStrategy(new PreconditionFailedResponseInterceptor(host, authorizationService, prompt, - new OAuth2ErrorResponseInterceptor(host, authorizationService, prompt))); + new OAuth2ErrorResponseInterceptor(host, authorizationService))); configuration.addInterceptorLast(authorizationService); configuration.addInterceptorLast(new HttpRequestInterceptor() { @Override @@ -219,7 +219,7 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal LocaleFactory.localizedString("Your DRACOON environment is outdated and no longer works with this application. Please contact your administrator.", "SDS")); } } - final Credentials credentials = host.getCredentials(); + final Credentials credentials; // The provided token is valid for two hours, every usage resets this period to two full hours again. Logging off invalidates the token. switch(SDSProtocol.Authorization.valueOf(host.getProtocol().getAuthorization())) { case oauth: @@ -234,8 +234,10 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal log.warn(String.format("Failure to parse software version %s", version)); } } - authorizationService.authorize(host, prompt, cancel); + credentials = authorizationService.validate(); break; + default: + credentials = host.getCredentials(); } final UserAccount account; try { diff --git a/dropbox/src/main/java/ch/cyberduck/core/dropbox/DropboxSession.java b/dropbox/src/main/java/ch/cyberduck/core/dropbox/DropboxSession.java index eab838136a4..e39b5eb462c 100644 --- a/dropbox/src/main/java/ch/cyberduck/core/dropbox/DropboxSession.java +++ b/dropbox/src/main/java/ch/cyberduck/core/dropbox/DropboxSession.java @@ -79,7 +79,7 @@ protected CustomDbxRawClientV2 connect(final Proxy proxy, final HostKeyCallback .withRedirectUri(host.getProtocol().getOAuthRedirectUrl()) .withParameter("token_access_type", "offline"); configuration.addInterceptorLast(authorizationService); - configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService, prompt)); + configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService)); if(new HostPreferences(host).getBoolean("dropbox.limit.requests.enable")) { configuration.addInterceptorLast(new RateLimitingHttpRequestInterceptor(new DefaultHttpRateLimiter( new HostPreferences(host).getInteger("dropbox.limit.requests.second") @@ -94,13 +94,12 @@ protected CustomDbxRawClientV2 connect(final Proxy proxy, final HostKeyCallback @Override public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - authorizationService.authorize(host, prompt, cancel); try { + final Credentials credentials = authorizationService.validate(); final FullAccount account = new DbxUserUsersRequests(client).getCurrentAccount(); if(log.isDebugEnabled()) { log.debug(String.format("Authenticated as user %s", account)); } - final Credentials credentials = host.getCredentials(); credentials.setUsername(account.getEmail()); switch(account.getAccountType()) { // The features listed below are only available to customers on Dropbox Professional, Standard, Advanced, and Enterprise. diff --git a/eue/src/main/java/ch/cyberduck/core/eue/EueSession.java b/eue/src/main/java/ch/cyberduck/core/eue/EueSession.java index a86a4c980c2..50caf8a77f3 100644 --- a/eue/src/main/java/ch/cyberduck/core/eue/EueSession.java +++ b/eue/src/main/java/ch/cyberduck/core/eue/EueSession.java @@ -22,7 +22,6 @@ import ch.cyberduck.core.HostKeyCallback; import ch.cyberduck.core.ListService; import ch.cyberduck.core.LoginCallback; -import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.Path; import ch.cyberduck.core.PathNormalizer; import ch.cyberduck.core.UrlProvider; @@ -31,7 +30,6 @@ import ch.cyberduck.core.eue.io.swagger.client.api.UserInfoApi; import ch.cyberduck.core.eue.io.swagger.client.model.UserSharesModel; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.InteroperabilityException; import ch.cyberduck.core.exception.NotfoundException; import ch.cyberduck.core.features.*; import ch.cyberduck.core.http.DefaultHttpRateLimiter; @@ -110,7 +108,7 @@ public void process(final HttpRequest request, final HttpContext context) { }).build(), host, prompt) .withRedirectUri(host.getProtocol().getOAuthRedirectUrl() ); - configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService, prompt)); + configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService)); configuration.addInterceptorLast(authorizationService); configuration.addInterceptorLast(new HttpRequestInterceptor() { @Override @@ -173,17 +171,7 @@ public void progress(final Integer seconds) { @Override public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - try { - authorizationService.refresh(authorizationService.authorize(host, prompt, cancel)); - } - catch(InteroperabilityException e) { - // Perm.INVALID_GRANT - log.warn(String.format("Failure %s refreshing OAuth tokens", e)); - // Reset OAuth Tokens - host.getCredentials().setOauth(OAuthTokens.EMPTY); - authorizationService.authorize(host, prompt, cancel - ); - } + final Credentials credentials = authorizationService.validate(); try { final StringBuilder url = new StringBuilder(); url.append(host.getProtocol().getScheme().toString()).append("://"); @@ -213,7 +201,6 @@ public void login(final Proxy proxy, final LoginCallback prompt, final CancelCal throw new DefaultHttpResponseExceptionMappingService().map(new HttpResponseException( response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase())); } - final Credentials credentials = host.getCredentials(); credentials.setUsername(new UserInfoApi(new EueApiClient(this)) .userinfoGet(null, null).getAccount().getOsServiceId()); if(StringUtils.isNotBlank(host.getProperty("pacs.url"))) { diff --git a/googledrive/src/main/java/ch/cyberduck/core/googledrive/DriveSession.java b/googledrive/src/main/java/ch/cyberduck/core/googledrive/DriveSession.java index 6484828351b..e957e152094 100644 --- a/googledrive/src/main/java/ch/cyberduck/core/googledrive/DriveSession.java +++ b/googledrive/src/main/java/ch/cyberduck/core/googledrive/DriveSession.java @@ -27,7 +27,6 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ConnectionCanceledException; import ch.cyberduck.core.exception.HostParserException; -import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.features.*; import ch.cyberduck.core.http.DefaultHttpRateLimiter; import ch.cyberduck.core.http.HttpSession; @@ -75,7 +74,7 @@ protected Drive connect(final Proxy proxy, final HostKeyCallback callback, final authorizationService = new OAuth2RequestInterceptor(builder.build(ProxyFactory.get().find(host.getProtocol().getOAuthAuthorizationUrl()), this, prompt).build(), host, prompt) .withRedirectUri(host.getProtocol().getOAuthRedirectUrl()); configuration.addInterceptorLast(authorizationService); - configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService, prompt)); + configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService)); if(new HostPreferences(host).getBoolean("googledrive.limit.requests.enable")) { configuration.addInterceptorLast(new RateLimitingHttpRequestInterceptor(new DefaultHttpRateLimiter( new HostPreferences(host).getInteger("googledrive.limit.requests.second") @@ -106,8 +105,7 @@ public void initialize(final HttpRequest request) { @Override public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - authorizationService.authorize(host, prompt, cancel); - final Credentials credentials = host.getCredentials(); + final Credentials credentials = authorizationService.validate(); final About about; try { about = client.about().get().setFields("user").execute(); diff --git a/googlestorage/src/main/java/ch/cyberduck/core/googlestorage/GoogleStorageSession.java b/googlestorage/src/main/java/ch/cyberduck/core/googlestorage/GoogleStorageSession.java index bc3f4f9148b..d3fb16e4b51 100644 --- a/googlestorage/src/main/java/ch/cyberduck/core/googlestorage/GoogleStorageSession.java +++ b/googlestorage/src/main/java/ch/cyberduck/core/googlestorage/GoogleStorageSession.java @@ -15,6 +15,7 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.Credentials; import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostKeyCallback; @@ -26,7 +27,6 @@ import ch.cyberduck.core.cdn.DistributionConfiguration; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ConnectionCanceledException; -import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.features.*; import ch.cyberduck.core.http.HttpSession; import ch.cyberduck.core.http.UserAgentHttpRequestInitializer; @@ -70,18 +70,17 @@ protected Storage connect(final Proxy proxy, final HostKeyCallback key, final Lo authorizationService = new OAuth2RequestInterceptor(builder.build(ProxyFactory.get().find(host.getProtocol().getOAuthAuthorizationUrl()), this, prompt).build(), host, prompt) .withRedirectUri(host.getProtocol().getOAuthRedirectUrl()); configuration.addInterceptorLast(authorizationService); - configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService, prompt)); + configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService)); transport = new ApacheHttpTransport(configuration.build()); final UseragentProvider ua = new PreferencesUseragentProvider(); return new Storage.Builder(transport, new GsonFactory(), new UserAgentHttpRequestInitializer(ua)) - .setApplicationName(ua.get()) - .build(); + .setApplicationName(ua.get()) + .build(); } @Override - public void login(final Proxy proxy, final LoginCallback prompt, - final CancelCallback cancel) throws BackgroundException { - authorizationService.authorize(host, prompt, cancel); + public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { + authorizationService.validate(); } @Override diff --git a/hubic/src/main/java/ch/cyberduck/core/hubic/HubicSession.java b/hubic/src/main/java/ch/cyberduck/core/hubic/HubicSession.java index 175bd2ab95b..bab8e5b999f 100644 --- a/hubic/src/main/java/ch/cyberduck/core/hubic/HubicSession.java +++ b/hubic/src/main/java/ch/cyberduck/core/hubic/HubicSession.java @@ -15,6 +15,7 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.Credentials; import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostKeyCallback; @@ -23,7 +24,6 @@ import ch.cyberduck.core.cdn.DistributionConfiguration; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ConnectionCanceledException; -import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.oauth.OAuth2ErrorResponseInterceptor; import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; import ch.cyberduck.core.openstack.SwiftExceptionMappingService; @@ -57,18 +57,18 @@ protected Client connect(final Proxy proxy, final HostKeyCallback key, final Log authorizationService = new OAuth2RequestInterceptor(configuration.build(), host, prompt) .withRedirectUri(host.getProtocol().getOAuthRedirectUrl()); configuration.addInterceptorLast(authorizationService); - configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService, prompt)); + configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService)); return new Client(configuration.build()); } @Override public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - final OAuthTokens tokens = authorizationService.authorize(host, prompt, cancel); + final Credentials credentials = authorizationService.validate(); try { if(log.isInfoEnabled()) { - log.info(String.format("Attempt authentication with %s", tokens)); + log.info(String.format("Attempt authentication with %s", credentials.getOauth())); } - client.authenticate(new HubicAuthenticationRequest(tokens.getAccessToken()), new HubicAuthenticationResponseHandler()); + client.authenticate(new HubicAuthenticationRequest(credentials.getOauth().getAccessToken()), new HubicAuthenticationResponseHandler()); } catch(GenericException e) { throw new SwiftExceptionMappingService().map(e); diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java index c57aeae6c5d..e5a6f7bb376 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2AuthorizationService.java @@ -23,6 +23,7 @@ import ch.cyberduck.core.LoginCallback; import ch.cyberduck.core.LoginOptions; import ch.cyberduck.core.OAuthTokens; +import ch.cyberduck.core.PasswordCallback; import ch.cyberduck.core.PreferencesUseragentProvider; import ch.cyberduck.core.Profile; import ch.cyberduck.core.StringAppender; @@ -32,7 +33,6 @@ import ch.cyberduck.core.http.DefaultHttpResponseExceptionMappingService; import ch.cyberduck.core.http.UserAgentHttpRequestInitializer; import ch.cyberduck.core.preferences.PreferencesFactory; -import ch.cyberduck.core.threading.CancelCallback; import org.apache.commons.lang3.StringUtils; import org.apache.http.client.HttpClient; @@ -70,8 +70,11 @@ public class OAuth2AuthorizationService { private final JsonFactory json = new GsonFactory(); + private final Host host; + private final Credentials credentials; private final String tokenServerUrl; private final String authorizationServerUrl; + private final LoginCallback prompt; private final String clientid; private final String clientsecret; @@ -92,17 +95,22 @@ public class OAuth2AuthorizationService { public OAuth2AuthorizationService(final HttpClient client, final Host host, final String tokenServerUrl, final String authorizationServerUrl, - final String clientid, final String clientsecret, final List scopes, final boolean pkce, final LoginCallback prompt) throws LoginCanceledException { + final String clientid, final String clientsecret, final List scopes, final boolean pkce, + final LoginCallback prompt) throws LoginCanceledException { this(new ApacheHttpTransport(client), host, tokenServerUrl, authorizationServerUrl, clientid, clientsecret, scopes, pkce, prompt); } public OAuth2AuthorizationService(final HttpTransport transport, final Host host, final String tokenServerUrl, final String authorizationServerUrl, - final String clientid, final String clientsecret, final List scopes, final boolean pkce, final LoginCallback prompt) throws LoginCanceledException { + final String clientid, final String clientsecret, final List scopes, final boolean pkce, + final LoginCallback prompt) throws LoginCanceledException { this.transport = transport; + this.host = host; + this.credentials = host.getCredentials(); this.tokenServerUrl = tokenServerUrl; this.authorizationServerUrl = authorizationServerUrl; + this.prompt = prompt; this.clientid = prompt(host, prompt, Profile.OAUTH_CLIENT_ID_KEY, LocaleFactory.localizedString( Profile.OAUTH_CLIENT_ID_KEY, "Credentials"), clientid); this.clientsecret = prompt(host, prompt, Profile.OAUTH_CLIENT_SECRET_KEY, LocaleFactory.localizedString( @@ -111,38 +119,49 @@ public OAuth2AuthorizationService(final HttpTransport transport, final Host host this.pkce = pkce; } - public OAuthTokens authorize(final Host bookmark, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - final Credentials credentials = bookmark.getCredentials(); + /** + * Authorize when cached tokens expired otherwise return + * @return Tokens retrieved + */ + public Credentials validate() throws BackgroundException { final OAuthTokens saved = credentials.getOauth(); if(saved.validate()) { // Found existing tokens if(saved.isExpired()) { - log.warn(String.format("Refresh expired access tokens %s", saved)); + if(log.isWarnEnabled()) { + log.warn(String.format("Refresh expired access tokens %s", saved)); + } // Refresh expired access key try { - return credentials.withOauth(this.refresh(saved)).withSaved(new LoginOptions().keychain).getOauth(); + return credentials.withOauth(this.refresh(saved)); } catch(LoginFailureException e) { - log.warn(String.format("Failure refreshing tokens from %s for %s", saved, bookmark)); + log.warn(String.format("Failure refreshing tokens from %s for %s", saved, host)); // Continue with new OAuth 2 flow } } else { if(log.isDebugEnabled()) { - log.debug(String.format("Returned saved OAuth tokens %s for %s", saved, bookmark)); + log.debug(String.format("Returned saved OAuth tokens %s for %s", saved, host)); } - return saved; + return credentials; } } + return credentials.withOauth(this.authorize()); + } + + /** + * @return Tokens retrieved + */ + public OAuthTokens authorize() throws BackgroundException { if(log.isDebugEnabled()) { - log.debug(String.format("Start new OAuth flow for %s with missing access token", bookmark)); + log.debug(String.format("Start new OAuth flow for %s with missing access token", host)); } - final IdTokenResponse response; // Save access token, refresh token and id token switch(flowType) { case AuthorizationCode: - response = this.authorizeWithCode(bookmark, prompt); + response = this.authorizeWithCode(host, prompt); break; case PasswordGrant: response = this.authorizeWithPassword(credentials); @@ -150,11 +169,10 @@ public OAuthTokens authorize(final Host bookmark, final LoginCallback prompt, fi default: throw new LoginCanceledException(); } - return credentials.withOauth(new OAuthTokens( - response.getAccessToken(), response.getRefreshToken(), - null == response.getExpiresInSeconds() ? Long.MAX_VALUE : - System.currentTimeMillis() + response.getExpiresInSeconds() * 1000, response.getIdToken())) - .withSaved(new LoginOptions().keychain).getOauth(); + return new OAuthTokens( + response.getAccessToken(), response.getRefreshToken(), + null == response.getExpiresInSeconds() ? Long.MAX_VALUE : + System.currentTimeMillis() + response.getExpiresInSeconds() * 1000, response.getIdToken()); } private IdTokenResponse authorizeWithCode(final Host bookmark, final LoginCallback prompt) throws BackgroundException { @@ -372,7 +390,7 @@ public IdTokenResponse toTokenResponse() { /** * Prompt for value if missing */ - private static String prompt(final Host bookmark, final LoginCallback prompt, + private static String prompt(final Host bookmark, final PasswordCallback prompt, final String property, final String message, final String value) throws LoginCanceledException { if(null == value) { final Credentials input = prompt.prompt(bookmark, message, diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java index 80377502028..3c674dc7265 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2ErrorResponseInterceptor.java @@ -15,13 +15,8 @@ * GNU General Public License for more details. */ -import ch.cyberduck.core.DisabledCancelCallback; import ch.cyberduck.core.Host; -import ch.cyberduck.core.LoginCallback; -import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.InteroperabilityException; -import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.http.DisabledServiceUnavailableRetryStrategy; import org.apache.http.HttpResponse; @@ -35,16 +30,10 @@ public class OAuth2ErrorResponseInterceptor extends DisabledServiceUnavailableRe private static final int MAX_RETRIES = 1; - private final Host bookmark; private final OAuth2RequestInterceptor service; - private final LoginCallback prompt; - public OAuth2ErrorResponseInterceptor(final Host bookmark, - final OAuth2RequestInterceptor service, - final LoginCallback prompt) { - this.bookmark = bookmark; + public OAuth2ErrorResponseInterceptor(final Host host, final OAuth2RequestInterceptor service) { this.service = service; - this.prompt = prompt; } @Override @@ -53,7 +42,10 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun case HttpStatus.SC_UNAUTHORIZED: if(executionCount <= MAX_RETRIES) { try { - this.refresh(response); + if(log.isWarnEnabled()) { + log.warn(String.format("Attempt to refresh OAuth tokens for failure %s", response)); + } + service.save(service.refresh()); // Try again return true; } @@ -68,17 +60,4 @@ public boolean retryRequest(final HttpResponse response, final int executionCoun } return false; } - - protected OAuthTokens refresh(final HttpResponse response) throws BackgroundException { - OAuthTokens tokens; - try { - log.warn(String.format("Attempt to refresh OAuth tokens for failure %s", response)); - tokens = service.refresh(); - } - catch(LoginFailureException e) { - log.warn(String.format("Failure %s refreshing OAuth tokens", e)); - tokens = service.authorize(bookmark, prompt, new DisabledCancelCallback()); - } - return service.save(tokens); - } } diff --git a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2RequestInterceptor.java b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2RequestInterceptor.java index 9cb9e53317a..cbc0ec3f3bf 100644 --- a/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2RequestInterceptor.java +++ b/oauth/src/main/java/ch/cyberduck/core/oauth/OAuth2RequestInterceptor.java @@ -25,7 +25,7 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LocalAccessDeniedException; import ch.cyberduck.core.exception.LoginCanceledException; -import ch.cyberduck.core.threading.CancelCallback; +import ch.cyberduck.core.exception.LoginFailureException; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpException; @@ -73,17 +73,32 @@ public OAuth2RequestInterceptor(final HttpClient client, final Host host, final } @Override - public OAuthTokens authorize(final Host bookmark, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - return tokens = super.authorize(bookmark, prompt, cancel); + public OAuthTokens authorize() throws BackgroundException { + return tokens = super.authorize(); } + /** + * Refresh with cached refresh token + */ public OAuthTokens refresh() throws BackgroundException { return tokens = this.refresh(tokens); } + /** + * + * @param previous Refresh token + */ @Override public OAuthTokens refresh(final OAuthTokens previous) throws BackgroundException { - return tokens = super.refresh(previous); + try { + return tokens = super.refresh(previous); + } + catch(LoginFailureException e) { + if(log.isWarnEnabled()) { + log.warn(String.format("Failure %s refreshing OAuth tokens", e)); + } + return tokens = this.authorize(); + } } /** diff --git a/onedrive/src/main/java/ch/cyberduck/core/onedrive/GraphSession.java b/onedrive/src/main/java/ch/cyberduck/core/onedrive/GraphSession.java index 467e80b33af..0018096559b 100644 --- a/onedrive/src/main/java/ch/cyberduck/core/onedrive/GraphSession.java +++ b/onedrive/src/main/java/ch/cyberduck/core/onedrive/GraphSession.java @@ -26,7 +26,6 @@ import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ConnectionCanceledException; import ch.cyberduck.core.exception.HostParserException; -import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.features.*; import ch.cyberduck.core.http.HttpSession; import ch.cyberduck.core.oauth.OAuth2ErrorResponseInterceptor; @@ -140,7 +139,7 @@ public void process(final HttpRequest request, final HttpContext context) throws }.withRedirectUri(host.getProtocol().getOAuthRedirectUrl()) .withParameter("prompt", "select_account"); configuration.addInterceptorLast(authorizationService); - configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService, prompt)); + configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService)); final RequestExecutor executor = new GraphCommonsHttpRequestExecutor(configuration.build()) { @Override public void addAuthorizationHeader(final Set headers) { @@ -185,14 +184,13 @@ public String getEmailURL() { @Override public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { - authorizationService.authorize(host, prompt, cancel); + final Credentials credentials = authorizationService.validate(); try { user = Users.get(User.getCurrent(client), new ODataQuery().select(User.Select.values())); final String account = user.getUserPrincipalName(); if(log.isDebugEnabled()) { log.debug(String.format("Authenticated as user %s", account)); } - final Credentials credentials = host.getCredentials(); credentials.setUsername(account); } catch(OneDriveAPIException e) { diff --git a/owncloud/src/main/java/ch/cyberduck/core/owncloud/OwncloudSession.java b/owncloud/src/main/java/ch/cyberduck/core/owncloud/OwncloudSession.java index 8761ac9b5fb..83f6d07198f 100644 --- a/owncloud/src/main/java/ch/cyberduck/core/owncloud/OwncloudSession.java +++ b/owncloud/src/main/java/ch/cyberduck/core/owncloud/OwncloudSession.java @@ -22,7 +22,6 @@ import ch.cyberduck.core.dav.DAVSession; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ConnectionCanceledException; -import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.features.AttributesFinder; import ch.cyberduck.core.features.Delete; import ch.cyberduck.core.features.Home; @@ -69,7 +68,7 @@ protected HttpClientBuilder getConfiguration(final Proxy proxy, final LoginCallb .withFlowType(OAuth2AuthorizationService.FlowType.valueOf(host.getProtocol().getAuthorization())) .withRedirectUri(host.getProtocol().getOAuthRedirectUrl()); configuration.addInterceptorLast(authorizationService); - configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService, prompt)); + configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService)); } return configuration; } @@ -77,7 +76,7 @@ protected HttpClientBuilder getConfiguration(final Proxy proxy, final LoginCallb @Override public void login(final Proxy proxy, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { if(host.getProtocol().isOAuthConfigurable()) { - authorizationService.authorize(host, prompt, cancel); + authorizationService.validate(); } super.login(proxy, prompt, cancel); } diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 062aa648130..62d29e5b3f8 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -53,9 +53,10 @@ public class STSAssumeRoleAuthorizationService { private final AWSSecurityTokenService service; private final LoginCallback prompt; + private final Host bookmark; public STSAssumeRoleAuthorizationService(final Host bookmark, final X509TrustManager trust, final X509KeyManager key, final LoginCallback prompt) { - this(AWSSecurityTokenServiceClientBuilder + this(bookmark, AWSSecurityTokenServiceClientBuilder .standard() .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(bookmark.getProtocol().getSTSEndpoint(), null)) .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) @@ -64,12 +65,13 @@ public STSAssumeRoleAuthorizationService(final Host bookmark, final X509TrustMan .build(), prompt); } - public STSAssumeRoleAuthorizationService(final AWSSecurityTokenService service, final LoginCallback prompt) { + public STSAssumeRoleAuthorizationService(final Host bookmark, final AWSSecurityTokenService service, final LoginCallback prompt) { + this.bookmark = bookmark; this.service = service; this.prompt = prompt; } - public STSTokens authorize(final Host bookmark, final String sAMLAssertion) throws BackgroundException { + public STSTokens authorize(final String sAMLAssertion) throws BackgroundException { final AssumeRoleWithSAMLRequest request = new AssumeRoleWithSAMLRequest().withSAMLAssertion(sAMLAssertion); final HostPreferences preferences = new HostPreferences(bookmark); if(preferences.getInteger("s3.assumerole.durationseconds") != -1) { @@ -99,13 +101,12 @@ public STSTokens authorize(final Host bookmark, final String sAMLAssertion) thro } } - public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws BackgroundException { + public STSTokens authorize(final OAuthTokens oauth) throws BackgroundException { final AssumeRoleWithWebIdentityRequest request = new AssumeRoleWithWebIdentityRequest(); - final String token = oauth.getIdToken(); if(log.isDebugEnabled()) { log.debug(String.format("Assume role with OIDC Id token for %s", bookmark)); } - request.setWebIdentityToken(token); + request.setWebIdentityToken(oauth.getIdToken()); final HostPreferences preferences = new HostPreferences(bookmark); if(preferences.getInteger("s3.assumerole.durationseconds") != -1) { request.setDurationSeconds(preferences.getInteger("s3.assumerole.durationseconds")); @@ -134,10 +135,10 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws } final String sub; try { - sub = JWT.decode(token).getSubject(); + sub = JWT.decode(oauth.getIdToken()).getSubject(); } catch(JWTDecodeException e) { - log.warn(String.format("Failure %s decoding JWT %s", e, token)); + log.warn(String.format("Failure %s decoding JWT %s", e, oauth.getIdToken())); throw new LoginFailureException("Invalid JWT or JSON format in authentication token", e); } if(StringUtils.isNotBlank(preferences.getProperty("s3.assumerole.rolesessionname"))) { @@ -148,7 +149,7 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws request.setRoleSessionName(sub); } else { - log.warn(String.format("Missing subject in decoding JWT %s", token)); + log.warn(String.format("Missing subject in decoding JWT %s", oauth.getIdToken())); request.setRoleSessionName(new AsciiRandomStringService().random()); } } @@ -160,13 +161,10 @@ public STSTokens authorize(final Host bookmark, final OAuthTokens oauth) throws if(log.isDebugEnabled()) { log.debug(String.format("Received assume role identity result %s", result)); } - final Credentials credentials = bookmark.getCredentials(); - return credentials - .withUsername(sub) - .withTokens(new STSTokens(result.getCredentials().getAccessKeyId(), - result.getCredentials().getSecretAccessKey(), - result.getCredentials().getSessionToken(), - result.getCredentials().getExpiration().getTime())).getTokens(); + return new STSTokens(result.getCredentials().getAccessKeyId(), + result.getCredentials().getSecretAccessKey(), + result.getCredentials().getSessionToken(), + result.getCredentials().getExpiration().getTime()); } catch(AWSSecurityTokenServiceException e) { throw new STSExceptionMappingService().map(e); diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java index 929f6c6904d..57bb1216ac9 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -19,10 +19,11 @@ import ch.cyberduck.core.Host; import ch.cyberduck.core.HostPasswordStore; import ch.cyberduck.core.LoginCallback; +import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.PasswordStoreFactory; import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.ExpiredTokenException; +import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; import ch.cyberduck.core.s3.S3CredentialsStrategy; import ch.cyberduck.core.s3.S3Session; @@ -40,6 +41,9 @@ import java.io.IOException; +import com.auth0.jwt.JWT; +import com.auth0.jwt.exceptions.JWTDecodeException; + /** * Swap OIDC Id token for temporary security credentials */ @@ -58,7 +62,6 @@ public class STSAssumeRoleCredentialsRequestInterceptor extends STSAssumeRoleAut private final OAuth2RequestInterceptor oauth; private final S3Session session; private final Host host; - private final LoginCallback prompt; private final CancelCallback cancel; public STSAssumeRoleCredentialsRequestInterceptor(final OAuth2RequestInterceptor oauth, final S3Session session, @@ -68,19 +71,19 @@ public STSAssumeRoleCredentialsRequestInterceptor(final OAuth2RequestInterceptor this.oauth = oauth; this.session = session; this.host = session.getHost(); - this.prompt = prompt; this.cancel = cancel; } - public STSTokens refresh() throws BackgroundException { + public STSTokens refresh(final OAuthTokens oidc) throws BackgroundException { try { - return this.tokens = this.authorize(host, oauth.authorize(host, prompt, cancel)); + return this.tokens = this.authorize(oidc); } - catch(ExpiredTokenException e) { + catch(LoginFailureException e) { + // Expired STS tokens if(log.isWarnEnabled()) { log.warn(String.format("Failure %s authorizing. Retry with refreshed OAuth tokens", e)); } - return this.tokens = this.authorize(host, oauth.refresh()); + return this.tokens = this.authorize(oauth.refresh(oidc)); } } @@ -88,7 +91,7 @@ public STSTokens refresh() throws BackgroundException { public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { if(tokens.isExpired()) { try { - this.refresh(); + this.refresh(oauth.getTokens()); if(log.isInfoEnabled()) { log.info(String.format("Authorizing service request with STS tokens %s", tokens)); } @@ -105,6 +108,19 @@ public void process(final HttpRequest request, final HttpContext context) throws @Override public Credentials get() throws BackgroundException { // Get temporary credentials from STS using Web Identity (OIDC) token - return host.getCredentials().withTokens(this.refresh()); + final Credentials credentials = oauth.validate(); + final OAuthTokens identity = credentials.getOauth(); + final String token = identity.getIdToken(); + final String sub; + try { + sub = JWT.decode(token).getSubject(); + } + catch(JWTDecodeException e) { + throw new LoginFailureException("Invalid JWT or JSON format in authentication token", e); + } + final STSTokens tokens = this.refresh(identity); + return credentials + .withUsername(sub) + .withTokens(tokens); } } diff --git a/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java b/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java index 9970e72be43..0230dd674c4 100644 --- a/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java +++ b/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java @@ -12,7 +12,7 @@ import ch.cyberduck.core.LoginConnectionService; import ch.cyberduck.core.Profile; import ch.cyberduck.core.ProtocolFactory; -import ch.cyberduck.core.Session; +import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.TranscriptListener; import ch.cyberduck.core.cdn.DistributionConfiguration; import ch.cyberduck.core.exception.BackgroundException; @@ -112,32 +112,6 @@ public void testConnectSessionTokenStatic() throws Exception { assertThrows(ExpiredTokenException.class, () -> session.login(new DisabledProxyFinder().find(host.getHostname()), new DisabledLoginCallback(), new DisabledCancelCallback())); } - @Test - public void testConnectSessionTokenFromService() throws Exception { - final S3Protocol protocol = new S3Protocol() { - @Override - public boolean isTokenConfigurable() { - return true; - } - }; - final Host host = new Host(protocol, protocol.getDefaultHostname(), new Credentials( - PROPERTIES.get("s3.key"), PROPERTIES.get("s3.secret") - )); - final S3Session session = new S3Session(host); - assertNotNull(session.open(new DisabledProxyFinder().find(host.getHostname()), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback())); - assertTrue(session.isConnected()); - assertNotNull(session.getClient()); - session.login(new DisabledProxyFinder().find(host.getHostname()), new DisabledLoginCallback(), new DisabledCancelCallback()); - assertTrue(session.isConnected()); - session.close(); - assertFalse(session.isConnected()); - assertEquals(Session.State.closed, session.getState()); - session.open(new DisabledProxyFinder().find(host.getHostname()), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - assertTrue(session.isConnected()); - session.close(); - assertFalse(session.isConnected()); - } - @Test public void testConnectDefaultPath() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); @@ -157,7 +131,7 @@ public void testConnectDefaultPath() throws Exception { public void testCustomHostnameUnknown() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/S3 (HTTPS).cyberduckprofile")); + this.getClass().getResourceAsStream("/S3 (HTTPS).cyberduckprofile")); final Host host = new Host(profile, "testu.cyberduck.ch", new Credentials( PROPERTIES.get("s3.key"), "s" )); @@ -254,12 +228,12 @@ public void verify(final String hostname, final X509Certificate[] certs, final S super.verify(hostname, certs, cipher); } }, - new KeychainX509KeyManager(new DisabledCertificateIdentityCallback(), host, new DisabledCertificateStore())); + new KeychainX509KeyManager(new DisabledCertificateIdentityCallback(), host, new DisabledCertificateStore())); final LoginConnectionService c = new LoginConnectionService( - new DisabledLoginCallback(), - new DisabledHostKeyCallback(), - new DisabledPasswordStore(), - new DisabledProgressListener() + new DisabledLoginCallback(), + new DisabledHostKeyCallback(), + new DisabledPasswordStore(), + new DisabledProgressListener() ); c.connect(session, new DisabledCancelCallback()); assertTrue(verified.get()); diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index ca7ae9d2c71..130dedf9f79 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -104,6 +104,7 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException assertTrue(oauth.validate()); final STSTokens tokens = host.getCredentials().getTokens(); assertTrue(tokens.validate()); + host.getCredentials().reset(); Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); @@ -111,8 +112,6 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException Thread.sleep(MILLIS * OAUTH_TTL_SECS + LAG); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - assertNotEquals(oauth, host.getCredentials().getOauth()); - assertNotEquals(tokens, host.getCredentials().getTokens()); session.close(); } diff --git a/storegate/src/main/java/ch/cyberduck/core/storegate/StoregateSession.java b/storegate/src/main/java/ch/cyberduck/core/storegate/StoregateSession.java index f7b60079198..1c6b2bfe5cd 100644 --- a/storegate/src/main/java/ch/cyberduck/core/storegate/StoregateSession.java +++ b/storegate/src/main/java/ch/cyberduck/core/storegate/StoregateSession.java @@ -28,7 +28,6 @@ import ch.cyberduck.core.Scheme; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.ConnectionCanceledException; -import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.features.*; import ch.cyberduck.core.http.HttpSession; @@ -114,7 +113,7 @@ public void process(final HttpRequest request, final HttpContext context) { .withParameter("login_hint", preferences.getProperty("storegate.login.hint")); // Force login even if browser session already exists authorizationService.withParameter("prompt", "login"); - configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService, prompt)); + configuration.setServiceUnavailableRetryStrategy(new OAuth2ErrorResponseInterceptor(host, authorizationService)); configuration.addInterceptorLast(authorizationService); final CloseableHttpClient apache = configuration.build(); final StoregateApiClient client = new StoregateApiClient(apache); @@ -135,7 +134,7 @@ public void process(final HttpRequest request, final HttpContext context) { @Override public void login(final Proxy proxy, final LoginCallback controller, final CancelCallback cancel) throws BackgroundException { - authorizationService.authorize(host, controller, cancel); + final Credentials credentials = authorizationService.validate(); try { final HttpRequestBase request = new HttpPost( new HostUrlProvider().withUsername(false).withPath(true).get( @@ -174,7 +173,6 @@ public void login(final Proxy proxy, final LoginCallback controller, final Cance finally { EntityUtils.consume(response.getEntity()); } - final Credentials credentials = host.getCredentials(); // Get username final ExtendedUser me = new UsersApi(client).usersGetMe(); if(log.isDebugEnabled()) { From 2bfb964f5805fba8d47d1a4104ad70f60fa5590a Mon Sep 17 00:00:00 2001 From: David Kocher Date: Tue, 15 Aug 2023 10:57:26 +0200 Subject: [PATCH 78/84] Delete long running tests. --- ...AbstractAssumeRoleWithWebIdentityTest.java | 101 +++++++++++++-- ...RoleWithWebIdentityAuthenticationTest.java | 11 +- ...eTokenExpiryFailsBecauseOfLatencyTest.java | 84 ------------- ...CredentialsExpiredValidOAuthTokenTest.java | 99 --------------- .../core/sts/STSOAuthExpiredValidSTSTest.java | 93 -------------- .../ch/cyberduck/core/sts/STSTestSetup.java | 117 ------------------ .../keycloak/keycloak-realm.json | 2 +- 7 files changed, 102 insertions(+), 405 deletions(-) delete mode 100644 s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java delete mode 100644 s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java delete mode 100644 s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java delete mode 100644 s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java index f558afd6cf4..71917ebe619 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java @@ -16,7 +16,11 @@ */ import ch.cyberduck.core.Profile; +import ch.cyberduck.core.ProtocolFactory; +import ch.cyberduck.core.exception.AccessDeniedException; import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.s3.S3Protocol; +import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.test.TestcontainerTest; import org.apache.logging.log4j.LogManager; @@ -25,19 +29,30 @@ import org.junit.ClassRule; import org.junit.experimental.categories.Category; import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; -import static ch.cyberduck.core.sts.STSTestSetup.*; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; @Category(TestcontainerTest.class) public abstract class AbstractAssumeRoleWithWebIdentityTest { protected static final Logger log = LogManager.getLogger(AbstractAssumeRoleWithWebIdentityTest.class); - protected static final int MILLIS = 1000; - - // lag to wait after token expiry - protected static final int LAG = 10 * MILLIS; - - protected static int OAUTH_TTL_SECS = 30; + protected static final long MILLIS = 1000; + protected static int OAUTH_TTL_SECS = 5; protected static Profile profile = null; @@ -48,4 +63,76 @@ public abstract class AbstractAssumeRoleWithWebIdentityTest { public void setup() throws BackgroundException { profile = readProfile(); } + + public static DockerComposeContainer prepareDockerComposeContainer(final String keyCloakRealmTempFile) { + log.info("Preparing docker compose container..."); + return new DockerComposeContainer<>( + new File(AbstractAssumeRoleWithWebIdentityTest.class.getResource("/testcontainer/docker-compose.yml").getFile())) + .withEnv("KEYCLOAK_REALM_JSON", keyCloakRealmTempFile) + .withPull(false) + .withLocalCompose(true) + .withOptions("--compatibility") + .withExposedService("keycloak_1", 8080, Wait.forListeningPort()) + .withExposedService("minio_1", 9000, Wait.forListeningPort()); + } + + public static String getKeyCloakFile(Map replacements) { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + JsonElement je = new Gson().fromJson(new InputStreamReader(AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/testcontainer/keycloak/keycloak-realm.json")), JsonElement.class); + JsonObject jo = je.getAsJsonObject(); + + for(Map.Entry replacement : replacements.entrySet()) { + updateJsonValues(jo, replacement.getKey(), replacement.getValue()); + } + + String content = gson.toJson(jo); + try { + final Path tempFile = Files.createTempFile(null, null); + Files.write(tempFile, content.getBytes(StandardCharsets.UTF_8)); + return tempFile.toAbsolutePath().toString(); + + } + catch(IOException e) { + throw new RuntimeException(e); + } + } + + private static void updateJsonValues(JsonObject jsonObj, String key, String newVal) { + for(Map.Entry entry : jsonObj.entrySet()) { + JsonElement element = entry.getValue(); + if(element.isJsonArray()) { + updateJsonValues(element.getAsJsonArray(), key, newVal); + } + else if(element.isJsonObject()) { + updateJsonValues(element.getAsJsonObject(), key, newVal); + } + else if(entry.getKey().equals(key)) { + jsonObj.remove(key); + jsonObj.addProperty(key, newVal); + break; + } + } + } + + private static void updateJsonValues(JsonArray asJsonArray, String key, String newVal) { + for(int index = 0; index < asJsonArray.size(); index++) { + JsonElement element = asJsonArray.get(index); + if(element.isJsonArray()) { + updateJsonValues(element.getAsJsonArray(), key, newVal); + } + else if(element.isJsonObject()) { + updateJsonValues(element.getAsJsonObject(), key, newVal); + } + } + } + + public static String getKeyCloakFile() { + return getKeyCloakFile(Collections.emptyMap()); + } + + public static Profile readProfile() throws AccessDeniedException { + final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); + return new ProfilePlistReader(factory).read( + AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); + } } \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index 130dedf9f79..34820d5ae6e 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -100,16 +100,19 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); - final OAuthTokens oauth = host.getCredentials().getOauth(); + final Credentials credentials = host.getCredentials(); + final OAuthTokens oauth = credentials.getOauth(); assertTrue(oauth.validate()); - final STSTokens tokens = host.getCredentials().getTokens(); + final STSTokens tokens = credentials.getTokens(); assertTrue(tokens.validate()); - host.getCredentials().reset(); Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - Thread.sleep(MILLIS * OAUTH_TTL_SECS + LAG); + Thread.sleep(MILLIS * OAUTH_TTL_SECS); + assertTrue(credentials.getOauth().isExpired()); + assertTrue(credentials.getTokens().isExpired()); + assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); session.close(); diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java b/s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java deleted file mode 100644 index 7ab7d13e252..00000000000 --- a/s3/src/test/java/ch/cyberduck/core/sts/STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest.java +++ /dev/null @@ -1,84 +0,0 @@ -package ch.cyberduck.core.sts; - -/* - * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. - * https://cyberduck.io/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -import ch.cyberduck.core.Credentials; -import ch.cyberduck.core.DisabledCancelCallback; -import ch.cyberduck.core.DisabledHostKeyCallback; -import ch.cyberduck.core.DisabledLoginCallback; -import ch.cyberduck.core.Host; -import ch.cyberduck.core.HostUrlProvider; -import ch.cyberduck.core.OAuthTokens; -import ch.cyberduck.core.Path; -import ch.cyberduck.core.Profile; -import ch.cyberduck.core.STSTokens; -import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.proxy.DisabledProxyFinder; -import ch.cyberduck.core.s3.S3AccessControlListFeature; -import ch.cyberduck.core.s3.S3FindFeature; -import ch.cyberduck.core.s3.S3Session; -import ch.cyberduck.test.TestcontainerTest; - -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.testcontainers.containers.DockerComposeContainer; - -import java.util.EnumSet; - -import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.MILLIS; -import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.OAUTH_TTL_SECS; -import static ch.cyberduck.core.sts.STSTestSetup.*; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertTrue; - -@Category(TestcontainerTest.class) -public class STSBucketRequestBeforeTokenExpiryFailsBecauseOfLatencyTest { - - @ClassRule - public static DockerComposeContainer compose = prepareDockerComposeContainer( - getKeyCloakFile() - ); - - @Test - @Ignore("Time of network latency may vary and so the time needs to be adjusted manually") - public void testBucketRequestBeforeTokenExpiryFailsBecauseOfLatency() throws BackgroundException, InterruptedException { - final Profile profile = readProfile(); - - final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); - final S3Session session = new S3Session(host); - session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); - - final OAuthTokens oauth = host.getCredentials().getOauth(); - assertTrue(oauth.validate()); - final STSTokens tokens = host.getCredentials().getTokens(); - assertTrue(tokens.validate()); - - // Time of latency may vary and so the time needs to be adjusted accordingly - final int NETWORK_LATENCY = 1180; - final int wait = OAUTH_TTL_SECS * MILLIS - NETWORK_LATENCY; - Thread.sleep(wait); - - Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - - assertNotEquals(oauth, host.getCredentials().getOauth()); - assertNotEquals(tokens, host.getCredentials().getTokens()); - session.close(); - } -} \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java b/s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java deleted file mode 100644 index cca31f93c9c..00000000000 --- a/s3/src/test/java/ch/cyberduck/core/sts/STSCredentialsExpiredValidOAuthTokenTest.java +++ /dev/null @@ -1,99 +0,0 @@ -package ch.cyberduck.core.sts; - -/* - * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. - * https://cyberduck.io/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -import ch.cyberduck.core.Credentials; -import ch.cyberduck.core.DisabledCancelCallback; -import ch.cyberduck.core.DisabledHostKeyCallback; -import ch.cyberduck.core.DisabledLoginCallback; -import ch.cyberduck.core.Host; -import ch.cyberduck.core.HostUrlProvider; -import ch.cyberduck.core.OAuthTokens; -import ch.cyberduck.core.Path; -import ch.cyberduck.core.Profile; -import ch.cyberduck.core.STSTokens; -import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.preferences.HostPreferences; -import ch.cyberduck.core.proxy.DisabledProxyFinder; -import ch.cyberduck.core.s3.S3AccessControlListFeature; -import ch.cyberduck.core.s3.S3FindFeature; -import ch.cyberduck.core.s3.S3Session; -import ch.cyberduck.test.TestcontainerTest; - -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.testcontainers.containers.DockerComposeContainer; - -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Map; - -import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.LAG; -import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.MILLIS; -import static ch.cyberduck.core.sts.STSTestSetup.*; -import static org.junit.Assert.*; - -@Category(TestcontainerTest.class) -public class STSCredentialsExpiredValidOAuthTokenTest { - - /** - * Adjust OAuth token TTL in Keycloak: - * "access.token.lifespan": "930" - * "ssoSessionMaxLifespan": 1100, - */ - private static Map overrideKeycloakDefaults() { - Map m = new HashMap<>(); - m.put("access.token.lifespan", Integer.toString(930)); - m.put("ssoSessionMaxLifespan", Integer.toString(1100)); - return m; - } - - @ClassRule - public static DockerComposeContainer compose = prepareDockerComposeContainer( - getKeyCloakFile(overrideKeycloakDefaults()) - ); - - @Test - @Ignore("Takes 15 minutes, skip by default") - public void testSTSCredentialsExpiredValidOAuthToken() throws BackgroundException, InterruptedException { - final Profile profile = readProfile(); - // 900 secs = 15 min is mininmum value: https://min.io/docs/minio/linux/developers/security-token-service/AssumeRoleWithWebIdentity.html - final int assumeRoleDurationSeconds = 900; - final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); - host.setProperty("s3.assumerole.durationseconds", Integer.toString(assumeRoleDurationSeconds)); - - assertEquals(new HostPreferences(host).getInteger("s3.assumerole.durationseconds"), assumeRoleDurationSeconds); - final S3Session session = new S3Session(host); - session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); - - final OAuthTokens oauth = host.getCredentials().getOauth(); - assertTrue(oauth.validate()); - final STSTokens tokens = host.getCredentials().getTokens(); - assertTrue(tokens.validate()); - - Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - - Thread.sleep(MILLIS * assumeRoleDurationSeconds + LAG); - - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - assertEquals(oauth, host.getCredentials().getOauth()); - assertNotEquals(tokens, host.getCredentials().getTokens()); - } -} \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java b/s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java deleted file mode 100644 index ae25559eec1..00000000000 --- a/s3/src/test/java/ch/cyberduck/core/sts/STSOAuthExpiredValidSTSTest.java +++ /dev/null @@ -1,93 +0,0 @@ -package ch.cyberduck.core.sts; - -/* - * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. - * https://cyberduck.io/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -import ch.cyberduck.core.Credentials; -import ch.cyberduck.core.DisabledCancelCallback; -import ch.cyberduck.core.DisabledHostKeyCallback; -import ch.cyberduck.core.DisabledLoginCallback; -import ch.cyberduck.core.Host; -import ch.cyberduck.core.HostUrlProvider; -import ch.cyberduck.core.OAuthTokens; -import ch.cyberduck.core.Path; -import ch.cyberduck.core.Profile; -import ch.cyberduck.core.STSTokens; -import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.proxy.DisabledProxyFinder; -import ch.cyberduck.core.s3.S3AccessControlListFeature; -import ch.cyberduck.core.s3.S3FindFeature; -import ch.cyberduck.core.s3.S3Session; -import ch.cyberduck.test.TestcontainerTest; - -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.testcontainers.containers.DockerComposeContainer; - -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Map; - -import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.LAG; -import static ch.cyberduck.core.sts.AbstractAssumeRoleWithWebIdentityTest.MILLIS; -import static ch.cyberduck.core.sts.STSTestSetup.*; -import static org.junit.Assert.*; -import static org.junit.Assert.assertNotEquals; - -@Category(TestcontainerTest.class) -public class STSOAuthExpiredValidSTSTest { - - private static final int OAUTH_TTL_SECS = 5; - - private static Map overrideKeycloakDefaults() { - Map m = new HashMap<>(); - m.put("access.token.lifespan", Integer.toString(OAUTH_TTL_SECS)); - return m; - } - - @ClassRule - public static DockerComposeContainer compose = prepareDockerComposeContainer( - getKeyCloakFile(overrideKeycloakDefaults()) - ); - - @Test - public void testOAuthExpiry() throws BackgroundException, InterruptedException { - final Profile profile = readProfile(); - final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); - final S3Session session = new S3Session(host); - session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); - session.login(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledLoginCallback(), new DisabledCancelCallback()); - - final Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - - final OAuthTokens oauth = host.getCredentials().getOauth(); - assertTrue(oauth.validate()); - assertFalse(oauth.isExpired()); - final STSTokens tokens = host.getCredentials().getTokens(); - assertTrue(tokens.validate()); - - Thread.sleep(MILLIS * OAUTH_TTL_SECS + LAG); - - assertTrue(host.getCredentials().getOauth().isExpired()); - assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - - assertNotEquals(oauth, host.getCredentials().getOauth()); - assertNotEquals(tokens, host.getCredentials().getTokens()); - - session.close(); - } -} \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java b/s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java deleted file mode 100644 index c87653ef928..00000000000 --- a/s3/src/test/java/ch/cyberduck/core/sts/STSTestSetup.java +++ /dev/null @@ -1,117 +0,0 @@ -package ch.cyberduck.core.sts;/* - * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. - * https://cyberduck.io/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -import ch.cyberduck.core.Profile; -import ch.cyberduck.core.ProtocolFactory; -import ch.cyberduck.core.exception.AccessDeniedException; -import ch.cyberduck.core.s3.S3Protocol; -import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.testcontainers.containers.DockerComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; - -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - -public class STSTestSetup { - private static final Logger log = LogManager.getLogger(STSTestSetup.class); - - public static DockerComposeContainer prepareDockerComposeContainer(final String keyCloakRealmTempFile) { - log.info("Preparing docker compose container..."); - return new DockerComposeContainer<>( - new File(STSTestSetup.class.getResource("/testcontainer/docker-compose.yml").getFile())) - .withEnv("KEYCLOAK_REALM_JSON", keyCloakRealmTempFile) - .withPull(false) - .withLocalCompose(true) - .withOptions("--compatibility") - .withExposedService("keycloak_1", 8080, Wait.forListeningPort()) - .withExposedService("minio_1", 9000, Wait.forListeningPort()); - } - - public static String getKeyCloakFile(Map replacements) { - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - JsonElement je = new Gson().fromJson(new InputStreamReader(STSTestSetup.class.getResourceAsStream("/testcontainer/keycloak/keycloak-realm.json")), JsonElement.class); - JsonObject jo = je.getAsJsonObject(); - - for(Map.Entry replacement : replacements.entrySet()) { - updateJsonValues(jo, replacement.getKey(), replacement.getValue()); - } - - String content = gson.toJson(jo); - try { - final Path tempFile = Files.createTempFile(null, null); - Files.write(tempFile, content.getBytes(StandardCharsets.UTF_8)); - return tempFile.toAbsolutePath().toString(); - - } - catch(IOException e) { - throw new RuntimeException(e); - } - } - - private static void updateJsonValues(JsonObject jsonObj, String key, String newVal) { - for(Map.Entry entry : jsonObj.entrySet()) { - JsonElement element = entry.getValue(); - if(element.isJsonArray()) { - updateJsonValues(element.getAsJsonArray(), key, newVal); - } - else if(element.isJsonObject()) { - updateJsonValues(element.getAsJsonObject(), key, newVal); - } - else if(entry.getKey().equals(key)) { - jsonObj.remove(key); - jsonObj.addProperty(key, newVal); - break; - } - } - } - - private static void updateJsonValues(JsonArray asJsonArray, String key, String newVal) { - for(int index = 0; index < asJsonArray.size(); index++) { - JsonElement element = asJsonArray.get(index); - if(element.isJsonArray()) { - updateJsonValues(element.getAsJsonArray(), key, newVal); - } - else if(element.isJsonObject()) { - updateJsonValues(element.getAsJsonObject(), key, newVal); - } - } - } - - public static String getKeyCloakFile() { - return getKeyCloakFile(Collections.emptyMap()); - } - - public static Profile readProfile() throws AccessDeniedException { - final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); - return new ProfilePlistReader(factory).read( - STSTestSetup.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); - } -} diff --git a/s3/src/test/resources/testcontainer/keycloak/keycloak-realm.json b/s3/src/test/resources/testcontainer/keycloak/keycloak-realm.json index 8434c40622c..fd3cb7f825a 100644 --- a/s3/src/test/resources/testcontainer/keycloak/keycloak-realm.json +++ b/s3/src/test/resources/testcontainer/keycloak/keycloak-realm.json @@ -122,7 +122,7 @@ "frontchannelLogout": false, "protocol": "openid-connect", "attributes": { - "access.token.lifespan": "30", + "access.token.lifespan": "5", "saml.force.post.binding": "false", "saml.multivalued.roles": "false", "oauth2.device.authorization.grant.enabled": "false", From 72f28290c251d156b39d6d9f0d4e11cadc03ea88 Mon Sep 17 00:00:00 2001 From: chenkins Date: Tue, 15 Aug 2023 23:14:40 +0200 Subject: [PATCH 79/84] Bugfix STS testcontainer tests (BeforeClass annotation in abstract test class not working as expected). Signed-off-by: chenkins --- .../AbstractAssumeRoleWithWebIdentityTest.java | 9 --------- ...sumeRoleWithWebIdentityAuthenticationTest.java | 15 +++++++++++---- ...ssumeRoleWithWebIdentityAuthorizationTest.java | 12 ++++++++++++ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java index 71917ebe619..16020cd0796 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java @@ -18,15 +18,12 @@ import ch.cyberduck.core.Profile; import ch.cyberduck.core.ProtocolFactory; import ch.cyberduck.core.exception.AccessDeniedException; -import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.s3.S3Protocol; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.test.TestcontainerTest; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.junit.Before; -import org.junit.ClassRule; import org.junit.experimental.categories.Category; import org.testcontainers.containers.DockerComposeContainer; import org.testcontainers.containers.wait.strategy.Wait; @@ -56,13 +53,7 @@ public abstract class AbstractAssumeRoleWithWebIdentityTest { protected static Profile profile = null; - @ClassRule - public static DockerComposeContainer compose = prepareDockerComposeContainer(getKeyCloakFile()); - @Before - public void setup() throws BackgroundException { - profile = readProfile(); - } public static DockerComposeContainer prepareDockerComposeContainer(final String keyCloakRealmTempFile) { log.info("Preparing docker compose container..."); diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index 34820d5ae6e..5d052ccd719 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -24,12 +24,9 @@ import ch.cyberduck.core.HostUrlProvider; import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.Path; -import ch.cyberduck.core.Profile; import ch.cyberduck.core.ProtocolFactory; import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.ExpiredTokenException; -import ch.cyberduck.core.exception.InteroperabilityException; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.s3.S3AccessControlListFeature; @@ -37,12 +34,14 @@ import ch.cyberduck.core.s3.S3FindFeature; import ch.cyberduck.core.s3.S3Protocol; import ch.cyberduck.core.s3.S3Session; -import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.test.TestcontainerTest; import org.jets3t.service.security.AWSSessionCredentials; +import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.testcontainers.containers.DockerComposeContainer; import org.testcontainers.shaded.org.apache.commons.lang3.StringUtils; import java.util.Collections; @@ -55,6 +54,14 @@ @Category(TestcontainerTest.class) public class AssumeRoleWithWebIdentityAuthenticationTest extends AbstractAssumeRoleWithWebIdentityTest { + @ClassRule + public static DockerComposeContainer compose = prepareDockerComposeContainer(getKeyCloakFile()); + + @Before + public void setup() throws BackgroundException { + profile = readProfile(); + } + @Test public void testSuccessfulLogin() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java index 5505a444983..9ce57068266 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java @@ -37,8 +37,11 @@ import ch.cyberduck.core.transfer.TransferStatus; import ch.cyberduck.test.TestcontainerTest; +import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.testcontainers.containers.DockerComposeContainer; import java.util.Collections; import java.util.EnumSet; @@ -48,6 +51,15 @@ @Category(TestcontainerTest.class) public class AssumeRoleWithWebIdentityAuthorizationTest extends AbstractAssumeRoleWithWebIdentityTest { + @ClassRule + public static DockerComposeContainer compose = prepareDockerComposeContainer(getKeyCloakFile()); + + @Before + public void setup() throws BackgroundException { + profile = readProfile(); + } + + @Test public void testAuthorizationFindBucket() throws BackgroundException { final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); From 595c12ee0b2571040d8ecd66d7dcfcd22a4198d4 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Fri, 18 Aug 2023 09:40:12 +0200 Subject: [PATCH 80/84] Review test setup. --- ...3OidcProfileTest.java => ProfileTest.java} | 4 ++-- ...AbstractAssumeRoleWithWebIdentityTest.java | 21 ++-------------- ...RoleWithWebIdentityAuthenticationTest.java | 24 ++++++++++++------- ...eRoleWithWebIdentityAuthorizationTest.java | 19 ++++++++++----- 4 files changed, 32 insertions(+), 36 deletions(-) rename s3/src/test/java/ch/cyberduck/core/{sts/S3OidcProfileTest.java => ProfileTest.java} (96%) diff --git a/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java b/s3/src/test/java/ch/cyberduck/core/ProfileTest.java similarity index 96% rename from s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java rename to s3/src/test/java/ch/cyberduck/core/ProfileTest.java index 7d5bbb6b60c..2169f8236e6 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/S3OidcProfileTest.java +++ b/s3/src/test/java/ch/cyberduck/core/ProfileTest.java @@ -1,4 +1,4 @@ -package ch.cyberduck.core.sts; +package ch.cyberduck.core; /* * Copyright (c) 2002-2023 iterate GmbH. All rights reserved. @@ -29,7 +29,7 @@ import static org.junit.Assert.*; -public class S3OidcProfileTest { +public class ProfileTest { @Test public void testDefaultProfile() throws Exception { diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java index 16020cd0796..c2b346d26c5 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java @@ -15,11 +15,6 @@ * GNU General Public License for more details. */ -import ch.cyberduck.core.Profile; -import ch.cyberduck.core.ProtocolFactory; -import ch.cyberduck.core.exception.AccessDeniedException; -import ch.cyberduck.core.s3.S3Protocol; -import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.test.TestcontainerTest; import org.apache.logging.log4j.LogManager; @@ -35,7 +30,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Collections; -import java.util.HashSet; import java.util.Map; import com.google.gson.Gson; @@ -48,12 +42,7 @@ public abstract class AbstractAssumeRoleWithWebIdentityTest { protected static final Logger log = LogManager.getLogger(AbstractAssumeRoleWithWebIdentityTest.class); - protected static final long MILLIS = 1000; - protected static int OAUTH_TTL_SECS = 5; - - protected static Profile profile = null; - - + protected static final int OAUTH_TTL_MILLIS = 5000; public static DockerComposeContainer prepareDockerComposeContainer(final String keyCloakRealmTempFile) { log.info("Preparing docker compose container..."); @@ -117,13 +106,7 @@ else if(element.isJsonObject()) { } } - public static String getKeyCloakFile() { + protected static String getKeyCloakFile() { return getKeyCloakFile(Collections.emptyMap()); } - - public static Profile readProfile() throws AccessDeniedException { - final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); - return new ProfilePlistReader(factory).read( - AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); - } } \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index 5d052ccd719..71ac117cc68 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -24,6 +24,7 @@ import ch.cyberduck.core.HostUrlProvider; import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.Path; +import ch.cyberduck.core.Protocol; import ch.cyberduck.core.ProtocolFactory; import ch.cyberduck.core.STSTokens; import ch.cyberduck.core.exception.BackgroundException; @@ -34,10 +35,10 @@ import ch.cyberduck.core.s3.S3FindFeature; import ch.cyberduck.core.s3.S3Protocol; import ch.cyberduck.core.s3.S3Session; +import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.test.TestcontainerTest; import org.jets3t.service.security.AWSSessionCredentials; -import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -57,13 +58,10 @@ public class AssumeRoleWithWebIdentityAuthenticationTest extends AbstractAssumeR @ClassRule public static DockerComposeContainer compose = prepareDockerComposeContainer(getKeyCloakFile()); - @Before - public void setup() throws BackgroundException { - profile = readProfile(); - } - @Test public void testSuccessfulLogin() throws BackgroundException { + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); final S3Session session = new S3Session(host); session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); @@ -84,6 +82,8 @@ public void testSuccessfulLogin() throws BackgroundException { @Test public void testInvalidUserName() throws BackgroundException { + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("WrongUsername", "rouser")); final S3Session session = new S3Session(host); session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); @@ -93,6 +93,8 @@ public void testInvalidUserName() throws BackgroundException { @Test public void testInvalidPassword() throws BackgroundException { + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "invalidPassword")); final S3Session session = new S3Session(host); session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); @@ -102,6 +104,8 @@ public void testInvalidPassword() throws BackgroundException { @Test public void testTokenRefresh() throws BackgroundException, InterruptedException { + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); final S3Session session = new S3Session(host); session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); @@ -116,7 +120,7 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); assertTrue(new S3FindFeature(session, new S3AccessControlListFeature(session)).find(container)); - Thread.sleep(MILLIS * OAUTH_TTL_SECS); + Thread.sleep(OAUTH_TTL_MILLIS); assertTrue(credentials.getOauth().isExpired()); assertTrue(credentials.getTokens().isExpired()); @@ -132,7 +136,8 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException */ @Test public void testLoginInvalidOAuthTokensLogin() throws Exception { - final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Credentials credentials = new Credentials("rouser", "rouser") .withOauth(new OAuthTokens( "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJDQmpaYUNSeU5USmZqV0VmMU1fZXZLRVliMEdGLXU0QzhjZ3RZYnBtZUlFIn0.eyJleHAiOjE2OTE5OTc3MzUsImlhdCI6MTY5MTk5NzcwNSwianRpIjoiNDA1MGUxMGYtNzZjNC00MjYwLTk1YTctZTMyMTE2YTA3N2NlIiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL3JlYWxtcy9jeWJlcmR1Y2tyZWFsbSIsInN1YiI6IjMzNGRiZWIwLTE5NWQtNDJhMS1hMWQ2LTEyODFmMDBiZmIxZCIsInR5cCI6IkJlYXJlciIsImF6cCI6Im1pbmlvIiwic2Vzc2lvbl9zdGF0ZSI6IjNkZDY0MDVlLTNkMGMtNDVjOS05MTZkLTllYTNkNWY1ODVkYiIsInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIiwidXNlciJdfSwic2NvcGUiOiJvcGVuaWQgbWluaW8tYXV0aG9yaXphdGlvbiIsInNpZCI6IjNkZDY0MDVlLTNkMGMtNDVjOS05MTZkLTllYTNkNWY1ODVkYiIsInBvbGljeSI6WyJyZWFkb25seSJdfQ.uKxLmSW6j2EQEo86j0WZOKWgavhS8Ub7TjrnynUi4m1ls0SchvgCilVpzIzNdFL9Y7khiqxl7si5BezbTLPgwyh4GDgrHcJwBk5D6aOcaH6hYcAtcbOiu1KEyfj1O_lwvDCHb-J07TIEeuvquOs2nD7FxqafHjLe-3pL6JuTtBtlx8WKloO9PY-Dn-ntuyqikr7ysLcDBfFJda487cmeTADxiMQ_MmoidW3uGXn0Ps6vhRgteUQO5JTKMa7MT1PKMTY8iNnSdNVuhKkBodnkXMSo5JEt4veqR9Yh-WPT_XL8caUiGInYvHty-n6-yhGhNckrlvtmJc0dJsts4hi1Mw", @@ -156,7 +161,8 @@ public void testLoginInvalidOAuthTokensLogin() throws Exception { */ @Test public void testBucketListInvalidOAuthTokensList() throws Exception { - final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol()))); + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Credentials credentials = new Credentials("rouser", "rouser") .withOauth(OAuthTokens.EMPTY) .withTokens(new STSTokens( diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java index 9ce57068266..d2918360725 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java @@ -24,6 +24,8 @@ import ch.cyberduck.core.Host; import ch.cyberduck.core.HostUrlProvider; import ch.cyberduck.core.Path; +import ch.cyberduck.core.Protocol; +import ch.cyberduck.core.ProtocolFactory; import ch.cyberduck.core.exception.AccessDeniedException; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Delete; @@ -31,9 +33,11 @@ import ch.cyberduck.core.s3.S3AccessControlListFeature; import ch.cyberduck.core.s3.S3DefaultDeleteFeature; import ch.cyberduck.core.s3.S3FindFeature; +import ch.cyberduck.core.s3.S3Protocol; import ch.cyberduck.core.s3.S3ReadFeature; import ch.cyberduck.core.s3.S3Session; import ch.cyberduck.core.s3.S3TouchFeature; +import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; import ch.cyberduck.test.TestcontainerTest; @@ -45,6 +49,7 @@ import java.util.Collections; import java.util.EnumSet; +import java.util.HashSet; import static org.junit.Assert.*; @@ -54,14 +59,10 @@ public class AssumeRoleWithWebIdentityAuthorizationTest extends AbstractAssumeRo @ClassRule public static DockerComposeContainer compose = prepareDockerComposeContainer(getKeyCloakFile()); - @Before - public void setup() throws BackgroundException { - profile = readProfile(); - } - - @Test public void testAuthorizationFindBucket() throws BackgroundException { + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); final S3Session session = new S3Session(host); session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); @@ -73,6 +74,8 @@ public void testAuthorizationFindBucket() throws BackgroundException { @Test public void testAuthorizationUserReadAccessOnBucket() throws BackgroundException { + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); final S3Session session = new S3Session(host); session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); @@ -85,6 +88,8 @@ public void testAuthorizationUserReadAccessOnBucket() throws BackgroundException @Test public void testAuthorizationWritePermissionOnBucket() throws BackgroundException { + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); final S3Session session = new S3Session(host); session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); @@ -100,6 +105,8 @@ public void testAuthorizationWritePermissionOnBucket() throws BackgroundExceptio @Test public void testAuthorizationNoWritePermissionOnBucket() throws BackgroundException { + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); final S3Session session = new S3Session(host); session.open(new DisabledProxyFinder().find(new HostUrlProvider().get(host)), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); From 7e572ca5e7b0b8632a1d10f84a6980eaea1f41a4 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 21 Aug 2023 13:32:03 +0200 Subject: [PATCH 81/84] Formatting. --- core/src/main/java/ch/cyberduck/core/Credentials.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/ch/cyberduck/core/Credentials.java b/core/src/main/java/ch/cyberduck/core/Credentials.java index 479d38b5687..031d7b10d95 100644 --- a/core/src/main/java/ch/cyberduck/core/Credentials.java +++ b/core/src/main/java/ch/cyberduck/core/Credentials.java @@ -346,11 +346,11 @@ public boolean equals(final Object o) { } final Credentials that = (Credentials) o; return Objects.equals(user, that.user) && - Objects.equals(password, that.password) && + Objects.equals(password, that.password) && Objects.equals(token, that.token) && - Objects.equals(tokens, that.tokens) && - Objects.equals(identity, that.identity) && - Objects.equals(certificate, that.certificate); + Objects.equals(tokens, that.tokens) && + Objects.equals(identity, that.identity) && + Objects.equals(certificate, that.certificate); } @Override From 8150cf584eac2c93d1c3a74ec8da26d53f6e4402 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 21 Aug 2023 13:35:11 +0200 Subject: [PATCH 82/84] Review hash code. --- core/src/main/java/ch/cyberduck/core/Credentials.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/ch/cyberduck/core/Credentials.java b/core/src/main/java/ch/cyberduck/core/Credentials.java index 031d7b10d95..dd798328109 100644 --- a/core/src/main/java/ch/cyberduck/core/Credentials.java +++ b/core/src/main/java/ch/cyberduck/core/Credentials.java @@ -349,13 +349,14 @@ public boolean equals(final Object o) { Objects.equals(password, that.password) && Objects.equals(token, that.token) && Objects.equals(tokens, that.tokens) && + Objects.equals(oauth, that.oauth) && Objects.equals(identity, that.identity) && Objects.equals(certificate, that.certificate); } @Override public int hashCode() { - return Objects.hash(user, password, tokens, identity, certificate); + return Objects.hash(user, password, token, tokens, oauth, identity, certificate); } @Override From bdfdcf07bf7e8f7fb426c9aa614e561e77f71ad7 Mon Sep 17 00:00:00 2001 From: chenkins Date: Tue, 22 Aug 2023 06:58:50 +0200 Subject: [PATCH 83/84] Remove obsolete keycloak.json manipulation before running AssumeRoleWithWebIdentity testcontainer tests. Signed-off-by: chenkins --- ...AbstractAssumeRoleWithWebIdentityTest.java | 68 +------------------ ...RoleWithWebIdentityAuthenticationTest.java | 2 +- ...eRoleWithWebIdentityAuthorizationTest.java | 11 ++- 3 files changed, 7 insertions(+), 74 deletions(-) diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java index c2b346d26c5..42edf038e80 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AbstractAssumeRoleWithWebIdentityTest.java @@ -24,19 +24,6 @@ import org.testcontainers.containers.wait.strategy.Wait; import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import java.util.Map; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; @Category(TestcontainerTest.class) public abstract class AbstractAssumeRoleWithWebIdentityTest { @@ -44,11 +31,10 @@ public abstract class AbstractAssumeRoleWithWebIdentityTest { protected static final int OAUTH_TTL_MILLIS = 5000; - public static DockerComposeContainer prepareDockerComposeContainer(final String keyCloakRealmTempFile) { + public static DockerComposeContainer prepareDockerComposeContainer() { log.info("Preparing docker compose container..."); return new DockerComposeContainer<>( new File(AbstractAssumeRoleWithWebIdentityTest.class.getResource("/testcontainer/docker-compose.yml").getFile())) - .withEnv("KEYCLOAK_REALM_JSON", keyCloakRealmTempFile) .withPull(false) .withLocalCompose(true) .withOptions("--compatibility") @@ -56,57 +42,5 @@ public static DockerComposeContainer prepareDockerComposeContainer(final String .withExposedService("minio_1", 9000, Wait.forListeningPort()); } - public static String getKeyCloakFile(Map replacements) { - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - JsonElement je = new Gson().fromJson(new InputStreamReader(AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/testcontainer/keycloak/keycloak-realm.json")), JsonElement.class); - JsonObject jo = je.getAsJsonObject(); - - for(Map.Entry replacement : replacements.entrySet()) { - updateJsonValues(jo, replacement.getKey(), replacement.getValue()); - } - - String content = gson.toJson(jo); - try { - final Path tempFile = Files.createTempFile(null, null); - Files.write(tempFile, content.getBytes(StandardCharsets.UTF_8)); - return tempFile.toAbsolutePath().toString(); - - } - catch(IOException e) { - throw new RuntimeException(e); - } - } - - private static void updateJsonValues(JsonObject jsonObj, String key, String newVal) { - for(Map.Entry entry : jsonObj.entrySet()) { - JsonElement element = entry.getValue(); - if(element.isJsonArray()) { - updateJsonValues(element.getAsJsonArray(), key, newVal); - } - else if(element.isJsonObject()) { - updateJsonValues(element.getAsJsonObject(), key, newVal); - } - else if(entry.getKey().equals(key)) { - jsonObj.remove(key); - jsonObj.addProperty(key, newVal); - break; - } - } - } - private static void updateJsonValues(JsonArray asJsonArray, String key, String newVal) { - for(int index = 0; index < asJsonArray.size(); index++) { - JsonElement element = asJsonArray.get(index); - if(element.isJsonArray()) { - updateJsonValues(element.getAsJsonArray(), key, newVal); - } - else if(element.isJsonObject()) { - updateJsonValues(element.getAsJsonObject(), key, newVal); - } - } - } - - protected static String getKeyCloakFile() { - return getKeyCloakFile(Collections.emptyMap()); - } } \ No newline at end of file diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index 71ac117cc68..0f82f2e1c4a 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -56,7 +56,7 @@ public class AssumeRoleWithWebIdentityAuthenticationTest extends AbstractAssumeRoleWithWebIdentityTest { @ClassRule - public static DockerComposeContainer compose = prepareDockerComposeContainer(getKeyCloakFile()); + public static DockerComposeContainer compose = prepareDockerComposeContainer(); @Test public void testSuccessfulLogin() throws BackgroundException { diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java index d2918360725..a82f1597e43 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthorizationTest.java @@ -41,7 +41,6 @@ import ch.cyberduck.core.transfer.TransferStatus; import ch.cyberduck.test.TestcontainerTest; -import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -57,11 +56,11 @@ public class AssumeRoleWithWebIdentityAuthorizationTest extends AbstractAssumeRoleWithWebIdentityTest { @ClassRule - public static DockerComposeContainer compose = prepareDockerComposeContainer(getKeyCloakFile()); + public static DockerComposeContainer compose = prepareDockerComposeContainer(); @Test public void testAuthorizationFindBucket() throws BackgroundException { - final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); final S3Session session = new S3Session(host); @@ -74,7 +73,7 @@ public void testAuthorizationFindBucket() throws BackgroundException { @Test public void testAuthorizationUserReadAccessOnBucket() throws BackgroundException { - final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); final S3Session session = new S3Session(host); @@ -88,7 +87,7 @@ public void testAuthorizationUserReadAccessOnBucket() throws BackgroundException @Test public void testAuthorizationWritePermissionOnBucket() throws BackgroundException { - final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rawuser", "rawuser")); final S3Session session = new S3Session(host); @@ -105,7 +104,7 @@ public void testAuthorizationWritePermissionOnBucket() throws BackgroundExceptio @Test public void testAuthorizationNoWritePermissionOnBucket() throws BackgroundException { - final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( + final Protocol profile = new ProfilePlistReader(new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())))).read( AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("rouser", "rouser")); final S3Session session = new S3Session(host); From a999e869398e9ab2f2ce8a0a514f9ee67c193810 Mon Sep 17 00:00:00 2001 From: David Kocher Date: Mon, 28 Aug 2023 12:09:01 +0200 Subject: [PATCH 84/84] Merge token ivar. --- .../java/ch/cyberduck/core/Credentials.java | 40 ++++++++----------- ...Tokens.java => TemporaryAccessTokens.java} | 18 ++++----- .../auth/AWSSessionCredentialsRetriever.java | 4 +- .../core/s3/S3CredentialsConfigurator.java | 10 ++--- .../STSAssumeRoleAuthorizationService.java | 10 ++--- ...sumeRoleCredentialsRequestInterceptor.java | 8 ++-- .../ch/cyberduck/core/s3/S3SessionTest.java | 4 +- ...RoleWithWebIdentityAuthenticationTest.java | 12 +++--- 8 files changed, 49 insertions(+), 57 deletions(-) rename core/src/main/java/ch/cyberduck/core/{STSTokens.java => TemporaryAccessTokens.java} (80%) diff --git a/core/src/main/java/ch/cyberduck/core/Credentials.java b/core/src/main/java/ch/cyberduck/core/Credentials.java index dd798328109..2d836455503 100644 --- a/core/src/main/java/ch/cyberduck/core/Credentials.java +++ b/core/src/main/java/ch/cyberduck/core/Credentials.java @@ -38,8 +38,7 @@ public class Credentials implements Comparable { * The login password */ private String password = StringUtils.EMPTY; - private String token = StringUtils.EMPTY; - private STSTokens tokens = STSTokens.EMPTY; + private TemporaryAccessTokens tokens = TemporaryAccessTokens.EMPTY; private OAuthTokens oauth = OAuthTokens.EMPTY; /** @@ -73,7 +72,6 @@ public Credentials() { public Credentials(final Credentials copy) { this.user = copy.user; this.password = copy.password; - this.token = copy.token; this.tokens = copy.tokens; this.oauth = copy.oauth; this.identity = copy.identity; @@ -99,7 +97,7 @@ public Credentials(final String user, final String password) { public Credentials(final String user, final String password, final String token) { this.user = user; this.password = password; - this.token = token; + this.tokens = new TemporaryAccessTokens(token); } /** @@ -115,8 +113,7 @@ public void setUsername(final String user) { } public Credentials withUsername(final String user) { - this.user = user; - this.passed = false; + this.setUsername((user)); return this; } @@ -138,38 +135,35 @@ public void setPassword(final String password) { } public Credentials withPassword(final String password) { - this.password = password; - this.passed = false; + this.setPassword(password); return this; } public String getToken() { - return token; + return tokens.getSessionToken(); } public void setToken(final String token) { - this.token = token; + this.tokens = new TemporaryAccessTokens(token); this.passed = false; } public Credentials withToken(final String token) { - this.token = token; - this.passed = false; + this.setToken(token); return this; } - public STSTokens getTokens() { + public TemporaryAccessTokens getTokens() { return tokens; } - public void setTokens(final STSTokens tokens) { + public void setTokens(final TemporaryAccessTokens tokens) { this.tokens = tokens; this.passed = false; } - public Credentials withTokens(final STSTokens tokens) { - this.tokens = tokens; - this.passed = false; + public Credentials withTokens(final TemporaryAccessTokens tokens) { + this.setTokens(tokens); return this; } @@ -182,7 +176,7 @@ public void setOauth(final OAuthTokens oauth) { } public Credentials withOauth(final OAuthTokens oauth) { - this.oauth = oauth; + this.setOauth(oauth); return this; } @@ -203,7 +197,7 @@ public void setSaved(final boolean saved) { } public Credentials withSaved(final boolean saved) { - this.saved = saved; + this.setSaved(saved); return this; } @@ -230,7 +224,7 @@ public boolean isPasswordAuthentication() { } public boolean isTokenAuthentication() { - return StringUtils.isNotBlank(token); + return StringUtils.isNotBlank(tokens.getSessionToken()); } public boolean isOAuthAuthentication() { @@ -317,7 +311,7 @@ public boolean validate(final Protocol protocol, final LoginOptions options) { public void reset() { this.setPassword(StringUtils.EMPTY); this.setToken(StringUtils.EMPTY); - this.setTokens(STSTokens.EMPTY); + this.setTokens(TemporaryAccessTokens.EMPTY); this.setOauth(OAuthTokens.EMPTY); this.setIdentityPassphrase(StringUtils.EMPTY); } @@ -347,7 +341,6 @@ public boolean equals(final Object o) { final Credentials that = (Credentials) o; return Objects.equals(user, that.user) && Objects.equals(password, that.password) && - Objects.equals(token, that.token) && Objects.equals(tokens, that.tokens) && Objects.equals(oauth, that.oauth) && Objects.equals(identity, that.identity) && @@ -356,7 +349,7 @@ public boolean equals(final Object o) { @Override public int hashCode() { - return Objects.hash(user, password, token, tokens, oauth, identity, certificate); + return Objects.hash(user, password, tokens, oauth, identity, certificate); } @Override @@ -364,7 +357,6 @@ public String toString() { final StringBuilder sb = new StringBuilder("Credentials{"); sb.append("user='").append(user).append('\''); sb.append(", password='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(password)))).append('\''); - sb.append(", token='").append(StringUtils.repeat("*", Integer.min(8, StringUtils.length(token)))).append('\''); sb.append(", tokens='").append(tokens).append('\''); sb.append(", oauth='").append(oauth).append('\''); sb.append(", identity=").append(identity); diff --git a/core/src/main/java/ch/cyberduck/core/STSTokens.java b/core/src/main/java/ch/cyberduck/core/TemporaryAccessTokens.java similarity index 80% rename from core/src/main/java/ch/cyberduck/core/STSTokens.java rename to core/src/main/java/ch/cyberduck/core/TemporaryAccessTokens.java index 4757845dc29..b3a4ca9c228 100644 --- a/core/src/main/java/ch/cyberduck/core/STSTokens.java +++ b/core/src/main/java/ch/cyberduck/core/TemporaryAccessTokens.java @@ -22,21 +22,21 @@ /** * Temporary access credentials */ -public final class STSTokens { +public final class TemporaryAccessTokens { - public static final STSTokens EMPTY - = new STSTokens(null, null, null, Long.MAX_VALUE); + public static final TemporaryAccessTokens EMPTY + = new TemporaryAccessTokens(null, null, null, Long.MAX_VALUE); private final String accessKeyId; private final String secretAccessKey; private final String sessionToken; private final Long expiryInMilliseconds; - public STSTokens(final String sessionToken) { + public TemporaryAccessTokens(final String sessionToken) { this(null, null, sessionToken, -1L); } - public STSTokens(final String accessKeyId, final String secretAccessKey, final String sessionToken, final Long expiryInMilliseconds) { + public TemporaryAccessTokens(final String accessKeyId, final String secretAccessKey, final String sessionToken, final Long expiryInMilliseconds) { this.accessKeyId = accessKeyId; this.secretAccessKey = secretAccessKey; this.sessionToken = sessionToken; @@ -75,14 +75,14 @@ public boolean equals(final Object o) { if(o == null || getClass() != o.getClass()) { return false; } - final STSTokens stsTokens = (STSTokens) o; - if(!Objects.equals(accessKeyId, stsTokens.accessKeyId)) { + final TemporaryAccessTokens temporaryAccessTokens = (TemporaryAccessTokens) o; + if(!Objects.equals(accessKeyId, temporaryAccessTokens.accessKeyId)) { return false; } - if(!Objects.equals(secretAccessKey, stsTokens.secretAccessKey)) { + if(!Objects.equals(secretAccessKey, temporaryAccessTokens.secretAccessKey)) { return false; } - if(!Objects.equals(sessionToken, stsTokens.sessionToken)) { + if(!Objects.equals(sessionToken, temporaryAccessTokens.sessionToken)) { return false; } return true; diff --git a/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java b/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java index 6cefc315bd0..9ee35193fba 100644 --- a/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java +++ b/s3/src/main/java/ch/cyberduck/core/auth/AWSSessionCredentialsRetriever.java @@ -27,7 +27,7 @@ import ch.cyberduck.core.Path; import ch.cyberduck.core.PathNormalizer; import ch.cyberduck.core.ProtocolFactory; -import ch.cyberduck.core.STSTokens; +import ch.cyberduck.core.TemporaryAccessTokens; import ch.cyberduck.core.TranscriptListener; import ch.cyberduck.core.date.ISO8601DateParser; import ch.cyberduck.core.date.InvalidDateException; @@ -165,7 +165,7 @@ protected Credentials parse(final InputStream in) throws BackgroundException { } } reader.endObject(); - return new Credentials().withTokens(new STSTokens(key, secret, token, expiration != null ? expiration.getTime() : -1L)); + return new Credentials().withTokens(new TemporaryAccessTokens(key, secret, token, expiration != null ? expiration.getTime() : -1L)); } catch(MalformedJsonException e) { throw new InteroperabilityException("Invalid JSON response", e); diff --git a/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java b/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java index 8ff754af9fb..406842511b0 100644 --- a/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java +++ b/s3/src/main/java/ch/cyberduck/core/s3/S3CredentialsConfigurator.java @@ -24,13 +24,13 @@ import ch.cyberduck.core.LocaleFactory; import ch.cyberduck.core.LoginOptions; import ch.cyberduck.core.PasswordCallback; +import ch.cyberduck.core.TemporaryAccessTokens; import ch.cyberduck.core.aws.CustomClientConfiguration; import ch.cyberduck.core.exception.AccessDeniedException; import ch.cyberduck.core.exception.LoginCanceledException; import ch.cyberduck.core.ssl.ThreadLocalHostnameDelegatingTrustManager; import ch.cyberduck.core.ssl.X509KeyManager; import ch.cyberduck.core.ssl.X509TrustManager; -import ch.cyberduck.core.STSTokens; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -214,7 +214,7 @@ else if(!profiles.containsKey(basicProfile.getRoleSourceProfile())) { if(log.isDebugEnabled()) { log.debug(String.format("Set credentials from %s", assumeRoleResult)); } - credentials.setTokens(new STSTokens( + credentials.setTokens(new TemporaryAccessTokens( assumeRoleResult.getCredentials().getAccessKeyId(), assumeRoleResult.getCredentials().getSecretAccessKey(), assumeRoleResult.getCredentials().getSessionToken(), @@ -237,7 +237,7 @@ else if(!profiles.containsKey(basicProfile.getRoleSourceProfile())) { if(null == cached) { return credentials; } - return credentials.withTokens(new STSTokens( + return credentials.withTokens(new TemporaryAccessTokens( cached.accessKey, cached.secretKey, cached.sessionToken, Long.valueOf(cached.expiration))); } if(tokenCode != null) { @@ -265,7 +265,7 @@ else if(!profiles.containsKey(basicProfile.getRoleSourceProfile())) { if(log.isDebugEnabled()) { log.debug(String.format("Set credentials from %s", sessionTokenResult)); } - return credentials.withTokens(new STSTokens( + return credentials.withTokens(new TemporaryAccessTokens( sessionTokenResult.getCredentials().getAccessKeyId(), sessionTokenResult.getCredentials().getSecretAccessKey(), sessionTokenResult.getCredentials().getSessionToken(), @@ -279,7 +279,7 @@ else if(!profiles.containsKey(basicProfile.getRoleSourceProfile())) { if(log.isDebugEnabled()) { log.debug(String.format("Set credentials from profile %s", basicProfile.getProfileName())); } - return credentials.withTokens(new STSTokens( + return credentials.withTokens(new TemporaryAccessTokens( basicProfile.getAwsAccessIdKey(), basicProfile.getAwsSecretAccessKey(), basicProfile.getAwsSessionToken(), diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java index 62d29e5b3f8..9059900e9a5 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleAuthorizationService.java @@ -22,7 +22,7 @@ import ch.cyberduck.core.LoginCallback; import ch.cyberduck.core.LoginOptions; import ch.cyberduck.core.OAuthTokens; -import ch.cyberduck.core.STSTokens; +import ch.cyberduck.core.TemporaryAccessTokens; import ch.cyberduck.core.aws.CustomClientConfiguration; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LoginFailureException; @@ -71,7 +71,7 @@ public STSAssumeRoleAuthorizationService(final Host bookmark, final AWSSecurityT this.prompt = prompt; } - public STSTokens authorize(final String sAMLAssertion) throws BackgroundException { + public TemporaryAccessTokens authorize(final String sAMLAssertion) throws BackgroundException { final AssumeRoleWithSAMLRequest request = new AssumeRoleWithSAMLRequest().withSAMLAssertion(sAMLAssertion); final HostPreferences preferences = new HostPreferences(bookmark); if(preferences.getInteger("s3.assumerole.durationseconds") != -1) { @@ -89,7 +89,7 @@ public STSTokens authorize(final String sAMLAssertion) throws BackgroundExceptio log.debug(String.format("Received assume role identity result %s", result)); } final Credentials credentials = bookmark.getCredentials(); - final STSTokens tokens = new STSTokens(result.getCredentials().getAccessKeyId(), + final TemporaryAccessTokens tokens = new TemporaryAccessTokens(result.getCredentials().getAccessKeyId(), result.getCredentials().getSecretAccessKey(), result.getCredentials().getSessionToken(), result.getCredentials().getExpiration().getTime()); @@ -101,7 +101,7 @@ public STSTokens authorize(final String sAMLAssertion) throws BackgroundExceptio } } - public STSTokens authorize(final OAuthTokens oauth) throws BackgroundException { + public TemporaryAccessTokens authorize(final OAuthTokens oauth) throws BackgroundException { final AssumeRoleWithWebIdentityRequest request = new AssumeRoleWithWebIdentityRequest(); if(log.isDebugEnabled()) { log.debug(String.format("Assume role with OIDC Id token for %s", bookmark)); @@ -161,7 +161,7 @@ public STSTokens authorize(final OAuthTokens oauth) throws BackgroundException { if(log.isDebugEnabled()) { log.debug(String.format("Received assume role identity result %s", result)); } - return new STSTokens(result.getCredentials().getAccessKeyId(), + return new TemporaryAccessTokens(result.getCredentials().getAccessKeyId(), result.getCredentials().getSecretAccessKey(), result.getCredentials().getSessionToken(), result.getCredentials().getExpiration().getTime()); diff --git a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java index 57bb1216ac9..77095eeebfa 100644 --- a/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java +++ b/s3/src/main/java/ch/cyberduck/core/sts/STSAssumeRoleCredentialsRequestInterceptor.java @@ -21,7 +21,7 @@ import ch.cyberduck.core.LoginCallback; import ch.cyberduck.core.OAuthTokens; import ch.cyberduck.core.PasswordStoreFactory; -import ch.cyberduck.core.STSTokens; +import ch.cyberduck.core.TemporaryAccessTokens; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.oauth.OAuth2RequestInterceptor; @@ -53,7 +53,7 @@ public class STSAssumeRoleCredentialsRequestInterceptor extends STSAssumeRoleAut /** * Currently valid tokens */ - private STSTokens tokens = STSTokens.EMPTY; + private TemporaryAccessTokens tokens = TemporaryAccessTokens.EMPTY; private final HostPasswordStore store = PasswordStoreFactory.get(); /** @@ -74,7 +74,7 @@ public STSAssumeRoleCredentialsRequestInterceptor(final OAuth2RequestInterceptor this.cancel = cancel; } - public STSTokens refresh(final OAuthTokens oidc) throws BackgroundException { + public TemporaryAccessTokens refresh(final OAuthTokens oidc) throws BackgroundException { try { return this.tokens = this.authorize(oidc); } @@ -118,7 +118,7 @@ public Credentials get() throws BackgroundException { catch(JWTDecodeException e) { throw new LoginFailureException("Invalid JWT or JSON format in authentication token", e); } - final STSTokens tokens = this.refresh(identity); + final TemporaryAccessTokens tokens = this.refresh(identity); return credentials .withUsername(sub) .withTokens(tokens); diff --git a/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java b/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java index 0230dd674c4..a309479b54c 100644 --- a/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java +++ b/s3/src/test/java/ch/cyberduck/core/s3/S3SessionTest.java @@ -12,7 +12,7 @@ import ch.cyberduck.core.LoginConnectionService; import ch.cyberduck.core.Profile; import ch.cyberduck.core.ProtocolFactory; -import ch.cyberduck.core.STSTokens; +import ch.cyberduck.core.TemporaryAccessTokens; import ch.cyberduck.core.TranscriptListener; import ch.cyberduck.core.cdn.DistributionConfiguration; import ch.cyberduck.core.exception.BackgroundException; @@ -99,7 +99,7 @@ public X509Certificate[] getAcceptedIssuers() { public void testConnectSessionTokenStatic() throws Exception { final S3Protocol protocol = new S3Protocol(); final Host host = new Host(protocol, protocol.getDefaultHostname(), new Credentials() - .withTokens(new STSTokens( + .withTokens(new TemporaryAccessTokens( "ASIA5RMYTHDIR37CTCXI", "TsnhChH4FlBt7hql2KnzrwNizmktJnO8YzDQwFqx", "FQoDYXdzEN3//////////wEaDLAz85HLZTQ7zu6/OSKrAfwLewUMHKaswh5sXv50BgMwbeKfCoMATjagvM+KV9++z0I6rItmMectuYoEGCOcnWHKZxtvpZAGcjlvgEDPw1KRYu16riUnd2Yo3doskqAoH0dlL2nH0eoj0d81H5e6IjdlGCm1E3K3zQPFLfMbvn1tdDQR1HV8o9eslmxo54hWMY2M14EpZhcXQMlns0mfYLYHLEVvgpz/8xYjR0yKDxJlXSATEpXtowHtqSi8tL7aBQ==", diff --git a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java index 0f82f2e1c4a..4494ee17fa8 100644 --- a/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java +++ b/s3/src/test/java/ch/cyberduck/core/sts/AssumeRoleWithWebIdentityAuthenticationTest.java @@ -26,7 +26,7 @@ import ch.cyberduck.core.Path; import ch.cyberduck.core.Protocol; import ch.cyberduck.core.ProtocolFactory; -import ch.cyberduck.core.STSTokens; +import ch.cyberduck.core.TemporaryAccessTokens; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.proxy.DisabledProxyFinder; @@ -114,7 +114,7 @@ public void testTokenRefresh() throws BackgroundException, InterruptedException final Credentials credentials = host.getCredentials(); final OAuthTokens oauth = credentials.getOauth(); assertTrue(oauth.validate()); - final STSTokens tokens = credentials.getTokens(); + final TemporaryAccessTokens tokens = credentials.getTokens(); assertTrue(tokens.validate()); Path container = new Path("cyberduckbucket", EnumSet.of(Path.Type.directory, Path.Type.volume)); @@ -144,7 +144,7 @@ public void testLoginInvalidOAuthTokensLogin() throws Exception { "eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIxNzA0N2Q0NS0wMTVhLTQwYWItYjc5NS03Y2Y1ZDE2ZmFhMmQifQ.eyJleHAiOjE2OTE5OTk1MDUsImlhdCI6MTY5MTk5NzcwNSwianRpIjoiY2U4OGVlMjMtOTQ1Yi00YzlmLWExMjAtZjU2ODk0NzIwZDk0IiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwL3JlYWxtcy9jeWJlcmR1Y2tyZWFsbSIsImF1ZCI6Imh0dHA6Ly9sb2NhbGhvc3Q6ODA4MC9yZWFsbXMvY3liZXJkdWNrcmVhbG0iLCJzdWIiOiIzMzRkYmViMC0xOTVkLTQyYTEtYTFkNi0xMjgxZjAwYmZiMWQiLCJ0eXAiOiJSZWZyZXNoIiwiYXpwIjoibWluaW8iLCJzZXNzaW9uX3N0YXRlIjoiM2RkNjQwNWUtM2QwYy00NWM5LTkxNmQtOWVhM2Q1ZjU4NWRiIiwic2NvcGUiOiJvcGVuaWQgbWluaW8tYXV0aG9yaXphdGlvbiIsInNpZCI6IjNkZDY0MDVlLTNkMGMtNDVjOS05MTZkLTllYTNkNWY1ODVkYiJ9.iRFLFjU-Uyv81flgieBht2K2BSlM-67fe5unvqI9PXA", Long.MAX_VALUE, "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJDQmpaYUNSeU5USmZqV0VmMU1fZXZLRVliMEdGLXU0QzhjZ3RZYnBtZUlFIn0.eyJleHAiOjE2OTE5OTc3MzUsImlhdCI6MTY5MTk5NzcwNSwiYXV0aF90aW1lIjowLCJqdGkiOiJlYWZiNWE5NS1lYmY3LTQ0OTEtODAwYy0yZjU1NTk2MjQ0YzIiLCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjgwODAvcmVhbG1zL2N5YmVyZHVja3JlYWxtIiwiYXVkIjoibWluaW8iLCJzdWIiOiIzMzRkYmViMC0xOTVkLTQyYTEtYTFkNi0xMjgxZjAwYmZiMWQiLCJ0eXAiOiJJRCIsImF6cCI6Im1pbmlvIiwic2Vzc2lvbl9zdGF0ZSI6IjNkZDY0MDVlLTNkMGMtNDVjOS05MTZkLTllYTNkNWY1ODVkYiIsImF0X2hhc2giOiJWX1lIZTVpc0UzY0IyOGF4cXQzRGpnIiwic2lkIjoiM2RkNjQwNWUtM2QwYy00NWM5LTkxNmQtOWVhM2Q1ZjU4NWRiIiwicG9saWN5IjpbInJlYWRvbmx5Il19.bXjcBJY7H79O9rtYr3b_EpKuclaRRsWGIVm5SEesqMM3aIkGq6ikWNmoL4Ffy48Frx1E3UnvG5PQfd8C2-XgNg_9EnWyR1MkgxJ67xQOAT10E77wZ0YbFWYIcdOojR98rmh4_TGVeTaGwDMMQZzRMr0nQwfZP3TQ8ciRhor8svnkFkk3FBzT1rSJA0bJv181HyerQl0f_TnTEnr3UjmmFmDrNASxHoXbwqiE4L-qZBnNiz97jLxGULfyVn4CZUub53x0ka0KGnLeicFHDh1asiHMW18o9-BUh8cGp-Ywm7Xu_f_c8XokNjG8ls56Xp7g8rQ4-d3J0F0-TAgnn7xO1g")) - .withTokens(STSTokens.EMPTY); + .withTokens(TemporaryAccessTokens.EMPTY); final Host host = new Host(profile, profile.getDefaultHostname(), credentials); final S3Session session = new S3Session(host); assertNotNull(session.open(new DisabledProxyFinder().find(host.getHostname()), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback())); @@ -152,7 +152,7 @@ public void testLoginInvalidOAuthTokensLogin() throws Exception { assertNotNull(session.getClient()); session.login(new DisabledProxyFinder().find(host.getHostname()), new DisabledLoginCallback(), new DisabledCancelCallback()); assertNotEquals(OAuthTokens.EMPTY, credentials.getOauth()); - assertNotEquals(STSTokens.EMPTY, credentials.getTokens()); + assertNotEquals(TemporaryAccessTokens.EMPTY, credentials.getTokens()); } /** @@ -165,7 +165,7 @@ public void testBucketListInvalidOAuthTokensList() throws Exception { AbstractAssumeRoleWithWebIdentityTest.class.getResourceAsStream("/S3 (OIDC).cyberduckprofile")); final Credentials credentials = new Credentials("rouser", "rouser") .withOauth(OAuthTokens.EMPTY) - .withTokens(new STSTokens( + .withTokens(new TemporaryAccessTokens( "5K1AVE34L4U1SQ7QTMWM", "LfkexzCDPojZpdIoNLNvHxrUi1KI5yP3Yken+DGI", "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiI1SzFBVkUzNEw0VTFTUTdRVE1XTSIsImF0X2hhc2giOiJWX1lIZTVpc0UzY0IyOGF4cXQzRGpnIiwiYXVkIjoibWluaW8iLCJhdXRoX3RpbWUiOjAsImF6cCI6Im1pbmlvIiwiZXhwIjoxNjkxOTk3NzM1LCJpYXQiOjE2OTE5OTc3MDUsImlzcyI6Imh0dHA6Ly9sb2NhbGhvc3Q6ODA4MC9yZWFsbXMvY3liZXJkdWNrcmVhbG0iLCJqdGkiOiJlYWZiNWE5NS1lYmY3LTQ0OTEtODAwYy0yZjU1NTk2MjQ0YzIiLCJwb2xpY3kiOiJyZWFkb25seSIsInNlc3Npb25fc3RhdGUiOiIzZGQ2NDA1ZS0zZDBjLTQ1YzktOTE2ZC05ZWEzZDVmNTg1ZGIiLCJzaWQiOiIzZGQ2NDA1ZS0zZDBjLTQ1YzktOTE2ZC05ZWEzZDVmNTg1ZGIiLCJzdWIiOiIzMzRkYmViMC0xOTVkLTQyYTEtYTFkNi0xMjgxZjAwYmZiMWQiLCJ0eXAiOiJJRCJ9.HmyC7XuJw9XnsNUd2ZuGSVIPjnGHPpgbXX1HSbNJuhis1kUjhcrYY2HnQZ-uScoX57o_C3fF1eEv_t1kW2U6Rw", @@ -182,6 +182,6 @@ public void testBucketListInvalidOAuthTokensList() throws Exception { new S3BucketListService(session).list( new Path(String.valueOf(Path.DELIMITER), EnumSet.of(Path.Type.volume, Path.Type.directory)), new DisabledListProgressListener()); assertNotEquals(OAuthTokens.EMPTY, credentials.getOauth()); - assertNotEquals(STSTokens.EMPTY, credentials.getTokens()); + assertNotEquals(TemporaryAccessTokens.EMPTY, credentials.getTokens()); } } \ No newline at end of file