Skip to content

Commit

Permalink
Merge pull request #16473 from iterate-ch/bugfix/GH-16377
Browse files Browse the repository at this point in the history 
Fix #16377
  • Loading branch information
@dkocher
dkocher authored Nov 7, 2024
2 parents d9de9fa + 0233042 commit edfb2fa
Show file tree
Hide file tree
Showing 14 changed files with 120 additions and 102 deletions.
71 changes: 37 additions & 34 deletions s3/src/main/java/ch/cyberduck/core/s3/RequestEntityRestStorageService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,47 +143,50 @@ public HttpUriRequest setupConnection(final HTTP_METHOD method, final String buc
String endpoint = host.getHostname();
// Apply default configuration
if(S3Session.isAwsHostname(host.getHostname(), false)) {
if(StringUtils.isNotBlank(host.getRegion())) {
log.debug("Apply default region {} to endpoint", host.getRegion());
// Apply default region
endpoint = createRegionSpecificEndpoint(host, host.getRegion());
}
else {
if(StringUtils.isNotBlank(bucketName)) {
// Only for AWS set endpoint to region specific
if(preferences.getBoolean(String.format("s3.transferacceleration.%s.enable", bucketName))) {
// Already set to accelerated endpoint
log.debug("Use accelerated endpoint {}", S3TransferAccelerationService.S3_ACCELERATE_DUALSTACK_HOSTNAME);
endpoint = S3TransferAccelerationService.S3_ACCELERATE_DUALSTACK_HOSTNAME;
}
else {
// Only attempt to determine region specific endpoint if virtual host style requests are enabled
if(!this.getDisableDnsBuckets()) {
// Check if not already request to query bucket location
if(requestParameters == null || !requestParameters.containsKey("location")) {
try {
// Determine region for bucket using cache
final Location.Name region = new S3LocationFeature(session, regionEndpointCache).getLocation(bucketName);
if(Location.unknown == region) {
// Missing permission or not supported
log.warn("Failure determining bucket location for {}", bucketName);
endpoint = host.getHostname();
}
else {
log.debug("Determined region {} for bucket {}", region, bucketName);
endpoint = createRegionSpecificEndpoint(host, region.getIdentifier());
}
if(StringUtils.isNotBlank(bucketName)) {
// Only for AWS set endpoint to region specific
if(preferences.getBoolean(String.format("s3.transferacceleration.%s.enable", bucketName))) {
// Already set to accelerated endpoint
log.debug("Use accelerated endpoint {}", S3TransferAccelerationService.S3_ACCELERATE_DUALSTACK_HOSTNAME);
endpoint = S3TransferAccelerationService.S3_ACCELERATE_DUALSTACK_HOSTNAME;
}
else {
// Only attempt to determine region specific endpoint if virtual host style requests are enabled
if(!this.getDisableDnsBuckets()) {
// Check if not already request to query bucket location
if(requestParameters == null || !requestParameters.containsKey("location")) {
try {
// Determine region for bucket using cache
final Location.Name region = new S3LocationFeature(session, regionEndpointCache).getLocation(bucketName);
if(Location.unknown == region) {
// Missing permission or not supported
log.warn("Failure determining bucket location for {}", bucketName);
endpoint = host.getHostname();
}
catch(BackgroundException e) {
// Ignore failure reading location for bucket
log.error("Failure {} determining bucket location for {}", e, bucketName);
endpoint = createRegionSpecificEndpoint(host, preferences.getProperty("s3.location"));
else {
log.debug("Determined region {} for bucket {}", region, bucketName);
endpoint = createRegionSpecificEndpoint(host, region.getIdentifier());
}
}
catch(BackgroundException e) {
// Ignore failure reading location for bucket
log.error("Failure {} determining bucket location for {}", e, bucketName);
endpoint = createRegionSpecificEndpoint(host, preferences.getProperty("s3.location"));
}
}
}
}
}
else {
if(StringUtils.isNotBlank(host.getRegion())) {
// Only attempt to determine region specific endpoint if virtual host style requests are enabled
if(!this.getDisableDnsBuckets()) {
log.debug("Apply default region {} to endpoint", host.getRegion());
// Apply default region
endpoint = createRegionSpecificEndpoint(host, host.getRegion());
}
}
}
}
log.debug("Set endpoint to {}", endpoint);
// Virtual host style endpoint including bucket name
Expand Down
3 changes: 3 additions & 0 deletions s3/src/main/java/ch/cyberduck/core/s3/S3AWS4SignatureRequestInterceptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public void process(final HttpRequest request, final HttpContext context) throws
if(host != null) {
try {
region = SignatureUtils.awsRegionForRequest(new URI(host.toURI()));
log.debug("Determined region {} from URI {}", region, host.toURI());
}
catch(URISyntaxException e) {
throw new IOException(e);
Expand All @@ -95,9 +96,11 @@ public void process(final HttpRequest request, final HttpContext context) throws
}
if(null == region) {
region = session.getHost().getRegion();
log.debug("Determined region {} from {}", region, session.getHost());
}
if(null == region) {
region = new HostPreferences(session.getHost()).getProperty("s3.location");
log.debug("Determined region {} from defaults", region);
}
final HttpUriRequest message = (HttpUriRequest) request;
String requestPayloadHexSHA256Hash =
Expand Down
10 changes: 10 additions & 0 deletions s3/src/main/java/ch/cyberduck/core/s3/S3BucketRegionRedirectStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
import org.apache.http.HttpResponse;
import org.apache.http.ProtocolException;
import org.apache.http.client.RedirectException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.impl.client.DefaultRedirectStrategy;
Expand All @@ -39,6 +43,12 @@ public class S3BucketRegionRedirectStrategy extends DefaultRedirectStrategy {
private final Host host;

public S3BucketRegionRedirectStrategy(final RequestEntityRestStorageService service, final Host host) {
super(new String[] {
HttpHead.METHOD_NAME,
HttpGet.METHOD_NAME,
HttpPost.METHOD_NAME,
HttpPut.METHOD_NAME
});
this.service = service;
this.host = host;
}
Expand Down
22 changes: 6 additions & 16 deletions s3/src/main/java/ch/cyberduck/core/s3/S3LocationFeature.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,31 +61,16 @@ public Name getDefault() {

@Override
public Set<Name> getLocations() {
if(StringUtils.isNotBlank(session.getHost().getRegion())) {
final S3Region region = new S3Region(session.getHost().getRegion());
log.debug("Return single region {} set in bookmark", region);
return Collections.singleton(region);
}
if(StringUtils.isNotEmpty(RequestEntityRestStorageService.findBucketInHostname(session.getHost()))) {
log.debug("Return empty set for hostname {}", session.getHost());
// Connected to single bucket
return Collections.emptySet();
}
if(!S3Session.isAwsHostname(session.getHost().getHostname(), false)) {
if(new S3Protocol().getRegions().equals(session.getHost().getProtocol().getRegions())) {
// Return empty set for unknown provider
log.debug("Return empty set for unknown provider {}", session.getHost());
return Collections.emptySet();
}
}
return session.getHost().getProtocol().getRegions();
}

@Override
public Name getLocation(final Path file) throws BackgroundException {
if(StringUtils.isNotBlank(session.getHost().getRegion())) {
return new S3Region(session.getHost().getRegion());
}
final Path bucket = containerService.getContainer(file);
return this.getLocation(bucket.isRoot() ? StringUtils.EMPTY : bucket.getName());
}
Expand All @@ -105,7 +90,12 @@ protected Name getLocation(final String bucketname) throws BackgroundException {
final S3Region region;
if(StringUtils.isBlank(location)) {
log.warn("No region known for bucket {}", bucketname);
region = DEFAULT_REGION;
if(StringUtils.isNotBlank(session.getHost().getRegion())) {
region = new S3Region(session.getHost().getRegion());
}
else {
region = DEFAULT_REGION;
}
}
else {
switch(location) {
Expand Down
65 changes: 50 additions & 15 deletions s3/src/main/java/ch/cyberduck/core/s3/S3Session.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
*/

import ch.cyberduck.core.Credentials;
import ch.cyberduck.core.DisabledListProgressListener;
import ch.cyberduck.core.Host;
import ch.cyberduck.core.HostKeyCallback;
import ch.cyberduck.core.ListService;
Expand All @@ -28,15 +29,15 @@
import ch.cyberduck.core.PathContainerService;
import ch.cyberduck.core.Scheme;
import ch.cyberduck.core.UrlProvider;
import ch.cyberduck.core.auth.AWSCredentialsConfigurator;
import ch.cyberduck.core.auth.AWSSessionCredentialsRetriever;
import ch.cyberduck.core.aws.CustomClientConfiguration;
import ch.cyberduck.core.cdn.Distribution;
import ch.cyberduck.core.cdn.DistributionConfiguration;
import ch.cyberduck.core.cloudfront.CloudFrontDistributionConfigurationPreloader;
import ch.cyberduck.core.cloudfront.WebsiteCloudFrontDistributionConfiguration;
import ch.cyberduck.core.date.RFC822DateFormatter;
import ch.cyberduck.core.exception.AccessDeniedException;
import ch.cyberduck.core.exception.BackgroundException;
import ch.cyberduck.core.exception.InteroperabilityException;
import ch.cyberduck.core.exception.LoginCanceledException;
import ch.cyberduck.core.features.*;
import ch.cyberduck.core.http.CustomServiceUnavailableRetryStrategy;
Expand All @@ -49,7 +50,6 @@
import ch.cyberduck.core.preferences.PreferencesReader;
import ch.cyberduck.core.proxy.ProxyFinder;
import ch.cyberduck.core.restore.Glacier;
import ch.cyberduck.core.shared.DefaultHomeFinderService;
import ch.cyberduck.core.shared.DefaultPathHomeFeature;
import ch.cyberduck.core.shared.DelegatingHomeFeature;
import ch.cyberduck.core.shared.DisabledBulkFeature;
Expand All @@ -59,8 +59,10 @@
import ch.cyberduck.core.ssl.X509KeyManager;
import ch.cyberduck.core.ssl.X509TrustManager;
import ch.cyberduck.core.sts.STSAssumeRoleCredentialsRequestInterceptor;
import ch.cyberduck.core.sts.STSExceptionMappingService;
import ch.cyberduck.core.threading.CancelCallback;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
Expand All @@ -87,6 +89,12 @@
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException;
import com.amazonaws.services.securitytoken.model.GetCallerIdentityRequest;
import com.amazonaws.services.securitytoken.model.GetCallerIdentityResult;

import static com.amazonaws.services.s3.Headers.*;

public class S3Session extends HttpSession<RequestEntityRestStorageService> {
Expand Down Expand Up @@ -323,20 +331,47 @@ public void login(final LoginCallback prompt, final CancelCallback cancel) throw
log.warn("Skip verifying credentials with previous successful authentication event for {}", this);
return;
}
try {
final Path home = new DelegatingHomeFeature(new DefaultPathHomeFeature(host)).find();
final Location.Name location = new S3LocationFeature(S3Session.this, regions).getLocation(home);
log.debug("Retrieved region {}", location);
if(!Location.unknown.equals(location)) {
log.debug("Set default region to {} determined from {}", location, home);
//
host.setProperty("s3.location", location.getIdentifier());
final Path home = new DelegatingHomeFeature(new DefaultPathHomeFeature(host)).find();
if(S3Session.isAwsHostname(host.getHostname(), false)) {
if(StringUtils.isEmpty(RequestEntityRestStorageService.findBucketInHostname(host))) {
final CustomClientConfiguration configuration = new CustomClientConfiguration(host,
new ThreadLocalHostnameDelegatingTrustManager(trust, host.getHostname()), key);
final AWSSecurityTokenServiceClientBuilder builder = AWSSecurityTokenServiceClientBuilder.standard()
.withCredentials(AWSCredentialsConfigurator.toAWSCredentialsProvider(client.getProviderCredentials()))
.withClientConfiguration(configuration);
final AWSSecurityTokenService service = builder.build();
// Returns details about the IAM user or role whose credentials are used to call the operation.
// No permissions are required to perform this operation.
try {
final GetCallerIdentityResult identity = service.getCallerIdentity(new GetCallerIdentityRequest());
log.debug("Successfully verified credentials for {}", identity);
}
catch(AWSSecurityTokenServiceException e) {
throw new STSExceptionMappingService().map(e);
}
}
else {
final Location.Name location = new S3LocationFeature(this, regions).getLocation(home);
log.debug("Retrieved region {}", location);
if(!Location.unknown.equals(location)) {
log.debug("Set default region to {} determined from {}", location, home);
host.setProperty("s3.location", location.getIdentifier());
}
}
}
catch(AccessDeniedException | InteroperabilityException e) {
log.warn("Failure {} querying region", e.getMessage());
final Path home = new DefaultHomeFinderService(this).find();
log.debug("Retrieved {}", home);
else {
if(home.isRoot() && StringUtils.isEmpty(RequestEntityRestStorageService.findBucketInHostname(host))) {
log.debug("Skip querying region for {}", home);
new S3ListService(this, acl).list(home, new DisabledListProgressListener());
}
else {
final Location.Name location = new S3LocationFeature(this, regions).getLocation(home);
log.debug("Retrieved region {}", location);
if(!Location.unknown.equals(location)) {
log.debug("Set default region to {} determined from {}", location, home);
host.setProperty("s3.location", location.getIdentifier());
}
}
}
}

Expand Down
5 changes: 2 additions & 3 deletions s3/src/main/java/ch/cyberduck/core/sts/STSExceptionMappingService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,17 @@
import ch.cyberduck.core.exception.LoginFailureException;

import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException;
import com.amazonaws.services.securitytoken.model.InvalidIdentityTokenException;

public class STSExceptionMappingService implements ExceptionMappingService<AWSSecurityTokenServiceException> {

@Override
public BackgroundException map(final AWSSecurityTokenServiceException e) {
if(e instanceof com.amazonaws.services.securitytoken.model.ExpiredTokenException) {
if("RequestExpired".equals(e.getErrorCode())) {
// The web identity token that was passed is expired or is not valid. Get a new identity token from the identity
// provider and then retry the request.
return new ExpiredTokenException(e.getErrorMessage(), e);
}
if(e instanceof InvalidIdentityTokenException) {
if("InvalidClientTokenId".equals(e.getErrorCode())) {
// The web identity token that was passed could not be validated by Amazon Web Services. Get a new identity token from
// the identity provider and then retry the request.
return new ExpiredTokenException(e.getErrorMessage(), e);
Expand Down
2 changes: 1 addition & 1 deletion s3/src/test/java/ch/cyberduck/core/s3/RequestEntityRestStorageServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public void testSetupConnection() throws Exception {
@Test
public void testSetupConnectionVirtualHost() throws Exception {
final RequestEntityRestStorageService service = new RequestEntityRestStorageService(virtualhost, new HttpConnectionPoolBuilder(virtualhost.getHost(),
new ThreadLocalHostnameDelegatingTrustManager(new DisabledX509TrustManager(), session.getHost().getHostname()),
new ThreadLocalHostnameDelegatingTrustManager(new DisabledX509TrustManager(), virtualhost.getHost().getHostname()),
new DefaultX509KeyManager(), new DisabledProxyFinder()).build(new DisabledProxyFinder(), new DisabledTranscriptListener(), new DisabledLoginCallback()));
final RegionEndpointCache cache = service.getRegionEndpointCache();
cache.clear();
Expand Down
4 changes: 2 additions & 2 deletions s3/src/test/java/ch/cyberduck/core/s3/S3AccessControlListFeatureTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,15 @@ public void testWrite() throws Exception {
@Test
public void testWriteVirtualHostBucket() throws Exception {
final Path test = new Path(new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file));
new S3TouchFeature(virtualhost, new S3AccessControlListFeature(session)).touch(test, new TransferStatus());
new S3TouchFeature(virtualhost, new S3AccessControlListFeature(virtualhost)).touch(test, new TransferStatus());
final S3AccessControlListFeature f = new S3AccessControlListFeature(virtualhost);
final Acl acl = new Acl();
acl.addAll(new Acl.Owner("80b9982b7b08045ee86680cc47f43c84bf439494a89ece22b5330f8a49477cf6"), new Acl.Role(Acl.Role.FULL));
acl.addAll(new Acl.GroupUser(Acl.GroupUser.EVERYONE), new Acl.Role(Acl.Role.READ));
acl.addAll(new Acl.GroupUser(Acl.GroupUser.AUTHENTICATED), new Acl.Role(Acl.Role.READ));
f.setPermission(test, acl);
assertEquals(acl, f.getPermission(test));
new S3DefaultDeleteFeature(virtualhost, new S3AccessControlListFeature(session)).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
new S3DefaultDeleteFeature(virtualhost, new S3AccessControlListFeature(virtualhost)).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
}

@Test
Expand Down
Loading

0 comments on commit edfb2fa

Please sign in to comment.