Support accessing bucket via access point ARN (#2987)

* Support accessing bucket via access point ARN

* add ARN parser and validator to util

* implement s3UseArnRegion config

* implement validating access point ARN

* implement ARN builder utility

* implement populating uri from access point ARN

* re-organize setupRequestListeners()

* throw when client region is fips region because it is not supported

.changes/next-release/feature-Bucket-a1a45ab9.json

@@ -0,0 +1,5 @@
+{
+  "type": "feature",
+  "category": "Bucket",
+  "description": "Add support for S3 access point. Access Points provide a customizable way to access the objects in a bucket, with a unique hostname and access policy that enforces the specific permissions and network controls for any request made through the access point."
+}

lib/config.js

@@ -87,6 +87,10 @@ var PromisesDependency;
  *     request to global endpoints or 'us-east-1' regional endpoints. This config is only
  *     applicable to S3 client;
  *     Defaults to 'legacy'
+ * @!attribute s3UseArnRegion
+ *   @return [Boolean] whether to override the request region with the region inferred
+ *     from requested resource's ARN. Only available for S3 buckets
+ *     Defaults to `true`
  *
  * @!attribute useAccelerateEndpoint
  *   @note This configuration option is only compatible with S3 while accessing
@@ -248,6 +252,9 @@ AWS.Config = AWS.util.inherit({
    *   is set to 'us-east-1', whether to send s3 request to global endpoints or
    *   'us-east-1' regional endpoints. This config is only applicable to S3 client.
    *   Defaults to `legacy`
+   * @option options s3UseArnRegion [Boolean] whether to override the request region
+   *   with the region inferred from requested resource's ARN. Only available for S3 buckets
+   *   Defaults to `true`
    *
    * @option options retryDelayOptions [map] A set of options to configure
    *   the retry delay on retryable errors. Currently supported options are:
@@ -534,6 +541,7 @@ AWS.Config = AWS.util.inherit({
     s3BucketEndpoint: false,
     s3DisableBodySigning: true,
     s3UsEast1RegionalEndpoint: 'legacy',
+    s3UseArnRegion: undefined,
     computeChecksums: true,
     convertResponseTypes: true,
     correctClockSkew: false,

lib/region_config.js

@@ -66,7 +66,28 @@ function configureEndpoint(service) {
   }
 }
 
+function getEndpointSuffix(region) {
+  var regionRegexes = {
+    '^(us|eu|ap|sa|ca|me)\\-\\w+\\-\\d+$': 'amazonaws.com',
+    '^cn\\-\\w+\\-\\d+$': 'amazonaws.com.cn',
+    '^us\\-gov\\-\\w+\\-\\d+$': 'amazonaws.com',
+    '^us\\-iso\\-\\w+\\-\\d+$': 'c2s.ic.gov',
+    '^us\\-isob\\-\\w+\\-\\d+$': 'sc2s.sgov.gov'
+  };
+  var defaultSuffix = 'amazonaws.com';
+  var regexes = Object.keys(regionRegexes);
+  for (var i = 0; i < regexes.length; i++) {
+    var regionPattern = RegExp(regexes[i]);
+    var dnsSuffix = regionRegexes[regexes[i]];
+    if (regionPattern.test(region)) return dnsSuffix;
+  }
+  return defaultSuffix;
+}
+
 /**
  * @api private
  */
-module.exports = configureEndpoint;
+module.exports = {
+  configureEndpoint: configureEndpoint,
+  getEndpointSuffix: getEndpointSuffix
+};

lib/service.js

@@ -50,7 +50,7 @@ AWS.Service = inherit({
     if (config) this.config.update(config, true);
 
     this.validateService();
-    if (!this.config.endpoint) regionConfig(this);
+    if (!this.config.endpoint) regionConfig.configureEndpoint(this);
 
     this.config.endpoint = this.endpointFromTemplate(this.config.endpoint);
     this.setEndpoint(this.config.endpoint);

lib/services/s3.js

@@ -1,6 +1,7 @@
 var AWS = require('../core');
 var v4Credentials = require('../signers/v4_credentials');
 var resolveRegionalEndpointsFlag = require('../config_regional_endpoint');
+var regionUtil = require('../region_config');
 
 // Pull in managed upload extension
 require('../s3/managed_upload');
@@ -103,30 +104,40 @@ AWS.util.update(AWS.S3.prototype, {
   setupRequestListeners: function setupRequestListeners(request) {
     var prependListener = true;
     request.addListener('validate', this.validateScheme);
-    request.addListener('validate', this.validateBucketEndpoint);
-    request.addListener('validate', this.correctBucketRegionFromCache);
     request.addListener('validate', this.validateBucketName, prependListener);
     request.addListener('validate', this.optInUsEast1RegionalEndpoint, prependListener);
 
+    request.removeListener('validate',
+      AWS.EventListeners.Core.VALIDATE_REGION);
     request.addListener('build', this.addContentType);
-    request.addListener('build', this.populateURI);
     request.addListener('build', this.computeContentMd5);
     request.addListener('build', this.computeSseCustomerKeyMd5);
+    request.addListener('build', this.populateURI);
     request.addListener('afterBuild', this.addExpect100Continue);
-    request.removeListener('validate',
-      AWS.EventListeners.Core.VALIDATE_REGION);
     request.addListener('extractError', this.extractError);
-    request.onAsync('extractError', this.requestBucketRegion);
-    request.addListener('extractData', this.extractData);
     request.addListener('extractData', AWS.util.hoistPayloadMember);
+    request.addListener('extractData', this.extractData);
     request.addListener('beforePresign', this.prepareSignedUrl);
-    if (AWS.util.isBrowser()) {
-      request.onAsync('retry', this.reqRegionForNetworkingError);
-    }
     if (this.shouldDisableBodySigning(request))  {
       request.removeListener('afterBuild', AWS.EventListeners.Core.COMPUTE_SHA256);
       request.addListener('afterBuild', this.disableBodySigning);
     }
+    //deal with ARNs supplied to Bucket
+    if (this.isAccessPointApplicable(request)) {
+      request.removeListener('validate', this.validateBucketName);
+      request.addListener('validate', this.validateAccessPointArn, prependListener);
+      request.addListener('validate', this.validateArnRegion);
+      request.removeListener('build', this.populateURI);
+      request.addListener('build', this.populateUriFromAccessPoint);
+      return;
+    }
+    //listeners regarding region inference
+    request.addListener('validate', this.validateBucketEndpoint);
+    request.addListener('validate', this.correctBucketRegionFromCache);
+    request.onAsync('extractError', this.requestBucketRegion);
+    if (AWS.util.isBrowser()) {
+      request.onAsync('retry', this.reqRegionForNetworkingError);
+    }
   },
 
   /**
@@ -155,6 +166,156 @@ AWS.util.update(AWS.S3.prototype, {
     }
   },
 
+  /**
+   * @api private
+   */
+  isAccessPointApplicable: function hasBucketInParams(req) {
+    var inputShape = (req.service.api.operations[req.operation] || {}).input || {};
+    var inputMembers = inputShape.members || {};
+    if (
+      req.operation === 'createBucket' ||
+      !req.params.Bucket ||
+      !inputMembers.Bucket
+    ) return false;
+    if (!AWS.util.ARN.validate(req.params.Bucket)) return false;
+    return true;
+  },
+
+  /**
+   * Validate ARN supplied in Bucket parameter is a valid access point ARN
+   *
+   * @api private
+   */
+  validateAccessPointArn: function validateAccessPointArn(req) {
+    var parsedArn = AWS.util.ARN.parse(req.params.Bucket);
+    //avoid duplicated parsing in the future
+    req._parsedAccessPointArn = parsedArn;
+    var parsedArn = req._parsedAccessPointArn;
+    if (parsedArn.service !== 's3') {
+      throw AWS.util.error(new Error(), {
+        code: 'InvalidAccessPointARN',
+        message: 'expect \'s3\' in access point ARN service component'
+      });
+    }
+    if (!parsedArn.region) {
+      throw AWS.util.error(new Error(), {
+        code: 'InvalidAccessPointARN',
+        message: 'Access point ARN region is empty'
+      });
+    }
+    if (
+      parsedArn.resource.indexOf('accesspoint:') !== 0 &&
+      parsedArn.resource.indexOf('accesspoint/') !== 0
+    ) {
+      throw AWS.util.error(new Error(), {
+        code: 'InvalidAccessPointARN',
+        message: 'Access point ARN resource should begin with \'accesspoint/\''
+      });
+    }
+    var delimiter = parsedArn.resource['accesspoint'.length]; //can be ':' or '/'
+    if (parsedArn.resource.split(delimiter).length !== 2) {
+      throw AWS.util.error(new Error(), {
+        code: 'InvalidAccessPointARN',
+        message: 'Too many resource parameters in access point ARN'
+      });
+    }
+    var accessPoint = parsedArn.resource.split(delimiter)[1];
+    var accessPointPrefix = accessPoint + '-' + parsedArn.accountId;
+    if (!req.service.isDnsCompatible(accessPointPrefix) || accessPointPrefix.match(/\./)) {
+      throw AWS.util.error(new Error(), {
+        code: 'InvalidAccessPointARN',
+        message: 'Access point ARN is not DNS compatible. Got ' + accessPoint
+      });
+    }
+    //set parsed valid access point
+    req._parsedAccessPointArn.accessPoint = accessPoint;
+  },
+
+  /**
+   * @api private
+   */
+  validateArnRegion: function validateArnRegion(req) {
+    var useArnRegion = req.service.loadUseArnRegionConfig(req);
+    var regionFromArn = req._parsedAccessPointArn.region;
+    var clientRegion = req.service.config.region;
+    if (
+      clientRegion.indexOf('fips') >= 0 ||
+      regionFromArn.indexOf('fips') >= 0
+    ) {
+      throw AWS.util.error(new Error(), {
+        code: 'InvalidConfiguration',
+        message: 'Access point endpoint is not compatible with FIPS region'
+      });
+    }
+    if (!useArnRegion && regionFromArn !== clientRegion) {
+      throw AWS.util.error(new Error(), {
+        code: 'InvalidConfiguration',
+        message: 'Configured region conflicts with access point region'
+      });
+    } else if (
+      useArnRegion &&
+      regionUtil.getEndpointSuffix(regionFromArn) !== regionUtil.getEndpointSuffix(clientRegion)
+    ) {
+      throw AWS.util.error(new Error(), {
+        code: 'InvalidConfiguration',
+        message: 'Configured region and access point region not in same partition'
+      });
+    }
+    if (req.service.config.useAccelerateEndpoint) {
+      throw AWS.util.error(new Error(), {
+        code: 'InvalidConfiguration',
+        message: 'useAccelerateEndpoint config is not supported with access point ARN'
+      });
+    }
+  },
+
+  /**
+   * @api private
+   */
+  loadUseArnRegionConfig: function loadUseArnRegionConfig(req) {
+    var envName = 'AWS_S3_USE_ARN_REGION';
+    var configName = 's3_use_arn_region';
+    var useArnRegion = true;
+    var originalConfig = req.service._originalConfig || {};
+    if (req.service.config.s3UseArnRegion !== undefined) {
+      return req.service.config.s3UseArnRegion;
+    } else if (originalConfig.s3UseArnRegion !== undefined) {
+      useArnRegion = originalConfig.s3UseArnRegion === true;
+    } else if (AWS.util.isNode()) {
+      //load from environmental variable AWS_USE_ARN_REGION
+      if (process.env[envName]) {
+        var value = process.env[envName].trim().toLowerCase();
+        if (['false', 'true'].indexOf(value) < 0) {
+          throw AWS.util.error(new Error(), {
+            code: 'InvalidConfiguration',
+            message: envName + ' only accepts true or false. Got ' + process.env[envName],
+            retryable: false
+          });
+        }
+        useArnRegion = value === 'true';
+      } else {  //load from shared config property use_arn_region
+        var profiles = {};
+        var profile = {};
+        try {
+          profiles = AWS.util.getProfilesFromSharedConfig(AWS.util.iniLoader);
+          profile = profiles[process.env.AWS_PROFILE || AWS.util.defaultProfile];
+        } catch (e) {}
+        if (profile[configName]) {
+          if (['false', 'true'].indexOf(profile[configName].trim().toLowerCase()) < 0) {
+            throw AWS.util.error(new Error(), {
+              code: 'InvalidConfiguration',
+              message: configName + ' only accepts true or false. Got ' + profile[configName],
+              retryable: false
+            });
+          }
+          useArnRegion = profile[configName].trim().toLowerCase() === 'true';
+        }
+      }
+    }
+    req.service.config.s3UseArnRegion = useArnRegion;
+    return useArnRegion;
+  },
+
   /**
    * @api private
    */
@@ -283,6 +444,45 @@ AWS.util.update(AWS.S3.prototype, {
     }
   },
 
+  /**
+   * When user supply an access point ARN in the Bucket parameter, we need to
+   * populate the URI according to the ARN.
+   * @api private
+   */
+  populateUriFromAccessPoint: function populateUriFromAccessPoint(req) {
+    if (req.service._originalConfig.endpoint) {
+      throw AWS.util.error(new Error(), {
+        code: 'InvalidConfiguration',
+        message: 'Custom endpoint is not compatible with access point ARN'
+      });
+    }
+    if (req.service.config.s3ForcePathStyle) {
+      throw AWS.util.error(new Error(), {
+        code: 'InvalidConfiguration',
+        message: 'Cannot construct path-style endpoint with access point'
+      });
+    }
+    var accessPointArn = req._parsedAccessPointArn;
+    var serviceName = req.service.config.useDualstack ?
+      's3-accesspoint.dualstack':
+      's3-accesspoint';
+    var endpoint = req.httpRequest.endpoint;
+    var dnsSuffix = regionUtil.getEndpointSuffix(accessPointArn.region);
+    var useArnRegion = req.service.config.s3UseArnRegion;
+    endpoint.hostname = [
+      accessPointArn.accessPoint + '-' + accessPointArn.accountId,
+      serviceName,
+      useArnRegion ? accessPointArn.region : req.service.config.region,
+      dnsSuffix
+    ].join('.');
+    endpoint.host = endpoint.hostname;
+    var encodedArn = AWS.util.uriEscapePath(req.params.Bucket);
+    var path = req.httpRequest.path;
+    //remove the Bucket value from path
+    req.httpRequest.path = path.replace(new RegExp('/' + encodedArn), '');
+    req.httpRequest.region = accessPointArn.region; //region used to sign
+  },
+
   /**
    * Adds Expect: 100-continue header if payload is greater-or-equal 1MB
    * @api private
@@ -419,7 +619,7 @@ AWS.util.update(AWS.S3.prototype, {
     if (this.config.s3ForcePathStyle) return true;
     if (this.config.s3BucketEndpoint) return false;
 
-    if (this.dnsCompatibleBucketName(bucketName)) {
+    if (this.isDnsCompatible(bucketName)) {
       return (this.config.sslEnabled && bucketName.match(/\./)) ? true : false;
     } else {
       return true; // not dns compatible names must always use path style
@@ -432,7 +632,7 @@ AWS.util.update(AWS.S3.prototype, {
    *
    * @api private
    */
-  dnsCompatibleBucketName: function dnsCompatibleBucketName(bucketName) {
+  isDnsCompatible: function isDnsCompatible(bucketName) {
     var b = bucketName;
     var domain = new RegExp(/^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$/);
     var ipAddress = new RegExp(/(\d+\.){3}\d+/);
@@ -674,7 +874,7 @@ AWS.util.update(AWS.S3.prototype, {
     if (cachedRegion && cachedRegion !== request.httpRequest.region) {
       service.updateReqBucketRegion(request, cachedRegion);
       done();
-    } else if (!service.dnsCompatibleBucketName(bucket)) {
+    } else if (!service.isDnsCompatible(bucket)) {
       service.updateReqBucketRegion(request, 'us-east-1');
       if (bucketRegionCache[bucket] !== 'us-east-1') {
         bucketRegionCache[bucket] = 'us-east-1';

lib/util.js

@@ -975,6 +975,35 @@ var util = {
     return profiles;
   },
 
+  /**
+   * @api private
+   */
+  ARN: {
+    validate: function validateARN(str) {
+      return str && str.indexOf('arn:') === 0 && str.split(':').length >= 6;
+    },
+    parse: function parseARN(arn) {
+      var matched = arn.split(':');
+      return {
+        partition: matched[1],
+        service: matched[2],
+        region: matched[3],
+        accountId: matched[4],
+        resource: matched.slice(5).join(':')
+      };
+    },
+    build: function buildARN(arnObject) {
+      if (
+        arnObject.service === undefined ||
+        arnObject.region === undefined ||
+        arnObject.accountId === undefined ||
+        arnObject.resource === undefined
+      ) throw util.error(new Error('Input ARN object is invalid'));
+      return 'arn:'+ (arnObject.partition || 'aws') + ':' + arnObject.service +
+        ':' + arnObject.region + ':' + arnObject.accountId + ':' + arnObject.resource;
+    }
+  },
+
   /**
    * @api private
    */