feat(core): add ability to resolve mount targets using EFS API (#392)

packages/aws-rfdk/lib/core/lib/mountable-efs.ts

@@ -66,6 +66,18 @@ export interface MountableEfsProps {
    * @default No extra options.
    */
   readonly extraMountOptions?: string[];
+
+  /**
+   * If enabled, RFDK will add user-data to the instances mounting this EFS file-system that obtains the mount target
+   * IP address using AWS APIs and writes them to the system's `/etc/hosts` file to not require DNS lookups.
+   *
+   * If mounting EFS from instances in a VPC configured to not use the Amazon-provided DNS Route 53 Resolver server,
+   * then the EFS mount targets will not be resolvable using DNS (see
+   * https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html) and enabling this will work around that issue.
+   *
+   * @default false
+   */
+  readonly resolveMountTargetDnsWithApi?: boolean;
 }
 
 /**
@@ -157,11 +169,26 @@ export class MountableEfs implements IMountableLinuxFilesystem {
     }
     const mountOptionsStr: string = mountOptions.join(',');
 
+    const resolveMountTargetDnsWithApi = this.props.resolveMountTargetDnsWithApi ?? false;
+    if (resolveMountTargetDnsWithApi) {
+      const describeMountTargetResources = [
+        (this.props.filesystem.node.defaultChild as efs.CfnFileSystem).attrArn,
+      ];
+      if (this.props.accessPoint) {
+        describeMountTargetResources.push(this.props.accessPoint.accessPointArn);
+      }
+
+      target.grantPrincipal.addToPrincipalPolicy(new PolicyStatement({
+        resources: describeMountTargetResources,
+        actions: ['elasticfilesystem:DescribeMountTargets'],
+      }));
+    }
+
     target.userData.addCommands(
       'TMPDIR=$(mktemp -d)',
       'pushd "$TMPDIR"',
       `unzip ${mountScript}`,
-      `bash ./mountEfs.sh ${this.props.filesystem.fileSystemId} ${mountDir} ${mountOptionsStr}`,
+      `bash ./mountEfs.sh ${this.props.filesystem.fileSystemId} ${mountDir} ${resolveMountTargetDnsWithApi} ${mountOptionsStr}`,
       'popd',
       `rm -f ${mountScript}`,
     );

packages/aws-rfdk/lib/core/scripts/bash/metadataUtilities.sh

@@ -68,3 +68,10 @@ function get_region() {
     #  into: us-west-2
     echo $IDENTITY_DOC | tr ',' '\n' | tr -d '[",{}]' | grep 'region' | awk '{print $3}'
 }
+
+function get_availability_zone() {
+    # Get the availability zone that this instance is running within (ex: us-west-2b)
+    # Usage: $0 <token>
+    TOKEN=$1
+    curl -H "X-aws-ec2-metadata-token: $TOKEN" -v 'http://169.254.169.254/latest/meta-data/placement/availability-zone' 2> /dev/null
+}

packages/aws-rfdk/lib/core/scripts/bash/mountEfs.sh

@@ -12,13 +12,17 @@
 # Script arguments:
 #  $1 -- EFS Identifier (ex: efs-00000000000)
 #  $2 -- Mount path; directory that we mount the EFS to.
-#  $3 -- (optional) NFSv4 mount options for the EFS. 
+#  $3 -- whether to obtain the EFS mount target's IP address using the EFS API and persist this to
+#        the /etc/hosts file on the system. This allows the script to work when the mounting instance cannot resolve the
+#        mount target using DNS. A value of "true" (case sensitive) will  enable this feature. Any other value will
+#        is treated as being disabled.
+#  $4 -- (optional) NFSv4 mount options for the EFS. 
 
 set -xeu
 
-if test $# -lt 2
+if test $# -lt 3
 then
-  echo "Usage: $0 <file system ID> <mount path> [<mount options>]"
+  echo "Usage: $0 FILE_SYSTEM_ID MOUNT_PATH RESOLVE_MOUNT_POINT_USING_API [MOUNT_OPTIONS]"
   exit 1
 fi
 
@@ -31,12 +35,14 @@ authenticate_identity_document
 
 METADATA_TOKEN=$(get_metadata_token)
 AWS_REGION=$(get_region "${METADATA_TOKEN}")
+AVAILABILITY_ZONE_NAME=$(get_availability_zone "${METADATA_TOKEN}")
 
 FILESYSTEM_ID=$1
 MOUNT_PATH=$2
-MOUNT_OPTIONS="${3:-}"
+RESOLVE_MOUNTPOINT_IP_VIA_API=$3
+MOUNT_OPTIONS="${4:-}"
 
-sudo mkdir -p "${MOUNT_PATH}"
+mkdir -p "${MOUNT_PATH}"
 
 AMAZON_EFS_PACKAGE="amazon-efs-utils"
 if which yum
@@ -49,31 +55,128 @@ else
 fi
 
 function use_amazon_efs_mount() {
-  test -f "/sbin/mount.efs" || sudo "${PACKAGE_MANAGER}" install -y "${AMAZON_EFS_PACKAGE}"
+  test -f "/sbin/mount.efs" || "${PACKAGE_MANAGER}" install -y "${AMAZON_EFS_PACKAGE}"
   return $?
 }
 
 function use_nfs_mount() {
-  test -f "/sbin/mount.nfs4" || sudo "${PACKAGE_MANAGER}" install -y "${NFS_UTILS_PACKAGE}"
+  test -f "/sbin/mount.nfs4" || "${PACKAGE_MANAGER}" install -y "${NFS_UTILS_PACKAGE}"
   return $?
 }
 
+function resolve_mount_target_ip_via_api() {
+  local EFS_FS_ID=$1
+  local MNT_TARGET_RESOURCE_ID=$2
+  local AVAILABILITY_ZONE_NAME=$3
+  local AWS_REGION=$4
+  local MOUNT_POINT_IP=""
+
+  local FILTER_ARGUMENT=""
+  if [[ $MNT_TARGET_RESOURCE_ID == fs-* ]]
+  then
+    # Mounting without an access point
+    FILTER_ARGUMENT="--file-system-id ${MNT_TARGET_RESOURCE_ID}"
+  elif [[ $MNT_TARGET_RESOURCE_ID == fsap-* ]]
+  then
+    # Mounting with an access point
+    FILTER_ARGUMENT="--access-point-id ${MNT_TARGET_RESOURCE_ID}"
+  else
+    echo "Unsupported mount target resource: ${MNT_TARGET_RESOURCE_ID}"
+    return 1
+  fi
+
+  # We prioritize the mount target in the same availability zone as the mounting instance
+  # jq sorts with false first then true (https://stedolan.github.io/jq/manual/#sort,sort_by(path_expression), so we
+  # negate the condition in the sort_by(...) expression
+  MOUNT_POINT_JSON=$(aws efs describe-mount-targets \
+    --region "${AWS_REGION}"                      \
+    ${FILTER_ARGUMENT}                            \
+    | jq ".MountTargets | sort_by( .AvailabilityZoneName != \"${AVAILABILITY_ZONE_NAME}\" ) | .[0]"
+  )
+
+  if [[ -z "${MOUNT_POINT_JSON}" ]]
+  then
+    echo "Could not find mount target for ${MNT_TARGET_RESOURCE_ID}"
+    return 1
+  fi
+
+  MOUNT_POINT_IP=$(echo "${MOUNT_POINT_JSON}" | jq -r .IpAddress)
+  MOUNT_POINT_AZ=$(echo "${MOUNT_POINT_JSON}" | jq -r .AvailabilityZoneName )
+
+  if [[ "${MOUNT_POINT_AZ}" != "${AVAILABILITY_ZONE_NAME}" ]]
+  then
+    set +x
+    echo "------------------------------------------ WARNING ------------------------------------------"
+    echo "Could not find mount target for ${MNT_TARGET_RESOURCE_ID} matching the current availability"
+    echo "zone (${AVAILABILITY_ZONE_NAME}). Cross-AZ data charges will be applied. To reduce costs,"
+    echo "add a mount target for ${MNT_TARGET_RESOURCE_ID} in ${AVAILABILITY_ZONE_NAME}."
+    echo "------------------------------------------ WARNING ------------------------------------------"
+    set -x
+  fi
+
+  DNS_NAME="${EFS_FS_ID}.efs.${AWS_REGION}.amazonaws.com"
+
+  # Backup the old hosts file
+  cp /etc/hosts "/etc/hosts.rfdk-backup-$(date +%Y-%m-%dT%H:%M:%S)"
+  # Remove any existing entries for the target DNS name
+  sed -i -e "/${DNS_NAME}/d" /etc/hosts
+  # Write the resolved entry for the target DNS name
+  cat >> /etc/hosts <<EOF
+
+${MOUNT_POINT_IP} ${DNS_NAME} # Added by RFDK
+EOF
+}
+
+# Optionally resolve DNS using the EFS API
+if [[ $RESOLVE_MOUNTPOINT_IP_VIA_API == "true"  ]]
+then
+  # jq is used to query the JSON API response
+  if ! where jq > /dev/null 2>&1
+  then
+    "${PACKAGE_MANAGER}" install -y jq
+  fi
+
+  # Get access point ID if available, otherwise file system ID
+  MNT_TARGET_RESOURCE_ID=$FILESYSTEM_ID
+  # The access point is supplied as in the MOUNT_OPTIONS argument, which is a list of comma-separated fstab options.
+  # Here is a sample opts string containing an access point:
+  #
+  #   rw,iam,accesspoint=fsap-1234567890,fsc
+  #
+  # See https://docs.aws.amazon.com/efs/latest/ug/efs-mount-helper.html#mounting-access-points
+  #
+  # We extract that value from MOUNT_OPTIONS here:
+  ACCESS_POINT_MOUNT_OPT=$(echo "${MOUNT_OPTIONS}" | sed -e 's#,#\n#g' | grep 'accesspoint=') || true
+  if [[ ! -z "${ACCESS_POINT_MOUNT_OPT}" ]]; then
+    ACCESS_POINT_ID=$(echo "${ACCESS_POINT_MOUNT_OPT}" | cut -d= -f2)
+    MNT_TARGET_RESOURCE_ID="${ACCESS_POINT_ID}"
+  fi
+
+  # This feature is treated as a best-effort first choice but falls-back to a regular DNS lookup with a warning emitted
+  resolve_mount_target_ip_via_api \
+      "${FILESYSTEM_ID}"          \
+      "${MNT_TARGET_RESOURCE_ID}" \
+      "${AVAILABILITY_ZONE_NAME}" \
+      "${AWS_REGION}"             \
+    || echo "WARNING: Couldn't resolve EFS IP address using the EFS service API endpoint"
+fi
+
 # Attempt to mount the EFS file system
 
 # fstab may be missing a newline at end of file.
 if test $(tail -c 1 /etc/fstab | wc -l) -eq 0
 then
   # Newline was missing, so add one.
-  echo "" | sudo tee -a /etc/fstab
+  echo "" | tee -a /etc/fstab
 fi
 
 if use_amazon_efs_mount
 then
-  echo "${FILESYSTEM_ID}:/ ${MOUNT_PATH} efs defaults,tls,_netdev,${MOUNT_OPTIONS}" | sudo tee -a /etc/fstab
+  echo "${FILESYSTEM_ID}:/ ${MOUNT_PATH} efs defaults,tls,_netdev,${MOUNT_OPTIONS}" | tee -a /etc/fstab
   MOUNT_TYPE=efs
 elif use_nfs_mount
 then
-  echo "${FILESYSTEM_ID}.efs.${AWS_REGION}.amazonaws.com:/ ${MOUNT_PATH} nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport,_netdev,${MOUNT_OPTIONS} 0 0" | sudo tee -a /etc/fstab
+  echo "${FILESYSTEM_ID}.efs.${AWS_REGION}.amazonaws.com:/ ${MOUNT_PATH} nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport,_netdev,${MOUNT_OPTIONS} 0 0" | tee -a /etc/fstab
   MOUNT_TYPE=nfs4
 else
   echo "Could not find suitable mount helper to mount the Elastic File System: ${FILESYSTEM_ID}"
@@ -85,7 +188,7 @@ fi
 # only if unable to mount it after that.
 TRIES=0
 MAX_TRIES=20
-while test ${TRIES} -lt ${MAX_TRIES} && ! sudo mount -a -t ${MOUNT_TYPE}
+while test ${TRIES} -lt ${MAX_TRIES} && ! mount -a -t ${MOUNT_TYPE}
 do
   let TRIES=TRIES+1
   sleep 2

packages/aws-rfdk/lib/core/test/asset-constants.ts

@@ -19,7 +19,7 @@ export const CWA_ASSET_WINDOWS = {
 
 // mountEbsBlockVolume.sh + metadataUtilities.sh + ec2-certificates.crt
 export const MOUNT_EBS_SCRIPT_LINUX = {
-  Bucket: stringLike('AssetParameters*S3BucketD23CD539'),
+  Bucket: stringLike('AssetParameters*S3BucketD3D2B3C1'),
 };
 
 export const INSTALL_MONGODB_3_6_SCRIPT_LINUX = {

packages/aws-rfdk/lib/core/test/mountable-efs.test.ts

@@ -92,7 +92,7 @@ describe('Test MountableEFS', () => {
     expect(userData).toMatch(new RegExp(escapeTokenRegex(s3Copy)));
     expect(userData).toMatch(new RegExp(escapeTokenRegex('unzip /tmp/${Token[TOKEN.\\d+]}${Token[TOKEN.\\d+]}')));
     // Make sure we execute the script with the correct args
-    expect(userData).toMatch(new RegExp(escapeTokenRegex('bash ./mountEfs.sh ${Token[TOKEN.\\d+]} /mnt/efs/fs1 rw')));
+    expect(userData).toMatch(new RegExp(escapeTokenRegex('bash ./mountEfs.sh ${Token[TOKEN.\\d+]} /mnt/efs/fs1 false rw')));
   });
 
   test('assert Linux-only', () => {
@@ -129,7 +129,7 @@ describe('Test MountableEFS', () => {
     const userData = instance.userData.render();
 
     // THEN
-    expect(userData).toMatch(new RegExp(escapeTokenRegex('mountEfs.sh ${Token[TOKEN.\\d+]} /mnt/efs/fs1 r')));
+    expect(userData).toMatch(new RegExp(escapeTokenRegex('mountEfs.sh ${Token[TOKEN.\\d+]} /mnt/efs/fs1 false r')));
   });
 
   describe.each<[MountPermissions | undefined]>([
@@ -206,7 +206,7 @@ describe('Test MountableEFS', () => {
             expect.arrayContaining([
               expect.stringMatching(new RegExp('(\\n|^)bash \\./mountEfs.sh $')),
               stack.resolve(efsFS.fileSystemId),
-              ` ${mountPath} ${expectedMountMode},iam,accesspoint=`,
+              ` ${mountPath} false ${expectedMountMode},iam,accesspoint=`,
               stack.resolve(accessPoint.accessPointId),
               expect.stringMatching(/^\n/),
             ]),
@@ -257,7 +257,7 @@ describe('Test MountableEFS', () => {
     const userData = instance.userData.render();
 
     // THEN
-    expect(userData).toMatch(new RegExp(escapeTokenRegex('mountEfs.sh ${Token[TOKEN.\\d+]} /mnt/efs/fs1 rw,option1,option2')));
+    expect(userData).toMatch(new RegExp(escapeTokenRegex('mountEfs.sh ${Token[TOKEN.\\d+]} /mnt/efs/fs1 false rw,option1,option2')));
   });
 
   test('asset is singleton', () => {
@@ -286,4 +286,60 @@ describe('Test MountableEFS', () => {
     expect(matches).toHaveLength(2);
     expect(matches[0]).toBe(matches[1]);
   });
+
+  describe('resolves mount target using API', () => {
+    describe.each<[string, () => efs.AccessPoint | undefined]>([
+      ['with access point', () => {
+
+        return new efs.AccessPoint(stack, 'AccessPoint', {
+          fileSystem: efsFS,
+          posixUser: {
+            gid: '1',
+            uid: '1',
+          },
+        });
+      }],
+      ['without access point', () => undefined],
+    ])('%s', (_, getAccessPoint) => {
+      let accessPoint: efs.AccessPoint | undefined;
+
+      beforeEach(() => {
+        // GIVEN
+        accessPoint = getAccessPoint();
+        const mountable = new MountableEfs(efsFS, {
+          filesystem: efsFS,
+          accessPoint,
+          resolveMountTargetDnsWithApi: true,
+        });
+
+        // WHEN
+        mountable.mountToLinuxInstance(instance, {
+          location: '/mnt/efs',
+        });
+      });
+
+      test('grants DescribeMountTargets permission', () => {
+        const expectedResources = [
+          stack.resolve((efsFS.node.defaultChild as efs.CfnFileSystem).attrArn),
+        ];
+        if (accessPoint) {
+          expectedResources.push(stack.resolve(accessPoint?.accessPointArn));
+        }
+        cdkExpect(stack).to(haveResourceLike('AWS::IAM::Policy', {
+          PolicyDocument: objectLike({
+            Statement: arrayWith(
+              {
+                Action: 'elasticfilesystem:DescribeMountTargets',
+                Effect: 'Allow',
+                Resource: expectedResources.length == 1 ? expectedResources[0] : expectedResources,
+              },
+            ),
+          }),
+          Roles: arrayWith(
+            stack.resolve((instance.role.node.defaultChild as CfnResource).ref),
+          ),
+        }));
+      });
+    });
+  });
 });

packages/aws-rfdk/lib/deadline/test/repository.test.ts

@@ -433,7 +433,7 @@ test('repository mounts repository filesystem', () => {
   const userData = (repo.node.defaultChild as AutoScalingGroup).userData.render();
 
   // THEN
-  expect(userData).toMatch(new RegExp(escapeTokenRegex('mountEfs.sh ${Token[TOKEN.\\d+]} /mnt/efs/fs1 rw')));
+  expect(userData).toMatch(new RegExp(escapeTokenRegex('mountEfs.sh ${Token[TOKEN.\\d+]} /mnt/efs/fs1 false rw')));
 });
 
 test.each([
@@ -940,7 +940,7 @@ test('repository configure client instance', () => {
 
   // THEN
   // white-box testing. If we mount the filesystem, then we've called: setupDirectConnect()
-  expect(userData).toMatch(new RegExp(escapeTokenRegex('mountEfs.sh ${Token[TOKEN.\\d+]} /mnt/repository rw')));
+  expect(userData).toMatch(new RegExp(escapeTokenRegex('mountEfs.sh ${Token[TOKEN.\\d+]} /mnt/repository false rw')));
 
   // Make sure we added the DB connection args
   expect(userData).toMatch(/.*export -f configure_deadline_database.*/);