feat(core): make cloudwatch agent install optional

aws · Mar 10, 2021 · e594aec · e594aec
1 parent f84097e
commit e594aec
Show file tree

Hide file tree

Showing 17 changed files with 1,435 additions and 208 deletions.
diff --git a/THIRD-PARTY b/THIRD-PARTY
diff --git a/examples/deadline/All-In-AWS-Infrastructure-Basic/python/package/config.py b/examples/deadline/All-In-AWS-Infrastructure-Basic/python/package/config.py
@@ -32,10 +32,10 @@ def __init__(self):
         # to pin to. Some examples of pinned version values are "10", "10.1", or "10.1.12"
         self.deadline_version: Optional[str] = None
 
-        # A map of regions to Deadline Client Linux AMIs. As an example, the Linux Deadline 10.1.12.1 AMI ID
+        # A map of regions to Deadline Client Linux AMIs. As an example, the Linux Deadline 10.1.13.2 AMI ID
         # from us-west-2 is filled in. It can be used as-is, added to, or replaced. Ideally the version here
-        #  should match the one used for staging the render queue and usage based licensing recipes.
-        self.deadline_client_linux_ami_map: Mapping[str, str] = {'us-west-2': 'ami-039f0c1faba28b015'}
+        #  should match the one used for fetching the render queue and usage based licensing images.
+        self.deadline_client_linux_ami_map: Mapping[str, str] = {'us-west-2': 'ami-0237f13ce87af168e'}
 
         # A secret (in binary form) in SecretsManager that stores the UBL certificates in a .zip file.
         self.ubl_certificate_secret_arn: str =\

diff --git a/examples/deadline/All-In-AWS-Infrastructure-Basic/ts/bin/config.ts b/examples/deadline/All-In-AWS-Infrastructure-Basic/ts/bin/config.ts
@@ -31,11 +31,11 @@ class AppConfig {
   public readonly deadlineVersion?: string;
 
   /**
-   * A map of regions to Deadline Client Linux AMIs. As an example, the Linux Deadline 10.1.12.1 AMI ID from us-west-2
+   * A map of regions to Deadline Client Linux AMIs. As an example, the Linux Deadline 10.1.13.2 AMI ID from us-west-2
    * is filled in. It can be used as-is, added to, or replaced. Ideally the version here should match the one in
-   * package.json used for staging the render queue and usage based licensing recipes.
+   * package.json used for fetching the render queue and usage based licensing images.
    */
-  public readonly deadlineClientLinuxAmiMap: Record<string, string> = {['us-west-2']: 'ami-039f0c1faba28b015'};
+  public readonly deadlineClientLinuxAmiMap: Record<string, string> = {['us-west-2']: 'ami-0237f13ce87af168e'};
 
   /**
    * (Optional) A secret (in binary form) in SecretsManager that stores the UBL certificates in a .zip file.

diff --git a/packages/aws-rfdk/lib/core/lib/cloudwatch-agent.ts b/packages/aws-rfdk/lib/core/lib/cloudwatch-agent.ts
@@ -5,11 +5,12 @@
 
 import * as path from 'path';
 
-import {IGrantable} from '@aws-cdk/aws-iam';
-import {Bucket} from '@aws-cdk/aws-s3';
-import {StringParameter} from '@aws-cdk/aws-ssm';
-import {Construct} from '@aws-cdk/core';
-import {IScriptHost, ScriptAsset} from './script-assets';
+import { IGrantable } from '@aws-cdk/aws-iam';
+import { Bucket } from '@aws-cdk/aws-s3';
+import { StringParameter } from '@aws-cdk/aws-ssm';
+import { Construct, Stack } from '@aws-cdk/core';
+
+import { IScriptHost, ScriptAsset } from './script-assets';
 
 /**
  *  Properties for creating the resources needed for CloudWatch Agent configuration.
@@ -26,6 +27,13 @@ export interface CloudWatchAgentProps {
    * The host instance/ASG/fleet with a CloudWatch Agent to be configured.
    */
   readonly host: IScriptHost;
+
+  /**
+   * Whether or not we should attempt to install the CloudWatch agent
+   *
+   * @default true
+   */
+  readonly shouldInstallAgent?: boolean;
 }
 
 /**
@@ -70,6 +78,11 @@ export class CloudWatchAgent extends Construct {
    */
   private static readonly SKIP_CWAGENT_VALIDATION_FLAG = '-s';
 
+  /**
+   * The flag for configureCloudWatchAgent script to skip CloudWatch agent installer validation.
+   */
+  private static readonly INSTALL_CWAGENT_FLAG = '-i';
+
   /**
    * An S3 script asset that configures the CloudWatch agent.
    */
@@ -83,6 +96,8 @@ export class CloudWatchAgent extends Construct {
   constructor(scope: Construct, id: string, props: CloudWatchAgentProps) {
     super(scope, id);
 
+    const shouldInstallAgent = props.shouldInstallAgent ?? true;
+
     // Create the asset for the configuration script
     this.configurationScript = ScriptAsset.fromPathConvention(scope, 'CloudWatchConfigurationScriptAsset', {
       osType: props.host.osType,
@@ -97,7 +112,11 @@ export class CloudWatchAgent extends Construct {
     });
 
     this.grantRead(props.host);
-    this.configure(props.host, this.node.tryGetContext(CloudWatchAgent.SKIP_CWAGENT_VALIDATION_CTX_VAR) === 'TRUE');
+    this.configure(
+      props.host,
+      shouldInstallAgent,
+      this.node.tryGetContext(CloudWatchAgent.SKIP_CWAGENT_VALIDATION_CTX_VAR) === 'TRUE',
+    );
   }
 
   /**
@@ -114,18 +133,37 @@ export class CloudWatchAgent extends Construct {
    * This is done by adding UserData commands to the target host.
    *
    * @param host The host to configure the CloudWatch agent on
+   * @param shouldInstallAgent Attempts to install the CloudWatch agent on the instance if set to true.
    * @param skipValidation Skips the validation of the CloudWatch agent installer if set to true.
    */
-  private configure(host: IScriptHost, skipValidation: boolean) {
-    // Grant access to the required CloudWatch Agent installer files
-    const cloudWatchAgentBucket = Bucket.fromBucketArn(this, 'CloudWatchAgentBucket', 'arn:aws:s3:::amazoncloudwatch-agent');
-    cloudWatchAgentBucket.grantRead(host);
+  private configure(
+    host: IScriptHost,
+    shouldInstallAgent: boolean,
+    skipValidation: boolean,
+  ) {
+    const region = Stack.of(this).region;
+    if (shouldInstallAgent) {
+      // Grant access to the required CloudWatch Agent and GPG installer files.
+      const cloudWatchAgentBucket = Bucket.fromBucketArn(this, 'CloudWatchAgentBucket', `arn:aws:s3:::amazoncloudwatch-agent-${region}`);
+      cloudWatchAgentBucket.grantRead(host);
+      const gpgBucket = Bucket.fromBucketArn(this, 'GpgBucket', `arn:aws:s3:::rfdk-external-dependencies-${region}`);
+      gpgBucket.grantRead(host);
+    }
 
     const scriptArgs = [];
+
+    // Flags must be set before positional arguments for some scripts
+    if (shouldInstallAgent) {
+      scriptArgs.push(CloudWatchAgent.INSTALL_CWAGENT_FLAG);
+    }
     if (skipValidation) {
-      // Flags must be set before positional arguments for some scripts
       scriptArgs.push(CloudWatchAgent.SKIP_CWAGENT_VALIDATION_FLAG);
     }
+
+    // This assumes that the CloudWatch agent construct is always put in the same region as the instance or ASG
+    // using it, which should always hold true.
+    scriptArgs.push(region);
+
     scriptArgs.push(this.ssmParameterForConfig.parameterName);
 
     this.configurationScript.executeOn({

diff --git a/packages/aws-rfdk/lib/core/scripts/bash/configureCloudWatchAgent.sh b/packages/aws-rfdk/lib/core/scripts/bash/configureCloudWatchAgent.sh
@@ -6,49 +6,37 @@
 # This script downloads, installs and configures the cloudwatch agent. Must be run as sudo capable user.
 
 usage() {
-  echo "This script downloads, installs and configures the cloudwatch agent. Must be run as sudo capable user.
+  echo "This script downloads, installs and configures the CloudWatch agent. Must be run as sudo capable user.
 Arguments:
-  -s: [Flag] Skips the verification of the cloudwatch agent installer
-  \$1: SSM parameter name
-  
+  -i: [Flag] Attempts to install the CloudWatch agent. If this isn't set we skip right to configuring it.
+  -s: [Flag] Skips the verification of the CloudWatch agent installer. Only relevent if we are trying to install the agent.
+  \$1: region
+  \$2: SSM parameter name
+
 Note: Flags must appear before positional parameters"
   exit 1
 }
 
-# exit when any command fails
-set -xeuo pipefail
-
-# Parse options
-SKIP_VERIFICATION=false
-OPTIND=1 # Reset index for getopts in case of previous invocations
-while getopts "s" opt; do
-  case $opt in
-  s) SKIP_VERIFICATION=true ;;
-  \?) echo "ERROR: Unknown option specified"; usage ;;
-  esac
-done
-shift $((OPTIND - 1))
+# Don't exit if commands fail
+set -x
 
-# Parse positional arguments
-if (($# != 1))
-then
-  echo "ERROR: Invalid arguments"
-  usage
-fi
-SSM_PARAMETER_NAME="$1"
+function install_agent {
+  region="$1"
+  # Check if amazon-cloudwatch-agent is already installed
+  if rpm -qa | grep amazon-cloudwatch-agent
+  then
+    return
+  fi
 
-# Check if amazon-cloudwatch-agent is already installed
-if ! rpm -qa | grep amazon-cloudwatch-agent
-then
   TMPDIR=$(mktemp -d)
   pushd $TMPDIR 2>&1 > /dev/null
 
   # Download CloudWatch agent installer
-  aws s3api get-object --bucket amazoncloudwatch-agent --key amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm amazon-cloudwatch-agent.rpm
+  aws s3api get-object --region $region --bucket amazoncloudwatch-agent-$region --key amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm amazon-cloudwatch-agent.rpm
 
   if [ "$SKIP_VERIFICATION" = false ]
   then
-    aws s3api get-object --bucket amazoncloudwatch-agent --key assets/amazon-cloudwatch-agent.gpg amazon-cloudwatch-agent.gpg
+    aws s3api get-object --region $region --bucket amazoncloudwatch-agent-$region --key assets/amazon-cloudwatch-agent.gpg amazon-cloudwatch-agent.gpg
     GPG_IMPORT_OUT=$(gpg --no-default-keyring --keyring ./keyring.gpg --import amazon-cloudwatch-agent.gpg 2>&1)
     GPG_KEY=$(echo "${GPG_IMPORT_OUT}" | grep -Eow 'key [0-9A-F]+' | awk '{print $2}')
     GPG_FINGERPRINT_OUT=$(gpg --no-default-keyring --keyring ./keyring.gpg --fingerprint ${GPG_KEY} 2>&1)
@@ -57,15 +45,19 @@ then
     then
         # Key failed to verify. Alert AWS!!
         echo "ERROR: Key failed to verify."
-        exit 1
+        popd
+        rm -rf ${TMPDIR}
+        return
     fi
 
-    aws s3api get-object --bucket amazoncloudwatch-agent --key amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm.sig amazon-cloudwatch-agent.rpm.sig
+    aws s3api get-object --region $region --bucket amazoncloudwatch-agent-$region --key amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm.sig amazon-cloudwatch-agent.rpm.sig
     if ! gpg --no-default-keyring --keyring ./keyring.gpg --verify amazon-cloudwatch-agent.rpm.sig amazon-cloudwatch-agent.rpm 2>&1
     then
         # CloudWatch agent installer failed to verify. Alert AWS!!
         echo "ERROR: Agent installer failed to verify"
-        exit 1
+        popd
+        rm -rf ${TMPDIR}
+        return
     fi
   fi
 
@@ -74,9 +66,39 @@ then
 
   popd
   rm -rf ${TMPDIR}
-fi
+}
 
+echo "Starting CloudWatch installation and configuration script."
 
+# Parse options
+SKIP_VERIFICATION=false
+INSTALL_AGENT=false
+OPTIND=1 # Reset index for getopts in case of previous invocations
+while getopts "is" opt; do
+  case $opt in
+  i) INSTALL_AGENT=true ;;
+  s) SKIP_VERIFICATION=true ;;
+  \?) echo "ERROR: Unknown option specified"; usage ;;
+  esac
+done
+shift $((OPTIND - 1))
+
+# Parse positional arguments
+if (($# != 2))
+then
+  echo "ERROR: Invalid arguments"
+  usage
+fi
+REGION="$1"
+SSM_PARAMETER_NAME="$2"
+
+# Check if we want to attempt to install the agent
+if $INSTALL_AGENT
+then
+  install_agent $REGION
+else
+  echo "Skipped installation of CloudWatch agent"
+fi
 
 # starts the agent
 /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a append-config -m ec2 -c ssm:$SSM_PARAMETER_NAME -s