feat(s3-deployment): added Source.dataYaml helper function (#24579)

Add the `Source.dataYaml` helper function to deploy an object with the specified object converted to YAML format. Closes #24554. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
aws · Mar 29, 2023 · d969ddf · d969ddf
1 parent 72b3a95
commit d969ddf
Show file tree

Hide file tree

Showing 21 changed files with 421 additions and 87 deletions.
diff --git a/package.json b/package.json
@@ -119,6 +119,8 @@
       "@aws-cdk/core/minimatch/**",
       "@aws-cdk/core/table",
       "@aws-cdk/core/table/**",
+      "@aws-cdk/core/yaml",
+      "@aws-cdk/core/yaml/**",
       "@aws-cdk/cx-api/semver",
       "@aws-cdk/cx-api/semver/**",
       "@aws-cdk/pipelines/aws-sdk",

diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md
@@ -86,6 +86,8 @@ The following source types are supported for bucket deployments:
   (supports [deploy-time values](#data-with-deploy-time-values))
 - JSON data: `s3deploy.Source.jsonData('object-key.json', { json: 'object' })`
   (supports [deploy-time values](#data-with-deploy-time-values))
+- YAML data: `s3deploy.Source.yamlData('object-key.yaml', { yaml: 'object' })`
+  (supports [deploy-time values](#data-with-deploy-time-values))
 
 To create a source from a single file, you can pass `AssetOptions` to exclude
 all but a single file:
@@ -313,7 +315,7 @@ new s3deploy.BucketDeployment(this, 'DeployMeWithEfsStorage', {
 
 ## Data with deploy-time values
 
-The content passed to `Source.data()` or `Source.jsonData()` can include
+The content passed to `Source.data()`, `Source.jsonData()`, or `Source.yamlData()` can include
 references that will get resolved only during deployment.
 
 For example:

diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/source.ts b/packages/@aws-cdk/aws-s3-deployment/lib/source.ts
@@ -58,7 +58,8 @@ export interface ISource {
  *     Source.asset('/local/path/to/directory')
  *     Source.asset('/local/path/to/a/file.zip')
  *     Source.data('hello/world/file.txt', 'Hello, world!')
- *     Source.data('config.json', { baz: topic.topicArn })
+ *     Source.dataJson('config.json', { baz: topic.topicArn })
+ *     Source.dataYaml('config.yaml', { baz: topic.topicArn })
  *
  */
 export class Source {
@@ -125,6 +126,7 @@ export class Source {
    * will get resolved only during deployment.
    *
    * To store a JSON object use `Source.jsonData()`.
+   * To store YAML content use `Source.yamlData()`.
    *
    * @param objectKey The destination S3 object key (relative to the root of the
    * S3 deployment).
@@ -165,5 +167,22 @@ export class Source {
     };
   }
 
+  /**
+   * Deploys an object with the specified JSON object formatted as YAML into the bucket.
+   * The object can include deploy-time values (such as `snsTopic.topicArn`) that
+   * will get resolved only during deployment.
+   *
+   * @param objectKey The destination S3 object key (relative to the root of the
+   * S3 deployment).
+   * @param obj A JSON object.
+   */
+  public static yamlData(objectKey: string, obj: any): ISource {
+    return {
+      bind: (scope: Construct, context?: DeploymentSourceContext) => {
+        return Source.data(objectKey, Stack.of(scope).toYamlString(obj)).bind(scope, context);
+      },
+    };
+  }
+
   private constructor() { }
 }
diff --git a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts
@@ -1387,6 +1387,32 @@ test('Source.jsonData() can be used to create a file with a JSON object', () =>
   });
 });
 
+test('Source.yamlData() can be used to create a file with YAML content', () => {
+  const app = new cdk.App();
+  const stack = new cdk.Stack(app, 'Test');
+  const bucket = new s3.Bucket(stack, 'Bucket');
+
+  const config = {
+    foo: 'bar',
+    sub: {
+      hello: bucket.bucketArn,
+    },
+  };
+
+  new s3deploy.BucketDeployment(stack, 'DeployWithVpc3', {
+    sources: [s3deploy.Source.yamlData('app-config.yaml', config)],
+    destinationBucket: bucket,
+  });
+
+  const result = app.synth();
+  const output = readDataFile(result, 'app-config.yaml');
+  expect(output.trim()).toEqual([
+    'foo: bar',
+    'sub:',
+    '  hello: <<marker:0xbaba:0>>',
+  ].join('\n'));
+});
+
 test('can add sources with addSource', () => {
   const app = new cdk.App();
   const stack = new cdk.Stack(app, 'Test');

diff --git a/...ent/test/integ.bucket-deployment-data.js.snapshot/TestBucketDeploymentContent.assets.json b/...ent/test/integ.bucket-deployment-data.js.snapshot/TestBucketDeploymentContent.assets.json
@@ -1,28 +1,28 @@
 {
-  "version": "20.0.0",
+  "version": "31.0.0",
   "files": {
-    "39ee629321f531fffd853b944b2d6f3fa7b5276431c9a4fd4dc681303ab15080": {
+    "68b22621fff135f9e3f225bad7ff80fdf2f45c3d9910af601206a0d9b279933a": {
       "source": {
-        "path": "asset.39ee629321f531fffd853b944b2d6f3fa7b5276431c9a4fd4dc681303ab15080.zip",
+        "path": "asset.68b22621fff135f9e3f225bad7ff80fdf2f45c3d9910af601206a0d9b279933a.zip",
         "packaging": "file"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "39ee629321f531fffd853b944b2d6f3fa7b5276431c9a4fd4dc681303ab15080.zip",
+          "objectKey": "68b22621fff135f9e3f225bad7ff80fdf2f45c3d9910af601206a0d9b279933a.zip",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }
     },
-    "f98b78092dcdd31f5e6d47489beb5f804d4835ef86a8085d0a2053cb9ae711da": {
+    "2bc265c5e0569aeb24a6349c15bd54e76e845892376515e036627ab0cc70bb64": {
       "source": {
-        "path": "asset.f98b78092dcdd31f5e6d47489beb5f804d4835ef86a8085d0a2053cb9ae711da",
+        "path": "asset.2bc265c5e0569aeb24a6349c15bd54e76e845892376515e036627ab0cc70bb64",
         "packaging": "zip"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "f98b78092dcdd31f5e6d47489beb5f804d4835ef86a8085d0a2053cb9ae711da.zip",
+          "objectKey": "2bc265c5e0569aeb24a6349c15bd54e76e845892376515e036627ab0cc70bb64.zip",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }
@@ -53,28 +53,41 @@
         }
       }
     },
-    "0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a": {
+    "27eff729291aea0a2b33592996b9a764c233dc3387bd9cfd58c6f064073f177f": {
       "source": {
-        "path": "asset.0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a",
+        "path": "asset.27eff729291aea0a2b33592996b9a764c233dc3387bd9cfd58c6f064073f177f",
         "packaging": "zip"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a.zip",
+          "objectKey": "27eff729291aea0a2b33592996b9a764c233dc3387bd9cfd58c6f064073f177f.zip",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }
     },
-    "55549864d8c27c2125682e2e817e0a7fede6b77576ccd6e21178645f08472948": {
+    "939a4ab8b51f1a1cccb59d97f04c318ad41c1d404e666c158ca2810894bc5f5f": {
+      "source": {
+        "path": "asset.939a4ab8b51f1a1cccb59d97f04c318ad41c1d404e666c158ca2810894bc5f5f",
+        "packaging": "zip"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "939a4ab8b51f1a1cccb59d97f04c318ad41c1d404e666c158ca2810894bc5f5f.zip",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
+    },
+    "2961e8222a48394849f4466d2789ae256aa88adc4ccbf79feb35306b850c08dc": {
       "source": {
         "path": "TestBucketDeploymentContent.template.json",
         "packaging": "file"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "55549864d8c27c2125682e2e817e0a7fede6b77576ccd6e21178645f08472948.json",
+          "objectKey": "2961e8222a48394849f4466d2789ae256aa88adc4ccbf79feb35306b850c08dc.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }

diff --git a/...t/test/integ.bucket-deployment-data.js.snapshot/TestBucketDeploymentContent.template.json b/...t/test/integ.bucket-deployment-data.js.snapshot/TestBucketDeploymentContent.template.json
@@ -20,7 +20,7 @@
      "S3Bucket": {
       "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
      },
-     "S3Key": "39ee629321f531fffd853b944b2d6f3fa7b5276431c9a4fd4dc681303ab15080.zip"
+     "S3Key": "68b22621fff135f9e3f225bad7ff80fdf2f45c3d9910af601206a0d9b279933a.zip"
     },
     "Description": "/opt/awscli/aws"
    }
@@ -41,14 +41,18 @@
      {
       "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
      },
+     {
+      "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
+     },
      {
       "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
      }
     ],
     "SourceObjectKeys": [
      "d09271be89b6cb0398f793b40c1531fd9b076aa92ba80b5e436914b1808fe18d.zip",
      "0f14dedeaf4386031c978375cbda0f65d7b52b29452cabb8873eb8f0d0fa936b.zip",
-     "0d7be86c2a7d62be64fcbe2cbaa36c912a72d445022cc17c37af4f99f1b97a5a.zip"
+     "27eff729291aea0a2b33592996b9a764c233dc3387bd9cfd58c6f064073f177f.zip",
+     "939a4ab8b51f1a1cccb59d97f04c318ad41c1d404e666c158ca2810894bc5f5f.zip"
     ],
     "SourceMarkers": [
      {},
@@ -57,6 +61,14 @@
        "Ref": "Bucket83908E77"
       }
      },
+     {
+      "<<marker:0xbaba:0>>": {
+       "Fn::GetAtt": [
+        "Bucket83908E77",
+        "WebsiteURL"
+       ]
+      }
+     },
      {
       "<<marker:0xbaba:0>>": {
        "Fn::GetAtt": [
@@ -208,7 +220,7 @@
      "S3Bucket": {
       "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
      },
-     "S3Key": "f98b78092dcdd31f5e6d47489beb5f804d4835ef86a8085d0a2053cb9ae711da.zip"
+     "S3Key": "2bc265c5e0569aeb24a6349c15bd54e76e845892376515e036627ab0cc70bb64.zip"
     },
     "Role": {
      "Fn::GetAtt": [

diff --git a/...45022cc17c37af4f99f1b97a5a/my/config.json → ...d9cfd58c6f064073f177f/my-json/config.json b/...45022cc17c37af4f99f1b97a5a/my/config.json → ...d9cfd58c6f064073f177f/my-json/config.json
diff --git a/...4d4835ef86a8085d0a2053cb9ae711da/index.py → ...6e845892376515e036627ab0cc70bb64/index.py b/...4d4835ef86a8085d0a2053cb9ae711da/index.py → ...6e845892376515e036627ab0cc70bb64/index.py
@@ -49,6 +49,7 @@ def cfn_error(message=None):
             source_markers      = props.get('SourceMarkers', None)
             dest_bucket_name    = props['DestinationBucketName']
             dest_bucket_prefix  = props.get('DestinationBucketKeyPrefix', '')
+            extract             = props.get('Extract', 'true') == 'true'
             retain_on_delete    = props.get('RetainOnDelete', "true") == "true"
             distribution_id     = props.get('DistributionId', '')
             user_metadata       = props.get('UserMetadata', {})
@@ -113,14 +114,15 @@ def cfn_error(message=None):
             aws_command("s3", "rm", old_s3_dest, "--recursive")
 
         if request_type == "Update" or request_type == "Create":
-            s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers)
+            s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers, extract)
 
         if distribution_id:
             cloudfront_invalidate(distribution_id, distribution_paths)
 
         cfn_send(event, context, CFN_SUCCESS, physicalResourceId=physical_id, responseData={
             # Passing through the ARN sequences dependencees on the deployment
-            'DestinationBucketArn': props.get('DestinationBucketArn')
+            'DestinationBucketArn': props.get('DestinationBucketArn'),
+            'SourceObjectKeys': props.get('SourceObjectKeys'),
         })
     except KeyError as e:
         cfn_error("invalid request. Missing key %s" % str(e))
@@ -130,7 +132,7 @@ def cfn_error(message=None):
 
 #---------------------------------------------------------------------------------------------------
 # populate all files from s3_source_zips to a destination bucket
-def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers):
+def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers, extract):
     # list lengths are equal
     if len(s3_source_zips) != len(source_markers):
         raise Exception("'source_markers' and 's3_source_zips' must be the same length")
@@ -154,12 +156,16 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex
             s3_source_zip = s3_source_zips[i]
             markers       = source_markers[i]
 
-            archive=os.path.join(workdir, str(uuid4()))
-            logger.info("archive: %s" % archive)
-            aws_command("s3", "cp", s3_source_zip, archive)
-            logger.info("| extracting archive to: %s\n" % contents_dir)
-            logger.info("| markers: %s" % markers)
-            extract_and_replace_markers(archive, contents_dir, markers)
+            if extract:
+                archive=os.path.join(workdir, str(uuid4()))
+                logger.info("archive: %s" % archive)
+                aws_command("s3", "cp", s3_source_zip, archive)
+                logger.info("| extracting archive to: %s\n" % contents_dir)
+                logger.info("| markers: %s" % markers)
+                extract_and_replace_markers(archive, contents_dir, markers)
+            else:
+                logger.info("| copying archive to: %s\n" % contents_dir)
+                aws_command("s3", "cp", s3_source_zip, contents_dir)
 
         # sync from "contents" to destination
 
@@ -230,7 +236,6 @@ def aws_command(*args):
 def cfn_send(event, context, responseStatus, responseData={}, physicalResourceId=None, noEcho=False, reason=None):
 
     responseUrl = event['ResponseURL']
-    logger.info(responseUrl)
 
     responseBody = {}
     responseBody['Status'] = responseStatus
@@ -285,7 +290,7 @@ def extract_and_replace_markers(archive, contents_dir, markers):
         for file in zip.namelist():
             file_path = os.path.join(contents_dir, file)
             if os.path.isdir(file_path): continue
-            replace_markers(file_path, markers)    
+            replace_markers(file_path, markers)
 
 def replace_markers(filename, markers):
     # convert the dict of string markers to binary markers
@@ -300,4 +305,4 @@ def replace_markers(filename, markers):
 
     # # delete the original file and rename the new one to the original
     os.remove(filename)
-    os.rename(outfile, filename)
+    os.rename(outfile, filename)
diff --git a/...d6f3fa7b5276431c9a4fd4dc681303ab15080.zip → ...f80fdf2f45c3d9910af601206a0d9b279933a.zip b/...d6f3fa7b5276431c9a4fd4dc681303ab15080.zip → ...f80fdf2f45c3d9910af601206a0d9b279933a.zip
diff --git a/...sset.939a4ab8b51f1a1cccb59d97f04c318ad41c1d404e666c158ca2810894bc5f5f/my-yaml/config.yaml b/...sset.939a4ab8b51f1a1cccb59d97f04c318ad41c1d404e666c158ca2810894bc5f5f/my-yaml/config.yaml
@@ -0,0 +1 @@
+website_url: <<marker:0xbaba:0>>
diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/cdk.out b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/cdk.out
@@ -1 +1 @@
-{"version":"21.0.0"}
+{"version":"31.0.0"}
diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/integ.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-data.js.snapshot/integ.json
@@ -1,14 +1,12 @@
 {
-  "version": "21.0.0",
+  "version": "31.0.0",
   "testCases": {
-    "integ.bucket-deployment-data": {
+    "integ-test-bucket-deployment-data/DefaultTest": {
       "stacks": [
         "TestBucketDeploymentContent"
       ],
-      "diffAssets": false,
-      "stackUpdateWorkflow": true
+      "assertionStack": "integ-test-bucket-deployment-data/DefaultTest/DeployAssert",
+      "assertionStackName": "integtestbucketdeploymentdataDefaultTestDeployAssert6FF3075D"
     }
-  },
-  "synthContext": {},
-  "enableLookups": false
+  }
 }
diff --git a/...data.js.snapshot/integtestbucketdeploymentdataDefaultTestDeployAssert6FF3075D.assets.json b/...data.js.snapshot/integtestbucketdeploymentdataDefaultTestDeployAssert6FF3075D.assets.json
@@ -0,0 +1,19 @@
+{
+  "version": "31.0.0",
+  "files": {
+    "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": {
+      "source": {
+        "path": "integtestbucketdeploymentdataDefaultTestDeployAssert6FF3075D.template.json",
+        "packaging": "file"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
+    }
+  },
+  "dockerImages": {}
+}