-
Notifications
You must be signed in to change notification settings - Fork 249
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(new construct): aws-fargate-kinesisfirehose (#881)
* First Draft, README and architecture * Typos, errors * First Draft * Snapshot Update * Snapshot updates * Results of self-review * test-helper adjustments * Sync version numbers * Address review comments * Address review comments
- Loading branch information
Showing
26 changed files
with
10,289 additions
and
76 deletions.
There are no files selected for viewing
4 changes: 4 additions & 0 deletions
4
source/patterns/@aws-solutions-constructs/aws-fargate-kinesisfirehose/.eslintignore
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| lib/*.js | ||
| test/*.js | ||
| *.d.ts | ||
| coverage |
15 changes: 15 additions & 0 deletions
15
source/patterns/@aws-solutions-constructs/aws-fargate-kinesisfirehose/.gitignore
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| lib/*.js | ||
| test/*.js | ||
| *.js.map | ||
| *.d.ts | ||
| node_modules | ||
| *.generated.ts | ||
| dist | ||
| .jsii | ||
|
|
||
| .LAST_BUILD | ||
| .nyc_output | ||
| coverage | ||
| .nycrc | ||
| .LAST_PACKAGE | ||
| *.snk |
21 changes: 21 additions & 0 deletions
21
source/patterns/@aws-solutions-constructs/aws-fargate-kinesisfirehose/.npmignore
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| # Exclude typescript source and config | ||
| *.ts | ||
| tsconfig.json | ||
| coverage | ||
| .nyc_output | ||
| *.tgz | ||
| *.snk | ||
| *.tsbuildinfo | ||
|
|
||
| # Include javascript files and typescript declarations | ||
| !*.js | ||
| !*.d.ts | ||
|
|
||
| # Exclude jsii outdir | ||
| dist | ||
|
|
||
| # Include .jsii | ||
| !.jsii | ||
|
|
||
| # Include .jsii | ||
| !.jsii |
128 changes: 128 additions & 0 deletions
128
source/patterns/@aws-solutions-constructs/aws-fargate-kinesisfirehose/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,128 @@ | ||
| # aws-fargate-kinesisfirehose module | ||
| <!--BEGIN STABILITY BANNER--> | ||
|
|
||
| --- | ||
|
|
||
|  | ||
|
|
||
| --- | ||
| <!--END STABILITY BANNER--> | ||
|
|
||
| | **Reference Documentation**:| <span style="font-weight: normal">https://docs.aws.amazon.com/solutions/latest/constructs/</span>| | ||
| |:-------------|:-------------| | ||
| <div style="height:8px"></div> | ||
|
|
||
| | **Language** | **Package** | | ||
| |:-------------|-----------------| | ||
| | Python|`aws_solutions_constructs.aws_fargate_kinesisfirehose`| | ||
| | Typescript|`@aws-solutions-constructs/aws-fargate-kinesisfirehose`| | ||
| | Java|`software.amazon.awsconstructs.services.fargatekinesisfirehose`| | ||
|
|
||
| ## Overview | ||
| This AWS Solutions Construct deploys an AWS Fargate Service that can put records on an Amazon Firehose Delivery Stream. | ||
|
|
||
| Here is a minimal deployable pattern definition: | ||
|
|
||
| Typescript | ||
| ``` typescript | ||
| import { Construct } from 'constructs'; | ||
| import { Stack, StackProps } from 'aws-cdk-lib'; | ||
| import { FargateToKinesisFirehoseProps } from '@aws-solutions-constructs/aws-fargate-kinesisfirehose'; | ||
| import * as fargate from 'aws-cdk-lib/aws-fargate'; | ||
|
|
||
| // The construct requires an existing Firehose Delivery Stream, this can be created in raw CDK or extracted | ||
| // from a previously instantiated construct that created an Firehose Delivery Stream | ||
| const existingFirehoseDeliveryStream = previouslyCreatedKinesisFirehoseToS3Construct.kinesisFirehose; | ||
|
|
||
| new FargateToKinesisFirehose(this, 'FargateToKinesisFirehose', { | ||
| publicApi: true, | ||
| ecrRepositoryArn: "arn:aws:ecr:us-east-1:123456789012:repository/your-ecr-repo", | ||
| existingKinesisFirehose: existingFirehoseDeliveryStream | ||
| }); | ||
| ``` | ||
|
|
||
| Python | ||
| ``` python | ||
| from aws_solutions_constructs.aws_fargate_kinesisfirehose import FargateToKinesisFirehose | ||
| from aws_cdk import ( | ||
| aws_fargate as _fargate, | ||
| Stack | ||
| ) | ||
| from constructs import Construct | ||
|
|
||
| # The construct requires an existing Firehose Delivery Stream, this can be created in raw CDK or extracted | ||
| # from a previously instantiated construct that created an Firehose Delivery Stream | ||
| existingFirehoseDeliveryStream = previouslyCreatedKinesisFirehoseToS3Construct.kinesisFirehose; | ||
|
|
||
| FargateToKinesisFirehose(self, 'FargateToKinesisFirehose', | ||
| public_api=True, | ||
| ecr_repository_arn="arn:aws:ecr:us-east-1:123456789012:repository/your-ecr-repo", | ||
| existingKinesisFirehose=existingFirehoseDeliveryStream | ||
| ) | ||
| ``` | ||
|
|
||
| Java | ||
| ``` java | ||
| import software.constructs.Construct; | ||
| import software.amazon.awscdk.Stack; | ||
| import software.amazon.awscdk.StackProps; | ||
| import software.amazon.awscdk.services.fargate.*; | ||
| import software.amazon.awscdk.services.fargate.eventsources.*; | ||
| import software.amazon.awscdk.services.fargate.Runtime; | ||
| import software.amazon.awsconstructs.services.fargatekinesisfirehose.*; | ||
|
|
||
| // The construct requires an existing Firehose Delivery Stream, this can be created in raw CDK or extracted | ||
| // from a previously instantiated construct that created an Firehose Delivery Stream | ||
| existingFirehoseDeliveryStream = previouslyCreatedKinesisFirehoseToS3Construct.kinesisFirehose; | ||
|
|
||
| new FargateToKinesisFirehose(this, "FargateToKinesisFirehose", new FargateToKinesisFirehoseProps.Builder() | ||
| .publicApi(true) | ||
| .ecrRepositoryArn("arn:aws:ecr:us-east-1:123456789012:repository/your-ecr-repo") | ||
| .existingKinesisFirehose(existingFirehoseDeliveryStream) | ||
| .build()); | ||
| ``` | ||
|
|
||
| ## Pattern Construct Props | ||
|
|
||
| | **Name** | **Type** | **Description** | | ||
| |:-------------|:----------------|-----------------| | ||
| | publicApi | `boolean` | True if the VPC provisioned by this construct should contain Public/Private Subnets, otherwise False for the VPC to contain Isolated Subnets only. Note this property is ignored if an existing VPC is specified in the existingVpc property. If you are getting a container from a public repo, this must be true so the repo can be accessed from the network. | | ||
| | vpcProps? | [`ec2.VpcProps`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.VpcProps.html) | Optional custom properties for a new VPC the construct will create. Providing both this and `existingVpc` is an error. An Amazon Kinesis Firehose Interface Endpoint will be added to this VPC. | | ||
| | existingVpc? | [`ec2.IVpc`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.IVpc.html) | An existing VPC in which to deploy the Fargate Service. Providing both this and `vpcProps` is an error. If the client provides an existing Fargate Service in the `existingFargateServiceObject` property, this value must be the VPC where the service is running. An Amazon Kinesis Firehose Interface Endpoint will be added to this VPC. | | ||
| | clusterProps? | [`ecs.ClusterProps`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.ClusterProps.html) | Optional properties to create a new ECS cluster. To provide an existing cluster, use the cluster attribute of fargateServiceProps. | | ||
| | ecrRepositoryArn? | `string` | The arn of an ECR Repository containing the image to use to generate the containers. Either this or the image property of containerDefinitionProps must be provided. format: arn:aws:ecr:*region*:*account number*:repository/*Repository Name* | | ||
| | ecrImageVersion? | `string` | The version of the image to use from the repository. Defaults to 'Latest' | | ||
| | containerDefinitionProps? | [`ecs.ContainerDefinitionProps \| any`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.ContainerDefinitionProps.html) | Optional props to define the container created for the Fargate Service. (defaults found in fargate-defaults.ts) | | ||
| | fargateTaskDefinitionProps? | [`ecs.FargateTaskDefinitionProps \| any`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.FargateTaskDefinitionProps.html) | Optional props to define the Fargate Task Definition for this construct. (defaults found in fargate-defaults.ts) | | ||
| | fargateServiceProps? | [`ecs.FargateServiceProps \| any`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.FargateServiceProps.html) | Optional values to override default Fargate Task definition properties (fargate-defaults.ts). The construct will default to launching the service is the most isolated subnets available (precedence: Isolated, Private and Public). Override those and other defaults here. | | ||
| | existingFargateServiceObject? | [`ecs.FargateService`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.FargateService.html) | A Fargate Service already instantiated (probably by another Solutions Construct). If this is specified, then no props defining a new service can be provided, including: ecrImageVersion, containerDefinitionProps, fargateTaskDefinitionProps, ecrRepositoryArn, fargateServiceProps, clusterProps | | ||
| | existingContainerDefinitionObject? | [`ecs.ContainerDefinition`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.ContainerDefinition.html) | A container definition already instantiated as part of a Fargate service. This must be the container in the `existingFargateServiceObject`. | | ||
| |existingKinesisFirehose|[kinesisfirehose.CfnDeliveryStream](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_kinesisfirehose.CfnDeliveryStream.html)|An existing Kinesis Firehose Delivery Stream to which the Fargate container can put data. Note - the delivery stream construct must have already been created and have the deliveryStreamName set. This construct will *not* create a new Delivery Stream.| | ||
| |firehoseEnvironmentVariableName?|`string`|Optional Name for the Fargate container environment variable set to the name of the delivery stream. Default: FIREHOSE_DELIVERYSTREAM_NAME | | ||
|
|
||
| ## Pattern Properties | ||
|
|
||
| | **Name** | **Type** | **Description** | | ||
| |:-------------|:----------------|-----------------| | ||
| | vpc | [`ec2.IVpc`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.IVpc.html) | The new or existing VPC used by the construct. | | ||
| | fargateService | [`ecs.FargateService`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.FargateService.html) | The new or existing AWS Fargate service used by this construct. | | ||
| | container | [`ecs.ContainerDefinition`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.ContainerDefinition.html) | The container associated with the AWS Fargate service in the service property. | | ||
| |kinesisFirehose|[kinesisfirehose.CfnDeliveryStream](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_kinesisfirehose.CfnDeliveryStream.html)|The Kinesis Firehose Delivery Stream used by the construct.| | ||
|
|
||
| ## Default settings | ||
|
|
||
| Out of the box implementation of the Construct without any overrides will set the following defaults: | ||
|
|
||
| ### AWS Fargate Service | ||
| * An AWS Fargate Service running in the isolated subnets of a new VPC | ||
| * Minimally-permissive IAM role for the Fargate Service to put records on the Firehose Delivery Stream | ||
| * Sets an Environment Variable named FIREHOSE_DELIVERYSTREAM_NAME that holds the Firehose Delivery Stream Name, which is a required property of the Kinesis Firehose SDK when making calls to it | ||
|
|
||
| ### Amazon Firehose Delivery Stream | ||
| * This construct must be provided a configured Firehose Data Stream construct, it does not change this Stream. | ||
|
|
||
| ## Architecture | ||
|  | ||
|
|
||
| *** | ||
| © Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. |
Binary file added
BIN
+42.7 KB
...patterns/@aws-solutions-constructs/aws-fargate-kinesisfirehose/architecture.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
181 changes: 181 additions & 0 deletions
181
source/patterns/@aws-solutions-constructs/aws-fargate-kinesisfirehose/lib/index.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,181 @@ | ||
| /** | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance | ||
| * with the License. A copy of the License is located at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES | ||
| * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions | ||
| * and limitations under the License. | ||
| */ | ||
|
|
||
| import * as ec2 from "aws-cdk-lib/aws-ec2"; | ||
| import * as kinesisfirehose from "aws-cdk-lib/aws-kinesisfirehose"; | ||
| // Note: To ensure CDKv2 compatibility, keep the import statement for Construct separate | ||
| import { Construct } from "constructs"; | ||
| import * as defaults from "@aws-solutions-constructs/core"; | ||
| import * as ecs from "aws-cdk-lib/aws-ecs"; | ||
| import * as iam from "aws-cdk-lib/aws-iam"; | ||
|
|
||
| export interface FargateToKinesisFirehoseProps { | ||
| /** | ||
| * Optional custom properties for a VPC the construct will create. This VPC will | ||
| * be used by the new Fargate service the construct creates (that's | ||
| * why targetGroupProps can't include a VPC). Providing | ||
| * both this and existingVpc is an error. A Kinesis Firehose Interface | ||
| * endpoint will be included in this VPC. | ||
| * | ||
| * @default - none | ||
| */ | ||
| readonly vpcProps?: ec2.VpcProps; | ||
| /** | ||
| * An existing VPC in which to deploy the construct. Providing both this and | ||
| * vpcProps is an error. If the client provides an existing Fargate service, | ||
| * this value must be the VPC where the service is running. A Kinesis Interface | ||
| * endpoint will be added to this VPC. | ||
| * | ||
| * @default - none | ||
| */ | ||
| readonly existingVpc?: ec2.IVpc; | ||
| /** | ||
| * True if the VPC provisioned by this construct should contain Public/Private Subnets, | ||
| * otherwise False for the VPC to contain Isolated Subnets only. Note this property is | ||
| * ignored if an existing VPC is specified in the existingVpc property. If you are getting | ||
| * a container from a public repo, this must be true so the repo can be accessed from the | ||
| * network. | ||
| */ | ||
| readonly publicApi: boolean; | ||
| /** | ||
| * Optional properties to create a new ECS cluster | ||
| */ | ||
| readonly clusterProps?: ecs.ClusterProps; | ||
| /** | ||
| * The arn of an ECR Repository containing the image to use | ||
| * to generate the containers | ||
| * | ||
| * format: | ||
| * arn:aws:ecr:[region]:[account number]:repository/[Repository Name] | ||
| */ | ||
| readonly ecrRepositoryArn?: string; | ||
| /** | ||
| * The version of the image to use from the repository | ||
| * | ||
| * @default - 'latest' | ||
| */ | ||
| readonly ecrImageVersion?: string; | ||
| /* | ||
| * Optional props to define the container created for the Fargate Service | ||
| * | ||
| * defaults - fargate-defaults.ts | ||
| */ | ||
| readonly containerDefinitionProps?: ecs.ContainerDefinitionProps | any; | ||
| /* | ||
| * Optional props to define the Fargate Task Definition for this construct | ||
| * | ||
| * defaults - fargate-defaults.ts | ||
| */ | ||
| readonly fargateTaskDefinitionProps?: ecs.FargateTaskDefinitionProps | any; | ||
| /** | ||
| * Optional values to override default Fargate Task definition properties | ||
| * (fargate-defaults.ts). The construct will default to launching the service | ||
| * is the most isolated subnets available (precedence: Isolated, Private and | ||
| * Public). Override those and other defaults here. | ||
| * | ||
| * defaults - fargate-defaults.ts | ||
| */ | ||
| readonly fargateServiceProps?: ecs.FargateServiceProps | any; | ||
| /** | ||
| * A Fargate Service already instantiated (probably by another Solutions Construct). If | ||
| * this is specified, then no props defining a new service can be provided, including: | ||
| * existingImageObject, ecrImageVersion, containerDefintionProps, fargateTaskDefinitionProps, | ||
| * ecrRepositoryArn, fargateServiceProps, clusterProps, existingClusterInterface. If this value | ||
| * is provided, then existingContainerDefinitionObject must be provided as well. | ||
| * | ||
| * @default - none | ||
| */ | ||
| readonly existingFargateServiceObject?: ecs.FargateService; | ||
| /** | ||
| * An existing Kinesis Firehose Delivery Stream to which the Fargate container can put data. Note - the delivery stream | ||
| * construct must have already been created and have the deliveryStreamName set. This construct will *not* create a | ||
| * new Delivery Stream. | ||
| */ | ||
| readonly existingKinesisFirehose: kinesisfirehose.CfnDeliveryStream; | ||
| /** | ||
| * Optional Name for the container environment variable set to the bucket ARN. | ||
| * | ||
| * @default - FIREHOSE_DELIVERYSTREAM_NAME | ||
| */ | ||
| readonly firehoseEnvironmentVariableName?: string; | ||
| /* | ||
| * A container definition already instantiated as part of a Fargate service. This must | ||
| * be the container in the existingFargateServiceObject. | ||
| * | ||
| * @default - None | ||
| */ | ||
| readonly existingContainerDefinitionObject?: ecs.ContainerDefinition; | ||
| } | ||
|
|
||
| export class FargateToKinesisFirehose extends Construct { | ||
| public readonly vpc: ec2.IVpc; | ||
| public readonly service: ecs.FargateService; | ||
| public readonly container: ecs.ContainerDefinition; | ||
| public readonly kinesisFirehose: kinesisfirehose.CfnDeliveryStream; | ||
|
|
||
| constructor(scope: Construct, id: string, props: FargateToKinesisFirehoseProps) { | ||
| super(scope, id); | ||
| defaults.CheckProps(props); | ||
| defaults.CheckFargateProps(props); | ||
|
|
||
| if (!props.existingKinesisFirehose.deliveryStreamName) { | ||
| throw new Error('existingKinesisFirehose must have a defined deliveryStreamName'); | ||
| } | ||
|
|
||
| this.vpc = defaults.buildVpc(scope, { | ||
| existingVpc: props.existingVpc, | ||
| defaultVpcProps: props.publicApi ? defaults.DefaultPublicPrivateVpcProps() : defaults.DefaultIsolatedVpcProps(), | ||
| userVpcProps: props.vpcProps, | ||
| constructVpcProps: { enableDnsHostnames: true, enableDnsSupport: true } | ||
| }); | ||
|
|
||
| defaults.AddAwsServiceEndpoint(scope, this.vpc, defaults.ServiceEndpointTypes.KINESIS_FIREHOSE); | ||
|
|
||
| if (props.existingFargateServiceObject) { | ||
| this.service = props.existingFargateServiceObject; | ||
| // CheckFargateProps confirms that the container is provided | ||
| this.container = props.existingContainerDefinitionObject!; | ||
| } else { | ||
| [this.service, this.container] = defaults.CreateFargateService( | ||
| scope, | ||
| id, | ||
| this.vpc, | ||
| props.clusterProps, | ||
| props.ecrRepositoryArn, | ||
| props.ecrImageVersion, | ||
| props.fargateTaskDefinitionProps, | ||
| props.containerDefinitionProps, | ||
| props.fargateServiceProps | ||
| ); | ||
| } | ||
|
|
||
| this.kinesisFirehose = props.existingKinesisFirehose; | ||
|
|
||
| const taskPolicyStatement = new iam.PolicyStatement({ | ||
| actions: [ | ||
| "firehose:DeleteDeliveryStream", | ||
| "firehose:PutRecord", | ||
| "firehose:PutRecordBatch", | ||
| "firehose:UpdateDestination" | ||
| ], | ||
| resources: [this.kinesisFirehose.attrArn], | ||
| }); | ||
| this.service.taskDefinition.taskRole.addToPrincipalPolicy(taskPolicyStatement); | ||
|
|
||
| // Configure environment variables | ||
| const deliveryStreamEnvironmentVariableName = props.firehoseEnvironmentVariableName || 'FIREHOSE_DELIVERYSTREAM_NAME'; | ||
| // We can use ! because we checked for a stream name on props.existingKinesisFirehose at the top of this function | ||
| this.container.addEnvironment(deliveryStreamEnvironmentVariableName, this.kinesisFirehose!.deliveryStreamName!); | ||
|
|
||
| } | ||
| } |
Oops, something went wrong.