From ebbb57f9c668c54133b1fbbc024fff37990f2426 Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Sun, 4 Dec 2022 14:45:13 +0000 Subject: [PATCH 1/2] feat(glue): support glue version 4 https://aws.amazon.com/blogs/aws/new-aws-glue-4-0-new-and-updated-engines-more-data-formats-and-more/ closes #23220 --- .../@aws-cdk/aws-glue/lib/job-executable.ts | 7 +- .../aws-glue-job.assets.json | 6 +- .../aws-glue-job.template.json | 750 +++++++++- .../test/integ.job.js.snapshot/cdk.out | 2 +- .../test/integ.job.js.snapshot/integ.json | 2 +- .../test/integ.job.js.snapshot/manifest.json | 216 ++- .../test/integ.job.js.snapshot/tree.json | 1212 ++++++++++++++++- packages/@aws-cdk/aws-glue/test/integ.job.ts | 90 +- .../aws-glue/test/job-executable.test.ts | 6 +- 9 files changed, 2125 insertions(+), 166 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/lib/job-executable.ts b/packages/@aws-cdk/aws-glue/lib/job-executable.ts index 57e917575dd1b..092cdca11552d 100644 --- a/packages/@aws-cdk/aws-glue/lib/job-executable.ts +++ b/packages/@aws-cdk/aws-glue/lib/job-executable.ts @@ -29,6 +29,11 @@ export class GlueVersion { */ public static readonly V3_0 = new GlueVersion('3.0'); + /** + * Glue version using Spark 3.3.0 and Python 3.10 + */ + public static readonly V4_0 = new GlueVersion('4.0'); + /** * Custom Glue version * @param version custom version @@ -292,7 +297,7 @@ export class JobExecutable { if (config.language !== JobLanguage.PYTHON) { throw new Error('Python shell requires the language to be set to Python'); } - if ([GlueVersion.V0_9, GlueVersion.V2_0, GlueVersion.V3_0].includes(config.glueVersion)) { + if ([GlueVersion.V0_9, GlueVersion.V2_0, GlueVersion.V3_0, GlueVersion.V4_0].includes(config.glueVersion)) { throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support Python Shell`); } } diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.assets.json index ef7baf72c0933..29ae3d1255380 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -1,5 +1,5 @@ { - "version": "21.0.0", + "version": "22.0.0", "files": { "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { "source": { @@ -14,7 +14,7 @@ } } }, - "3d4fb124b6f262a7f5f621715b1437dcf5af3450121d622990f89e94fd55d8f5": { + "0985af21379e9d6e1cba091105ecb533ee38a96b4c738816daf17d951a0752b4": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "3d4fb124b6f262a7f5f621715b1437dcf5af3450121d622990f89e94fd55d8f5.json", + "objectKey": "0985af21379e9d6e1cba091105ecb533ee38a96b4c738816daf17d951a0752b4.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.template.json index 2828cc13dc81b..f7838294de6e4 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -1,6 +1,6 @@ { "Resources": { - "EtlJobServiceRole837F781B": { + "EtlJob20ServiceRoleD520CA20": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { @@ -31,7 +31,7 @@ ] } }, - "EtlJobServiceRoleDefaultPolicy8BFE343B": { + "EtlJob20ServiceRoleDefaultPolicy37D28060": { "Type": "AWS::IAM::Policy", "Properties": { "PolicyDocument": { @@ -53,7 +53,7 @@ "Resource": [ { "Fn::GetAtt": [ - "EtlJobSparkUIBucketBF23744B", + "EtlJob20SparkUIBucketFD07FBD8", "Arn" ] }, @@ -63,7 +63,7 @@ [ { "Fn::GetAtt": [ - "EtlJobSparkUIBucketBF23744B", + "EtlJob20SparkUIBucketFD07FBD8", "Arn" ] }, @@ -117,20 +117,20 @@ ], "Version": "2012-10-17" }, - "PolicyName": "EtlJobServiceRoleDefaultPolicy8BFE343B", + "PolicyName": "EtlJob20ServiceRoleDefaultPolicy37D28060", "Roles": [ { - "Ref": "EtlJobServiceRole837F781B" + "Ref": "EtlJob20ServiceRoleD520CA20" } ] } }, - "EtlJobSparkUIBucketBF23744B": { + "EtlJob20SparkUIBucketFD07FBD8": { "Type": "AWS::S3::Bucket", "UpdateReplacePolicy": "Retain", "DeletionPolicy": "Retain" }, - "EtlJob7FC88E45": { + "EtlJob20134354DC": { "Type": "AWS::Glue::Job", "Properties": { "Command": { @@ -151,7 +151,7 @@ }, "Role": { "Fn::GetAtt": [ - "EtlJobServiceRole837F781B", + "EtlJob20ServiceRoleD520CA20", "Arn" ] }, @@ -167,7 +167,7 @@ [ "s3://", { - "Ref": "EtlJobSparkUIBucketBF23744B" + "Ref": "EtlJob20SparkUIBucketFD07FBD8" } ] ] @@ -180,7 +180,7 @@ }, "GlueVersion": "2.0", "MaxRetries": 2, - "Name": "EtlJob", + "Name": "EtlJob2.0", "NotificationProperty": { "NotifyDelayAfter": 1 }, @@ -192,7 +192,7 @@ "WorkerType": "G.2X" } }, - "EtlJobSuccessMetricRuleA72A3EF6": { + "EtlJob20SuccessMetricRule1759F889": { "Type": "AWS::Events::Rule", "Properties": { "Description": { @@ -201,7 +201,7 @@ [ "Rule triggered when Glue job ", { - "Ref": "EtlJob7FC88E45" + "Ref": "EtlJob20134354DC" }, " is in SUCCEEDED state" ] @@ -218,7 +218,7 @@ "detail": { "jobName": [ { - "Ref": "EtlJob7FC88E45" + "Ref": "EtlJob20134354DC" } ], "state": [ @@ -229,7 +229,7 @@ "State": "ENABLED" } }, - "StreamingJobServiceRole1B4B8BF9": { + "StreamingJob20ServiceRole491E0FFF": { "Type": "AWS::IAM::Role", "Properties": { "AssumeRolePolicyDocument": { @@ -260,7 +260,7 @@ ] } }, - "StreamingJobServiceRoleDefaultPolicyA0CC4C68": { + "StreamingJob20ServiceRoleDefaultPolicy3CF74B2D": { "Type": "AWS::IAM::Policy", "Properties": { "PolicyDocument": { @@ -309,15 +309,15 @@ ], "Version": "2012-10-17" }, - "PolicyName": "StreamingJobServiceRoleDefaultPolicyA0CC4C68", + "PolicyName": "StreamingJob20ServiceRoleDefaultPolicy3CF74B2D", "Roles": [ { - "Ref": "StreamingJobServiceRole1B4B8BF9" + "Ref": "StreamingJob20ServiceRole491E0FFF" } ] } }, - "StreamingJob3783CC17": { + "StreamingJob20355B58C7": { "Type": "AWS::Glue::Job", "Properties": { "Command": { @@ -338,7 +338,7 @@ }, "Role": { "Fn::GetAtt": [ - "StreamingJobServiceRole1B4B8BF9", + "StreamingJob20ServiceRole491E0FFF", "Arn" ] }, @@ -348,7 +348,715 @@ "arg2": "value2" }, "GlueVersion": "2.0", - "Name": "StreamingJob", + "Name": "StreamingJob2.0", + "Tags": { + "key": "value" + } + } + }, + "EtlJob30ServiceRole8E675579": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "EtlJob30ServiceRoleDefaultPolicyE57ADD66": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "EtlJob30SparkUIBucket9D789346", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "EtlJob30SparkUIBucket9D789346", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "EtlJob30ServiceRoleDefaultPolicyE57ADD66", + "Roles": [ + { + "Ref": "EtlJob30ServiceRole8E675579" + } + ] + } + }, + "EtlJob30SparkUIBucket9D789346": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "EtlJob307A0F9470": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "glueetl", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, + "DefaultArguments": { + "--job-language": "python", + "--enable-continuous-cloudwatch-log": "true", + "--enable-continuous-log-filter": "true", + "--continuous-log-logStreamPrefix": "EtlJob", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "EtlJob30SparkUIBucket9D789346" + } + ] + ] + }, + "arg1": "value1", + "arg2": "value2" + }, + "ExecutionProperty": { + "MaxConcurrentRuns": 2 + }, + "GlueVersion": "3.0", + "MaxRetries": 2, + "Name": "EtlJob3.0", + "NotificationProperty": { + "NotifyDelayAfter": 1 + }, + "NumberOfWorkers": 10, + "Tags": { + "key": "value" + }, + "Timeout": 5, + "WorkerType": "G.2X" + } + }, + "EtlJob30SuccessMetricRuleF8870F8A": { + "Type": "AWS::Events::Rule", + "Properties": { + "Description": { + "Fn::Join": [ + "", + [ + "Rule triggered when Glue job ", + { + "Ref": "EtlJob307A0F9470" + }, + " is in SUCCEEDED state" + ] + ] + }, + "EventPattern": { + "source": [ + "aws.glue" + ], + "detail-type": [ + "Glue Job State Change", + "Glue Job Run Status" + ], + "detail": { + "jobName": [ + { + "Ref": "EtlJob307A0F9470" + } + ], + "state": [ + "SUCCEEDED" + ] + } + }, + "State": "ENABLED" + } + }, + "StreamingJob30ServiceRole443B2FDE": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "StreamingJob30ServiceRoleDefaultPolicy0C15D010": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "StreamingJob30ServiceRoleDefaultPolicy0C15D010", + "Roles": [ + { + "Ref": "StreamingJob30ServiceRole443B2FDE" + } + ] + } + }, + "StreamingJob30E005FBEB": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "gluestreaming", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, + "DefaultArguments": { + "--job-language": "python", + "arg1": "value1", + "arg2": "value2" + }, + "GlueVersion": "3.0", + "Name": "StreamingJob3.0", + "Tags": { + "key": "value" + } + } + }, + "EtlJob40ServiceRoleBDD9998A": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "EtlJob40ServiceRoleDefaultPolicy369BD98D": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "EtlJob40SparkUIBucket02F50B0D", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "EtlJob40SparkUIBucket02F50B0D", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "EtlJob40ServiceRoleDefaultPolicy369BD98D", + "Roles": [ + { + "Ref": "EtlJob40ServiceRoleBDD9998A" + } + ] + } + }, + "EtlJob40SparkUIBucket02F50B0D": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "EtlJob4046D61F0B": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "glueetl", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "EtlJob40ServiceRoleBDD9998A", + "Arn" + ] + }, + "DefaultArguments": { + "--job-language": "python", + "--enable-continuous-cloudwatch-log": "true", + "--enable-continuous-log-filter": "true", + "--continuous-log-logStreamPrefix": "EtlJob", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "EtlJob40SparkUIBucket02F50B0D" + } + ] + ] + }, + "arg1": "value1", + "arg2": "value2" + }, + "ExecutionProperty": { + "MaxConcurrentRuns": 2 + }, + "GlueVersion": "4.0", + "MaxRetries": 2, + "Name": "EtlJob4.0", + "NotificationProperty": { + "NotifyDelayAfter": 1 + }, + "NumberOfWorkers": 10, + "Tags": { + "key": "value" + }, + "Timeout": 5, + "WorkerType": "G.2X" + } + }, + "EtlJob40SuccessMetricRule00D3EF34": { + "Type": "AWS::Events::Rule", + "Properties": { + "Description": { + "Fn::Join": [ + "", + [ + "Rule triggered when Glue job ", + { + "Ref": "EtlJob4046D61F0B" + }, + " is in SUCCEEDED state" + ] + ] + }, + "EventPattern": { + "source": [ + "aws.glue" + ], + "detail-type": [ + "Glue Job State Change", + "Glue Job Run Status" + ], + "detail": { + "jobName": [ + { + "Ref": "EtlJob4046D61F0B" + } + ], + "state": [ + "SUCCEEDED" + ] + } + }, + "State": "ENABLED" + } + }, + "StreamingJob40ServiceRole034BDEBD": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "StreamingJob40ServiceRoleDefaultPolicy0667C434": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "StreamingJob40ServiceRoleDefaultPolicy0667C434", + "Roles": [ + { + "Ref": "StreamingJob40ServiceRole034BDEBD" + } + ] + } + }, + "StreamingJob40E284A782": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "gluestreaming", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "StreamingJob40ServiceRole034BDEBD", + "Arn" + ] + }, + "DefaultArguments": { + "--job-language": "python", + "arg1": "value1", + "arg2": "value2" + }, + "GlueVersion": "4.0", + "Name": "StreamingJob4.0", "Tags": { "key": "value" } diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/cdk.out index 8ecc185e9dbee..145739f539580 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"21.0.0"} \ No newline at end of file +{"version":"22.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/integ.json index bbdd18dcc3878..28e4fd8c94ff9 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "21.0.0", + "version": "22.0.0", "testCases": { "integ.job": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/manifest.json index ed5aa6d8556fd..9384baab89163 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/manifest.json @@ -1,12 +1,6 @@ { - "version": "21.0.0", + "version": "22.0.0", "artifacts": { - "Tree": { - "type": "cdk:tree", - "properties": { - "file": "tree.json" - } - }, "aws-glue-job.assets": { "type": "cdk:asset-manifest", "properties": { @@ -23,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/3d4fb124b6f262a7f5f621715b1437dcf5af3450121d622990f89e94fd55d8f5.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/0985af21379e9d6e1cba091105ecb533ee38a96b4c738816daf17d951a0752b4.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -39,52 +33,148 @@ "aws-glue-job.assets" ], "metadata": { - "/aws-glue-job/EtlJob/ServiceRole/Resource": [ + "/aws-glue-job/EtlJob2.0/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob20ServiceRoleD520CA20" + } + ], + "/aws-glue-job/EtlJob2.0/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob20ServiceRoleDefaultPolicy37D28060" + } + ], + "/aws-glue-job/EtlJob2.0/SparkUIBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob20SparkUIBucketFD07FBD8" + } + ], + "/aws-glue-job/EtlJob2.0/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob20134354DC" + } + ], + "/aws-glue-job/EtlJob2.0/SuccessMetricRule/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob20SuccessMetricRule1759F889" + } + ], + "/aws-glue-job/StreamingJob2.0/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob20ServiceRole491E0FFF" + } + ], + "/aws-glue-job/StreamingJob2.0/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob20ServiceRoleDefaultPolicy3CF74B2D" + } + ], + "/aws-glue-job/StreamingJob2.0/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob20355B58C7" + } + ], + "/aws-glue-job/EtlJob3.0/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob30ServiceRole8E675579" + } + ], + "/aws-glue-job/EtlJob3.0/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob30ServiceRoleDefaultPolicyE57ADD66" + } + ], + "/aws-glue-job/EtlJob3.0/SparkUIBucket/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob30SparkUIBucket9D789346" + } + ], + "/aws-glue-job/EtlJob3.0/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob307A0F9470" + } + ], + "/aws-glue-job/EtlJob3.0/SuccessMetricRule/Resource": [ { "type": "aws:cdk:logicalId", - "data": "EtlJobServiceRole837F781B" + "data": "EtlJob30SuccessMetricRuleF8870F8A" } ], - "/aws-glue-job/EtlJob/ServiceRole/DefaultPolicy/Resource": [ + "/aws-glue-job/StreamingJob3.0/ServiceRole/Resource": [ { "type": "aws:cdk:logicalId", - "data": "EtlJobServiceRoleDefaultPolicy8BFE343B" + "data": "StreamingJob30ServiceRole443B2FDE" } ], - "/aws-glue-job/EtlJob/SparkUIBucket/Resource": [ + "/aws-glue-job/StreamingJob3.0/ServiceRole/DefaultPolicy/Resource": [ { "type": "aws:cdk:logicalId", - "data": "EtlJobSparkUIBucketBF23744B" + "data": "StreamingJob30ServiceRoleDefaultPolicy0C15D010" } ], - "/aws-glue-job/EtlJob/Resource": [ + "/aws-glue-job/StreamingJob3.0/Resource": [ { "type": "aws:cdk:logicalId", - "data": "EtlJob7FC88E45" + "data": "StreamingJob30E005FBEB" } ], - "/aws-glue-job/EtlJob/SuccessMetricRule/Resource": [ + "/aws-glue-job/EtlJob4.0/ServiceRole/Resource": [ { "type": "aws:cdk:logicalId", - "data": "EtlJobSuccessMetricRuleA72A3EF6" + "data": "EtlJob40ServiceRoleBDD9998A" } ], - "/aws-glue-job/StreamingJob/ServiceRole/Resource": [ + "/aws-glue-job/EtlJob4.0/ServiceRole/DefaultPolicy/Resource": [ { "type": "aws:cdk:logicalId", - "data": "StreamingJobServiceRole1B4B8BF9" + "data": "EtlJob40ServiceRoleDefaultPolicy369BD98D" } ], - "/aws-glue-job/StreamingJob/ServiceRole/DefaultPolicy/Resource": [ + "/aws-glue-job/EtlJob4.0/SparkUIBucket/Resource": [ { "type": "aws:cdk:logicalId", - "data": "StreamingJobServiceRoleDefaultPolicyA0CC4C68" + "data": "EtlJob40SparkUIBucket02F50B0D" } ], - "/aws-glue-job/StreamingJob/Resource": [ + "/aws-glue-job/EtlJob4.0/Resource": [ { "type": "aws:cdk:logicalId", - "data": "StreamingJob3783CC17" + "data": "EtlJob4046D61F0B" + } + ], + "/aws-glue-job/EtlJob4.0/SuccessMetricRule/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob40SuccessMetricRule00D3EF34" + } + ], + "/aws-glue-job/StreamingJob4.0/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob40ServiceRole034BDEBD" + } + ], + "/aws-glue-job/StreamingJob4.0/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob40ServiceRoleDefaultPolicy0667C434" + } + ], + "/aws-glue-job/StreamingJob4.0/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob40E284A782" } ], "/aws-glue-job/ShellJob/ServiceRole/Resource": [ @@ -134,9 +224,87 @@ "type": "aws:cdk:logicalId", "data": "CheckBootstrapVersion" } + ], + "EtlJobServiceRole837F781B": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJobServiceRole837F781B", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } + ], + "EtlJobServiceRoleDefaultPolicy8BFE343B": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJobServiceRoleDefaultPolicy8BFE343B", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } + ], + "EtlJobSparkUIBucketBF23744B": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJobSparkUIBucketBF23744B", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } + ], + "EtlJob7FC88E45": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJob7FC88E45", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } + ], + "EtlJobSuccessMetricRuleA72A3EF6": [ + { + "type": "aws:cdk:logicalId", + "data": "EtlJobSuccessMetricRuleA72A3EF6", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } + ], + "StreamingJobServiceRole1B4B8BF9": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJobServiceRole1B4B8BF9", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } + ], + "StreamingJobServiceRoleDefaultPolicyA0CC4C68": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJobServiceRoleDefaultPolicyA0CC4C68", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } + ], + "StreamingJob3783CC17": [ + { + "type": "aws:cdk:logicalId", + "data": "StreamingJob3783CC17", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } ] }, "displayName": "aws-glue-job" + }, + "Tree": { + "type": "cdk:tree", + "properties": { + "file": "tree.json" + } } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/tree.json index 2ca14ca23a148..c601e3512008c 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue/test/integ.job.js.snapshot/tree.json @@ -4,29 +4,1081 @@ "id": "App", "path": "", "children": { - "Tree": { - "id": "Tree", - "path": "Tree", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.1.92" - } - }, "aws-glue-job": { "id": "aws-glue-job", "path": "aws-glue-job", "children": { - "EtlJob": { - "id": "EtlJob", - "path": "aws-glue-job/EtlJob", + "EtlJob2.0": { + "id": "EtlJob2.0", + "path": "aws-glue-job/EtlJob2.0", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-glue-job/EtlJob2.0/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/EtlJob2.0/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "@aws-cdk/core.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob2.0/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-glue-job/EtlJob2.0/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob2.0/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "EtlJob20SparkUIBucketFD07FBD8", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "EtlJob20SparkUIBucketFD07FBD8", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "EtlJob20ServiceRoleDefaultPolicy37D28060", + "roles": [ + { + "Ref": "EtlJob20ServiceRoleD520CA20" + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.Role", + "version": "0.0.0" + } + }, + "SparkUIBucket": { + "id": "SparkUIBucket", + "path": "aws-glue-job/EtlJob2.0/SparkUIBucket", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob2.0/SparkUIBucket/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::S3::Bucket", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-s3.CfnBucket", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-s3.Bucket", + "version": "0.0.0" + } + }, + "Code93a4952ea654434aca8481fb2bc2a836": { + "id": "Code93a4952ea654434aca8481fb2bc2a836", + "path": "aws-glue-job/EtlJob2.0/Code93a4952ea654434aca8481fb2bc2a836", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job/EtlJob2.0/Code93a4952ea654434aca8481fb2bc2a836/Stage", + "constructInfo": { + "fqn": "@aws-cdk/core.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job/EtlJob2.0/Code93a4952ea654434aca8481fb2bc2a836/AssetBucket", + "constructInfo": { + "fqn": "@aws-cdk/aws-s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-s3-assets.Asset", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob2.0/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "glueetl", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "role": { + "Fn::GetAtt": [ + "EtlJob20ServiceRoleD520CA20", + "Arn" + ] + }, + "defaultArguments": { + "--job-language": "python", + "--enable-continuous-cloudwatch-log": "true", + "--enable-continuous-log-filter": "true", + "--continuous-log-logStreamPrefix": "EtlJob", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "EtlJob20SparkUIBucketFD07FBD8" + } + ] + ] + }, + "arg1": "value1", + "arg2": "value2" + }, + "executionProperty": { + "maxConcurrentRuns": 2 + }, + "glueVersion": "2.0", + "maxRetries": 2, + "name": "EtlJob2.0", + "notificationProperty": { + "notifyDelayAfter": 1 + }, + "numberOfWorkers": 10, + "tags": { + "key": "value" + }, + "timeout": 5, + "workerType": "G.2X" + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.CfnJob", + "version": "0.0.0" + } + }, + "SuccessMetricRule": { + "id": "SuccessMetricRule", + "path": "aws-glue-job/EtlJob2.0/SuccessMetricRule", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob2.0/SuccessMetricRule/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Events::Rule", + "aws:cdk:cloudformation:props": { + "description": { + "Fn::Join": [ + "", + [ + "Rule triggered when Glue job ", + { + "Ref": "EtlJob20134354DC" + }, + " is in SUCCEEDED state" + ] + ] + }, + "eventPattern": { + "source": [ + "aws.glue" + ], + "detail-type": [ + "Glue Job State Change", + "Glue Job Run Status" + ], + "detail": { + "jobName": [ + { + "Ref": "EtlJob20134354DC" + } + ], + "state": [ + "SUCCEEDED" + ] + } + }, + "state": "ENABLED" + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-events.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-events.Rule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.Job", + "version": "0.0.0" + } + }, + "StreamingJob2.0": { + "id": "StreamingJob2.0", + "path": "aws-glue-job/StreamingJob2.0", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-glue-job/StreamingJob2.0/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/StreamingJob2.0/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "@aws-cdk/core.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob2.0/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-glue-job/StreamingJob2.0/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob2.0/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "StreamingJob20ServiceRoleDefaultPolicy3CF74B2D", + "roles": [ + { + "Ref": "StreamingJob20ServiceRole491E0FFF" + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.Role", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob2.0/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "gluestreaming", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "role": { + "Fn::GetAtt": [ + "StreamingJob20ServiceRole491E0FFF", + "Arn" + ] + }, + "defaultArguments": { + "--job-language": "python", + "arg1": "value1", + "arg2": "value2" + }, + "glueVersion": "2.0", + "name": "StreamingJob2.0", + "tags": { + "key": "value" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.CfnJob", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.Job", + "version": "0.0.0" + } + }, + "EtlJob3.0": { + "id": "EtlJob3.0", + "path": "aws-glue-job/EtlJob3.0", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-glue-job/EtlJob3.0/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/EtlJob3.0/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "@aws-cdk/core.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob3.0/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-glue-job/EtlJob3.0/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob3.0/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "EtlJob30SparkUIBucket9D789346", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "EtlJob30SparkUIBucket9D789346", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "EtlJob30ServiceRoleDefaultPolicyE57ADD66", + "roles": [ + { + "Ref": "EtlJob30ServiceRole8E675579" + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.Role", + "version": "0.0.0" + } + }, + "SparkUIBucket": { + "id": "SparkUIBucket", + "path": "aws-glue-job/EtlJob3.0/SparkUIBucket", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob3.0/SparkUIBucket/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::S3::Bucket", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-s3.CfnBucket", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-s3.Bucket", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob3.0/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "glueetl", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, + "defaultArguments": { + "--job-language": "python", + "--enable-continuous-cloudwatch-log": "true", + "--enable-continuous-log-filter": "true", + "--continuous-log-logStreamPrefix": "EtlJob", + "--enable-spark-ui": "true", + "--spark-event-logs-path": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "EtlJob30SparkUIBucket9D789346" + } + ] + ] + }, + "arg1": "value1", + "arg2": "value2" + }, + "executionProperty": { + "maxConcurrentRuns": 2 + }, + "glueVersion": "3.0", + "maxRetries": 2, + "name": "EtlJob3.0", + "notificationProperty": { + "notifyDelayAfter": 1 + }, + "numberOfWorkers": 10, + "tags": { + "key": "value" + }, + "timeout": 5, + "workerType": "G.2X" + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.CfnJob", + "version": "0.0.0" + } + }, + "SuccessMetricRule": { + "id": "SuccessMetricRule", + "path": "aws-glue-job/EtlJob3.0/SuccessMetricRule", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/EtlJob3.0/SuccessMetricRule/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Events::Rule", + "aws:cdk:cloudformation:props": { + "description": { + "Fn::Join": [ + "", + [ + "Rule triggered when Glue job ", + { + "Ref": "EtlJob307A0F9470" + }, + " is in SUCCEEDED state" + ] + ] + }, + "eventPattern": { + "source": [ + "aws.glue" + ], + "detail-type": [ + "Glue Job State Change", + "Glue Job Run Status" + ], + "detail": { + "jobName": [ + { + "Ref": "EtlJob307A0F9470" + } + ], + "state": [ + "SUCCEEDED" + ] + } + }, + "state": "ENABLED" + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-events.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-events.Rule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.Job", + "version": "0.0.0" + } + }, + "StreamingJob3.0": { + "id": "StreamingJob3.0", + "path": "aws-glue-job/StreamingJob3.0", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-glue-job/StreamingJob3.0/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/StreamingJob3.0/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "@aws-cdk/core.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob3.0/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-glue-job/StreamingJob3.0/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob3.0/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "StreamingJob30ServiceRoleDefaultPolicy0C15D010", + "roles": [ + { + "Ref": "StreamingJob30ServiceRole443B2FDE" + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.Role", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job/StreamingJob3.0/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "gluestreaming", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, + "defaultArguments": { + "--job-language": "python", + "arg1": "value1", + "arg2": "value2" + }, + "glueVersion": "3.0", + "name": "StreamingJob3.0", + "tags": { + "key": "value" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.CfnJob", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.Job", + "version": "0.0.0" + } + }, + "EtlJob4.0": { + "id": "EtlJob4.0", + "path": "aws-glue-job/EtlJob4.0", "children": { "ServiceRole": { "id": "ServiceRole", - "path": "aws-glue-job/EtlJob/ServiceRole", + "path": "aws-glue-job/EtlJob4.0/ServiceRole", "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/EtlJob4.0/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "@aws-cdk/core.Resource", + "version": "0.0.0" + } + }, "Resource": { "id": "Resource", - "path": "aws-glue-job/EtlJob/ServiceRole/Resource", + "path": "aws-glue-job/EtlJob4.0/ServiceRole/Resource", "attributes": { "aws:cdk:cloudformation:type": "AWS::IAM::Role", "aws:cdk:cloudformation:props": { @@ -65,11 +1117,11 @@ }, "DefaultPolicy": { "id": "DefaultPolicy", - "path": "aws-glue-job/EtlJob/ServiceRole/DefaultPolicy", + "path": "aws-glue-job/EtlJob4.0/ServiceRole/DefaultPolicy", "children": { "Resource": { "id": "Resource", - "path": "aws-glue-job/EtlJob/ServiceRole/DefaultPolicy/Resource", + "path": "aws-glue-job/EtlJob4.0/ServiceRole/DefaultPolicy/Resource", "attributes": { "aws:cdk:cloudformation:type": "AWS::IAM::Policy", "aws:cdk:cloudformation:props": { @@ -92,7 +1144,7 @@ "Resource": [ { "Fn::GetAtt": [ - "EtlJobSparkUIBucketBF23744B", + "EtlJob40SparkUIBucket02F50B0D", "Arn" ] }, @@ -102,7 +1154,7 @@ [ { "Fn::GetAtt": [ - "EtlJobSparkUIBucketBF23744B", + "EtlJob40SparkUIBucket02F50B0D", "Arn" ] }, @@ -156,10 +1208,10 @@ ], "Version": "2012-10-17" }, - "policyName": "EtlJobServiceRoleDefaultPolicy8BFE343B", + "policyName": "EtlJob40ServiceRoleDefaultPolicy369BD98D", "roles": [ { - "Ref": "EtlJobServiceRole837F781B" + "Ref": "EtlJob40ServiceRoleBDD9998A" } ] } @@ -183,11 +1235,11 @@ }, "SparkUIBucket": { "id": "SparkUIBucket", - "path": "aws-glue-job/EtlJob/SparkUIBucket", + "path": "aws-glue-job/EtlJob4.0/SparkUIBucket", "children": { "Resource": { "id": "Resource", - "path": "aws-glue-job/EtlJob/SparkUIBucket/Resource", + "path": "aws-glue-job/EtlJob4.0/SparkUIBucket/Resource", "attributes": { "aws:cdk:cloudformation:type": "AWS::S3::Bucket", "aws:cdk:cloudformation:props": {} @@ -203,35 +1255,9 @@ "version": "0.0.0" } }, - "Codeb6774d6d1e75972579cd7f65b2bccbf8": { - "id": "Codeb6774d6d1e75972579cd7f65b2bccbf8", - "path": "aws-glue-job/EtlJob/Codeb6774d6d1e75972579cd7f65b2bccbf8", - "children": { - "Stage": { - "id": "Stage", - "path": "aws-glue-job/EtlJob/Codeb6774d6d1e75972579cd7f65b2bccbf8/Stage", - "constructInfo": { - "fqn": "@aws-cdk/core.AssetStaging", - "version": "0.0.0" - } - }, - "AssetBucket": { - "id": "AssetBucket", - "path": "aws-glue-job/EtlJob/Codeb6774d6d1e75972579cd7f65b2bccbf8/AssetBucket", - "constructInfo": { - "fqn": "@aws-cdk/aws-s3.BucketBase", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-s3-assets.Asset", - "version": "0.0.0" - } - }, "Resource": { "id": "Resource", - "path": "aws-glue-job/EtlJob/Resource", + "path": "aws-glue-job/EtlJob4.0/Resource", "attributes": { "aws:cdk:cloudformation:type": "AWS::Glue::Job", "aws:cdk:cloudformation:props": { @@ -253,7 +1279,7 @@ }, "role": { "Fn::GetAtt": [ - "EtlJobServiceRole837F781B", + "EtlJob40ServiceRoleBDD9998A", "Arn" ] }, @@ -269,7 +1295,7 @@ [ "s3://", { - "Ref": "EtlJobSparkUIBucketBF23744B" + "Ref": "EtlJob40SparkUIBucket02F50B0D" } ] ] @@ -280,9 +1306,9 @@ "executionProperty": { "maxConcurrentRuns": 2 }, - "glueVersion": "2.0", + "glueVersion": "4.0", "maxRetries": 2, - "name": "EtlJob", + "name": "EtlJob4.0", "notificationProperty": { "notifyDelayAfter": 1 }, @@ -301,11 +1327,11 @@ }, "SuccessMetricRule": { "id": "SuccessMetricRule", - "path": "aws-glue-job/EtlJob/SuccessMetricRule", + "path": "aws-glue-job/EtlJob4.0/SuccessMetricRule", "children": { "Resource": { "id": "Resource", - "path": "aws-glue-job/EtlJob/SuccessMetricRule/Resource", + "path": "aws-glue-job/EtlJob4.0/SuccessMetricRule/Resource", "attributes": { "aws:cdk:cloudformation:type": "AWS::Events::Rule", "aws:cdk:cloudformation:props": { @@ -315,7 +1341,7 @@ [ "Rule triggered when Glue job ", { - "Ref": "EtlJob7FC88E45" + "Ref": "EtlJob4046D61F0B" }, " is in SUCCEEDED state" ] @@ -332,7 +1358,7 @@ "detail": { "jobName": [ { - "Ref": "EtlJob7FC88E45" + "Ref": "EtlJob4046D61F0B" } ], "state": [ @@ -360,17 +1386,25 @@ "version": "0.0.0" } }, - "StreamingJob": { - "id": "StreamingJob", - "path": "aws-glue-job/StreamingJob", + "StreamingJob4.0": { + "id": "StreamingJob4.0", + "path": "aws-glue-job/StreamingJob4.0", "children": { "ServiceRole": { "id": "ServiceRole", - "path": "aws-glue-job/StreamingJob/ServiceRole", + "path": "aws-glue-job/StreamingJob4.0/ServiceRole", "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/StreamingJob4.0/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "@aws-cdk/core.Resource", + "version": "0.0.0" + } + }, "Resource": { "id": "Resource", - "path": "aws-glue-job/StreamingJob/ServiceRole/Resource", + "path": "aws-glue-job/StreamingJob4.0/ServiceRole/Resource", "attributes": { "aws:cdk:cloudformation:type": "AWS::IAM::Role", "aws:cdk:cloudformation:props": { @@ -409,11 +1443,11 @@ }, "DefaultPolicy": { "id": "DefaultPolicy", - "path": "aws-glue-job/StreamingJob/ServiceRole/DefaultPolicy", + "path": "aws-glue-job/StreamingJob4.0/ServiceRole/DefaultPolicy", "children": { "Resource": { "id": "Resource", - "path": "aws-glue-job/StreamingJob/ServiceRole/DefaultPolicy/Resource", + "path": "aws-glue-job/StreamingJob4.0/ServiceRole/DefaultPolicy/Resource", "attributes": { "aws:cdk:cloudformation:type": "AWS::IAM::Policy", "aws:cdk:cloudformation:props": { @@ -463,10 +1497,10 @@ ], "Version": "2012-10-17" }, - "policyName": "StreamingJobServiceRoleDefaultPolicyA0CC4C68", + "policyName": "StreamingJob40ServiceRoleDefaultPolicy0667C434", "roles": [ { - "Ref": "StreamingJobServiceRole1B4B8BF9" + "Ref": "StreamingJob40ServiceRole034BDEBD" } ] } @@ -490,7 +1524,7 @@ }, "Resource": { "id": "Resource", - "path": "aws-glue-job/StreamingJob/Resource", + "path": "aws-glue-job/StreamingJob4.0/Resource", "attributes": { "aws:cdk:cloudformation:type": "AWS::Glue::Job", "aws:cdk:cloudformation:props": { @@ -512,7 +1546,7 @@ }, "role": { "Fn::GetAtt": [ - "StreamingJobServiceRole1B4B8BF9", + "StreamingJob40ServiceRole034BDEBD", "Arn" ] }, @@ -521,8 +1555,8 @@ "arg1": "value1", "arg2": "value2" }, - "glueVersion": "2.0", - "name": "StreamingJob", + "glueVersion": "4.0", + "name": "StreamingJob4.0", "tags": { "key": "value" } @@ -547,6 +1581,14 @@ "id": "ServiceRole", "path": "aws-glue-job/ShellJob/ServiceRole", "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/ShellJob/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "@aws-cdk/core.Resource", + "version": "0.0.0" + } + }, "Resource": { "id": "Resource", "path": "aws-glue-job/ShellJob/ServiceRole/Resource", @@ -726,6 +1768,14 @@ "id": "ServiceRole", "path": "aws-glue-job/ShellJob39/ServiceRole", "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job/ShellJob39/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "@aws-cdk/core.Resource", + "version": "0.0.0" + } + }, "Resource": { "id": "Resource", "path": "aws-glue-job/ShellJob39/ServiceRole/Resource", @@ -896,12 +1946,36 @@ "fqn": "@aws-cdk/aws-glue.Job", "version": "0.0.0" } + }, + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-glue-job/BootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-glue-job/CheckBootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnRule", + "version": "0.0.0" + } } }, "constructInfo": { "fqn": "@aws-cdk/core.Stack", "version": "0.0.0" } + }, + "Tree": { + "id": "Tree", + "path": "Tree", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.1.168" + } } }, "constructInfo": { diff --git a/packages/@aws-cdk/aws-glue/test/integ.job.ts b/packages/@aws-cdk/aws-glue/test/integ.job.ts index 2aaa812f9e977..417ede1793054 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.job.ts @@ -23,51 +23,53 @@ const stack = new cdk.Stack(app, 'aws-glue-job'); const script = glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')); -const etlJob = new glue.Job(stack, 'EtlJob', { - jobName: 'EtlJob', - executable: glue.JobExecutable.pythonEtl({ - glueVersion: glue.GlueVersion.V2_0, - pythonVersion: glue.PythonVersion.THREE, - script, - }), - workerType: glue.WorkerType.G_2X, - workerCount: 10, - maxConcurrentRuns: 2, - maxRetries: 2, - timeout: cdk.Duration.minutes(5), - notifyDelayAfter: cdk.Duration.minutes(1), - defaultArguments: { - arg1: 'value1', - arg2: 'value2', - }, - sparkUI: { - enabled: true, - }, - continuousLogging: { - enabled: true, - quiet: true, - logStreamPrefix: 'EtlJob', - }, - tags: { - key: 'value', - }, -}); -etlJob.metricSuccess(); +[glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { + const etlJob = new glue.Job(stack, 'EtlJob' + glueVersion.name, { + jobName: 'EtlJob' + glueVersion.name, + executable: glue.JobExecutable.pythonEtl({ + pythonVersion: glue.PythonVersion.THREE, + glueVersion, + script, + }), + workerType: glue.WorkerType.G_2X, + workerCount: 10, + maxConcurrentRuns: 2, + maxRetries: 2, + timeout: cdk.Duration.minutes(5), + notifyDelayAfter: cdk.Duration.minutes(1), + defaultArguments: { + arg1: 'value1', + arg2: 'value2', + }, + sparkUI: { + enabled: true, + }, + continuousLogging: { + enabled: true, + quiet: true, + logStreamPrefix: 'EtlJob', + }, + tags: { + key: 'value', + }, + }); + etlJob.metricSuccess(); -new glue.Job(stack, 'StreamingJob', { - jobName: 'StreamingJob', - executable: glue.JobExecutable.pythonStreaming({ - glueVersion: glue.GlueVersion.V2_0, - pythonVersion: glue.PythonVersion.THREE, - script, - }), - defaultArguments: { - arg1: 'value1', - arg2: 'value2', - }, - tags: { - key: 'value', - }, + new glue.Job(stack, 'StreamingJob' + glueVersion.name, { + jobName: 'StreamingJob' + glueVersion.name, + executable: glue.JobExecutable.pythonStreaming({ + pythonVersion: glue.PythonVersion.THREE, + glueVersion, + script, + }), + defaultArguments: { + arg1: 'value1', + arg2: 'value2', + }, + tags: { + key: 'value', + }, + }); }); new glue.Job(stack, 'ShellJob', { diff --git a/packages/@aws-cdk/aws-glue/test/job-executable.test.ts b/packages/@aws-cdk/aws-glue/test/job-executable.test.ts index d7416911df914..6de1ad9859509 100644 --- a/packages/@aws-cdk/aws-glue/test/job-executable.test.ts +++ b/packages/@aws-cdk/aws-glue/test/job-executable.test.ts @@ -11,6 +11,8 @@ describe('GlueVersion', () => { test('.V3_0 should set the name correctly', () => expect(glue.GlueVersion.V3_0.name).toEqual('3.0')); + test('.V4_0 should set the name correctly', () => expect(glue.GlueVersion.V4_0.name).toEqual('4.0')); + test('of(customVersion) should set the name correctly', () => expect(glue.GlueVersion.of('CustomVersion').name).toEqual('CustomVersion')); }); @@ -74,7 +76,7 @@ describe('JobExecutable', () => { })).toThrow(/extraPythonFiles is not supported for languages other than JobLanguage.PYTHON/); }); - [glue.GlueVersion.V0_9, glue.GlueVersion.V2_0, glue.GlueVersion.V3_0].forEach((glueVersion) => { + [glue.GlueVersion.V0_9, glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { test(`with JobType.PYTHON_SHELL and GlueVersion ${glueVersion} should throw`, () => { expect(() => glue.JobExecutable.of({ type: glue.JobType.PYTHON_SHELL, @@ -99,7 +101,7 @@ describe('JobExecutable', () => { }); }); - [glue.GlueVersion.V2_0, glue.GlueVersion.V3_0].forEach((glueVersion) => { + [glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { test(`with PythonVersion.TWO and GlueVersion ${glueVersion} should throw`, () => { expect(() => glue.JobExecutable.of({ type: glue.JobType.ETL, From 8a21558294761ce7a7536826c348d6a98b4d72a5 Mon Sep 17 00:00:00 2001 From: Ahmed Kamel Date: Sun, 4 Dec 2022 15:01:54 +0000 Subject: [PATCH 2/2] force an update to README --- packages/@aws-cdk/aws-glue/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index 60dbd498d99f2..31030b9af93a0 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -45,7 +45,7 @@ An ETL job processes data in batches using Apache Spark. declare const bucket: s3.Bucket; new glue.Job(this, 'ScalaSparkEtlJob', { executable: glue.JobExecutable.scalaEtl({ - glueVersion: glue.GlueVersion.V2_0, + glueVersion: glue.GlueVersion.V4_0, script: glue.Code.fromBucket(bucket, 'src/com/example/HelloWorld.scala'), className: 'com.example.HelloWorld', extraJars: [glue.Code.fromBucket(bucket, 'jars/HelloWorld.jar')], @@ -61,7 +61,7 @@ A Streaming job is similar to an ETL job, except that it performs ETL on data st ```ts new glue.Job(this, 'PythonSparkStreamingJob', { executable: glue.JobExecutable.pythonStreaming({ - glueVersion: glue.GlueVersion.V2_0, + glueVersion: glue.GlueVersion.V4_0, pythonVersion: glue.PythonVersion.THREE, script: glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')), }),