From 3dcbbb950b59a9aadde784193f33b2318ff8eda5 Mon Sep 17 00:00:00 2001 From: Luca Pizzini Date: Mon, 7 Aug 2023 09:22:33 +0200 Subject: [PATCH] fix(glue-alpha): fixed Glue Job synth time validation checks --- .../aws-glue-alpha/lib/job-executable.ts | 28 +- packages/@aws-cdk/aws-glue-alpha/lib/job.ts | 15 +- ...9be7858a12b228a2ae6e5c10faccd9097b1e855.py | 1 + .../aws-glue-job-python-shell.assets.json | 32 ++ .../aws-glue-job-python-shell.template.json | 290 ++++++++++ ...efaultTestDeployAssert453D25B7.assets.json | 19 + ...aultTestDeployAssert453D25B7.template.json | 36 ++ .../cdk.out | 1 + .../integ.json | 12 + .../manifest.json | 141 +++++ .../tree.json | 503 ++++++++++++++++++ .../test/integ.job-python-shell.ts | 65 +++ .../test/job-executable.test.ts | 102 +++- .../@aws-cdk/aws-glue-alpha/test/job.test.ts | 11 + 14 files changed, 1239 insertions(+), 17 deletions(-) create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.assets.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.template.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.template.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/cdk.out create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/integ.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/manifest.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/tree.json create mode 100644 packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.ts diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts index c121f69193b16..15cb5757e88b3 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts @@ -350,38 +350,40 @@ export class JobExecutable { private config: JobExecutableConfig; private constructor(config: JobExecutableConfig) { - if (JobType.PYTHON_SHELL === config.type) { + const glueVersion = config.glueVersion.name; + const type = config.type.name; + if (JobType.PYTHON_SHELL.name === type) { if (config.language !== JobLanguage.PYTHON) { throw new Error('Python shell requires the language to be set to Python'); } - if ([GlueVersion.V0_9, GlueVersion.V3_0, GlueVersion.V4_0].includes(config.glueVersion)) { - throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support Python Shell`); + if ([GlueVersion.V0_9.name, GlueVersion.V4_0.name].includes(glueVersion)) { + throw new Error(`Specified GlueVersion ${glueVersion} does not support Python Shell`); } } - if (JobType.RAY === config.type) { + if (JobType.RAY.name === type) { if (config.language !== JobLanguage.PYTHON) { throw new Error('Ray requires the language to be set to Python'); } - if ([GlueVersion.V0_9, GlueVersion.V1_0, GlueVersion.V2_0, GlueVersion.V3_0].includes(config.glueVersion)) { - throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support Ray`); + if ([GlueVersion.V0_9.name, GlueVersion.V1_0.name, GlueVersion.V2_0.name, GlueVersion.V3_0.name].includes(glueVersion)) { + throw new Error(`Specified GlueVersion ${glueVersion} does not support Ray`); } } - if (config.extraJarsFirst && [GlueVersion.V0_9, GlueVersion.V1_0].includes(config.glueVersion)) { - throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support extraJarsFirst`); + if (config.extraJarsFirst && [GlueVersion.V0_9.name, GlueVersion.V1_0.name].includes(glueVersion)) { + throw new Error(`Specified GlueVersion ${glueVersion} does not support extraJarsFirst`); } - if (config.pythonVersion === PythonVersion.TWO && ![GlueVersion.V0_9, GlueVersion.V1_0].includes(config.glueVersion)) { - throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support PythonVersion ${config.pythonVersion}`); + if (config.pythonVersion === PythonVersion.TWO && ![GlueVersion.V0_9.name, GlueVersion.V1_0.name].includes(glueVersion)) { + throw new Error(`Specified GlueVersion ${glueVersion} does not support PythonVersion ${config.pythonVersion}`); } if (JobLanguage.PYTHON !== config.language && config.extraPythonFiles) { throw new Error('extraPythonFiles is not supported for languages other than JobLanguage.PYTHON'); } - if (config.pythonVersion === PythonVersion.THREE_NINE && config.type !== JobType.PYTHON_SHELL && config.type !== JobType.RAY) { + if (config.pythonVersion === PythonVersion.THREE_NINE && type !== JobType.PYTHON_SHELL.name && type !== JobType.RAY.name) { throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell and Ray'); } - if (config.pythonVersion === PythonVersion.THREE && config.type === JobType.RAY) { + if (config.pythonVersion === PythonVersion.THREE && type === JobType.RAY.name) { throw new Error('Specified PythonVersion PythonVersion.THREE is not supported for Ray'); } - if (config.runtime === undefined && config.type === JobType.RAY) { + if (config.runtime === undefined && type === JobType.RAY.name) { throw new Error('Runtime is required for Ray jobs.'); } this.config = config; diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index 7b5c6a1cdf7c4..6962f801aa3a7 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -719,11 +719,20 @@ export class Job extends JobBase { } } - if (props.maxCapacity !== undefined && (props.workerType && props.workerCount !== undefined)) { + let maxCapacity = props.maxCapacity; + if (maxCapacity !== undefined && (props.workerType && props.workerCount !== undefined)) { throw new Error('maxCapacity cannot be used when setting workerType and workerCount'); } - if (props.maxCapacity !== undefined && ![GlueVersion.V0_9, GlueVersion.V1_0].includes(executable.glueVersion)) { - throw new Error('maxCapacity cannot be used when GlueVersion 2.0 or later'); + if (executable.type !== JobType.PYTHON_SHELL) { + if (maxCapacity !== undefined && ![GlueVersion.V0_9, GlueVersion.V1_0].includes(executable.glueVersion)) { + throw new Error('maxCapacity cannot be used when GlueVersion 2.0 or later'); + } + } else { + // max capacity validation for python shell jobs (defaults to 0.0625) + maxCapacity = maxCapacity ?? 0.0625; + if (maxCapacity !== 0.0625 && maxCapacity !== 1) { + throw new Error(`maxCapacity value must be either 0.0625 or 1 for JobType.PYTHON_SHELL jobs, received ${maxCapacity}`); + } } if ((!props.workerType && props.workerCount !== undefined) || (props.workerType && props.workerCount === undefined)) { throw new Error('Both workerType and workerCount must be set'); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py new file mode 100644 index 0000000000000..e75154b7c390f --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py @@ -0,0 +1 @@ +print("hello world") \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.assets.json new file mode 100644 index 0000000000000..17b109b19285f --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.assets.json @@ -0,0 +1,32 @@ +{ + "version": "33.0.0", + "files": { + "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { + "source": { + "path": "asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "13432a74ca6cfada399f4d2b33385964f66c49aeeb01c5f0cefec52560a4dffa": { + "source": { + "path": "aws-glue-job-python-shell.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "13432a74ca6cfada399f4d2b33385964f66c49aeeb01c5f0cefec52560a4dffa.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.template.json new file mode 100644 index 0000000000000..dece180ae8219 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.template.json @@ -0,0 +1,290 @@ +{ + "Resources": { + "ShellJobServiceRoleCF97BC4B": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "ShellJobServiceRoleDefaultPolicy7F22D315": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "ShellJobServiceRoleDefaultPolicy7F22D315", + "Roles": [ + { + "Ref": "ShellJobServiceRoleCF97BC4B" + } + ] + } + }, + "ShellJob42E81F95": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "pythonshell", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "DefaultArguments": { + "--job-language": "python", + "arg1": "value1", + "arg2": "value2" + }, + "GlueVersion": "1.0", + "MaxCapacity": 0.0625, + "Name": "ShellJob", + "Role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, + "Tags": { + "key": "value" + } + } + }, + "ShellJob39ServiceRole2F6F3768": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "ShellJob39ServiceRoleDefaultPolicy38A33919": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "ShellJob39ServiceRoleDefaultPolicy38A33919", + "Roles": [ + { + "Ref": "ShellJob39ServiceRole2F6F3768" + } + ] + } + }, + "ShellJob390C141361": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "pythonshell", + "PythonVersion": "3.9", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "DefaultArguments": { + "--job-language": "python", + "arg1": "value1", + "arg2": "value2" + }, + "GlueVersion": "3.0", + "MaxCapacity": 1, + "Name": "ShellJob39", + "Role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, + "Tags": { + "key": "value" + } + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets.json new file mode 100644 index 0000000000000..fcf891c433efb --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets.json @@ -0,0 +1,19 @@ +{ + "version": "33.0.0", + "files": { + "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { + "source": { + "path": "awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.template.json new file mode 100644 index 0000000000000..ad9d0fb73d1dd --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.template.json @@ -0,0 +1,36 @@ +{ + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/cdk.out new file mode 100644 index 0000000000000..560dae10d018f --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/cdk.out @@ -0,0 +1 @@ +{"version":"33.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/integ.json new file mode 100644 index 0000000000000..89660486d806d --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/integ.json @@ -0,0 +1,12 @@ +{ + "version": "33.0.0", + "testCases": { + "aws-glue-job-python-shell-integ-test/DefaultTest": { + "stacks": [ + "aws-glue-job-python-shell" + ], + "assertionStack": "aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert", + "assertionStackName": "awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/manifest.json new file mode 100644 index 0000000000000..93cd15ece1b08 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/manifest.json @@ -0,0 +1,141 @@ +{ + "version": "33.0.0", + "artifacts": { + "aws-glue-job-python-shell.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "aws-glue-job-python-shell.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "aws-glue-job-python-shell": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "aws-glue-job-python-shell.template.json", + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/13432a74ca6cfada399f4d2b33385964f66c49aeeb01c5f0cefec52560a4dffa.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "aws-glue-job-python-shell.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "aws-glue-job-python-shell.assets" + ], + "metadata": { + "/aws-glue-job-python-shell/ShellJob/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "ShellJobServiceRoleCF97BC4B" + } + ], + "/aws-glue-job-python-shell/ShellJob/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "ShellJobServiceRoleDefaultPolicy7F22D315" + } + ], + "/aws-glue-job-python-shell/ShellJob/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "ShellJob42E81F95" + } + ], + "/aws-glue-job-python-shell/ShellJob39/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "ShellJob39ServiceRole2F6F3768" + } + ], + "/aws-glue-job-python-shell/ShellJob39/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "ShellJob39ServiceRoleDefaultPolicy38A33919" + } + ], + "/aws-glue-job-python-shell/ShellJob39/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "ShellJob390C141361" + } + ], + "/aws-glue-job-python-shell/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/aws-glue-job-python-shell/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "aws-glue-job-python-shell" + }, + "awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.template.json", + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "awsgluejobpythonshellintegtestDefaultTestDeployAssert453D25B7.assets" + ], + "metadata": { + "/aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert" + }, + "Tree": { + "type": "cdk:tree", + "properties": { + "file": "tree.json" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/tree.json new file mode 100644 index 0000000000000..05905851160a8 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.js.snapshot/tree.json @@ -0,0 +1,503 @@ +{ + "version": "tree-0.1", + "tree": { + "id": "App", + "path": "", + "children": { + "aws-glue-job-python-shell": { + "id": "aws-glue-job-python-shell", + "path": "aws-glue-job-python-shell", + "children": { + "ShellJob": { + "id": "ShellJob", + "path": "aws-glue-job-python-shell/ShellJob", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-glue-job-python-shell/ShellJob/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job-python-shell/ShellJob/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job-python-shell/ShellJob/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-glue-job-python-shell/ShellJob/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job-python-shell/ShellJob/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "ShellJobServiceRoleDefaultPolicy7F22D315", + "roles": [ + { + "Ref": "ShellJobServiceRoleCF97BC4B" + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "Code8835353412338ec0bac0ee05542d1c16": { + "id": "Code8835353412338ec0bac0ee05542d1c16", + "path": "aws-glue-job-python-shell/ShellJob/Code8835353412338ec0bac0ee05542d1c16", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job-python-shell/ShellJob/Code8835353412338ec0bac0ee05542d1c16/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job-python-shell/ShellJob/Code8835353412338ec0bac0ee05542d1c16/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job-python-shell/ShellJob/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "pythonshell", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "defaultArguments": { + "--job-language": "python", + "arg1": "value1", + "arg2": "value2" + }, + "glueVersion": "1.0", + "maxCapacity": 0.0625, + "name": "ShellJob", + "role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, + "tags": { + "key": "value" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" + } + }, + "ShellJob39": { + "id": "ShellJob39", + "path": "aws-glue-job-python-shell/ShellJob39", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-glue-job-python-shell/ShellJob39/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-glue-job-python-shell/ShellJob39/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job-python-shell/ShellJob39/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-glue-job-python-shell/ShellJob39/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-glue-job-python-shell/ShellJob39/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "ShellJob39ServiceRoleDefaultPolicy38A33919", + "roles": [ + { + "Ref": "ShellJob39ServiceRole2F6F3768" + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-glue-job-python-shell/ShellJob39/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "pythonshell", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3.9" + }, + "defaultArguments": { + "--job-language": "python", + "arg1": "value1", + "arg2": "value2" + }, + "glueVersion": "3.0", + "maxCapacity": 1, + "name": "ShellJob39", + "role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, + "tags": { + "key": "value" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" + } + }, + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-glue-job-python-shell/BootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-glue-job-python-shell/CheckBootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" + } + }, + "aws-glue-job-python-shell-integ-test": { + "id": "aws-glue-job-python-shell-integ-test", + "path": "aws-glue-job-python-shell-integ-test", + "children": { + "DefaultTest": { + "id": "DefaultTest", + "path": "aws-glue-job-python-shell-integ-test/DefaultTest", + "children": { + "Default": { + "id": "Default", + "path": "aws-glue-job-python-shell-integ-test/DefaultTest/Default", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.69" + } + }, + "DeployAssert": { + "id": "DeployAssert", + "path": "aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert", + "children": { + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert/BootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-glue-job-python-shell-integ-test/DefaultTest/DeployAssert/CheckBootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", + "version": "0.0.0" + } + }, + "Tree": { + "id": "Tree", + "path": "Tree", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.2.69" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.App", + "version": "0.0.0" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.ts new file mode 100644 index 0000000000000..4e283327f0ad4 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job-python-shell.ts @@ -0,0 +1,65 @@ +import * as integ from '@aws-cdk/integ-tests-alpha'; +import * as path from 'path'; +import * as cdk from 'aws-cdk-lib'; +import * as glue from '../lib'; + +/** + * To verify the ability to run jobs created in this test + * + * Run the job using + * `aws glue start-job-run --region us-east-1 --job-name ` + * This will return a runId + * + * Get the status of the job run using + * `aws glue get-job-run --region us-east-1 --job-name --run-id ` + * + * For example, to test the ShellJob + * - Run: `aws glue start-job-run --region us-east-1 --job-name ShellJob` + * - Get Status: `aws glue get-job-run --region us-east-1 --job-name ShellJob --run-id ` + * - Check output: `aws logs get-log-events --region us-east-1 --log-group-name "/aws-glue/python-jobs/output" --log-stream-name ">` which should show "hello world" + */ +const app = new cdk.App(); + +const stack = new cdk.Stack(app, 'aws-glue-job-python-shell'); + +const script = glue.Code.fromAsset(path.join(__dirname, 'job-script/hello_world.py')); + +new glue.Job(stack, 'ShellJob', { + jobName: 'ShellJob', + executable: glue.JobExecutable.pythonShell({ + glueVersion: glue.GlueVersion.V1_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + defaultArguments: { + arg1: 'value1', + arg2: 'value2', + }, + tags: { + key: 'value', + }, + maxCapacity: 0.0625, +}); + +new glue.Job(stack, 'ShellJob39', { + jobName: 'ShellJob39', + executable: glue.JobExecutable.pythonShell({ + glueVersion: glue.GlueVersion.V3_0, + pythonVersion: glue.PythonVersion.THREE_NINE, + script, + }), + defaultArguments: { + arg1: 'value1', + arg2: 'value2', + }, + tags: { + key: 'value', + }, + maxCapacity: 1.0, +}); + +new integ.IntegTest(app, 'aws-glue-job-python-shell-integ-test', { + testCases: [stack], +}); + +app.synth(); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts index ea575f1a022d4..28c6225c542b6 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts @@ -67,6 +67,24 @@ describe('JobExecutable', () => { })).toThrow(/Python shell requires the language to be set to Python/); }); + test('with JobType.of("pythonshell") and a language other than JobLanguage.PYTHON should throw', () => { + expect(() => glue.JobExecutable.of({ + glueVersion: glue.GlueVersion.V3_0, + type: glue.JobType.of('pythonshell'), + language: glue.JobLanguage.SCALA, + script, + })).toThrow(/Python shell requires the language to be set to Python/); + }); + + test('with JobType.of("glueray") and a language other than JobLanguage.PYTHON should throw', () => { + expect(() => glue.JobExecutable.of({ + glueVersion: glue.GlueVersion.V4_0, + type: glue.JobType.of('glueray'), + language: glue.JobLanguage.SCALA, + script, + })).toThrow(/Ray requires the language to be set to Python/); + }); + test('with JobType.RAY and a language other than JobLanguage.PYTHON should throw', () => { expect(() => glue.JobExecutable.of({ glueVersion: glue.GlueVersion.V4_0, @@ -87,7 +105,7 @@ describe('JobExecutable', () => { })).toThrow(/extraPythonFiles is not supported for languages other than JobLanguage.PYTHON/); }); - [glue.GlueVersion.V0_9, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { + [glue.GlueVersion.V0_9, glue.GlueVersion.V4_0].forEach((glueVersion) => { test(`with JobType.PYTHON_SHELL and GlueVersion ${glueVersion} should throw`, () => { expect(() => glue.JobExecutable.of({ type: glue.JobType.PYTHON_SHELL, @@ -99,6 +117,42 @@ describe('JobExecutable', () => { }); }); + [glue.GlueVersion.V0_9, glue.GlueVersion.V4_0].forEach((glueVersion) => { + test(`with JobType.PYTHON_SHELL and GlueVersion.of("${glueVersion.name}") should throw`, () => { + expect(() => glue.JobExecutable.of({ + type: glue.JobType.PYTHON_SHELL, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.TWO, + script, + glueVersion: glue.GlueVersion.of(glueVersion.name), + })).toThrow(`Specified GlueVersion ${glueVersion.name} does not support Python Shell`); + }); + }); + + [glue.GlueVersion.V0_9, glue.GlueVersion.V1_0, glue.GlueVersion.V2_0, glue.GlueVersion.V3_0].forEach((glueVersion) => { + test(`with JobType.RAY and GlueVersion ${glueVersion} should throw`, () => { + expect(() => glue.JobExecutable.of({ + type: glue.JobType.RAY, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.TWO, + script, + glueVersion, + })).toThrow(`Specified GlueVersion ${glueVersion.name} does not support Ray`); + }); + }); + + [glue.GlueVersion.V0_9, glue.GlueVersion.V1_0, glue.GlueVersion.V2_0, glue.GlueVersion.V3_0].forEach((glueVersion) => { + test(`with JobType.of("glueray") and GlueVersion ${glueVersion} should throw`, () => { + expect(() => glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.TWO, + script, + glueVersion, + })).toThrow(`Specified GlueVersion ${glueVersion.name} does not support Ray`); + }); + }); + [glue.GlueVersion.V0_9, glue.GlueVersion.V1_0].forEach((glueVersion) => { test(`with extraJarsFirst set and GlueVersion ${glueVersion.name} should throw`, () => { expect(() => glue.JobExecutable.of({ @@ -112,6 +166,19 @@ describe('JobExecutable', () => { }); }); + [glue.GlueVersion.V0_9, glue.GlueVersion.V1_0].forEach((glueVersion) => { + test(`with extraJarsFirst set and GlueVersion.of("${glueVersion.name}") should throw`, () => { + expect(() => glue.JobExecutable.of({ + type: glue.JobType.ETL, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.TWO, + extraJarsFirst: true, + script, + glueVersion: glue.GlueVersion.of(glueVersion.name), + })).toThrow(`Specified GlueVersion ${glueVersion.name} does not support extraJarsFirst`); + }); + }); + [glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { test(`with PythonVersion.TWO and GlueVersion ${glueVersion} should throw`, () => { expect(() => glue.JobExecutable.of({ @@ -124,6 +191,18 @@ describe('JobExecutable', () => { }); }); + [glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { + test(`with PythonVersion.TWO and GlueVersion.of("${glueVersion.name}") should throw`, () => { + expect(() => glue.JobExecutable.of({ + type: glue.JobType.ETL, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.TWO, + script, + glueVersion: glue.GlueVersion.of(glueVersion.name), + })).toThrow(`Specified GlueVersion ${glueVersion.name} does not support PythonVersion 2`); + }); + }); + test('with PythonVersion set to PythonVersion.THREE_NINE and JobType etl should throw', () => { expect(() => glue.JobExecutable.of({ type: glue.JobType.ETL, @@ -144,6 +223,16 @@ describe('JobExecutable', () => { })).toBeDefined(); }); + test('with PythonVersion PythonVersion.THREE_NINE and JobType.of("pythonshell") should succeed', () => { + expect(glue.JobExecutable.of({ + type: glue.JobType.of('pythonshell'), + glueVersion: glue.GlueVersion.V1_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + script, + })).toBeDefined(); + }); + test('with PythonVersion PythonVersion.THREE_NINE and JobType ray should succeed', () => { expect(glue.JobExecutable.of({ type: glue.JobType.RAY, @@ -154,5 +243,16 @@ describe('JobExecutable', () => { script, })).toBeDefined(); }); + + test('with PythonVersion PythonVersion.THREE_NINE and JobTypeof("glueray") should succeed', () => { + expect(glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + runtime: glue.Runtime.RAY_TWO_FOUR, + script, + })).toBeDefined(); + }); }); }); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts index cdb5a838fba09..81245f43929af 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts @@ -1075,6 +1075,17 @@ describe('Job', () => { })).toThrow('maxCapacity cannot be used when GlueVersion 2.0 or later'); }); + test('maxCapacity with Python Shell jobs validation', () => { + expect(() => new glue.Job(stack, 'Job', { + executable: glue.JobExecutable.pythonShell({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), + maxCapacity: 10, + })).toThrow(/maxCapacity value must be either 0.0625 or 1 for JobType.PYTHON_SHELL jobs/); + }); + test('workerType without workerCount should throw', () => { expect(() => new glue.Job(stack, 'Job', { executable: glue.JobExecutable.pythonEtl({