fix(glue): synth time validation does not work in Python/Java/C#/Go (#…

…26650) Fixes some issues on Glue Job synth time validation: - `maxCapacity` can be specified for PythonShell jobs with Glue Version 2.0+ and should be either 0.0625 or 1 (defaults to 0.0625) - Validation was not working when using `GlueVersion.of()` and `JobType.of()` - PythonShell jobs supports GlueVersion 3.0 Closes #26620. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
aws · Aug 7, 2023 · dba8cf3 · dba8cf3
1 parent 972a06f
commit dba8cf3
Show file tree

Hide file tree

Showing 14 changed files with 1,239 additions and 17 deletions.
diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts
@@ -350,38 +350,40 @@ export class JobExecutable {
   private config: JobExecutableConfig;
 
   private constructor(config: JobExecutableConfig) {
-    if (JobType.PYTHON_SHELL === config.type) {
+    const glueVersion = config.glueVersion.name;
+    const type = config.type.name;
+    if (JobType.PYTHON_SHELL.name === type) {
       if (config.language !== JobLanguage.PYTHON) {
         throw new Error('Python shell requires the language to be set to Python');
       }
-      if ([GlueVersion.V0_9, GlueVersion.V3_0, GlueVersion.V4_0].includes(config.glueVersion)) {
-        throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support Python Shell`);
+      if ([GlueVersion.V0_9.name, GlueVersion.V4_0.name].includes(glueVersion)) {
+        throw new Error(`Specified GlueVersion ${glueVersion} does not support Python Shell`);
       }
     }
-    if (JobType.RAY === config.type) {
+    if (JobType.RAY.name === type) {
       if (config.language !== JobLanguage.PYTHON) {
         throw new Error('Ray requires the language to be set to Python');
       }
-      if ([GlueVersion.V0_9, GlueVersion.V1_0, GlueVersion.V2_0, GlueVersion.V3_0].includes(config.glueVersion)) {
-        throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support Ray`);
+      if ([GlueVersion.V0_9.name, GlueVersion.V1_0.name, GlueVersion.V2_0.name, GlueVersion.V3_0.name].includes(glueVersion)) {
+        throw new Error(`Specified GlueVersion ${glueVersion} does not support Ray`);
       }
     }
-    if (config.extraJarsFirst && [GlueVersion.V0_9, GlueVersion.V1_0].includes(config.glueVersion)) {
-      throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support extraJarsFirst`);
+    if (config.extraJarsFirst && [GlueVersion.V0_9.name, GlueVersion.V1_0.name].includes(glueVersion)) {
+      throw new Error(`Specified GlueVersion ${glueVersion} does not support extraJarsFirst`);
     }
-    if (config.pythonVersion === PythonVersion.TWO && ![GlueVersion.V0_9, GlueVersion.V1_0].includes(config.glueVersion)) {
-      throw new Error(`Specified GlueVersion ${config.glueVersion.name} does not support PythonVersion ${config.pythonVersion}`);
+    if (config.pythonVersion === PythonVersion.TWO && ![GlueVersion.V0_9.name, GlueVersion.V1_0.name].includes(glueVersion)) {
+      throw new Error(`Specified GlueVersion ${glueVersion} does not support PythonVersion ${config.pythonVersion}`);
     }
     if (JobLanguage.PYTHON !== config.language && config.extraPythonFiles) {
       throw new Error('extraPythonFiles is not supported for languages other than JobLanguage.PYTHON');
     }
-    if (config.pythonVersion === PythonVersion.THREE_NINE && config.type !== JobType.PYTHON_SHELL && config.type !== JobType.RAY) {
+    if (config.pythonVersion === PythonVersion.THREE_NINE && type !== JobType.PYTHON_SHELL.name && type !== JobType.RAY.name) {
       throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell and Ray');
     }
-    if (config.pythonVersion === PythonVersion.THREE && config.type === JobType.RAY) {
+    if (config.pythonVersion === PythonVersion.THREE && type === JobType.RAY.name) {
       throw new Error('Specified PythonVersion PythonVersion.THREE is not supported for Ray');
     }
-    if (config.runtime === undefined && config.type === JobType.RAY) {
+    if (config.runtime === undefined && type === JobType.RAY.name) {
       throw new Error('Runtime is required for Ray jobs.');
     }
     this.config = config;

diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts
@@ -719,11 +719,20 @@ export class Job extends JobBase {
       }
     }
 
-    if (props.maxCapacity !== undefined && (props.workerType && props.workerCount !== undefined)) {
+    let maxCapacity = props.maxCapacity;
+    if (maxCapacity !== undefined && (props.workerType && props.workerCount !== undefined)) {
       throw new Error('maxCapacity cannot be used when setting workerType and workerCount');
     }
-    if (props.maxCapacity !== undefined && ![GlueVersion.V0_9, GlueVersion.V1_0].includes(executable.glueVersion)) {
-      throw new Error('maxCapacity cannot be used when GlueVersion 2.0 or later');
+    if (executable.type !== JobType.PYTHON_SHELL) {
+      if (maxCapacity !== undefined && ![GlueVersion.V0_9, GlueVersion.V1_0].includes(executable.glueVersion)) {
+        throw new Error('maxCapacity cannot be used when GlueVersion 2.0 or later');
+      }
+    } else {
+      // max capacity validation for python shell jobs (defaults to 0.0625)
+      maxCapacity = maxCapacity ?? 0.0625;
+      if (maxCapacity !== 0.0625 && maxCapacity !== 1) {
+        throw new Error(`maxCapacity value must be either 0.0625 or 1 for JobType.PYTHON_SHELL jobs, received ${maxCapacity}`);
+      }
     }
     if ((!props.workerType && props.workerCount !== undefined) || (props.workerType && props.workerCount === undefined)) {
       throw new Error('Both workerType and workerCount must be set');

diff --git a/...ell.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py b/...ell.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py
@@ -0,0 +1 @@
+print("hello world")
diff --git a/...-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.assets.json b/...-glue-alpha/test/integ.job-python-shell.js.snapshot/aws-glue-job-python-shell.assets.json
@@ -0,0 +1,32 @@
+{
+  "version": "33.0.0",
+  "files": {
+    "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": {
+      "source": {
+        "path": "asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py",
+        "packaging": "file"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
+    },
+    "13432a74ca6cfada399f4d2b33385964f66c49aeeb01c5f0cefec52560a4dffa": {
+      "source": {
+        "path": "aws-glue-job-python-shell.template.json",
+        "packaging": "file"
+      },
+      "destinations": {
+        "current_account-current_region": {
+          "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+          "objectKey": "13432a74ca6cfada399f4d2b33385964f66c49aeeb01c5f0cefec52560a4dffa.json",
+          "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+        }
+      }
+    }
+  },
+  "dockerImages": {}
+}