AWS CodeBuild GitHub Job Pruner

tests/ci/cdk/app.py

@@ -3,7 +3,7 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0 OR ISC
 
-from aws_cdk import core
+from aws_cdk import Environment, App
 
 # from cdk.bm_framework_stack import BmFrameworkStack
 from cdk.aws_lc_mac_arm_ci_stack import AwsLcMacArmCIStack
@@ -17,10 +17,10 @@
 from util.metadata import AWS_ACCOUNT, AWS_REGION, LINUX_X86_ECR_REPO, LINUX_AARCH_ECR_REPO, WINDOWS_X86_ECR_REPO
 
 # Initialize app.
-app = core.App()
+app = App()
 
 # Initialize env.
-env = core.Environment(account=AWS_ACCOUNT, region=AWS_REGION)
+env = Environment(account=AWS_ACCOUNT, region=AWS_REGION)
 
 # Define AWS ECR stacks.
 # ECR holds the docker images, which are pre-built to accelerate the code builds/tests of git pull requests.

tests/ci/cdk/cdk.context.json

@@ -1 +1,11 @@
-{}
+{
+  "acknowledged-issue-numbers": [
+    19836
+  ],
+  "availability-zones:account=431229050501:region=us-west-2": [
+    "us-west-2a",
+    "us-west-2b",
+    "us-west-2c",
+    "us-west-2d"
+  ]
+}

tests/ci/cdk/cdk.json

@@ -1,7 +1,4 @@
 {
   "app": "python3 app.py",
-  "context": {
-    "@aws-cdk/core:enableStackNameDuplicates": "true",
-    "aws-cdk:enableDiffNoFail": "true"
-  }
+  "context": {}
 }

tests/ci/cdk/cdk/aws_lc_analytics_stack.py

@@ -1,18 +1,20 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0 OR ISC
 
-from aws_cdk import core, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs
+from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs
+from constructs import Construct
 
+from cdk.components import PruneStaleGitHubBuilds
 from util.iam_policies import code_build_publish_metrics_in_json
 from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME
 from util.build_spec_loader import BuildSpecLoader
 
 
-class AwsLcGitHubAnalyticsStack(core.Stack):
+class AwsLcGitHubAnalyticsStack(Stack):
     """Define a stack used to batch execute AWS-LC tests in GitHub."""
 
     def __init__(self,
-                 scope: core.Construct,
+                 scope: Construct,
                  id: str,
                  spec_file_path: str,
                  **kwargs) -> None:
@@ -46,9 +48,11 @@ def __init__(self,
             project_name=id,
             source=git_hub_source,
             role=role,
-            timeout=core.Duration.minutes(120),
+            timeout=Duration.minutes(120),
             environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.LARGE,
                                                    privileged=True,
                                                    build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
             build_spec=BuildSpecLoader.load(spec_file_path))
         analytics.enable_batch_builds()
+
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics)

tests/ci/cdk/cdk/aws_lc_android_ci_stack.py

@@ -1,19 +1,23 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0 OR ISC
 
-from aws_cdk import core, aws_codebuild as codebuild, aws_iam as iam
+from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam
+from constructs import Construct
+
+from cdk.components import PruneStaleGitHubBuilds
 from util.iam_policies import code_build_batch_policy_in_json, device_farm_access_policy_in_json
 from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME
 from util.build_spec_loader import BuildSpecLoader
 
 
-class AwsLcAndroidCIStack(core.Stack):
+class AwsLcAndroidCIStack(Stack):
     """Define a stack used to batch execute AWS-LC tests in GitHub."""
+
     # The Device Farm resource used to in this CI spec, must be manually created.
     # TODO: Automate Device Farm creation with cdk script.
 
     def __init__(self,
-                 scope: core.Construct,
+                 scope: Construct,
                  id: str,
                  spec_file_path: str,
                  **kwargs) -> None:
@@ -52,9 +56,11 @@ def __init__(self,
             project_name=id,
             source=git_hub_source,
             role=role,
-            timeout=core.Duration.minutes(180),
+            timeout=Duration.minutes(180),
             environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.SMALL,
                                                    privileged=False,
                                                    build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
+
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)

tests/ci/cdk/cdk/aws_lc_github_ci_stack.py

@@ -1,17 +1,20 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0 OR ISC
 
-from aws_cdk import core, aws_codebuild as codebuild, aws_iam as iam
+from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_s3_assets
+from constructs import Construct
+
+from cdk.components import PruneStaleGitHubBuilds
 from util.iam_policies import code_build_batch_policy_in_json
-from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME
+from util.metadata import CAN_AUTOLOAD, GITHUB_REPO_OWNER, GITHUB_REPO_NAME
 from util.build_spec_loader import BuildSpecLoader
 
 
-class AwsLcGitHubCIStack(core.Stack):
+class AwsLcGitHubCIStack(Stack):
     """Define a stack used to batch execute AWS-LC tests in GitHub."""
 
     def __init__(self,
-                 scope: core.Construct,
+                 scope: Construct,
                  id: str,
                  spec_file_path: str,
                  **kwargs) -> None:
@@ -47,9 +50,11 @@ def __init__(self,
             project_name=id,
             source=git_hub_source,
             role=role,
-            timeout=core.Duration.minutes(180),
+            timeout=Duration.minutes(180),
             environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.SMALL,
                                                    privileged=False,
                                                    build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
+
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)

tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py

@@ -1,20 +1,22 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0 OR ISC
 
-from aws_cdk import core, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs
+from aws_cdk import Duration, Size, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs
+from constructs import Construct
 
+from cdk.components import PruneStaleGitHubBuilds
 from util.ecr_util import ecr_arn
 from util.iam_policies import code_build_batch_policy_in_json, \
     code_build_publish_metrics_in_json
 from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_REPO_OWNER, GITHUB_REPO_NAME
 from util.build_spec_loader import BuildSpecLoader
 
 
-class AwsLcGitHubFuzzCIStack(core.Stack):
+class AwsLcGitHubFuzzCIStack(Stack):
     """Define a stack used to batch execute AWS-LC tests in GitHub."""
 
     def __init__(self,
-                 scope: core.Construct,
+                 scope: Construct,
                  id: str,
                  spec_file_path: str,
                  **kwargs) -> None:
@@ -47,7 +49,7 @@ def __init__(self,
 
         # Create the VPC for EFS and CodeBuild
         public_subnet = ec2.SubnetConfiguration(name="PublicFuzzingSubnet", subnet_type=ec2.SubnetType.PUBLIC)
-        private_subnet = ec2.SubnetConfiguration(name="PrivateFuzzingSubnet", subnet_type=ec2.SubnetType.PRIVATE)
+        private_subnet = ec2.SubnetConfiguration(name="PrivateFuzzingSubnet", subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS)
 
         # Create a VPC with a single public and private subnet in a single AZ. This is to avoid the elastic IP limit
         # being used up by a bunch of idle NAT gateways
@@ -69,7 +71,7 @@ def __init__(self,
             description="Allow all traffic inside security group"
         )
 
-        efs_subnet_selection = ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE)
+        efs_subnet_selection = ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS)
 
         # Create the EFS to store the corpus and logs. EFS allows new filesystems to burst to 100 MB/s for the first 2
         # TB of data read/written, after that the rate is limited based on the size of the filesystem. As of late
@@ -90,7 +92,7 @@ def __init__(self,
             vpc_subnets=efs_subnet_selection,
             performance_mode=efs.PerformanceMode.GENERAL_PURPOSE,
             throughput_mode=efs.ThroughputMode.PROVISIONED,
-            provisioned_throughput_per_second=core.Size.mebibytes(100),
+            provisioned_throughput_per_second=Size.mebibytes(100),
         )
 
         # Define CodeBuild.
@@ -100,7 +102,7 @@ def __init__(self,
             project_name=id,
             source=git_hub_source,
             role=role,
-            timeout=core.Duration.minutes(120),
+            timeout=Duration.minutes(120),
             environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.LARGE,
                                                    privileged=True,
                                                    build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
@@ -123,3 +125,5 @@ def __init__(self,
           "MountPoint": "/efs_fuzzing_root",
           "Type": "EFS"
         }])
+
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=fuzz_codebuild)

tests/ci/cdk/cdk/aws_lc_mac_arm_ci_stack.py

@@ -5,18 +5,21 @@
 import boto3
 
 from botocore.exceptions import ClientError
-from aws_cdk import core, aws_ec2 as ec2, aws_codebuild as codebuild, aws_iam as iam, aws_s3 as s3, aws_logs as logs
+from aws_cdk import CfnTag, Duration, Stack, Tags, aws_ec2 as ec2, aws_codebuild as codebuild, aws_iam as iam, aws_s3 as s3, aws_logs as logs
+from constructs import Construct
+
+from cdk.components import PruneStaleGitHubBuilds
 from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_REPO_OWNER, GITHUB_REPO_NAME
 from util.iam_policies import code_build_batch_policy_in_json, ec2_policies_in_json, ssm_policies_in_json, s3_read_write_policy_in_json
 from util.build_spec_loader import BuildSpecLoader
 
 # detailed documentation can be found here: https://docs.aws.amazon.com/cdk/api/latest/docs/aws-ec2-readme.html
 
-class AwsLcMacArmCIStack(core.Stack):
+class AwsLcMacArmCIStack(Stack):
     """Define a stack used to create a CodeBuild instance on which to execute the AWS-LC m1 ci ec2 instance"""
 
     def __init__(self,
-                 scope: core.Construct,
+                 scope: Construct,
                  id: str,
                  spec_file_path: str,
                  **kwargs) -> None:
@@ -60,13 +63,15 @@ def __init__(self,
             project_name=id,
             source=git_hub_source,
             role=codebuild_role,
-            timeout=core.Duration.minutes(120),
+            timeout=Duration.minutes(120),
             environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.SMALL,
                                                    privileged=False,
                                                    build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
 
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+
         # S3 bucket for testing internal fixes.
         s3_read_write_policy = iam.PolicyDocument.from_json(s3_read_write_policy_in_json("aws-lc-codebuild"))
         ec2_inline_policies = {"s3_read_write_policy": s3_read_write_policy}
@@ -97,7 +102,7 @@ def __init__(self,
                                 availability_zone="us-west-2a",
                                 auto_placement="off",
                                 instance_type="mac2.metal")
-        core.Tags.of(cfn_host).add("Name", "{}-dedicated-host".format(id))
+        Tags.of(cfn_host).add("Name", "{}-dedicated-host".format(id))
 
         # AMI is for M1 MacOS Monterey.
         ami_id="ami-084c6ab9d03ad4d46"
@@ -110,7 +115,7 @@ def __init__(self,
                         instance_type="mac2.metal",
                         security_group_ids=[security_group.security_group_id],
                         subnet_id=selection.subnet_ids[0],
-                        tags=[core.CfnTag(key="Name",value="aws-lc-ci-macos-arm-ec2-instance")])
+                        tags=[CfnTag(key="Name",value="aws-lc-ci-macos-arm-ec2-instance")])
 
         # Define logs for SSM.
         logs.LogGroup(self, "{}-cw-logs".format(id), log_group_name=CLOUDWATCH_LOGS)

tests/ci/cdk/cdk/bm_framework_stack.py

@@ -5,7 +5,10 @@
 import boto3
 
 from botocore.exceptions import ClientError
-from aws_cdk import core, aws_ec2 as ec2, aws_codebuild as codebuild, aws_iam as iam, aws_s3 as s3, aws_logs as logs
+from aws_cdk import Duration, Stack, aws_ec2 as ec2, aws_codebuild as codebuild, aws_iam as iam, aws_s3 as s3, aws_logs as logs
+from constructs import Construct
+
+from cdk.components import PruneStaleGitHubBuilds
 from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_REPO_OWNER, GITHUB_REPO_NAME
 from util.iam_policies import code_build_batch_policy_in_json, s3_read_write_policy_in_json, \
     ec2_bm_framework_policies_in_json, ssm_bm_framework_policies_in_json, s3_bm_framework_policies_in_json, \
@@ -14,11 +17,11 @@
 
 # detailed documentation can be found here: https://docs.aws.amazon.com/cdk/api/latest/docs/aws-ec2-readme.html
 
-class BmFrameworkStack(core.Stack):
+class BmFrameworkStack(Stack):
     """Define a stack used to create a CodeBuild instance on which to execute the AWS-LC benchmarking framework"""
 
     def __init__(self,
-                 scope: core.Construct,
+                 scope: Construct,
                  id: str,
                  spec_file_path: str,
                  **kwargs) -> None:
@@ -72,13 +75,15 @@ def __init__(self,
             project_name=id,
             source=git_hub_source,
             role=codebuild_role,
-            timeout=core.Duration.minutes(120),
+            timeout=Duration.minutes(120),
             environment=codebuild.BuildEnvironment(compute_type=codebuild.ComputeType.SMALL,
                                                    privileged=False,
                                                    build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
 
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+
         # use boto3 to determine if a bucket with the name that we want exists, and if it doesn't, create it
         s3_res = boto3.resource('s3')
         prod_bucket = s3_res.Bucket(S3_PROD_BUCKET)

tests/ci/cdk/cdk/components.py

@@ -0,0 +1,47 @@
+import pathlib
+
+from aws_cdk import aws_codebuild as codebuild, aws_lambda as lambda_, aws_ecr as ecr, aws_secretsmanager as sm, \
+    aws_events as events, aws_events_targets as events_targets, aws_iam as iam, Duration
+
+from constructs import Construct
+from util.metadata import GITHUB_REPO_OWNER, GITHUB_TOKEN_SECRET_NAME
+
+
+class PruneStaleGitHubBuilds(Construct):
+    def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject) -> None:
+        super().__init__(scope, id)
+
+        github_token_secret = sm.Secret.from_secret_name_v2(scope=self,
+                                                            id="{}-GitHubToken".format(id),
+                                                            secret_name=GITHUB_TOKEN_SECRET_NAME)
+
+        lambda_function = lambda_.Function(scope=self,
+                                           id="LambdaFunction",
+                                           code=lambda_.Code.from_asset_image(
+                                               directory=str(pathlib.Path().joinpath("..", "lambda")),
+                                               target="purge-stale-builds"),
+                                           handler=lambda_.Handler.FROM_IMAGE,
+                                           runtime=lambda_.Runtime.FROM_IMAGE,
+                                           environment={
+                                               "CODEBUILD_PROJECT_NAME": project.project_name,
+                                               "GITHUB_REPO_OWNER": GITHUB_REPO_OWNER,
+                                               "GITHUB_TOKEN_SECRET_NAME": github_token_secret.secret_name,
+                                               "RUST_LOG": "info",
+                                           })
+
+        github_token_secret.grant_read(lambda_function)
+
+        lambda_function.add_to_role_policy(
+            iam.PolicyStatement(effect=iam.Effect.ALLOW,
+                                actions=[
+                                    "codebuild:BatchGetBuildBatches",
+                                    "codebuild:ListBuildBatchesForProject",
+                                    "codebuild:StopBuild",
+                                ],
+                                resources=[project.project_arn]))
+
+        events.Rule(scope=self, id="PurgeEventRule",
+                    description="Purge stale GitHub codebuild jobs (once per minute)",
+                    enabled=True,
+                    schedule=events.Schedule.rate(Duration.minutes(1)),
+                    targets=[events_targets.LambdaFunction(handler=lambda_function)])

tests/ci/cdk/cdk/ecr_stack.py

@@ -1,13 +1,14 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0 OR ISC
 
-from aws_cdk import core, aws_ecr as ecr, aws_iam as iam
+from aws_cdk import Stack, aws_ecr as ecr, aws_iam as iam
+from constructs import Construct
 
 
-class EcrStack(core.Stack):
+class EcrStack(Stack):
     """Define a stack of ECR to store pre-built Docker Images."""
 
-    def __init__(self, scope: core.Construct, id: str, repo_name: str, **kwargs) -> None:
+    def __init__(self, scope: Construct, id: str, repo_name: str, **kwargs) -> None:
         super().__init__(scope, id, **kwargs)
 
         ecr.Repository(scope=self, id=id, repository_name=repo_name).grant_pull_push(