Stale GitHub CodeBuild Job Pruner

aws · Apr 25, 2023 · 67330b2 · 67330b2
1 parent c09e6f9
commit 67330b2
Show file tree

Hide file tree

Showing 15 changed files with 2,837 additions and 2 deletions.
diff --git a/tests/ci/cdk/cdk.context.json b/tests/ci/cdk/cdk.context.json
@@ -1 +1,11 @@
-{}
+{
+  "acknowledged-issue-numbers": [
+    19836
+  ],
+  "availability-zones:account=431229050501:region=us-west-2": [
+    "us-west-2a",
+    "us-west-2b",
+    "us-west-2c",
+    "us-west-2d"
+  ]
+}
diff --git a/tests/ci/cdk/cdk/aws_lc_analytics_stack.py b/tests/ci/cdk/cdk/aws_lc_analytics_stack.py
@@ -4,6 +4,7 @@
 from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs
 from constructs import Construct
 
+from cdk.components import PruneStaleGitHubBuilds
 from util.iam_policies import code_build_publish_metrics_in_json
 from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME
 from util.build_spec_loader import BuildSpecLoader
@@ -53,3 +54,5 @@ def __init__(self,
                                                    build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
             build_spec=BuildSpecLoader.load(spec_file_path))
         analytics.enable_batch_builds()
+
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics)
diff --git a/tests/ci/cdk/cdk/aws_lc_android_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_android_ci_stack.py
@@ -3,13 +3,16 @@
 
 from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam
 from constructs import Construct
+
+from cdk.components import PruneStaleGitHubBuilds
 from util.iam_policies import code_build_batch_policy_in_json, device_farm_access_policy_in_json
 from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME
 from util.build_spec_loader import BuildSpecLoader
 
 
 class AwsLcAndroidCIStack(Stack):
     """Define a stack used to batch execute AWS-LC tests in GitHub."""
+
     # The Device Farm resource used to in this CI spec, must be manually created.
     # TODO: Automate Device Farm creation with cdk script.
 
@@ -59,3 +62,5 @@ def __init__(self,
                                                    build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
+
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
diff --git a/tests/ci/cdk/cdk/aws_lc_github_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_github_ci_stack.py
@@ -3,6 +3,8 @@
 
 from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_s3_assets
 from constructs import Construct
+
+from cdk.components import PruneStaleGitHubBuilds
 from util.iam_policies import code_build_batch_policy_in_json
 from util.metadata import CAN_AUTOLOAD, GITHUB_REPO_OWNER, GITHUB_REPO_NAME
 from util.build_spec_loader import BuildSpecLoader
@@ -54,3 +56,5 @@ def __init__(self,
                                                    build_image=codebuild.LinuxBuildImage.STANDARD_4_0),
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
+
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
diff --git a/tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py
@@ -4,6 +4,7 @@
 from aws_cdk import Duration, Size, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs
 from constructs import Construct
 
+from cdk.components import PruneStaleGitHubBuilds
 from util.ecr_util import ecr_arn
 from util.iam_policies import code_build_batch_policy_in_json, \
     code_build_publish_metrics_in_json
@@ -124,3 +125,5 @@ def __init__(self,
           "MountPoint": "/efs_fuzzing_root",
           "Type": "EFS"
         }])
+
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=fuzz_codebuild)
diff --git a/tests/ci/cdk/cdk/aws_lc_mac_arm_ci_stack.py b/tests/ci/cdk/cdk/aws_lc_mac_arm_ci_stack.py
@@ -7,6 +7,8 @@
 from botocore.exceptions import ClientError
 from aws_cdk import CfnTag, Duration, Stack, Tags, aws_ec2 as ec2, aws_codebuild as codebuild, aws_iam as iam, aws_s3 as s3, aws_logs as logs
 from constructs import Construct
+
+from cdk.components import PruneStaleGitHubBuilds
 from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_REPO_OWNER, GITHUB_REPO_NAME
 from util.iam_policies import code_build_batch_policy_in_json, ec2_policies_in_json, ssm_policies_in_json, s3_read_write_policy_in_json
 from util.build_spec_loader import BuildSpecLoader
@@ -68,6 +70,8 @@ def __init__(self,
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
 
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+
         # S3 bucket for testing internal fixes.
         s3_read_write_policy = iam.PolicyDocument.from_json(s3_read_write_policy_in_json("aws-lc-codebuild"))
         ec2_inline_policies = {"s3_read_write_policy": s3_read_write_policy}

diff --git a/tests/ci/cdk/cdk/bm_framework_stack.py b/tests/ci/cdk/cdk/bm_framework_stack.py
@@ -7,6 +7,8 @@
 from botocore.exceptions import ClientError
 from aws_cdk import Duration, Stack, aws_ec2 as ec2, aws_codebuild as codebuild, aws_iam as iam, aws_s3 as s3, aws_logs as logs
 from constructs import Construct
+
+from cdk.components import PruneStaleGitHubBuilds
 from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_REPO_OWNER, GITHUB_REPO_NAME
 from util.iam_policies import code_build_batch_policy_in_json, s3_read_write_policy_in_json, \
     ec2_bm_framework_policies_in_json, ssm_bm_framework_policies_in_json, s3_bm_framework_policies_in_json, \
@@ -80,6 +82,8 @@ def __init__(self,
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
 
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+
         # use boto3 to determine if a bucket with the name that we want exists, and if it doesn't, create it
         s3_res = boto3.resource('s3')
         prod_bucket = s3_res.Bucket(S3_PROD_BUCKET)

diff --git a/tests/ci/cdk/cdk/components.py b/tests/ci/cdk/cdk/components.py
@@ -0,0 +1,47 @@
+import pathlib
+
+from aws_cdk import aws_codebuild as codebuild, aws_lambda as lambda_, aws_ecr as ecr, aws_secretsmanager as sm, \
+    aws_events as events, aws_events_targets as events_targets, aws_iam as iam, Duration
+
+from constructs import Construct
+from util.metadata import GITHUB_REPO_OWNER, GITHUB_TOKEN_SECRET_NAME, LINUX_X86_ECR_REPO
+
+
+class PruneStaleGitHubBuilds(Construct):
+    def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject) -> None:
+        super().__init__(scope, id)
+
+        github_token_secret = sm.Secret.from_secret_name_v2(scope=self,
+                                                            id="{}-GitHubToken".format(id),
+                                                            secret_name=GITHUB_TOKEN_SECRET_NAME)
+
+        lambda_function = lambda_.Function(scope=self,
+                                           id="LambdaFunction",
+                                           code=lambda_.Code.from_asset_image(
+                                               directory=str(pathlib.Path().joinpath("..", "lambda")),
+                                               target="purge-stale-builds"),
+                                           handler=lambda_.Handler.FROM_IMAGE,
+                                           runtime=lambda_.Runtime.FROM_IMAGE,
+                                           environment={
+                                               "CODEBUILD_PROJECT_NAME": project.project_name,
+                                               "GITHUB_REPO_OWNER": GITHUB_REPO_OWNER,
+                                               "GITHUB_TOKEN_SECRET_NAME": github_token_secret.secret_name,
+                                               "RUST_LOG": "info",
+                                           })
+
+        github_token_secret.grant_read(lambda_function)
+
+        lambda_function.add_to_role_policy(
+            iam.PolicyStatement(effect=iam.Effect.ALLOW,
+                                actions=[
+                                    "codebuild:BatchGetBuildBatches",
+                                    "codebuild:ListBuildBatchesForProject",
+                                    "codebuild:StopBuild",
+                                ],
+                                resources=[project.project_arn]))
+
+        events.Rule(scope=self, id="PurgeEventRule",
+                    description="Purge stale GitHub codebuild jobs (once per minute)",
+                    enabled=True,
+                    schedule=events.Schedule.rate(Duration.minutes(1)),
+                    targets=[events_targets.LambdaFunction(handler=lambda_function)])
diff --git a/tests/ci/cdk/util/metadata.py b/tests/ci/cdk/util/metadata.py
@@ -22,6 +22,7 @@
 GITHUB_REPO_OWNER = EnvUtil.get("GITHUB_REPO_OWNER", "aws")
 GITHUB_REPO_NAME = EnvUtil.get("GITHUB_REPO_NAME", "aws-lc")
 GITHUB_SOURCE_VERSION = EnvUtil.get("GITHUB_SOURCE_VERSION", "main")
+GITHUB_TOKEN_SECRET_NAME = EnvUtil.get("GITHUB_TOKEN_SECRET_NAME", "aws-lc/ci/github/token")
 
 # Used when AWS CDK defines resources for Windows docker image build.
 S3_BUCKET_NAME = EnvUtil.get("S3_FOR_WIN_DOCKER_IMG_BUILD", "aws-lc-windows-docker-image-build")

diff --git a/tests/ci/lambda/.dockerignore b/tests/ci/lambda/.dockerignore
@@ -0,0 +1 @@
+/target