diff --git a/.github/scripts/acquire-build-image b/.github/scripts/acquire-build-image index 772e3a3aab..ec2ed027b1 100755 --- a/.github/scripts/acquire-build-image +++ b/.github/scripts/acquire-build-image @@ -11,6 +11,7 @@ import subprocess import sys import time import unittest +import base64 REMOTE_BASE_IMAGE_NAME = "public.ecr.aws/w0m4q9l7/github-awslabs-smithy-rs-ci" LOCAL_BASE_IMAGE_NAME = "smithy-rs-base-image" @@ -41,7 +42,8 @@ class Platform(Enum): # Script context class Context: - def __init__(self, start_path, script_path, tools_path, user_id, image_tag, allow_local_build, github_actions): + def __init__(self, start_path, script_path, tools_path, user_id, image_tag, allow_local_build, github_actions, + encrypted_docker_password, docker_passphrase): self.start_path = start_path self.script_path = script_path self.tools_path = tools_path @@ -50,6 +52,8 @@ class Context: self.image_tag = image_tag self.allow_local_build = allow_local_build self.github_actions = github_actions + self.encrypted_docker_password = encrypted_docker_password + self.docker_passphrase = docker_passphrase @staticmethod def default(): @@ -60,6 +64,9 @@ class Context: image_tag = get_cmd_output("./docker-image-hash", cwd=script_path)[1] allow_local_build = os.getenv("ALLOW_LOCAL_BUILD") != "false" github_actions = os.getenv("GITHUB_ACTIONS") == "true" + encrypted_docker_password = os.getenv("ENCRYPTED_DOCKER_PASSWORD") + docker_passphrase = os.getenv("DOCKER_LOGIN_TOKEN_PASSPHRASE") + print(f"Start path: {start_path}") print(f"Script path: {script_path}") print(f"Tools path: {tools_path}") @@ -67,7 +74,9 @@ class Context: print(f"Required base image tag: {image_tag}") print(f"Allow local build: {allow_local_build}") print(f"Running in GitHub Actions: {github_actions}") - return Context(start_path, script_path, tools_path, user_id, image_tag, allow_local_build, github_actions) + return Context(start_path=start_path, script_path=script_path, tools_path=tools_path, user_id=user_id, + image_tag=image_tag, allow_local_build=allow_local_build, github_actions=github_actions, + encrypted_docker_password=encrypted_docker_password, docker_passphrase=docker_passphrase) def output_contains_any(stdout, stderr, messages): @@ -76,7 +85,6 @@ def output_contains_any(stdout, stderr, messages): return True return False - # Mockable shell commands class Shell: # Returns the platform that this script is running on @@ -91,6 +99,9 @@ class Shell: (status, _, _) = get_cmd_output(f"docker inspect \"{image_name}:{image_tag}\"", check=False) return status == 0 + def docker_login(self, password): + get_cmd_output("docker login --username AWS --password-stdin public.ecr.aws", input=password.encode('utf-8')) + # Pulls the requested `image_name` with `image_tag`. Returns `DockerPullResult`. def docker_pull(self, image_name, image_tag): (status, stdout, stderr) = get_cmd_output(f"docker pull \"{image_name}:{image_tag}\"", check=False) @@ -102,7 +113,7 @@ class Shell: print("-------------------") not_found_messages = ["not found: manifest unknown"] - throttle_messages = ["toomanyrequests: Rate exceeded", "toomanyrequests: Data limit exceeded"] + throttle_messages = ["toomanyrequests:"] retryable_messages = ["net/http: TLS handshake timeout"] if status == 0: return DockerPullResult.SUCCESS @@ -160,17 +171,39 @@ def run(command, cwd=None): # Returns (status, output) from a shell command -def get_cmd_output(command, cwd=None, check=True): +def get_cmd_output(command, cwd=None, check=True, **kwargs): + if isinstance(command, str): + command = shlex.split(command) + result = subprocess.run( - shlex.split(command), + command, capture_output=True, - check=check, - cwd=cwd + check=False, + cwd=cwd, + **kwargs ) - return (result.returncode, result.stdout.decode("utf-8").strip(), result.stderr.decode("utf-8").strip()) + stdout = result.stdout.decode("utf-8").strip() + stderr = result.stderr.decode("utf-8").strip() + if check and result.returncode != 0: + raise Exception(f"failed to run '{command}.\n{stdout}\n{stderr}") + + return result.returncode, stdout, stderr + + +def decrypt_and_login(shell, secret, passphrase): + decoded = base64.b64decode(secret, validate=True) + if not passphrase: + raise Exception("a secret was set but no passphrase was set (or it was empty)") + (code, password, err) = get_cmd_output( + ["gpg", "--decrypt", "--batch", "--quiet", "--passphrase", passphrase, "--output", "-"], + input=decoded) + shell.docker_login(password) + print("Docker login success!") def acquire_build_image(context=Context.default(), shell=Shell()): + if context.encrypted_docker_password is not None: + decrypt_and_login(shell, context.encrypted_docker_password, context.docker_passphrase) # If the image doesn't already exist locally, then look remotely if not shell.docker_image_exists_locally(LOCAL_BASE_IMAGE_NAME, context.image_tag): announce("Base image not found locally.") @@ -211,15 +244,18 @@ def acquire_build_image(context=Context.default(), shell=Shell()): class SelfTest(unittest.TestCase): - def test_context(self, allow_local_build=False, github_actions=False): + def test_context(self, github_actions=False, allow_local_build=False, encrypted_docker_password=None, + docker_passphrase=None): return Context( start_path="/tmp/test/start-path", script_path="/tmp/test/script-path", tools_path="/tmp/test/tools-path", user_id="123", image_tag="someimagetag", + encrypted_docker_password=encrypted_docker_password, + docker_passphrase=docker_passphrase, + github_actions=github_actions, allow_local_build=allow_local_build, - github_actions=github_actions ) def mock_shell(self): @@ -231,6 +267,7 @@ class SelfTest(unittest.TestCase): shell.docker_pull = MagicMock() shell.docker_save = MagicMock() shell.docker_tag = MagicMock() + shell.docker_login = MagicMock() return shell def test_retry_architecture_mismatch(self): @@ -247,6 +284,13 @@ class SelfTest(unittest.TestCase): ) ) + def test_docker_login(self): + shell = self.mock_shell() + acquire_build_image(self.test_context( + encrypted_docker_password="jA0ECQMCvYU/JxsX3g/70j0BxbLLW8QaFWWb/DqY9gPhTuEN/xdYVxaoDnV6Fha+lAWdT7xN0qZr5DHPBalLfVvvM1SEXRBI8qnfXyGI", + docker_passphrase="secret"), shell) + shell.docker_login.assert_called_with("payload") + def test_retry_immediate_success(self): shell = self.mock_shell() shell.docker_pull.side_effect = [DockerPullResult.SUCCESS] @@ -374,7 +418,7 @@ class SelfTest(unittest.TestCase): shell.docker_image_exists_locally.assert_called_once() shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG) - shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path") + shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path/ci-build") # When: # - the base image doesn't exist locally @@ -391,10 +435,10 @@ class SelfTest(unittest.TestCase): self.assertEqual(0, acquire_build_image(context, shell)) shell.docker_image_exists_locally.assert_called_once() - shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path") + shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path/ci-build") shell.docker_save.assert_not_called() shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG) - shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path") + shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path/ci-build") # When: # - the base image doesn't exist locally @@ -411,10 +455,10 @@ class SelfTest(unittest.TestCase): self.assertEqual(0, acquire_build_image(context, shell)) shell.docker_image_exists_locally.assert_called_once() - shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path") + shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path/ci-build") shell.docker_save.assert_not_called() shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG) - shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path") + shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path/ci-build") # When: # - the base image doesn't exist locally @@ -431,14 +475,14 @@ class SelfTest(unittest.TestCase): self.assertEqual(0, acquire_build_image(context, shell)) shell.docker_image_exists_locally.assert_called_once() - shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path") + shell.docker_build_base_image.assert_called_with("someimagetag", "/tmp/test/tools-path/ci-build") shell.docker_save.assert_called_with( LOCAL_BASE_IMAGE_NAME, "someimagetag", "/tmp/test/start-path/smithy-rs-base-image" ) shell.docker_tag.assert_called_with(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG) - shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path") + shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path/ci-build") # When: # - the base image doesn't exist locally @@ -478,7 +522,7 @@ class SelfTest(unittest.TestCase): call(REMOTE_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, "someimagetag"), call(LOCAL_BASE_IMAGE_NAME, "someimagetag", LOCAL_BASE_IMAGE_NAME, LOCAL_TAG) ]) - shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path") + shell.docker_build_build_image.assert_called_with("123", "/tmp/test/tools-path/ci-build") def main(): diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 96c73db029..c1b55a2584 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -16,13 +16,45 @@ env: ecr_repository: public.ecr.aws/w0m4q9l7/github-awslabs-smithy-rs-ci jobs: + # This job will, if possible, save a docker login password to the job outputs. The token will + # be encrypted with the passphrase stored as a GitHub secret. The login password expires after 12h. + # The login password is encrypted with the repo secret DOCKER_LOGIN_TOKEN_PASSPHRASE + save-docker-login-token: + outputs: + docker-login-password: ${{ steps.set-token.outputs.docker-login-password }} + permissions: + id-token: write + contents: read + continue-on-error: true + name: Save a docker login token + runs-on: ubuntu-latest + steps: + - name: Attempt to load a docker login password + uses: aws-actions/configure-aws-credentials@v1-node16 + with: + role-to-assume: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }} + role-session-name: GitHubActions + aws-region: us-west-2 + - name: Save the docker login password to the output + id: set-token + run: | + ENCRYPTED_PAYLOAD=$( + gpg --symmetric --batch --passphrase "${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}" --output - <(aws ecr-public get-login-password --region us-east-1) | base64 -w0 + ) + echo "docker-login-password=$ENCRYPTED_PAYLOAD" >> $GITHUB_OUTPUT + + # This job detects if the PR made changes to build tools. If it did, then it builds a new # build Docker image. Otherwise, it downloads a build image from Public ECR. In both cases, # it uploads the image as a build artifact for other jobs to download and use. acquire-base-image: name: Acquire Base Image + needs: save-docker-login-token if: ${{ github.event.pull_request.head.repo.full_name == 'awslabs/smithy-rs' }} runs-on: ubuntu-latest + env: + ENCRYPTED_DOCKER_PASSWORD: ${{ needs.save-docker-login-token.outputs.docker-login-password }} + DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }} permissions: id-token: write contents: read @@ -50,11 +82,16 @@ jobs: # Run shared CI after the Docker build image has either been rebuilt or found in ECR ci: - needs: acquire-base-image + needs: + - save-docker-login-token + - acquire-base-image if: ${{ github.event.pull_request.head.repo.full_name == 'awslabs/smithy-rs' }} uses: ./.github/workflows/ci.yml with: run_sdk_examples: true + secrets: + ENCRYPTED_DOCKER_PASSWORD: ${{ needs.save-docker-login-token.outputs.docker-login-password }} + DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }} # The PR bot requires a Docker build image, so make it depend on the `acquire-base-image` job. pr_bot: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b18987b0c..774c4f3c7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,18 @@ on: required: false type: string default: '' + secrets: + # the docker login password for ECR. This is + ENCRYPTED_DOCKER_PASSWORD: + required: false + DOCKER_LOGIN_TOKEN_PASSPHRASE: + required: false env: rust_version: 1.62.1 rust_toolchain_components: clippy,rustfmt + ENCRYPTED_DOCKER_PASSWORD: ${{ secrets.ENCRYPTED_DOCKER_PASSWORD }} + DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }} jobs: # The `generate` job runs scripts that produce artifacts that are required by the `test` job,