From da741dca5ba3f93be4c5b831ddf6ef12e52b035e Mon Sep 17 00:00:00 2001 From: ysaito1001 Date: Wed, 25 Sep 2024 08:30:43 -0500 Subject: [PATCH] Add workflows to manually/weekly update the runtime lockfiles and the SDK lockfile (#3844) ## Motivation and Context This PR introduces GitHub workflows to automate the process of running `cargo update` on lockfiles and creating PRs in this repository. - Scheduled workflow: This workflow runs weekly to ensure dependencies are updated to the latest semver-compliant versions. - Manual workflow: This workflow provides the same functionality but can be triggered on-demand. It includes an option to force updates on [known broken dependencies](https://github.com/smithy-lang/smithy-rs/blob/6b42eb5ca00a2dc9c46562452e495a2ec2e43d0f/aws/sdk/build.gradle.kts#L503-L504). ## Testing - Did NOT run a scheduled workflow, assuming that's a thin wrapper around what has been verified. We can afford a "see what happens and fix if necessary" approach once this PR is merged into main. - Manually triggered a workflow, successfully opening PRs with updated lockfiles ([ex1](https://github.com/smithy-lang/smithy-rs/pull/3842), [ex2](https://github.com/smithy-lang/smithy-rs/pull/3843)). - Manually triggered a workflow, forcing updates on broken dependencies (didn't open a PR to avoid noise, but confirmed `minicbor` was [updated to 0.24.4](https://github.com/smithy-lang/smithy-rs/blob/088cbe9f52695be0b24f4d0941667cf29631e785/rust-runtime/Cargo.lock#L2245-L2246)). ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- .github/workflows/manual-update-lockfiles.yml | 34 +++++ .../pull-request-updating-lockfiles.yml | 121 ++++++++++++++++++ .github/workflows/update-lockfiles.yml | 23 ++++ aws/sdk/build.gradle.kts | 4 +- tools/ci-build/runtime-versioner/Cargo.lock | 2 +- tools/ci-build/runtime-versioner/Cargo.toml | 2 +- .../runtime-versioner/src/command/audit.rs | 9 +- tools/ci-scripts/cargo-update-lockfiles | 47 +++++++ 8 files changed, 235 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/manual-update-lockfiles.yml create mode 100644 .github/workflows/pull-request-updating-lockfiles.yml create mode 100644 .github/workflows/update-lockfiles.yml create mode 100755 tools/ci-scripts/cargo-update-lockfiles diff --git a/.github/workflows/manual-update-lockfiles.yml b/.github/workflows/manual-update-lockfiles.yml new file mode 100644 index 0000000000..b87b509130 --- /dev/null +++ b/.github/workflows/manual-update-lockfiles.yml @@ -0,0 +1,34 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Update lockfiles manually +run-name: ${{ github.workflow }} (${{ inputs.base_branch }}) +on: + workflow_dispatch: + inputs: + base_branch: + description: The name of the branch on which to run `cargo update` for lockfiles + required: true + type: string + force_update_on_broken_dependencies: + description: When true, it forces `cargo update` to update broken dependencies to the latest semver-compatible versions, without downgrading them to the last known working versions + required: true + type: boolean + default: false + +concurrency: + group: ${{ github.workflow }}-${{ inputs.base_branch }} + cancel-in-progress: true + +jobs: + cargo-update-runtime-lockfiles-and-sdk-lockfile: + name: Run cargo update on the runtime lockfiles and the SDK lockfile + if: ${{ github.event_name == 'workflow_dispatch' }} + uses: ./.github/workflows/pull-request-updating-lockfiles.yml + with: + base_branch: ${{ inputs.base_branch }} + force_update_on_broken_dependencies: ${{ inputs.force_update_on_broken_dependencies }} + secrets: + DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }} + SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }} + RELEASE_AUTOMATION_BOT_PAT: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }} diff --git a/.github/workflows/pull-request-updating-lockfiles.yml b/.github/workflows/pull-request-updating-lockfiles.yml new file mode 100644 index 0000000000..28a04b9b77 --- /dev/null +++ b/.github/workflows/pull-request-updating-lockfiles.yml @@ -0,0 +1,121 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# This is a shared workflow used by both `update-lockfiles.yml` and `manual-update-lockfiles.yml`. + +name: Pull Request for Updating Lockfiles +on: + workflow_call: + inputs: + base_branch: + description: The name of the branch on which to run `cargo update` for lockfiles + required: true + type: string + force_update_on_broken_dependencies: + description: When true, it forces `cargo update` to update broken dependencies to the latest semver-compatible versions, without downgrading them to the last known working versions + required: true + type: boolean + secrets: + DOCKER_LOGIN_TOKEN_PASSPHRASE: + required: true + SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN: + required: true + RELEASE_AUTOMATION_BOT_PAT: + required: true + +env: + ecr_repository: public.ecr.aws/w0m4q9l7/github-awslabs-smithy-rs-ci + +jobs: + save-docker-login-token: + name: Save a docker login token + timeout-minutes: 10 + outputs: + docker-login-password: ${{ steps.set-token.outputs.docker-login-password }} + permissions: + id-token: write + contents: read + continue-on-error: true + runs-on: ubuntu-latest + steps: + - name: Attempt to load a docker login password + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }} + role-session-name: GitHubActions + aws-region: us-west-2 + - name: Save the docker login password to the output + id: set-token + run: | + ENCRYPTED_PAYLOAD=$( + gpg --symmetric --batch --passphrase "${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}" --output - <(aws ecr-public get-login-password --region us-east-1) | base64 -w0 + ) + echo "docker-login-password=$ENCRYPTED_PAYLOAD" >> $GITHUB_OUTPUT + + acquire-base-image: + name: Acquire Base Image + needs: save-docker-login-token + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + ENCRYPTED_DOCKER_PASSWORD: ${{ needs.save-docker-login-token.outputs.docker-login-password }} + DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }} + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v4 + with: + path: smithy-rs + - name: Acquire base image + id: acquire + env: + DOCKER_BUILDKIT: 1 + run: ./smithy-rs/.github/scripts/acquire-build-image + - name: Acquire credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }} + role-session-name: GitHubActions + aws-region: us-west-2 + - name: Upload image + run: | + IMAGE_TAG="$(./smithy-rs/.github/scripts/docker-image-hash)" + docker tag "smithy-rs-base-image:${IMAGE_TAG}" "${{ env.ecr_repository }}:${IMAGE_TAG}" + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws + docker push "${{ env.ecr_repository }}:${IMAGE_TAG}" + + create-pull-request-for-updating-lockfiles: + name: Create a Pull Request for updating lockfiles + needs: + - acquire-base-image + runs-on: ubuntu-latest + steps: + - name: Checkout smithy-rs + uses: actions/checkout@v4 + with: + path: smithy-rs + token: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }} + - name: Create branch name for updating lockfiles + id: branch-name-for-updating-lockfiles + shell: bash + run: | + branch_name="update-all-lockfiles-$(date +%s)" + echo "branch_name=${branch_name}" > $GITHUB_OUTPUT + - name: Cargo update all lockfiles + uses: ./smithy-rs/.github/actions/docker-build + with: + action: cargo-update-lockfiles + action-arguments: ${{ inputs.base_branch }} ${{ steps.branch-name-for-updating-lockfiles.outputs.branch_name }} ${{ inputs.force_update_on_broken_dependencies }} + - name: Create pull request + working-directory: smithy-rs + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }} + run: | + gh pr create \ + --title 'Run `cargo update` on the runtime lockfiles and the SDK lockfile' \ + --body 'If CI fails, commit the necessary fixes to this PR until all checks pass. If required, update entries in [crateNameToLastKnownWorkingVersions](https://github.com/smithy-lang/smithy-rs/blob/6b42eb5ca00a2dc9c46562452e495a2ec2e43d0f/aws/sdk/build.gradle.kts#L503-L504).' \ + --base ${{ inputs.base_branch }} \ + --head ${{ steps.branch-name-for-updating-lockfiles.outputs.branch_name }} \ + --label "needs-sdk-review" diff --git a/.github/workflows/update-lockfiles.yml b/.github/workflows/update-lockfiles.yml new file mode 100644 index 0000000000..28fe0bee8f --- /dev/null +++ b/.github/workflows/update-lockfiles.yml @@ -0,0 +1,23 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Update lockfiles scheduled +run-name: ${{ github.workflow }} +on: + schedule: + # Runs 22:00 UTC every Tuesday + - cron: 0 22 * * 2 + +jobs: + cargo-update-runtime-lockfiles-and-sdk-lockfile: + name: Run cargo update on the runtime lockfiles and the SDK lockfile + # Don't run on forked repositories + if: github.repository == 'smithy-lang/smithy-rs' + uses: ./.github/workflows/pull-request-updating-lockfiles.yml + with: + base_branch: main + force_update_on_broken_dependencies: false + secrets: + DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }} + SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }} + RELEASE_AUTOMATION_BOT_PAT: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }} diff --git a/aws/sdk/build.gradle.kts b/aws/sdk/build.gradle.kts index 8178850875..370e1dedb6 100644 --- a/aws/sdk/build.gradle.kts +++ b/aws/sdk/build.gradle.kts @@ -523,8 +523,10 @@ val downgradeAwsSdkLockfile = registerDowngradeFor(outputDir.asFile, "AwsSdk") fun Project.registerCargoUpdateFor( dir: File, name: String, + dependsOn: List = emptyList(), ): TaskProvider { return tasks.register("cargoUpdate${name}Lockfile") { + dependsOn(dependsOn) workingDir(dir) environment("RUSTFLAGS", "--cfg aws_sdk_unstable") commandLine("cargo", "update") @@ -532,7 +534,7 @@ fun Project.registerCargoUpdateFor( } } -val cargoUpdateAwsConfigLockfile = registerCargoUpdateFor(awsConfigPath, "AwsConfig") +val cargoUpdateAwsConfigLockfile = registerCargoUpdateFor(awsConfigPath, "AwsConfig", listOf("assemble")) val cargoUpdateAwsRuntimeLockfile = registerCargoUpdateFor(awsRustRuntimePath, "AwsRustRuntime") val cargoUpdateSmithyRuntimeLockfile = registerCargoUpdateFor(rustRuntimePath, "RustRuntime") diff --git a/tools/ci-build/runtime-versioner/Cargo.lock b/tools/ci-build/runtime-versioner/Cargo.lock index d71d9abb2d..22c07b9fcf 100644 --- a/tools/ci-build/runtime-versioner/Cargo.lock +++ b/tools/ci-build/runtime-versioner/Cargo.lock @@ -892,7 +892,7 @@ dependencies = [ [[package]] name = "runtime-versioner" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", "camino", diff --git a/tools/ci-build/runtime-versioner/Cargo.toml b/tools/ci-build/runtime-versioner/Cargo.toml index 4628dafdd4..0fcae4af36 100644 --- a/tools/ci-build/runtime-versioner/Cargo.toml +++ b/tools/ci-build/runtime-versioner/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "runtime-versioner" -version = "0.1.0" +version = "0.1.1" authors = ["AWS Rust SDK Team "] description = "Tool that manages runtime crate versions." edition = "2021" diff --git a/tools/ci-build/runtime-versioner/src/command/audit.rs b/tools/ci-build/runtime-versioner/src/command/audit.rs index 51d5ea7a67..ea4703be51 100644 --- a/tools/ci-build/runtime-versioner/src/command/audit.rs +++ b/tools/ci-build/runtime-versioner/src/command/audit.rs @@ -139,12 +139,13 @@ impl RuntimeCrate { .output() .with_context(|| format!("failed to git diff {}", self.name))?; let output = String::from_utf8(status.stdout)?; + // When run during a release, this file is replaced with it's actual contents. + // This breaks this git-based comparison and incorrectly requires a version bump. + // Temporary fix to allow the build to succeed. + let lines_to_ignore = &["aws-config/clippy.toml", "aws-config/Cargo.lock"]; let changed_files = output .lines() - // When run during a release, this file is replaced with it's actual contents. - // This breaks this git-based comparison and incorrectly requires a version bump. - // Temporary fix to allow the build to succeed. - .filter(|line| !line.contains("aws-config/clippy.toml")) + .filter(|line| !lines_to_ignore.iter().any(|ignore| line.contains(ignore))) .collect::>(); Ok(!changed_files.is_empty()) } diff --git a/tools/ci-scripts/cargo-update-lockfiles b/tools/ci-scripts/cargo-update-lockfiles new file mode 100755 index 0000000000..927ccc2c4e --- /dev/null +++ b/tools/ci-scripts/cargo-update-lockfiles @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# + +set -eux + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " + exit 1 +fi + +base_branch=$1 +branch_name_for_updating_lockfiles=$2 +force_update_on_broken_dependencies=$3 + +SMITHY_RS_DIR="$(pwd)/smithy-rs" + +pushd "${SMITHY_RS_DIR}" + +git config --local user.name "AWS SDK Rust Bot" +git config --local user.email "aws-sdk-rust-primary@amazon.com" + +git fetch --unshallow +git checkout "${base_branch}" +git checkout -b "${branch_name_for_updating_lockfiles}" + +if [[ "${force_update_on_broken_dependencies}" == "true" ]] +then + ./gradlew -Paws.sdk.force.update.broken.dependencies aws:sdk:cargoUpdateAllLockfiles +else + ./gradlew aws:sdk:cargoUpdateAllLockfiles +fi + +git add aws/rust-runtime/Cargo.lock \ + aws/rust-runtime/aws-config/Cargo.lock \ + aws/sdk/Cargo.lock \ + rust-runtime/Cargo.lock + +git diff --staged --quiet || \ + git commit \ + -m "Run cargo update on the runtime lockfiles and the SDK lockfile" + +git push origin HEAD + +popd