Add workflows to manually/weekly update the runtime lockfiles and the…

… SDK lockfile (#3844) ## Motivation and Context This PR introduces GitHub workflows to automate the process of running `cargo update` on lockfiles and creating PRs in this repository. - Scheduled workflow: This workflow runs weekly to ensure dependencies are updated to the latest semver-compliant versions. - Manual workflow: This workflow provides the same functionality but can be triggered on-demand. It includes an option to force updates on [known broken dependencies](https://github.com/smithy-lang/smithy-rs/blob/6b42eb5ca00a2dc9c46562452e495a2ec2e43d0f/aws/sdk/build.gradle.kts#L503-L504). ## Testing - Did NOT run a scheduled workflow, assuming that's a thin wrapper around what has been verified. We can afford a "see what happens and fix if necessary" approach once this PR is merged into main. - Manually triggered a workflow, successfully opening PRs with updated lockfiles ([ex1](#3842), [ex2](#3843)). - Manually triggered a workflow, forcing updates on broken dependencies (didn't open a PR to avoid noise, but confirmed `minicbor` was [updated to 0.24.4](https://github.com/smithy-lang/smithy-rs/blob/088cbe9f52695be0b24f4d0941667cf29631e785/rust-runtime/Cargo.lock#L2245-L2246)). ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._
smithy-lang · Sep 25, 2024 · da741dc · da741dc
1 parent 6ec8db1
commit da741dc
Show file tree

Hide file tree

Showing 8 changed files with 235 additions and 7 deletions.
diff --git a/.github/workflows/manual-update-lockfiles.yml b/.github/workflows/manual-update-lockfiles.yml
@@ -0,0 +1,34 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: Update lockfiles manually
+run-name: ${{ github.workflow }} (${{ inputs.base_branch }})
+on:
+  workflow_dispatch:
+    inputs:
+      base_branch:
+        description: The name of the branch on which to run `cargo update` for lockfiles
+        required: true
+        type: string
+      force_update_on_broken_dependencies:
+        description: When true, it forces `cargo update` to update broken dependencies to the latest semver-compatible versions, without downgrading them to the last known working versions
+        required: true
+        type: boolean
+        default: false
+
+concurrency:
+  group: ${{ github.workflow }}-${{ inputs.base_branch }}
+  cancel-in-progress: true
+
+jobs:
+  cargo-update-runtime-lockfiles-and-sdk-lockfile:
+    name: Run cargo update on the runtime lockfiles and the SDK lockfile
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    uses: ./.github/workflows/pull-request-updating-lockfiles.yml
+    with:
+      base_branch: ${{ inputs.base_branch }}
+      force_update_on_broken_dependencies: ${{ inputs.force_update_on_broken_dependencies }}
+    secrets:
+      DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}
+      SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }}
+      RELEASE_AUTOMATION_BOT_PAT: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }}
diff --git a/.github/workflows/pull-request-updating-lockfiles.yml b/.github/workflows/pull-request-updating-lockfiles.yml
@@ -0,0 +1,121 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# This is a shared workflow used by both `update-lockfiles.yml` and `manual-update-lockfiles.yml`.
+
+name: Pull Request for Updating Lockfiles
+on:
+  workflow_call:
+    inputs:
+      base_branch:
+        description: The name of the branch on which to run `cargo update` for lockfiles
+        required: true
+        type: string
+      force_update_on_broken_dependencies:
+        description: When true, it forces `cargo update` to update broken dependencies to the latest semver-compatible versions, without downgrading them to the last known working versions
+        required: true
+        type: boolean
+    secrets:
+      DOCKER_LOGIN_TOKEN_PASSPHRASE:
+        required: true
+      SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN:
+        required: true
+      RELEASE_AUTOMATION_BOT_PAT:
+        required: true
+
+env:
+  ecr_repository: public.ecr.aws/w0m4q9l7/github-awslabs-smithy-rs-ci
+
+jobs:
+  save-docker-login-token:
+    name: Save a docker login token
+    timeout-minutes: 10
+    outputs:
+      docker-login-password: ${{ steps.set-token.outputs.docker-login-password }}
+    permissions:
+      id-token: write
+      contents: read
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    steps:
+    - name: Attempt to load a docker login password
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }}
+        role-session-name: GitHubActions
+        aws-region: us-west-2
+    - name: Save the docker login password to the output
+      id: set-token
+      run: |
+        ENCRYPTED_PAYLOAD=$(
+          gpg --symmetric --batch --passphrase "${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}" --output - <(aws ecr-public get-login-password --region us-east-1) | base64 -w0
+        )
+        echo "docker-login-password=$ENCRYPTED_PAYLOAD" >> $GITHUB_OUTPUT
+
+  acquire-base-image:
+    name: Acquire Base Image
+    needs: save-docker-login-token
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      ENCRYPTED_DOCKER_PASSWORD: ${{ needs.save-docker-login-token.outputs.docker-login-password }}
+      DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        path: smithy-rs
+    - name: Acquire base image
+      id: acquire
+      env:
+        DOCKER_BUILDKIT: 1
+      run: ./smithy-rs/.github/scripts/acquire-build-image
+    - name: Acquire credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }}
+        role-session-name: GitHubActions
+        aws-region: us-west-2
+    - name: Upload image
+      run: |
+        IMAGE_TAG="$(./smithy-rs/.github/scripts/docker-image-hash)"
+        docker tag "smithy-rs-base-image:${IMAGE_TAG}" "${{ env.ecr_repository }}:${IMAGE_TAG}"
+        aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+        docker push "${{ env.ecr_repository }}:${IMAGE_TAG}"
+
+  create-pull-request-for-updating-lockfiles:
+    name: Create a Pull Request for updating lockfiles
+    needs:
+    - acquire-base-image
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout smithy-rs
+      uses: actions/checkout@v4
+      with:
+        path: smithy-rs
+        token: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }}
+    - name: Create branch name for updating lockfiles
+      id: branch-name-for-updating-lockfiles
+      shell: bash
+      run: |
+        branch_name="update-all-lockfiles-$(date +%s)"
+        echo "branch_name=${branch_name}" > $GITHUB_OUTPUT
+    - name: Cargo update all lockfiles
+      uses: ./smithy-rs/.github/actions/docker-build
+      with:
+        action: cargo-update-lockfiles
+        action-arguments: ${{ inputs.base_branch }} ${{ steps.branch-name-for-updating-lockfiles.outputs.branch_name }} ${{ inputs.force_update_on_broken_dependencies }}
+    - name: Create pull request
+      working-directory: smithy-rs
+      shell: bash
+      env:
+        GITHUB_TOKEN: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }}
+      run: |
+        gh pr create \
+          --title 'Run `cargo update` on the runtime lockfiles and the SDK lockfile' \
+          --body 'If CI fails, commit the necessary fixes to this PR until all checks pass. If required, update entries in [crateNameToLastKnownWorkingVersions](https://github.com/smithy-lang/smithy-rs/blob/6b42eb5ca00a2dc9c46562452e495a2ec2e43d0f/aws/sdk/build.gradle.kts#L503-L504).' \
+          --base ${{ inputs.base_branch }} \
+          --head ${{ steps.branch-name-for-updating-lockfiles.outputs.branch_name }} \
+          --label "needs-sdk-review"
diff --git a/.github/workflows/update-lockfiles.yml b/.github/workflows/update-lockfiles.yml
@@ -0,0 +1,23 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: Update lockfiles scheduled
+run-name: ${{ github.workflow }}
+on:
+  schedule:
+    # Runs 22:00 UTC every Tuesday
+  - cron: 0 22 * * 2
+
+jobs:
+  cargo-update-runtime-lockfiles-and-sdk-lockfile:
+    name: Run cargo update on the runtime lockfiles and the SDK lockfile
+    # Don't run on forked repositories
+    if: github.repository == 'smithy-lang/smithy-rs'
+    uses: ./.github/workflows/pull-request-updating-lockfiles.yml
+    with:
+      base_branch: main
+      force_update_on_broken_dependencies: false
+    secrets:
+      DOCKER_LOGIN_TOKEN_PASSPHRASE: ${{ secrets.DOCKER_LOGIN_TOKEN_PASSPHRASE }}
+      SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN: ${{ secrets.SMITHY_RS_PUBLIC_ECR_PUSH_ROLE_ARN }}
+      RELEASE_AUTOMATION_BOT_PAT: ${{ secrets.RELEASE_AUTOMATION_BOT_PAT }}
diff --git a/aws/sdk/build.gradle.kts b/aws/sdk/build.gradle.kts
@@ -523,16 +523,18 @@ val downgradeAwsSdkLockfile = registerDowngradeFor(outputDir.asFile, "AwsSdk")
 fun Project.registerCargoUpdateFor(
     dir: File,
     name: String,
+    dependsOn: List<String> = emptyList(),
 ): TaskProvider<Exec> {
     return tasks.register<Exec>("cargoUpdate${name}Lockfile") {
+        dependsOn(dependsOn)
         workingDir(dir)
         environment("RUSTFLAGS", "--cfg aws_sdk_unstable")
         commandLine("cargo", "update")
         finalizedBy("downgrade${name}Lockfile")
     }
 }
 
-val cargoUpdateAwsConfigLockfile = registerCargoUpdateFor(awsConfigPath, "AwsConfig")
+val cargoUpdateAwsConfigLockfile = registerCargoUpdateFor(awsConfigPath, "AwsConfig", listOf("assemble"))
 val cargoUpdateAwsRuntimeLockfile = registerCargoUpdateFor(awsRustRuntimePath, "AwsRustRuntime")
 val cargoUpdateSmithyRuntimeLockfile = registerCargoUpdateFor(rustRuntimePath, "RustRuntime")
 

diff --git a/tools/ci-build/runtime-versioner/Cargo.lock b/tools/ci-build/runtime-versioner/Cargo.lock
diff --git a/tools/ci-build/runtime-versioner/Cargo.toml b/tools/ci-build/runtime-versioner/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "runtime-versioner"
-version = "0.1.0"
+version = "0.1.1"
 authors = ["AWS Rust SDK Team <aws-sdk-rust@amazon.com>"]
 description = "Tool that manages runtime crate versions."
 edition = "2021"

diff --git a/tools/ci-build/runtime-versioner/src/command/audit.rs b/tools/ci-build/runtime-versioner/src/command/audit.rs
@@ -139,12 +139,13 @@ impl RuntimeCrate {
             .output()
             .with_context(|| format!("failed to git diff {}", self.name))?;
         let output = String::from_utf8(status.stdout)?;
+        // When run during a release, this file is replaced with it's actual contents.
+        // This breaks this git-based comparison and incorrectly requires a version bump.
+        // Temporary fix to allow the build to succeed.
+        let lines_to_ignore = &["aws-config/clippy.toml", "aws-config/Cargo.lock"];
         let changed_files = output
             .lines()
-            // When run during a release, this file is replaced with it's actual contents.
-            // This breaks this git-based comparison and incorrectly requires a version bump.
-            // Temporary fix to allow the build to succeed.
-            .filter(|line| !line.contains("aws-config/clippy.toml"))
+            .filter(|line| !lines_to_ignore.iter().any(|ignore| line.contains(ignore)))
             .collect::<Vec<_>>();
         Ok(!changed_files.is_empty())
     }

diff --git a/tools/ci-scripts/cargo-update-lockfiles b/tools/ci-scripts/cargo-update-lockfiles
@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -eux
+
+if [ "$#" -ne 3 ]; then
+    echo "Usage: $0 <base-branch> <branch-name-for-updating-lockfiles> <whether forcing update on broken dependencies (true/false)>"
+    exit 1
+fi
+
+base_branch=$1
+branch_name_for_updating_lockfiles=$2
+force_update_on_broken_dependencies=$3
+
+SMITHY_RS_DIR="$(pwd)/smithy-rs"
+
+pushd "${SMITHY_RS_DIR}"
+
+git config --local user.name "AWS SDK Rust Bot"
+git config --local user.email "aws-sdk-rust-primary@amazon.com"
+
+git fetch --unshallow
+git checkout "${base_branch}"
+git checkout -b "${branch_name_for_updating_lockfiles}"
+
+if [[ "${force_update_on_broken_dependencies}" == "true" ]]
+then
+  ./gradlew -Paws.sdk.force.update.broken.dependencies aws:sdk:cargoUpdateAllLockfiles
+else
+  ./gradlew aws:sdk:cargoUpdateAllLockfiles
+fi
+
+git add aws/rust-runtime/Cargo.lock \
+    aws/rust-runtime/aws-config/Cargo.lock \
+    aws/sdk/Cargo.lock \
+    rust-runtime/Cargo.lock
+
+git diff --staged --quiet || \
+    git commit \
+        -m "Run cargo update on the runtime lockfiles and the SDK lockfile"
+
+git push origin HEAD
+
+popd