ci: remote multiplatform builder

.github/workflows/ci.yml

@@ -32,40 +32,20 @@ concurrency:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
-  # Name these so they look less similar than AMD/ARM; omit 64 as uninformative.
-  X86_PLATFORM: linux/amd64
-  ARM_PLATFORM: linux/arm64/v8
 
 jobs:
-  platforms:
-    runs-on: ubuntu-latest
-    outputs:
-      platforms: '${{ steps.platforms.outputs.platforms }}'
-    steps:
-      - name: Compute Docker platforms
-        id: platforms
-        run: |
-          if ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }}; then
-            # JSON-encoded list consisting only of the default platform.
-            platforms='["'"$X86_PLATFORM"'"]'
-          else
-            platforms='["$X86_PLATFORM","$ARM_PLATFORM"]'
-          fi
-          echo "platforms=$platforms" >> $GITHUB_OUTPUT
-
   # see https://docs.docker.com/build/ci/github-actions/test-before-push/
   test-proposals:
-    needs: [platforms]
-    # UNTIL https://github.com/Agoric/agoric-3-proposals/issues/2
-    timeout-minutes: 120
-    strategy:
-      matrix:
-        platform: ${{ fromJSON(needs.platforms.outputs.platforms) }}
-    # Run on our own self-hosted ARM64 machine if the platform is ARMish.
-    runs-on: ${{ contains(matrix.platform, '/arm') && fromJSON('["self-hosted","Linux","ARM64"]') || 'ubuntu-latest' }}
+    permissions:
+      # allow issuing OIDC tokens for this workflow run
+      id-token: write
+      # allow at least reading the repo contents, add other permissions if necessary
+      contents: read
+      # to push the resulting images
+      packages: write
+    runs-on: 'ubuntu-latest'
     steps:
       - name: free up disk space
-        if: ${{ !contains(matrix.platform, '/arm') }}
         run: |
           # Workaround to provide additional free space for testing.
           #   https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
@@ -82,11 +62,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - uses: depot/setup-action@v1
+        with:
+          oidc: true # to set DEPOT_TOKEN for later steps
 
-      - name: Set up QEMU for cross-platform builds
-        uses: docker/setup-qemu-action@v3
+      # make Docker's CLI use depot to build
+      - run: depot configure-docker
 
       - name: Log in to the Container registry
         uses: docker/login-action@v3
@@ -102,24 +83,6 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
-      - name: Compute Docker tags
-        id: docker-tags
-        run: |
-          sep=
-          # A list of comma-separated tags to merge in the final image.
-          SUFFIXED=
-          # Our platform, replacing slashes with underscores.
-          uarch=$(echo "${{ matrix.platform }}" | tr / _)
-          for TAG in ${{ steps.meta.outputs.tags }}; do
-            SUFFIXED="$SUFFIXED$sep$TAG-$uarch"
-            if test -z "$sep"; then
-              # The first tag (suffixed with our architecture) is the one we build.
-              sep=,
-              echo "tag=$TAG-$uarch" | tee -a $GITHUB_OUTPUT
-            fi
-          done
-          echo "tags=$SUFFIXED" | tee -a $GITHUB_OUTPUT
-
       - name: Setup synthetic-chain
         run: |
           # The .ts scripts depend upon this
@@ -128,25 +91,15 @@ jobs:
           corepack enable || sudo corepack enable
           yarn install
 
-      - run: docker system df
-      - run: docker buildx du --verbose
-      - run: df -h
-
       # Test before pushing the images.
+      # Note this builds each proposal testing image sequentially. It does that so it can delete the image
+      # after it passees, freeing up disk space. It has the added benefit of making the build more robust
+      # to races in the build graph that BuildKit is prone to, especially with multiplatform.
       - name: Build and run proposal tests
-        if: ${{ matrix.platform == env.X86_PLATFORM }}
         run: yarn test
 
-      - run: docker system df
-      - run: docker buildx du --verbose
-      - run: df -h
-
-      # Build a "use" image for each proposal. This uses Docker Bake's
-      # matrix feature. We could have each "use" image built in a different runner
-      # by including https://github.com/docker/bake-action?tab=readme-ov-file#list-targets
-      # in the GHA matrix, but that wouldn't be able to resolve the DAG of what to build first.
       - name: Push proposal "use" images
-        uses: docker/bake-action@v4
+        uses: depot/bake-action@v1
         # If we pushed from PRs, each one would overwrite main's (e.g. use-upgrade-8)
         # To push PR "use" images we'll need to qualify the tag (e.g. use-upgrade-8-pr-2).
         if: ${{ github.event_name != 'pull_request' }}
@@ -156,75 +109,7 @@ jobs:
             ./docker-bake.hcl
             ${{ steps.meta.outputs.bake-file }}
           targets: use
-
-      - run: docker system df
-      - run: docker buildx du --verbose
-      - run: df -h
-
-      - name: Build and push default image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          platforms: ${{ matrix.platform }}
-          # push to registry on every repo push. A PR #2 will push with tag `pr-2` and `main` will have tag `main`.
-          # See https://github.com/docker/metadata-action?tab=readme-ov-file#basic.
           push: true
-          tags: ${{ steps.docker-tags.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-
-      - run: docker system df
-      - run: docker buildx du --verbose
-      - run: df -h
-
-  # Merge the default image from each platform into one multi-arch image,
-  # then publish that multiarch image.
-  docker-publish-multiarch:
-    needs: [test-proposals, platforms]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          buildkitd-flags: --debug
-
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          registry: ${{ env.REGISTRY }}
-      - name: Compute tags
-        id: docker-tags
-        run: |
-          echo "tags=${{ steps.meta.outputs.tags }}" >> $GITHUB_OUTPUT
-
-      - name: Push multiarch image
-        run: |
-          set -ex
-          # Push all tags, comprised of all architectures, to the registry.
-          for TAG in ${{ steps.docker-tags.outputs.tags }}; do
-            sources=
-            for ARCH in ${{ join(fromJson(needs.platforms.outputs.platforms), ' ') }}; do
-              uarch=$(echo "$ARCH" | tr / _)
-              BUILD_TAG="$TAG-$uarch"
-              sources="$sources $BUILD_TAG"
-            done
-            docker buildx imagetools create --tag "$TAG"$sources
-          done
 
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'

depot.json

@@ -0,0 +1 @@
+{"id":"bqgtmlhmh8"}

package.json

@@ -12,7 +12,11 @@
     "@agoric/synthetic-chain": "workspace:*"
   },
   "agoricSyntheticChain": {
-    "fromTag": null
+    "fromTag": null,
+    "platforms": [
+      "linux/amd64",
+      "linux/arm64"
+    ]
   },
   "license": "Apache-2.0",
   "packageManager": "yarn@4.0.2",

packages/synthetic-chain/cli.ts

@@ -41,7 +41,9 @@ const proposals = match
 const [cmd] = positionals;
 
 // TODO consider a lib like Commander for auto-gen help
-const usage = `USAGE:
+const USAGE = `USAGE:
+prepare-build   - generate Docker build configs
+
 build           - build the synthetic-chain "use" images
 
 test [--debug]  - build the "test" images and run them
@@ -50,6 +52,15 @@ test -m <name>  - target a particular proposal by substring match
 doctor          - diagnostics and quick fixes
 `;
 
+const EXPLAIN_MULTIPLATFORM = `
+ERROR: docker exporter does not currently support exporting manifest lists
+
+Multiple platforms are configured but Docker does not support multiplatform in one builder.
+Until https://github.com/docker/roadmap/issues/371, attempting it will error as above.
+
+Instead use a builder that supports multiplatform such as depot.dev.
+`;
+
 /**
  * Put into places files that building depends upon.
  */
@@ -58,7 +69,7 @@ const prepareDockerBuild = () => {
   // copy and generate files of the build context that aren't in the build contents
   execSync(`cp -r ${path.resolve(cliPath, '..', 'docker-bake.hcl')} .`);
   writeDockerfile(allProposals, buildConfig.fromTag);
-  writeBakefileProposals(allProposals);
+  writeBakefileProposals(allProposals, buildConfig.platforms);
   // copy and generate files to include in the build
   execSync(`cp -r ${path.resolve(cliPath, '..', 'upgrade-test-scripts')} .`);
   buildProposalSubmissions(proposals);
@@ -70,8 +81,16 @@ const prepareDockerBuild = () => {
 };
 
 switch (cmd) {
+  case 'prepare-build':
+    prepareDockerBuild();
+    break;
   case 'build': {
     prepareDockerBuild();
+    // do not encapsulate running Depot. It's a special case which the user should understand.
+    if (buildConfig.platforms) {
+      console.error(EXPLAIN_MULTIPLATFORM);
+      process.exit(1);
+    }
     bakeTarget('use', values.dry);
     break;
   }
@@ -103,5 +122,5 @@ switch (cmd) {
     runDoctor(allProposals);
     break;
   default:
-    console.log(usage);
+    console.log(USAGE);
 }

packages/synthetic-chain/docker-bake.hcl

@@ -14,9 +14,11 @@ group "default" {
   ]
 }
 
+// Images to use the result of all local proposals, optionally built multi-platform
 target "use" {
   inherits = ["docker-metadata-action"]
   name = "use-${proposal}"
+  platforms = PLATFORMS
   matrix = {
     proposal = PROPOSALS
   }
@@ -29,6 +31,7 @@ target "use" {
   target = "use-${proposal}"
 }
 
+// Image to test the result of a proposal, always current platform
 target "test" {
   name = "test-${proposal}"
   matrix = {

packages/synthetic-chain/src/cli/build.ts

@@ -3,13 +3,16 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { ProposalInfo } from './proposals.js';
 
+export type Platform = 'linux/amd64' | 'linux/arm64';
+
 export type AgoricSyntheticChainConfig = {
   /**
    * The agoric-3-proposals tag to build the agoric synthetic chain from.
    * If `null`, the chain is built from an ag0 genesis.
    * Defaults to `main`, which containing all passed proposals
    */
   fromTag: string | null;
+  platforms?: Platform[];
 };
 
 const defaultConfig: AgoricSyntheticChainConfig = {
@@ -66,7 +69,7 @@ export const buildProposalSubmissions = (proposals: ProposalInfo[]) => {
 
 /**
  * Bake images using the docker buildx bake command.
- * 
+ *
  * Note this uses `--load` which pushes the completed images to the builder,
  * consuming 2-3 GB per image.
  * @see {@link https://docs.docker.com/engine/reference/commandline/buildx_build/#load}

packages/synthetic-chain/src/cli/dockerfileGen.ts

@@ -3,14 +3,14 @@
 
 import fs from 'node:fs';
 import {
-  lastPassedProposal,
   type CoreEvalProposal,
   type ProposalInfo,
   type SoftwareUpgradeProposal,
   encodeUpgradeInfo,
   imageNameForProposal,
   isPassed,
 } from './proposals.js';
+import { Platform } from './build.ts';
 
 /**
  * Templates for Dockerfile stages
@@ -182,9 +182,15 @@ ENTRYPOINT ./start_agd.sh
   },
 };
 
-export function writeBakefileProposals(allProposals: ProposalInfo[]) {
+export function writeBakefileProposals(
+  allProposals: ProposalInfo[],
+  platforms?: Platform[],
+) {
   const json = {
     variable: {
+      PLATFORMS: {
+        default: platforms || null,
+      },
       PROPOSALS: {
         default: allProposals.map(p => p.proposalName),
       },