Consolidate multi-arch docker builds (#7902)

PrefectHQ · Jan 26, 2023 · 64e974b · 64e974b
1 parent 18ddcaf
commit 64e974b
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 340 deletions.
diff --git a/.github/workflows/docker-builds.yaml b/.github/workflows/docker-builds.yaml
@@ -100,7 +100,7 @@ jobs:
             ${{ ( endsWith(matrix.flavor, 'conda') && 'BASE_IMAGE=prefect-conda' ) || '' }}
           tags: ${{ join(steps.metadata-dev.outputs.tags) }},${{ join(steps.metadata-prod.outputs.tags) }}
           labels: ${{ steps.metadata-dev.outputs.labels }}
-          push: false
+          push: true
           pull: true
           # multi-stage cache manifests (mode=max) need a separate tag because they are not usable for runtime ref https://github.com/moby/buildkit#export-cache
           cache-from: type=registry,ref=prefecthq/prefect-dev:buildcache-python${{ matrix.python-version }}${{ matrix.flavor }}

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -1,15 +1,11 @@
 name: Release prefect
 
-# Note: Conda support for 3.11 is pending. See https://github.com/ContinuumIO/anaconda-issues/issues/13082
-
 on:
-  push:
-    tags:
-      - "[0-9]+.[0-9]+.[0-9]+"
-      - "[0-9]+.[0-9]+rc[0-9]+"
-      - "[0-9]+.[0-9]+[ab][0-9]+"
+  release:
+    types: [released, prereleased]
+
 jobs:
-  build-release:
+  build-pypi-dists:
     name: Build Python package
 
     runs-on: ubuntu-latest
@@ -44,166 +40,36 @@ jobs:
 
       - name: Build a binary wheel and a source tarball
         run: |
-          python setup.py sdist
-          python setup.py bdist_wheel
+          python setup.py sdist bdist_wheel
 
       - name: Publish build artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: built-package
+          name: pypi-dists
           path: "./dist"
 
-  build-docker-images:
-    name: Build Docker images
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version:
-          - "3.7"
-          - "3.8"
-          - "3.9"
-          - "3.10"
-          - "3.11"
-
-    steps:
-      # Not needed until we want multiarchecture builds
-      # - name: Set up QEMU
-      #   uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Generate image metadata
-        id: metadata
-        uses: docker/metadata-action@v4
-        with:
-          images: prefecthq/prefect
-          tags: |
-            type=pep440,pattern={{version}},suffix=-python${{ matrix.python-version }}
-            type=pep440,pattern={{major}}.{{minor}},suffix=-python${{ matrix.python-version }}
-            type=pep440,pattern={{major}},suffix=-python${{ matrix.python-version }}
-            type=sha,suffix=-python${{ matrix.python-version }}
-            type=raw,value=2-latest,enable=${{ matrix.python-version == '3.10'}}
-          flavor: |
-            latest=false
-
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Build image
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          build-args: PYTHON_VERSION=${{ matrix.python-version }}
-          tags: ${{ steps.metadata.outputs.tags }}
-          labels: ${{ steps.metadata.outputs.labels }}
-          outputs: type=docker,dest=/tmp/image.tar
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      - name: Test docker image
-        run: |
-          docker load --input /tmp/image.tar
-          docker run --rm prefecthq/prefect:sha-${GITHUB_SHA::7}-python${{ matrix.python-version }} prefect version
-
-      - name: Upload image artifact
-        uses: actions/upload-artifact@v3
-        # Images are stashed for publish after approval is given
-        with:
-          name: release-image-${{ github.sha }}-${{ matrix.python-version }}
-          path: /tmp/image.tar
-
-  build-conda-docker-images:
-    name: Build Conda Docker images
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version:
-          - "3.7"
-          - "3.8"
-          - "3.9"
-          - "3.10"
-          # Not yet supported, see note at top
-          # - "3.11"
-
-    steps:
-      # Not needed until we want multiarchecture builds
-      # - name: Set up QEMU
-      #   uses: docker/setup-qemu-action@v1
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Generate image metadata
-        id: metadata
-        uses: docker/metadata-action@v4
-        with:
-          images: prefecthq/prefect
-          tags: |
-            type=pep440,pattern={{version}},suffix=-python${{ matrix.python-version }}-conda
-            type=pep440,pattern={{major}}.{{minor}},suffix=-python${{ matrix.python-version }}-conda
-            type=pep440,pattern={{major}},suffix=-python${{ matrix.python-version }}-conda
-            type=sha,suffix=-python${{ matrix.python-version }}-conda
-            type=raw,value=2-latest-conda,enable=${{ matrix.python-version == '3.10'}}
-          flavor: |
-            latest=false
-
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Build image
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          build-args: |
-            PYTHON_VERSION=${{ matrix.python-version }}
-            BASE_IMAGE=prefect-conda
-          tags: ${{ steps.metadata.outputs.tags }}
-          labels: ${{ steps.metadata.outputs.labels }}
-          outputs: type=docker,dest=/tmp/image-conda.tar
-          cache-from: type=gha
-          # We do not cache Conda image layers because they very big and slow to upload
-          # cache-to: type=gha,mode=max
-
-      - name: Test docker image
-        run: |
-          docker load --input /tmp/image-conda.tar
-          docker run --rm prefecthq/prefect:sha-${GITHUB_SHA::7}-python${{ matrix.python-version }}-conda prefect version
-
-      - name: Upload image artifact
-        uses: actions/upload-artifact@v3
-        # Images are stashed for publish after approval is given
-        with:
-          name: release-image-${{ github.sha }}-${{ matrix.python-version }}-conda
-          path: /tmp/image-conda.tar
-
-  publish-release:
+  publish-pypi-dists:
     name: Publish to PyPI
     environment: "prod"
-    needs: [build-release]
+    needs: [build-pypi-dists]
     runs-on: ubuntu-latest
 
     steps:
       - name: Download build artifacts
         uses: actions/download-artifact@v3
         with:
-          name: built-package
+          name: pypi-dists
           path: "./dist"
 
       - name: Publish distribution to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
-
           name: ci
 
   publish-docs:
     name: Publish documentation
-    needs: [build-release]
+    needs: [build-pypi-dists]
     environment: "prod"
     runs-on: ubuntu-latest
 
@@ -241,95 +107,3 @@ jobs:
           gh pr merge --rebase --delete-branch
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  publish-docker-images:
-    name: Publish to DockerHub
-    needs: [build-docker-images, build-conda-docker-images, build-release]
-    environment: "prod"
-
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download image artifacts for Python 3.7
-        uses: actions/download-artifact@v3
-        with:
-          name: release-image-${{ github.sha }}-3.7
-          path: /tmp/3.7/
-
-      - name: Download image artifacts for Python 3.8
-        uses: actions/download-artifact@v3
-        with:
-          name: release-image-${{ github.sha }}-3.8
-          path: /tmp/3.8/
-
-      - name: Download image artifacts for Python 3.9
-        uses: actions/download-artifact@v3
-        with:
-          name: release-image-${{ github.sha }}-3.9
-          path: /tmp/3.9/
-
-      - name: Download image artifacts for Python 3.10
-        uses: actions/download-artifact@v3
-        with:
-          name: release-image-${{ github.sha }}-3.10
-          path: /tmp/3.10/
-
-      - name: Download image artifacts for Python 3.11
-        uses: actions/download-artifact@v3
-        with:
-          name: release-image-${{ github.sha }}-3.11
-          path: /tmp/3.11/
-
-      - name: Download conda image artifacts for Python 3.7
-        uses: actions/download-artifact@v3
-        with:
-          name: release-image-${{ github.sha }}-3.7-conda
-          path: /tmp/3.7/
-
-      - name: Download conda image artifacts for Python 3.8
-        uses: actions/download-artifact@v3
-        with:
-          name: release-image-${{ github.sha }}-3.8-conda
-          path: /tmp/3.8/
-
-      - name: Download conda image artifacts for Python 3.9
-        uses: actions/download-artifact@v3
-        with:
-          name: release-image-${{ github.sha }}-3.9-conda
-          path: /tmp/3.9/
-
-      - name: Download conda image artifacts for Python 3.10
-        uses: actions/download-artifact@v3
-        with:
-          name: release-image-${{ github.sha }}-3.10-conda
-          path: /tmp/3.10/
-
-      # Not yet available, see note at top
-      # - name: Download conda image artifacts for Python 3.11
-      #   uses: actions/download-artifact@v3
-      #   with:
-      #     name: release-image-${{ github.sha }}-3.11-conda
-      #     path: /tmp/3.11/
-
-      - name: Load images
-        run: |
-          docker load --input /tmp/3.7/image.tar
-          docker load --input /tmp/3.8/image.tar
-          docker load --input /tmp/3.9/image.tar
-          docker load --input /tmp/3.10/image.tar
-          docker load --input /tmp/3.11/image.tar
-          docker load --input /tmp/3.7/image-conda.tar
-          docker load --input /tmp/3.8/image-conda.tar
-          docker load --input /tmp/3.9/image-conda.tar
-          docker load --input /tmp/3.10/image-conda.tar
-          # Not yet available, see note at top
-          # docker load --input /tmp/3.11/image-conda.tar
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Push images
-        run: |
-          docker image push --all-tags prefecthq/prefect