diff --git a/.github/workflows/build-and-publish-images.yaml b/.github/workflows/build-and-publish-images.yaml new file mode 100644 index 00000000000..d7e9503b67f --- /dev/null +++ b/.github/workflows/build-and-publish-images.yaml @@ -0,0 +1,60 @@ +# Reusable workflows for publishing Katib images. +name: Build And Publish Images + +on: + workflow_call: + inputs: + component-name: + required: true + type: string + platforms: + required: true + type: string + dockerfile: + required: true + type: string + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + +jobs: + build-and-publish: + name: Publish Image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Docker Login + # Trigger workflow only for kubeflow/katib repository with specific branch (master, release-.*) or tag (v.*). + if: >- + github.repository == 'kubeflow/katib' && + (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v')) + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Publish Component ${{ inputs.component-name }} + # Trigger workflow only for kubeflow/katib repository with specific branch (master, release-.*) or tag (v.*). + if: >- + github.repository == 'kubeflow/katib' && + (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-') || startsWith(github.ref, 'refs/tags/v')) + id: publish + uses: ./.github/workflows/template-publish-image + with: + image: docker.io/kubeflowkatib/${{ inputs.component-name }} + dockerfile: ${{ inputs.dockerfile }} + platforms: ${{ inputs.platforms }} + push: true + + - name: Test Build For Component ${{ inputs.component-name }} + if: steps.publish.outcome == 'skipped' + uses: ./.github/workflows/template-publish-image + with: + image: docker.io/kubeflowkatib/${{ inputs.component-name }} + dockerfile: ${{ inputs.dockerfile }} + platforms: ${{ inputs.platforms }} + push: false diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index bd63e2b5cf3..9cf7c1f05fb 100755 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -20,4 +20,3 @@ jobs: - name: Check YAML run: make yamllint - diff --git a/.github/workflows/publish-algorithm-images.yaml b/.github/workflows/publish-algorithm-images.yaml index 8e7e74379a6..b6f13e2035e 100644 --- a/.github/workflows/publish-algorithm-images.yaml +++ b/.github/workflows/publish-algorithm-images.yaml @@ -1,29 +1,20 @@ name: Publish AutoML Algorithm Images on: - push: - branches: - - master - -env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + - push + - pull_request jobs: algorithm: name: Publish Image - # Trigger workflow only for kubeflow/katib repository. - if: github.repository == 'kubeflow/katib' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Publish Component ${{ matrix.component-name }} - uses: ./.github/workflows/template-publish-image - with: - image: docker.io/kubeflowkatib/${{ matrix.component-name }} - dockerfile: ${{ matrix.dockerfile }} + uses: ./.github/workflows/build-and-publish-images.yaml + with: + component-name: ${{ matrix.component-name }} + platforms: linux/amd64,linux/arm64 + dockerfile: ${{ matrix.dockerfile }} + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} strategy: fail-fast: false diff --git a/.github/workflows/publish-core-images.yaml b/.github/workflows/publish-core-images.yaml index af0ff9a46f0..18d892159e8 100644 --- a/.github/workflows/publish-core-images.yaml +++ b/.github/workflows/publish-core-images.yaml @@ -1,29 +1,21 @@ name: Publish Katib Core Images on: - push: - branches: - - master - -env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + - push + - pull_request jobs: core: name: Publish Image - # Trigger workflow only for kubeflow/katib repository. - if: github.repository == 'kubeflow/katib' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 + uses: ./.github/workflows/build-and-publish-images.yaml + with: + component-name: ${{ matrix.component-name }} + platforms: linux/amd64,linux/arm64 + dockerfile: ${{ matrix.dockerfile }} + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Publish Component ${{ matrix.component-name }} - uses: ./.github/workflows/template-publish-image - with: - image: docker.io/kubeflowkatib/${{ matrix.component-name }} - dockerfile: ${{ matrix.dockerfile }} strategy: fail-fast: false diff --git a/.github/workflows/publish-trial-images.yaml b/.github/workflows/publish-trial-images.yaml index 949afbe4229..860739a8df1 100644 --- a/.github/workflows/publish-trial-images.yaml +++ b/.github/workflows/publish-trial-images.yaml @@ -1,49 +1,49 @@ name: Publish Trial Images on: - push: - branches: - - master - -env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + - push + - pull_request jobs: trial: name: Publish Image - # Trigger workflow only for kubeflow/katib repository. - if: github.repository == 'kubeflow/katib' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Publish Trial ${{ matrix.trial-name }} - uses: ./.github/workflows/template-publish-image - with: - image: docker.io/kubeflowkatib/${{ matrix.trial-name }} - dockerfile: ${{ matrix.dockerfile }} + uses: ./.github/workflows/build-and-publish-images.yaml + with: + component-name: ${{ matrix.trial-name }} + platforms: ${{ matrix.platforms }} + dockerfile: ${{ matrix.dockerfile }} + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} strategy: fail-fast: false matrix: include: - trial-name: mxnet-mnist + platforms: linux/amd64,linux/arm64 dockerfile: examples/v1beta1/trial-images/mxnet-mnist/Dockerfile - trial-name: pytorch-mnist-cpu + platforms: linux/amd64,linux/arm64 dockerfile: examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.cpu - trial-name: pytorch-mnist-gpu + platforms: linux/amd64 dockerfile: examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.gpu - trial-name: tf-mnist-with-summaries + platforms: linux/amd64,linux/arm64 dockerfile: examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile - trial-name: enas-cnn-cifar10-gpu + platforms: linux/amd64 dockerfile: examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu - trial-name: enas-cnn-cifar10-cpu + platforms: linux/amd64,linux/arm64 dockerfile: examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu - trial-name: darts-cnn-cifar10-cpu + platforms: linux/amd64,linux/arm64 dockerfile: examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.cpu - trial-name: darts-cnn-cifar10-gpu + platforms: linux/amd64 dockerfile: examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.gpu - trial-name: simple-pbt + platforms: linux/amd64,linux/arm64 dockerfile: examples/v1beta1/trial-images/simple-pbt/Dockerfile diff --git a/.github/workflows/template-e2e-test/action.yaml b/.github/workflows/template-e2e-test/action.yaml index 342eb5428e5..ce483e7a795 100644 --- a/.github/workflows/template-e2e-test/action.yaml +++ b/.github/workflows/template-e2e-test/action.yaml @@ -1,23 +1,25 @@ -# Template for e2e tests. +# Composite action for e2e tests. +name: Run E2E Test +description: Run e2e test using the minikube cluster inputs: experiments: required: true - type: string + description: comma delimited experiment name training-operator: required: false - type: boolean - default: false + description: whether to deploy training-operator or not + default: "false" trial-images: required: true - type: string + description: comma delimited trial image name katib-ui: required: true - type: boolean - default: false + description: whether to deploy katib-ui or not + default: "false" database-type: required: false - type: string + description: mysql or postgres default: mysql runs: diff --git a/.github/workflows/template-publish-image/action.yaml b/.github/workflows/template-publish-image/action.yaml index 8bb73b3da94..bf14c6cb392 100644 --- a/.github/workflows/template-publish-image/action.yaml +++ b/.github/workflows/template-publish-image/action.yaml @@ -1,25 +1,30 @@ -# Template run for publishing Katib images. +# Composite action for publishing Katib images. +name: Build And Publish Container Images +description: Build MultiPlatform Supporting Container Images inputs: image: required: true - type: string + description: image tag dockerfile: required: true - type: string + description: path for dockerfile + platforms: + required: true + description: linux/amd64 or linux/amd64,linux/arm64 + push: + required: true + description: whether to push container images or not runs: using: composite steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set Up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Docker Login - uses: docker/login-action@v1 - with: - username: ${{ env.DOCKERHUB_USERNAME }} - password: ${{ env.DOCKERHUB_TOKEN }} - - name: Add Docker Tags id: meta uses: docker/metadata-action@v3 @@ -34,8 +39,8 @@ runs: with: context: . file: ${{ inputs.dockerfile }} - push: true + push: ${{ inputs.push }} tags: ${{ steps.meta.outputs.tags }} cache-from: type=gha cache-to: type=gha,mode=max - platforms: linux/amd64 + platforms: ${{ inputs.platforms }} diff --git a/.github/workflows/template-setup-e2e-test/action.yaml b/.github/workflows/template-setup-e2e-test/action.yaml index 630efa3fc13..3c525843b49 100644 --- a/.github/workflows/template-setup-e2e-test/action.yaml +++ b/.github/workflows/template-setup-e2e-test/action.yaml @@ -1,9 +1,11 @@ -# Template for e2e tests. +# Composite action to setup e2e tests. +name: Setup E2E Test +description: setup env for e2e test using the minikube cluster inputs: kubernetes-version: required: true - type: string + description: kubernetes version runs: using: composite