Use composite run

kubeflow · Nov 27, 2021 · bc60821 · bc60821
1 parent 44523a9
commit bc60821
Show file tree

Hide file tree

Showing 5 changed files with 142 additions and 251 deletions.
diff --git a/.github/workflows/publish-algorithm-images.yaml b/.github/workflows/publish-algorithm-images.yaml
@@ -5,95 +5,44 @@ on:
     branches:
       - master
 
-jobs:
-  # Suggestion Images.
-  suggestion-hyperopt:
-    name: Suggestion Hyperopt
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/suggestion-hyperopt
-      dockerfile: cmd/suggestion/hyperopt/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-  suggestion-chocolate:
-    name: Suggestion Chocolate
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/suggestion-chocolate
-      dockerfile: cmd/suggestion/chocolate/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-  suggestion-hyperband:
-    name: Suggestion Hyperband
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/suggestion-hyperband
-      dockerfile: cmd/suggestion/hyperband/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-  suggestion-skopt:
-    name: Suggestion Skopt
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/suggestion-skopt
-      dockerfile: cmd/suggestion/skopt/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-  suggestion-goptuna:
-    name: Suggestion Goptuna
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/suggestion-goptuna
-      dockerfile: cmd/suggestion/goptuna/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
-  suggestion-optuna:
-    name: Suggestion Optuna
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/suggestion-optuna
-      dockerfile: cmd/suggestion/optuna/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-  suggestion-enas:
-    name: Suggestion ENAS
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/suggestion-enas
-      dockerfile: cmd/suggestion/nas/enas/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-  suggestion-darts:
-    name: Suggestion DARTS
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/suggestion-darts
-      dockerfile: cmd/suggestion/nas/darts/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-  # Early Stopping Images.
-  earlystopping-medianstop:
-    name: Early Stopping MedianStop
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/earlystopping-medianstop
-      dockerfile: cmd/earlystopping/medianstop/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+jobs:
+  algorithm:
+    name: Publish Image
+    # Trigger workflow only for kubeflow/katib repository.
+    if: github.repository == 'kubeflow/katib'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Publish Component ${{ matrix.component-name }}
+        uses: ./.github/workflows/template-publish-image
+        with:
+          image: docker.io/kubeflowkatib/${{ matrix.component-name }}
+          dockerfile: ${{ matrix.dockerfile }}
+
+    strategy:
+      matrix:
+        include:
+          - component-name: suggestion-hyperopt
+            dockerfile: cmd/suggestion/hyperopt/v1beta1/Dockerfile
+          - component-name: suggestion-chocolate
+            dockerfile: cmd/suggestion/chocolate/v1beta1/Dockerfile
+          - component-name: suggestion-hyperband
+            dockerfile: cmd/suggestion/hyperband/v1beta1/Dockerfile
+          - component-name: suggestion-skopt
+            dockerfile: cmd/suggestion/skopt/v1beta1/Dockerfile
+          - component-name: suggestion-goptuna
+            dockerfile: cmd/suggestion/goptuna/v1beta1/Dockerfile
+          - component-name: suggestion-optuna
+            dockerfile: cmd/suggestion/optuna/v1beta1/Dockerfile
+          - component-name: suggestion-enas
+            dockerfile: cmd/suggestion/nas/enas/v1beta1/Dockerfile
+          - component-name: suggestion-darts
+            dockerfile: cmd/suggestion/nas/darts/v1beta1/Dockerfile
+          - component-name: earlystopping-medianstop
+            dockerfile: cmd/earlystopping/medianstop/v1beta1/Dockerfile
diff --git a/.github/workflows/publish-core-images.yaml b/.github/workflows/publish-core-images.yaml
@@ -5,65 +5,38 @@ on:
     branches:
       - master
 
-jobs:
-  # Katib Core Images.
-  controller:
-    name: Controller
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/katib-controller
-      dockerfile: cmd/katib-controller/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-  db-manager:
-    name: DB Manager
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/katib-db-manager
-      dockerfile: cmd/db-manager/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
-  ui:
-    name: UI
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/katib-ui
-      dockerfile: cmd/new-ui/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-  cert-generator:
-    name: Cert Generator
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/cert-generator
-      dockerfile: cmd/cert-generator/v1beta1/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+jobs:
+  core:
+    name: Publish Image
+    # Trigger workflow only for kubeflow/katib repository.
+    if: github.repository == 'kubeflow/katib'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
 
-  # Metrics Collector Images.
-  metrics-collector-file:
-    name: File Metrics Collector
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/file-metrics-collector
-      dockerfile: cmd/metricscollector/v1beta1/file-metricscollector/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Publish Component ${{ matrix.component-name }}
+        uses: ./.github/workflows/template-publish-image
+        with:
+          image: docker.io/kubeflowkatib/${{ matrix.component-name }}
+          dockerfile: ${{ matrix.dockerfile }}
 
-  metrics-collector-tfevent:
-    name: TFEvent Metrics Collector
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/tfevent-metrics-collector
-      dockerfile: cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+    strategy:
+      matrix:
+        include:
+          - component-name: katib-controller
+            dockerfile: cmd/katib-controller/v1beta1/Dockerfile
+          - component-name: katib-db-manager
+            dockerfile: cmd/db-manager/v1beta1/Dockerfile
+          - component-name: katib-ui
+            dockerfile: cmd/new-ui/v1beta1/Dockerfile
+          - component-name: cert-generator
+            dockerfile: cmd/cert-generator/v1beta1/Dockerfile
+          - component-name: file-metrics-collector
+            dockerfile: cmd/metricscollector/v1beta1/file-metricscollector/Dockerfile
+          - component-name: tfevent-metrics-collector
+            dockerfile: cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile
diff --git a/.github/workflows/publish-trial-images.yaml b/.github/workflows/publish-trial-images.yaml
@@ -5,53 +5,36 @@ on:
     branches:
       - master
 
-jobs:
-  trial-mxnet-mnist:
-    name: Trial MXNet MNIST
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/mxnet-mnist
-      dockerfile: examples/v1beta1/trial-images/mxnet-mnist/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
-
-  trial-pytorch-mnist:
-    name: Trial PyTorch MNIST
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/pytorch-mnist
-      dockerfile: examples/v1beta1/trial-images/pytorch-mnist/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
-  trial-enas-cnn-cifar10-gpu:
-    name: Trial ENAS CNN CIFAR10 for GPU
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/enas-cnn-cifar10-gpu
-      dockerfile: examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+jobs:
+  trial:
+    name: Publish Image
+    # Trigger workflow only for kubeflow/katib repository.
+    if: github.repository == 'kubeflow/katib'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
 
-  trial-enas-cnn-cifar10-cpu:
-    name: Trial ENAS CNN CIFAR10 for CPU
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/enas-cnn-cifar10-cpu
-      dockerfile: examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Publish Trial ${{ matrix.trial-name }}
+        uses: ./.github/workflows/template-publish-image
+        with:
+          image: docker.io/kubeflowkatib/${{ matrix.trial-name }}
+          dockerfile: ${{ matrix.dockerfile }}
 
-  trial-darts-cnn-cifar10:
-    name: Trial DARTS CNN CIFAR10
-    uses: kubeflow/katib/.github/workflows/template-publish-image.yaml@master
-    with:
-      image: docker.io/kubeflowkatib/darts-cnn-cifar10
-      dockerfile: examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile
-    secrets:
-      dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
-      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+    strategy:
+      matrix:
+        include:
+          - trial-name: mxnet-mnist
+            dockerfile: examples/v1beta1/trial-images/mxnet-mnist/Dockerfile
+          - trial-name: pytorch-mnist
+            dockerfile: examples/v1beta1/trial-images/pytorch-mnist/Dockerfile
+          - trial-name: enas-cnn-cifar10-gpu
+            dockerfile: examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu
+          - trial-name: enas-cnn-cifar10-cpu
+            dockerfile: examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu
+          - trial-name: darts-cnn-cifar10
+            dockerfile: examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile
diff --git a/.github/workflows/template-publish-image.yaml b/.github/workflows/template-publish-image.yaml