[Test] Reduce Katib GitHub Action Runs (#2036)

* [Test] Reduce Katib GitHub Action Runs * Add cancel-in-progress flag * Use single job for Charmed Katib * Add cancel-in-progress for all actions except publish * Bump ubuntu to 20.04 for Charmed tests
kubeflow · Nov 28, 2022 · b123dbf · b123dbf
1 parent 831e1d3
commit b123dbf
Show file tree

Hide file tree

Showing 19 changed files with 137 additions and 158 deletions.
diff --git a/...hub/workflows/darts-cifar10-e2e-test.yaml → ...hub/workflows/e2e-test-darts-cifar10.yaml b/...hub/workflows/darts-cifar10-e2e-test.yaml → ...hub/workflows/e2e-test-darts-cifar10.yaml
@@ -1,8 +1,13 @@
 name: E2E Test with darts-cnn-cifar10
+
 on:
   pull_request:
     paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
+      - "pkg/new-ui/v1beta1/frontend/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -26,12 +31,10 @@ jobs:
           experiments: ${{ matrix.experiments }}
           # Comma Delimited
           trial-images: darts-cnn-cifar10-cpu
-          database-type: ${{ matrix.database-type }}
 
     strategy:
       fail-fast: false
       matrix:
         kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"]
         # Comma Delimited
         experiments: ["darts-cpu"]
-        database-type: ["mysql", "postgres"]
diff --git a/.github/workflows/enas-cifar10-e2e-test.yaml → .github/workflows/e2e-test-enas-cifar10.yaml b/.github/workflows/enas-cifar10-e2e-test.yaml → .github/workflows/e2e-test-enas-cifar10.yaml
@@ -1,8 +1,13 @@
 name: E2E Test with enas-cnn-cifar10
+
 on:
   pull_request:
     paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
+      - "pkg/new-ui/v1beta1/frontend/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -26,12 +31,10 @@ jobs:
           experiments: ${{ matrix.experiments }}
           # Comma Delimited
           trial-images: enas-cnn-cifar10-cpu
-          database-type: ${{ matrix.database-type }}
 
     strategy:
       fail-fast: false
       matrix:
         kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"]
         # Comma Delimited
         experiments: ["enas-cpu"]
-        database-type: ["mysql", "postgres"]
diff --git a/.github/workflows/mxnet-mnist-e2e-test.yaml → .github/workflows/e2e-test-mxnet-mnist.yaml b/.github/workflows/mxnet-mnist-e2e-test.yaml → .github/workflows/e2e-test-mxnet-mnist.yaml
@@ -1,8 +1,13 @@
 name: E2E Test with mxnet-mnist
+
 on:
   pull_request:
     paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
+      - "pkg/new-ui/v1beta1/frontend/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -26,7 +31,6 @@ jobs:
           experiments: ${{ matrix.experiments }}
           # Comma Delimited
           trial-images: mxnet-mnist
-          database-type: ${{ matrix.database-type }}
 
     strategy:
       fail-fast: false
@@ -35,9 +39,6 @@ jobs:
         # Comma Delimited
         experiments:
           # suggestion-hyperopt
-          - "random,tpe,never-resume"
-          - "median-stop,from-volume-resume"
+          - "never-resume,from-volume-resume,median-stop"
           # others
-          - "grid,bayesian-optimization,tpe"
-          - "multivariate-tpe,cma-es,hyperband"
-        database-type: ["mysql", "postgres"]
+          - "grid,bayesian-optimization,tpe,multivariate-tpe,cma-es,hyperband"
diff --git a/...hub/workflows/pytorch-mnist-e2e-test.yaml → ...hub/workflows/e2e-test-pytorch-mnist.yaml b/...hub/workflows/pytorch-mnist-e2e-test.yaml → ...hub/workflows/e2e-test-pytorch-mnist.yaml
@@ -1,8 +1,13 @@
 name: E2E Test with pytorch-mnist
+
 on:
   pull_request:
     paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
+      - "pkg/new-ui/v1beta1/frontend/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -27,7 +32,6 @@ jobs:
           training-operator: true
           # Comma Delimited
           trial-images: pytorch-mnist-cpu
-          database-type: ${{ matrix.database-type }}
 
     strategy:
       fail-fast: false
@@ -37,4 +41,3 @@ jobs:
         experiments:
           - "file-metrics-collector,pytorchjob-mnist"
           - "median-stop-with-json-format,file-metrics-collector-with-json-format"
-        database-type: ["mysql", "postgres"]
diff --git a/.github/workflows/simple-pbt-e2e-test.yaml → .github/workflows/e2e-test-simple-pbt.yaml b/.github/workflows/simple-pbt-e2e-test.yaml → .github/workflows/e2e-test-simple-pbt.yaml
@@ -1,8 +1,13 @@
 name: E2E Test with simple-pbt
+
 on:
   pull_request:
     paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
+      - "pkg/new-ui/v1beta1/frontend/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -26,7 +31,6 @@ jobs:
           experiments: ${{ matrix.experiments }}
           # Comma Delimited
           trial-images: simple-pbt
-          database-type: ${{ matrix.database-type }}
 
     strategy:
       fail-fast: false
@@ -35,4 +39,3 @@ jobs:
         kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"]
         # Comma Delimited
         experiments: ["simple-pbt"]
-        database-type: ["mysql", "postgres"]
diff --git a/...ows/tf-mnist-with-summaries-e2e-test.yaml → ...ows/e2e-test-tf-mnist-with-summaries.yaml b/...ows/tf-mnist-with-summaries-e2e-test.yaml → ...ows/e2e-test-tf-mnist-with-summaries.yaml
@@ -1,8 +1,13 @@
 name: E2E Test with tf-mnist-with-summaries
+
 on:
   pull_request:
     paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
+      - "pkg/new-ui/v1beta1/frontend/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -27,12 +32,10 @@ jobs:
           training-operator: true
           # Comma Delimited
           trial-images: tf-mnist-with-summaries
-          database-type: ${{ matrix.database-type }}
 
     strategy:
       fail-fast: false
       matrix:
         kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"]
         # Comma Delimited
         experiments: ["tfjob-mnist-with-summaries"]
-        database-type: ["mysql", "postgres"]
diff --git a/.github/workflows/e2e-test-ui-random-search-postgres.yaml b/.github/workflows/e2e-test-ui-random-search-postgres.yaml
@@ -0,0 +1,38 @@
+name: E2E Test with Katib UI, random search, and postgres
+
+on:
+  - pull_request
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  e2e:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 120
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Test Env
+        uses: ./.github/workflows/template-setup-e2e-test
+        with:
+          kubernetes-version: ${{ matrix.kubernetes-version }}
+
+      - name: Run e2e test with ${{ matrix.experiments }} experiments
+        uses: ./.github/workflows/template-e2e-test
+        with:
+          experiments: random
+          # Comma Delimited
+          trial-images: mxnet-mnist
+          katib-ui: true
+          database-type: postgres
+
+    strategy:
+      fail-fast: false
+      matrix:
+        kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"]
diff --git a/.github/workflows/katib-ui-e2e-test.yaml b/.github/workflows/katib-ui-e2e-test.yaml
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
diff --git a/.github/workflows/publish-algorithm-images.yaml b/.github/workflows/publish-algorithm-images.yaml
@@ -2,11 +2,9 @@ name: Publish AutoML Algorithm Images
 
 on:
   push:
-    paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
   pull_request:
     paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
+      - "pkg/new-ui/v1beta1/frontend/**"
 
 jobs:
   algorithm:

diff --git a/.github/workflows/publish-core-images.yaml b/.github/workflows/publish-core-images.yaml
@@ -1,12 +1,8 @@
 name: Publish Katib Core Images
 
 on:
-  push:
-    paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
-  pull_request:
-    paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
+  - push
+  - pull_request
 
 jobs:
   core:

diff --git a/.github/workflows/publish-katib-ui-image.yaml b/.github/workflows/publish-katib-ui-image.yaml
diff --git a/.github/workflows/publish-trial-images.yaml b/.github/workflows/publish-trial-images.yaml
@@ -2,11 +2,9 @@ name: Publish Trial Images
 
 on:
   push:
-    paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
   pull_request:
     paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
+      - "pkg/new-ui/v1beta1/frontend/**"
 
 jobs:
   trial:

diff --git a/.github/workflows/test-charmed-katib.yaml b/.github/workflows/test-charmed-katib.yaml
@@ -1,17 +1,18 @@
 name: Charmed Katib
 
 on:
-  push:
-    paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
   pull_request:
     paths-ignore:
-      - 'pkg/new-ui/v1beta1/frontend/**'
+      - "pkg/new-ui/v1beta1/frontend/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-latest
+  charmed:
+    name: Lint and Test
+    runs-on: ubuntu-20.04
 
     steps:
       - name: Check out code
@@ -21,38 +22,25 @@ jobs:
         run: |
           set -eux
           sudo apt update
+          sudo apt install -y python3-pip
           sudo apt install python3-setuptools
           sudo pip3 install black flake8
+          sudo snap install juju --classic
+          sudo snap install juju-bundle --classic
+          sudo snap install juju-wait --classic
+          sudo pip3 install charmcraft==1.3.1
 
       - name: Check black
         run: black --check operators/*/src
 
       - name: Check flake8
         run: cd operators && flake8 ./katib*/src
 
-  build:
-    name: Test
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Check out repo
-        uses: actions/checkout@v3
-
       - uses: balchua/microk8s-actions@v0.2.2
         with:
           channel: "1.21/stable"
           addons: '["dns", "storage", "rbac"]'
 
-      - name: Install dependencies
-        run: |
-          set -eux
-          sudo apt update
-          sudo apt install -y python3-pip
-          sudo snap install juju --classic
-          sudo snap install juju-bundle --classic
-          sudo snap install juju-wait --classic
-          sudo pip3 install charmcraft==1.3.1
-
       - name: Set Up Docker Buildx
         uses: docker/setup-buildx-action@v2