From 7be9493b9abfc4a9584950ae40963983d91b7863 Mon Sep 17 00:00:00 2001 From: Julian Labatut Date: Mon, 13 Mar 2023 19:42:06 +0100 Subject: [PATCH] [CI] Reduce redundancy by using matrix strategy on workflows (#3514) * ci: reduce redundancy by using matrix strategy on Windows and Linux workflows (#3514) * update CHANGELOG.md (#3514) * ci: fix overriding matrix for Linux x86 (#3514) * ci: update os checks in preparation for darwin (#3514) Signed-off-by: Julian Labatut Co-authored-by: Miki Signed-off-by: David Sinclair --- .github/workflows/build_and_test_workflow.yml | 320 ++++++------------ CHANGELOG.md | 1 + 2 files changed, 96 insertions(+), 225 deletions(-) diff --git a/.github/workflows/build_and_test_workflow.yml b/.github/workflows/build_and_test_workflow.yml index 7abc1e639e7d..5daef79cc8b9 100644 --- a/.github/workflows/build_and_test_workflow.yml +++ b/.github/workflows/build_and_test_workflow.yml @@ -6,12 +6,12 @@ name: Build and test # trigger on every commit push and PR for all branches except pushes for backport branches on: push: - branches: [ '**', '!backport/**' ] + branches: ['**', '!backport/**'] paths-ignore: - '**/*.md' - 'docs/**' pull_request: - branches: [ '**' ] + branches: ['**'] paths-ignore: - '**/*.md' - 'docs/**' @@ -27,81 +27,28 @@ env: OSD_SNAPSHOT_SKIP_VERIFY_CHECKSUM: true jobs: - build-lint-test-linux: - runs-on: ubuntu-latest - container: - image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 - options: --user 1001 - name: Build and Verify on Linux - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version-file: '.nvmrc' - registry-url: 'https://registry.npmjs.org' - - - name: Setup Yarn - run: | - npm uninstall -g yarn - npm i -g yarn@1.22.10 - yarn config set network-timeout 1000000 -g - - - name: Configure Yarn Cache - run: echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $GITHUB_ENV - - - name: Initialize Yarn Cache - uses: actions/cache@v3 - with: - path: ${{ env.YARN_CACHE_LOCATION }} - key: yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - yarn- - - - name: Run bootstrap - run: yarn osd bootstrap - - - name: Run linter - id: linter - run: yarn lint - - - name: Validate NOTICE file - id: notice-validate - run: yarn notice:validate - - - name: Run unit tests with coverage - id: unit-tests - run: yarn test:jest:ci:coverage - - - name: Run mocha tests with coverage - id: mocha-tests - run: yarn test:mocha:coverage - - - name: Upload Code Coverage - id: upload-code-coverage - uses: codecov/codecov-action@v3 - with: - directory: ./target/opensearch-dashboards-coverage - flags: Linux - - - name: Run integration tests - id: integration-tests - run: yarn test:jest_integration:ci - - build-lint-test-windows: - runs-on: windows-latest - name: Build and Verify on Windows + build-lint-test: + name: Build and Verify on ${{ matrix.name }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + include: + - os: ubuntu-latest + name: Linux + - os: windows-latest + name: Windows + runs-on: ${{ matrix.os }} steps: - - name: Configure git's autocrlf + - name: Configure git's autocrlf (Windows only) + if: matrix.os == 'windows-latest' run: | git config --global core.autocrlf false - name: Checkout code uses: actions/checkout@v3 - - name: Setup JDK + - name: Setup JDK (Windows only) + if: matrix.os == 'windows-latest' uses: actions/setup-java@v3 with: java-version: '11' @@ -119,7 +66,12 @@ jobs: npm i -g yarn@1.22.10 yarn config set network-timeout 1000000 -g - - name: Configure Yarn Cache + - name: Configure Yarn Cache (Linux) + if: matrix.os != 'windows-latest' + run: echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $GITHUB_ENV + + - name: Configure Yarn Cache (Windows) + if: matrix.os == 'windows-latest' run: | echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $env:GITHUB_ENV echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" @@ -132,7 +84,12 @@ jobs: restore-keys: | yarn- - - name: Run bootstrap + - name: Run bootstrap (Linux) + if: matrix.os != 'windows-latest' + run: yarn osd bootstrap + + - name: Run bootstrap (Windows) + if: matrix.os == 'windows-latest' run: yarn osd bootstrap || yarn osd bootstrap - name: Run linter @@ -156,93 +113,37 @@ jobs: uses: codecov/codecov-action@v3 with: directory: ./target/opensearch-dashboards-coverage - flags: Windows + flags: ${{ matrix.name }} - name: Run integration tests id: integration-tests run: yarn test:jest_integration:ci - functional-tests-linux: - runs-on: ubuntu-latest - container: - image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 - options: --user 1001 - name: Run functional tests on Linux + functional-tests: + name: Run functional tests on ${{ matrix.name }} (ciGroup${{ matrix.group }}) strategy: matrix: - group: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 ] - steps: - - run: echo Running functional tests for ciGroup${{ matrix.group }} - - - name: Checkout code - uses: actions/checkout@v3 - - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version-file: '.nvmrc' - registry-url: 'https://registry.npmjs.org' - - - name: Setup Yarn - run: | - npm uninstall -g yarn - npm i -g yarn@1.22.10 - yarn config set network-timeout 1000000 -g - - - name: Configure Yarn Cache - run: echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $GITHUB_ENV - - - name: Initialize Yarn Cache - uses: actions/cache@v3 - with: - path: ${{ env.YARN_CACHE_LOCATION }} - key: yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - yarn- - - - name: Setup chromedriver - run: node scripts/upgrade_chromedriver.js - - - name: Run bootstrap - run: yarn osd bootstrap - - - name: Build plugins - run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 10 - - - name: Run CI test group ${{ matrix.group }} - id: ftr-tests - run: node scripts/functional_tests.js --config test/functional/config.js --include ciGroup${{ matrix.group }} - env: - CI_GROUP: ciGroup${{ matrix.group }} - CI_PARALLEL_PROCESS_NUMBER: ciGroup${{ matrix.group }} - JOB: ci${{ matrix.group }} - CACHE_DIR: ciGroup${{ matrix.group }} - - - uses: actions/upload-artifact@v3 - if: failure() - with: - name: failure-artifacts - path: | - test/*/failure_debug/ - test/*/screenshots/ - - functional-tests-windows: - runs-on: windows-latest - name: Run functional tests on Windows - strategy: - matrix: - group: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 ] + os: [ubuntu-latest, windows-latest] + group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] + include: + - os: ubuntu-latest + name: Linux + - os: windows-latest + name: Windows + runs-on: ${{ matrix.os }} steps: - run: echo Running functional tests for ciGroup${{ matrix.group }} - - name: Configure git's autocrlf + - name: Configure git's autocrlf (Windows only) + if: matrix.os == 'windows-latest' run: | git config --global core.autocrlf false - name: Checkout code uses: actions/checkout@v3 - - name: Setup JDK + - name: Setup JDK (Windows only) + if: matrix.os == 'windows-latest' uses: actions/setup-java@v3 with: java-version: '11' @@ -260,7 +161,12 @@ jobs: npm i -g yarn@1.22.10 yarn config set network-timeout 1000000 -g - - name: Configure Yarn Cache + - name: Configure Yarn Cache (Linux) + if: matrix.os != 'windows-latest' + run: echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $GITHUB_ENV + + - name: Configure Yarn Cache (Windows) + if: matrix.os == 'windows-latest' run: | echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $env:GITHUB_ENV echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" @@ -276,7 +182,12 @@ jobs: - name: Setup chromedriver run: node scripts/upgrade_chromedriver.js - - name: Run bootstrap + - name: Run bootstrap (Linux) + if: matrix.os != 'windows-latest' + run: yarn osd bootstrap + + - name: Run bootstrap (Windows) + if: matrix.os == 'windows-latest' run: yarn osd bootstrap || yarn osd bootstrap - name: Build plugins @@ -299,91 +210,33 @@ jobs: test/*/failure_debug/ test/*/screenshots/ - build-min-artifact-tests-linux: - runs-on: ubuntu-latest - container: - image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 - options: --user 1001 - name: Build min release artifacts on Linux - defaults: - run: - working-directory: ./artifacts + build-min-artifact-tests: + name: Build min release artifacts on ${{ matrix.name }} strategy: matrix: include: - - name: Linux x64 + - os: ubuntu-latest + name: Linux x64 ext: tar.gz suffix: linux-x64 script: build-platform --linux --skip-os-packages - - name: Linux ARM64 + - os: ubuntu-latest + name: Linux ARM64 ext: tar.gz suffix: linux-arm64 script: build-platform --linux-arm --skip-os-packages - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - path: ./artifacts - - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version-file: './artifacts/.nvmrc' - registry-url: 'https://registry.npmjs.org' - - - name: Setup Yarn - run: | - npm uninstall -g yarn - npm i -g yarn@1.22.10 - yarn config set network-timeout 1000000 -g - - - name: Configure Yarn Cache - run: echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $GITHUB_ENV - - - name: Initialize Yarn Cache - uses: actions/cache@v3 - with: - path: ${{ env.YARN_CACHE_LOCATION }} - key: yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - yarn- - - - name: Get package version - run: | - echo "VERSION=$(yarn --silent pkg-version)" >> $GITHUB_ENV - - - name: Get artifact build name - run: | - echo "ARTIFACT_BUILD_NAME=opensearch-dashboards-${{ env.VERSION }}-${{ matrix.suffix }}.${{ matrix.ext }}" >> $GITHUB_ENV - - - name: Run bootstrap - run: yarn osd bootstrap - - - name: Build `${{ matrix.name }}` - run: yarn ${{ matrix.script }} --release - - - uses: actions/upload-artifact@v3 - if: success() - with: - name: ${{ matrix.suffix }}-${{ env.VERSION }} - path: ./artifacts/target/${{ env.ARTIFACT_BUILD_NAME }} - retention-days: 1 - - build-min-artifact-tests-windows: - runs-on: windows-latest - name: Build min release artifacts on Windows - defaults: - run: - working-directory: artifacts - strategy: - matrix: - include: - - name: Windows x64 + - os: windows-latest + name: Windows x64 ext: zip suffix: windows-x64 script: build-platform --windows --skip-os-packages + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: ./artifacts steps: - - name: Configure git's autocrlf + - name: Configure git's autocrlf (Windows only) + if: matrix.os == 'windows-latest' run: | git config --global core.autocrlf false working-directory: . @@ -391,12 +244,12 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - path: artifacts + path: ./artifacts - name: Setup Node uses: actions/setup-node@v3 with: - node-version-file: 'artifacts/.nvmrc' + node-version-file: './artifacts/.nvmrc' registry-url: 'https://registry.npmjs.org' - name: Setup Yarn @@ -405,7 +258,12 @@ jobs: npm i -g yarn@1.22.10 yarn config set network-timeout 1000000 -g - - name: Configure Yarn Cache + - name: Configure Yarn Cache (Linux) + if: matrix.os != 'windows-latest' + run: echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $GITHUB_ENV + + - name: Configure Yarn Cache (Windows) + if: matrix.os == 'windows-latest' run: | echo "YARN_CACHE_LOCATION=$(yarn cache dir)" >> $env:GITHUB_ENV echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" @@ -418,11 +276,23 @@ jobs: restore-keys: | yarn- - - name: Get package version + - name: Get package version (Linux) + if: matrix.os != 'windows-latest' + run: | + echo "VERSION=$(yarn --silent pkg-version)" >> $GITHUB_ENV + + - name: Get artifact build name (Linux) + if: matrix.os != 'windows-latest' + run: | + echo "ARTIFACT_BUILD_NAME=opensearch-dashboards-${{ env.VERSION }}-${{ matrix.suffix }}.${{ matrix.ext }}" >> $GITHUB_ENV + + - name: Get package version (Windows) + if: matrix.os == 'windows-latest' run: | echo "VERSION=$(yarn --silent pkg-version)" >> $env:GITHUB_ENV - - name: Get artifact build name + - name: Get artifact build name (Windows) + if: matrix.os == 'windows-latest' run: | echo "ARTIFACT_BUILD_NAME=opensearch-dashboards-${{ env.VERSION }}-${{ matrix.suffix }}.${{ matrix.ext }}" >> $env:GITHUB_ENV @@ -436,11 +306,11 @@ jobs: if: success() with: name: ${{ matrix.suffix }}-${{ env.VERSION }} - path: artifacts/target/${{ env.ARTIFACT_BUILD_NAME }} + path: ./artifacts/target/${{ env.ARTIFACT_BUILD_NAME }} retention-days: 1 bwc-tests: - needs: [build-min-artifact-tests-linux] + needs: [build-min-artifact-tests] runs-on: ubuntu-latest container: image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 @@ -451,7 +321,7 @@ jobs: working-directory: ./artifacts strategy: matrix: - version: [ osd-2.0.0, osd-2.1.0, osd-2.2.0, osd-2.3.0, osd-2.4.0, osd-2.5.0 ] + version: [osd-2.0.0, osd-2.1.0, osd-2.2.0, osd-2.3.0, osd-2.4.0, osd-2.5.0] steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 041fb3b05c13..6178d1836203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Improve yarn's performance in workflows by caching yarn's cache folder ([#3194](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3194)) - Fix detection of Chrome's version on Darwin during CI ([#3296](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3296)) - Upgrade yarn version to be compatible with @openearch-project/opensearch ([#3443](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3443)) +- [CI] Reduce redundancy by using matrix strategy on Windows and Linux workflows ([#3514](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3514)) ### 📝 Documentation