Onboard jenkins prod docker images to github actions (#1025)

* Onboard jenkins prod docker images to github actions Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Add more Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Add more Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> --------- Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
opensearch-project · Oct 31, 2023 · ef33e4f · ef33e4f
1 parent 6aad0be
commit ef33e4f
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 137 deletions.
diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml
@@ -9,7 +9,13 @@ on:
       - "**"
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
   multi-node-test:
+    needs: Get-CI-Image-Tag
     env:
       TEST_FILTER: ${{ matrix.test_filter }}
     strategy:
@@ -23,6 +29,13 @@ jobs:
             test_filter: -PexcludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
     # This job runs on Linux
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK
@@ -34,7 +47,9 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       - name: Run integration tests with multi node config
-        run: ./gradlew integTest -PnumNodes=3 ${{ env.TEST_FILTER }}
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew integTest -PnumNodes=3 ${{ env.TEST_FILTER }}"
       - name: Upload failed logs
         uses: actions/upload-artifact@v2
         if: failure()

diff --git a/.github/workflows/release-workflow.yml b/.github/workflows/release-workflow.yml
diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml
@@ -9,9 +9,22 @@ on:
       - "**"
 
 jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
   security-test:
+    needs: Get-CI-Image-Tag
     # This job runs on Linux
     runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK
@@ -24,7 +37,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Run integration tests
         run: | 
-          ./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT'
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT'"
       - name: Upload failed logs
         uses: actions/upload-artifact@v2
         if: failure()

diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml
@@ -8,7 +8,73 @@ on:
       - "**"
 
 jobs:
-  test-and-build:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  test-and-build-linux:
+    needs: Get-CI-Image-Tag
+    env:
+      TEST_FILTER: ${{ matrix.test_filter }}
+    strategy:
+      # This setting says that all jobs should finish, even if one fails
+      fail-fast: false
+      # This starts three jobs, setting these environment variables uniquely for the different jobs
+      matrix:
+        java: [11, 17]
+        feature: [ism, non-ism]
+        include:
+          - feature: ism
+            test_filter: -PincludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
+          - feature: non-ism
+            test_filter: -PexcludeTests="org.opensearch.indexmanagement.indexstatemanagement*"
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v2
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: ${{ matrix.java }}
+      # build index management
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      # This is a hack, but this step creates a link to the X: mounted drive, which makes the path
+      # short enough to work on Windows
+      - name: Build with Gradle
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew build ${{ env.TEST_FILTER }}"
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: logs
+          path: build/testclusters/integTest-*/logs/*
+      - name: Create Artifact Path
+        run: |
+          mkdir -p index-management-artifacts
+          cp ./build/distributions/*.zip index-management-artifacts
+      - name: Uploads coverage
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+      # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: index-management-plugin-ubuntu-latest
+          path: index-management-artifacts
+
+  test-and-build-windows-macos:
     env:
       BUILD_ARGS: ${{ matrix.os_build_args }}
       WORKING_DIR: ${{ matrix.working_directory }}.
@@ -19,7 +85,7 @@ jobs:
       # This starts three jobs, setting these environment variables uniquely for the different jobs
       matrix:
         java: [11, 17]
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [windows-latest, macos-latest]
         feature: [ism, non-ism]
         include:
           - os: windows-latest
@@ -53,21 +119,10 @@ jobs:
         run: ./gradlew build ${{ env.BUILD_ARGS }} ${{ env.TEST_FILTER }}
         env:
           _JAVA_OPTIONS: ${{ matrix.os_java_options }}
-      - name: Upload failed logs
-        uses: actions/upload-artifact@v2
-        if: ${{ failure() && matrix.os == 'ubuntu-latest' }}
-        with:
-          name: logs
-          path: build/testclusters/integTest-*/logs/*
       - name: Create Artifact Path
         run: |
           mkdir -p index-management-artifacts
           cp ./build/distributions/*.zip index-management-artifacts
-      - name: Uploads coverage
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
       # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
       - name: Upload Artifacts
         uses: actions/upload-artifact@v1