Add Windows Support to BWC Tests

Signed-off-by: Naveen <navtat@amazon.com>
opensearch-project · Jan 12, 2023 · 66f1594 · 66f1594
1 parent 8a2aa04
commit 66f1594
Show file tree

Hide file tree

Showing 4 changed files with 123 additions and 13 deletions.
diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml
@@ -14,11 +14,21 @@ jobs:
     strategy:
       matrix:
         java: [ 11, 17 ]
+        os: [windows-latest, ubuntu-latest]
         bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0-SNAPSHOT" ]
         opensearch_version : [ "3.0.0-SNAPSHOT" ]
+        exclude:
+          - os: windows-latest
+            bwc_version: "2.0.1"
+          - os: windows-latest
+            bwc_version: "2.1.0"
+          - os: windows-latest
+            bwc_version: "2.2.1"
+          - os: windows-latest
+            bwc_version: "2.3.0"
 
     name: k-NN Restart-Upgrade BWC Tests
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     env:
       BWC_VERSION_RESTART_UPGRADE: ${{ matrix.bwc_version }}
 
@@ -31,12 +41,43 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
 
-      - name: Install dependencies
+      - name: Install dependencies on ubuntu
+        if: startsWith(matrix.os,'ubuntu')
         run: |
           sudo apt-get install libopenblas-dev gfortran -y
 
+      - name: Install MinGW Using Scoop
+        if: startsWith(matrix.os,'windows')
+        run: |
+          iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
+          scoop bucket add main
+          scoop install mingw
+
+      - name: Add MinGW to PATH
+        if: startsWith(matrix.os,'windows')
+        run: |
+          echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH
+          refreshenv
+
+      - name: Download OpenBLAS
+        if: startsWith(matrix.os,'windows')
+        run: |
+          curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip
+          mkdir OpenBLAS
+          Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\
+          mkdir ./src/main/resources/windowsDependencies
+          cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/
+          rm .\OpenBLAS-0.3.21-x64.zip
+          rm -r .\OpenBLAS\
 
-      - name: Run k-NN Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }}
+      - name: Run k-NN Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} Windows
+        if: (startsWith(matrix.os,'windows') && (startsWith(matrix.bwc_version, '1.3') || startsWith(matrix.bwc_version, '2.4') || startsWith(matrix.bwc_version, '2.5') || startsWith(matrix.bwc_version, '2.6') || startsWith(matrix.bwc_version, '3.')))
+        run: |
+          echo "Running restart-upgrade backwards compatibility tests ..."
+          ./gradlew :qa:restart-upgrade:testRestartUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}'
+
+      - name: Run k-NN Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} Ubuntu
+        if: startsWith(matrix.os,'ubuntu')
         run: |
           echo "Running restart-upgrade backwards compatibility tests ..."
           ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE
@@ -46,11 +87,12 @@ jobs:
     strategy:
       matrix:
         java: [ 11, 17 ]
+        os: [windows-latest, ubuntu-latest]
         bwc_version: [ "2.5.0-SNAPSHOT" ]
         opensearch_version: [ "3.0.0-SNAPSHOT" ]
 
     name: k-NN Rolling-Upgrade BWC Tests
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     env:
       BWC_VERSION_ROLLING_UPGRADE: ${{ matrix.bwc_version }}
 
@@ -63,11 +105,43 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
 
-      - name: Install dependencies
+      - name: Install dependencies on ubuntu
+        if: startsWith(matrix.os,'ubuntu')
         run: |
           sudo apt-get install libopenblas-dev gfortran -y
 
-      - name: Run k-NN Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }}
+      - name: Install MinGW Using Scoop
+        if: startsWith(matrix.os,'windows')
+        run: |
+          iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
+          scoop bucket add main
+          scoop install mingw
+
+      - name: Add MinGW to PATH
+        if: startsWith(matrix.os,'windows')
+        run: |
+          echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH
+          refreshenv
+
+      - name: Download OpenBLAS
+        if: startsWith(matrix.os,'windows')
+        run: |
+          curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip
+          mkdir OpenBLAS
+          Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\
+          mkdir ./src/main/resources/windowsDependencies
+          cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/
+          rm .\OpenBLAS-0.3.21-x64.zip
+          rm -r .\OpenBLAS\
+
+      - name: Run k-NN Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} Windows
+        if: (startsWith(matrix.os,'windows') && (startsWith(matrix.bwc_version, '1.3') || startsWith(matrix.bwc_version, '2.4') || startsWith(matrix.bwc_version, '2.5') || startsWith(matrix.bwc_version, '2.6') || startsWith(matrix.bwc_version, '3.')))
+        run: |
+          echo "Running rolling-upgrade backwards compatibility tests ..."
+          ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}'
+
+      - name: Run k-NN Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} Ubuntu
+        if: startsWith(matrix.os,'ubuntu')
         run: |
           echo "Running rolling-upgrade backwards compatibility tests ..."
           ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE
diff --git a/qa/build.gradle b/qa/build.gradle
@@ -1,3 +1,6 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+import java.nio.file.Files
+
 /*
  *  Copyright OpenSearch Contributors
  *  SPDX-License-Identifier: Apache-2.0
@@ -33,18 +36,34 @@ String default_bwc_version = System.getProperty("bwc.version")
 String knn_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version)
 boolean isSnapshot = knn_bwc_version.contains("-SNAPSHOT")
 String knn_bwc_version_no_qualifier = isSnapshot ? knn_bwc_version - "-SNAPSHOT" : knn_bwc_version
+
+String os_platform = "linux"
+String artifact_type = "tar"
+String file_ext = "tar.gz"
+
+if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+    os_platform = "windows"
+    artifact_type = "zip"
+    file_ext = "zip"
+}
+
 // Task to pull k-NN plugin from archive
 task pullBwcPlugin {
     doFirst {
-        delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-*")
+        File[] tempFiles = tmp_dir.listFiles()
+        for (File child : tempFiles) {
+            if (child.exists() && child.toString().contains("opensearch-")) {
+                Files.delete(child.toPath());
+            }
+        }
     }
 
     doLast {
         ext{
             if (isSnapshot) {
-                srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${knn_bwc_version_no_qualifier}/latest/linux/x64/tar/dist/opensearch/opensearch-${knn_bwc_version_no_qualifier}-linux-x64.tar.gz"
+                srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${knn_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${knn_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"
             } else {
-                srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${knn_bwc_version}/opensearch-${knn_bwc_version}-linux-x64.tar.gz"
+                srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${knn_bwc_version}/opensearch-${knn_bwc_version}-${os_platform}-x64.${file_ext}"
             }
         }
         ant.get(
@@ -53,14 +72,18 @@ task pullBwcPlugin {
                 httpusecaches: false
         )
         copy {
-            from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-linux-x64.tar.gz"))
+            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+                from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
+            } else {
+                from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
+            }
             into tmp_dir.absolutePath
         }
         copy {
             from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}", "plugins", "opensearch-knn"))
             into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn")
         }
-        delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-linux-x64.tar.gz")
+        delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")
     }
 }
 

diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle
@@ -4,6 +4,7 @@
  */
 
 import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
+import org.apache.tools.ant.taskdefs.condition.Os
 
 apply from : "$rootDir/qa/build.gradle"
 
@@ -23,7 +24,13 @@ testClusters {
         setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
         setting 'http.content_type.required', 'true'
         environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
-        systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies")
+            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
+        } else {
+            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
+        }
+
     }
 }
 

diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
@@ -4,6 +4,7 @@
  */
 
 import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
+import org.apache.tools.ant.taskdefs.condition.Os
 
 apply from : "$rootDir/qa/build.gradle"
 
@@ -23,7 +24,12 @@ testClusters {
         setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
         setting 'http.content_type.required', 'true'
         environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
-        systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies")
+            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
+        } else {
+            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
+        }
     }
 }