From 5247114eb29bf704b4e5459b07e6f15c4109370a Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 21 Sep 2022 21:46:55 +0000 Subject: [PATCH] Add windows build script changes here Signed-off-by: Peter Zhu --- .github/workflows/CI.yml | 12 ++--------- scripts/build.sh | 43 ++++++++++++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e2ea9f21b8..c7d7e28e5a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -16,10 +16,9 @@ jobs: strategy: matrix: java: [11, 17] - os: [ubuntu-latest, macos-latest] name: Build and Test k-NN Plugin - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Checkout k-NN @@ -30,17 +29,10 @@ jobs: with: java-version: ${{ matrix.java }} - - name: Install dependencies on ubuntu - if: startsWith(matrix.os,'ubuntu') + - name: Install dependencies run: | sudo apt-get install libopenblas-dev gfortran -y - - name: Install dependencies on macos - if: startsWith(matrix.os, 'macos') - run: | - brew reinstall gcc - export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran - - name: Run build run: | ./gradlew build diff --git a/scripts/build.sh b/scripts/build.sh index 2ff2bc2b6f..a9d9898623 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -75,7 +75,17 @@ work_dir=$PWD git submodule update --init -- jni/external/nmslib git submodule update --init -- jni/external/faiss -# Build knn libs +# Setup compile time dependency for Windows only +# As Linux version already have OpenBlas in the runner +if [ "$PLATFORM" = "windows" ]; then + openBlasVersion="0.3.21" + openBlasFile="openblas_${openBlasVersion}" + curl -SL https://github.com/xianyi/OpenBLAS/releases/download/v${openBlasVersion}/OpenBLAS-${openBlasVersion}-x64.zip -o ${openBlasFile}.zip + unzip -j -o ${openBlasFile}.zip bin/libopenblas.dll -d ./src/main/resources/windowsDependencies + rm -rf ${openBlasFile}.zip +fi + +# Setup knnlib build params for all platforms cd jni # For x64, generalize arch so library is compatible for processors without simd instruction extensions @@ -95,30 +105,41 @@ if [ "$JAVA_HOME" = "" ]; then echo "SET JAVA_HOME=$JAVA_HOME" fi -cmake . -make opensearchknn_faiss opensearchknn_nmslib - +# Build k-NN lib and plugin through gradle tasks cd $work_dir -./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +# Gradle build is used here to replace gradle assemble due to build will also call cmake and make before generating jars +./gradlew build --no-daemon --refresh-dependencies -x integTest -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER # Add lib to zip zipPath=$(find "$(pwd)" -path \*build/distributions/*.zip) distributions="$(dirname "${zipPath}")" mkdir $distributions/lib -cp ./jni/release/libopensearchknn* $distributions/lib - -# Copy libomp to be packaged with the lib contents -ompPath=$(ldconfig -p | grep libgomp | cut -d ' ' -f 4) -cp $ompPath $distributions/lib +libPrefix="libopensearchknn" +if [ "$PLATFORM" = "windows" ]; then + libPrefix="opensearchknn" + cp -v ./src/main/resources/windowsDependencies/libopenblas.dll $distributions/lib + + # Have to define $MINGW_BIN either in ENV VAR or User Provided Var + cp -v "$MINGW_BIN/libgcc_s_seh-1.dll" $distributions/lib + cp -v "$MINGW_BIN/libwinpthread-1.dll" $distributions/lib + cp -v "$MINGW_BIN/libstdc++-6.dll" $distributions/lib + cp -v "$MINGW_BIN/libgomp-1.dll" $distributions/lib +else + ompPath=$(ldconfig -p | grep libgomp | cut -d ' ' -f 4) + cp -v $ompPath $distributions/lib +fi +cp -v ./jni/release/${libPrefix}* $distributions/lib +ls -l $distributions/lib +# Add lib directory to the k-NN plugin zip cd $distributions zip -ur $zipPath lib cd $work_dir echo "COPY ${distributions}/*.zip" mkdir -p $OUTPUT/plugins -cp ${distributions}/*.zip $OUTPUT/plugins +cp -v ${distributions}/*.zip $OUTPUT/plugins mkdir -p $OUTPUT/maven/org/opensearch cp -r ./build/local-staging-repo/org/opensearch/. $OUTPUT/maven/org/opensearch