Skip to content

Commit

Permalink
Add windows build script changes here
Browse files Browse the repository at this point in the history
Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
  • Loading branch information
naveentatikonda authored and peterzhuamazon committed Oct 27, 2022
1 parent 515dcdb commit 5247114
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 21 deletions.
12 changes: 2 additions & 10 deletions .github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,9 @@ jobs:
strategy:
matrix:
java: [11, 17]
os: [ubuntu-latest, macos-latest]

name: Build and Test k-NN Plugin
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest

steps:
- name: Checkout k-NN
Expand All @@ -30,17 +29,10 @@ jobs:
with:
java-version: ${{ matrix.java }}

- name: Install dependencies on ubuntu
if: startsWith(matrix.os,'ubuntu')
- name: Install dependencies
run: |
sudo apt-get install libopenblas-dev gfortran -y
- name: Install dependencies on macos
if: startsWith(matrix.os, 'macos')
run: |
brew reinstall gcc
export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran
- name: Run build
run: |
./gradlew build
Expand Down
43 changes: 32 additions & 11 deletions scripts/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,17 @@ work_dir=$PWD
git submodule update --init -- jni/external/nmslib
git submodule update --init -- jni/external/faiss

# Build knn libs
# Setup compile time dependency for Windows only
# As Linux version already have OpenBlas in the runner
if [ "$PLATFORM" = "windows" ]; then
openBlasVersion="0.3.21"
openBlasFile="openblas_${openBlasVersion}"
curl -SL https://github.com/xianyi/OpenBLAS/releases/download/v${openBlasVersion}/OpenBLAS-${openBlasVersion}-x64.zip -o ${openBlasFile}.zip
unzip -j -o ${openBlasFile}.zip bin/libopenblas.dll -d ./src/main/resources/windowsDependencies
rm -rf ${openBlasFile}.zip
fi

# Setup knnlib build params for all platforms
cd jni

# For x64, generalize arch so library is compatible for processors without simd instruction extensions
Expand All @@ -95,30 +105,41 @@ if [ "$JAVA_HOME" = "" ]; then
echo "SET JAVA_HOME=$JAVA_HOME"
fi

cmake .
make opensearchknn_faiss opensearchknn_nmslib

# Build k-NN lib and plugin through gradle tasks
cd $work_dir
./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
# Gradle build is used here to replace gradle assemble due to build will also call cmake and make before generating jars
./gradlew build --no-daemon --refresh-dependencies -x integTest -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER

# Add lib to zip
zipPath=$(find "$(pwd)" -path \*build/distributions/*.zip)
distributions="$(dirname "${zipPath}")"
mkdir $distributions/lib
cp ./jni/release/libopensearchknn* $distributions/lib

# Copy libomp to be packaged with the lib contents
ompPath=$(ldconfig -p | grep libgomp | cut -d ' ' -f 4)
cp $ompPath $distributions/lib
libPrefix="libopensearchknn"
if [ "$PLATFORM" = "windows" ]; then
libPrefix="opensearchknn"
cp -v ./src/main/resources/windowsDependencies/libopenblas.dll $distributions/lib

# Have to define $MINGW_BIN either in ENV VAR or User Provided Var
cp -v "$MINGW_BIN/libgcc_s_seh-1.dll" $distributions/lib
cp -v "$MINGW_BIN/libwinpthread-1.dll" $distributions/lib
cp -v "$MINGW_BIN/libstdc++-6.dll" $distributions/lib
cp -v "$MINGW_BIN/libgomp-1.dll" $distributions/lib
else
ompPath=$(ldconfig -p | grep libgomp | cut -d ' ' -f 4)
cp -v $ompPath $distributions/lib
fi
cp -v ./jni/release/${libPrefix}* $distributions/lib
ls -l $distributions/lib

# Add lib directory to the k-NN plugin zip
cd $distributions
zip -ur $zipPath lib
cd $work_dir

echo "COPY ${distributions}/*.zip"
mkdir -p $OUTPUT/plugins
cp ${distributions}/*.zip $OUTPUT/plugins
cp -v ${distributions}/*.zip $OUTPUT/plugins

mkdir -p $OUTPUT/maven/org/opensearch
cp -r ./build/local-staging-repo/org/opensearch/. $OUTPUT/maven/org/opensearch

0 comments on commit 5247114

Please sign in to comment.