diff --git a/.github/workflows/android-cpu.yml b/.github/workflows/android-cpu.yml index b9ce3af7..e2165980 100755 --- a/.github/workflows/android-cpu.yml +++ b/.github/workflows/android-cpu.yml @@ -25,4 +25,4 @@ jobs: steps: - uses: actions/checkout@v3 - name: build - run: ./script/android_build.sh + run: ./script/android_build.sh \ No newline at end of file diff --git a/.github/workflows/model-test.yml b/.github/workflows/model-test.yml new file mode 100644 index 00000000..bd92eb19 --- /dev/null +++ b/.github/workflows/model-test.yml @@ -0,0 +1,82 @@ +name: model-test +on: + push: + branches: + - master + - 'feature/**' + paths: + - 'src/**' + - '.github/workflows/model-test.yml' + pull_request: + branches: [master] + paths: + - 'src/**' + - '.github/workflows/model-test.yml' + +jobs: + llm-build: + name: ${{ matrix.os }}-build + env: + PACAGE_DIR: ${{ matrix.os }}-package + PACAGE_FILE: ${{ matrix.os }}-package.zip + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v3 + # linux and macos + - name: linux-macos-build-pack + if: matrix.os != 'windows-latest' + run: | + ./script/build.sh + mkdir $PACAGE_DIR + ./script/package.sh $PACAGE_DIR + zip -r $PACAGE_FILE $PACAGE_DIR + # windows + - name: windows-build-pack + if: matrix.os == 'windows-latest' + run: | + .\script\windows_build.ps1 + mkdir $PACAGE_DIR + .\script\package.ps1 $PACAGE_DIR + 7z a -r $PACAGE_FILE $PACAGE_DIR + # upload + - name: upload-zip + uses: actions/upload-artifact@v3 + with: + path: ./*.zip + + model-test: + needs: llm-build + name: ${{ matrix.model }}-${{ matrix.os }}-test + runs-on: ${{ matrix.os }} + env: + PACAGE_DIR: ${{ matrix.os }}-package + PACAGE_FILE: ${{ matrix.os }}-package.zip + MODEL_NAME: ${{ matrix.model }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + model: [chatglm-6b, chatglm2-6b, codegeex2-6b, qwen-7b-chat, baichuan2-7b-chat, llama2-7b-chat] + + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: workspace + - name: linux-macos-test + if: matrix.os != 'windows-latest' + run: | + cd workspace + unzip $PACAGE_FILE + cd $PACAGE_DIR + ./script/model_test.sh $MODEL_NAME + - name: windows-test + if: matrix.os == 'windows-latest' + run: | + cd workspace + 7z x $PACAGE_FILE + cd $PACAGE_DIR + ./script/model_test.ps1 $MODEL_NAME \ No newline at end of file diff --git a/android/gradle/wrapper/gradle-wrapper.jar b/android/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..e708b1c0 Binary files /dev/null and b/android/gradle/wrapper/gradle-wrapper.jar differ diff --git a/android/gradle/wrapper/gradle-wrapper.properties b/android/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..da9702f9 --- /dev/null +++ b/android/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/android/gradlew b/android/gradlew new file mode 100755 index 00000000..cccdd3d5 --- /dev/null +++ b/android/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/script/android_app_build.sh b/script/android_app_build.sh new file mode 100755 index 00000000..d5e48d40 --- /dev/null +++ b/script/android_app_build.sh @@ -0,0 +1,26 @@ +# 1. clone MNN +git clone https://github.com/alibaba/MNN.git --depth=1 + +# 2. build MNN +cd MNN/project/android +mkdir build +cd build +../build_64.sh +cd ../../../.. + +# 3. copy headers and libs +cp -r MNN/include/MNN include +cp MNN/project/android/build/libMNN.so MNN/project/android/build/libMNN_Express.so libs + +# 4. build mnn-llm android +mkdir android_build +cd android_build +cmake .. \ +-DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ +-DANDROID_STL=c++_static \ +-DANDROID_ABI="arm64-v8a" \ +-DANDROID_NATIVE_API_LEVEL=android-21 \ +-DCMAKE_BUILD_TYPE=Release \ +-DBUILD_FOR_ANDROID=ON +make -j4 +cd .. \ No newline at end of file diff --git a/script/android_build.sh b/script/android_build.sh index 04a76030..d5e48d40 100755 --- a/script/android_build.sh +++ b/script/android_build.sh @@ -1,5 +1,5 @@ # 1. clone MNN -git clone https://github.com/alibaba/MNN.git -b 2.6.3 --depth=1 +git clone https://github.com/alibaba/MNN.git --depth=1 # 2. build MNN cd MNN/project/android diff --git a/script/windows_build.ps1 b/script/build.ps1 similarity index 90% rename from script/windows_build.ps1 rename to script/build.ps1 index 3a6ab075..77f5f76f 100644 --- a/script/windows_build.ps1 +++ b/script/build.ps1 @@ -1,5 +1,5 @@ # 1. clone MNN -git clone https://github.com/alibaba/MNN.git -b 2.7.1 --depth=1 +git clone https://github.com/alibaba/MNN.git --depth=1 # 2. build MNN cd MNN diff --git a/script/macos_build.sh b/script/build.sh similarity index 62% rename from script/macos_build.sh rename to script/build.sh index 47b48d41..b9428088 100755 --- a/script/macos_build.sh +++ b/script/build.sh @@ -1,5 +1,5 @@ # 1. clone MNN -git clone https://github.com/alibaba/MNN.git -b 2.7.1 --depth=1 +git clone https://github.com/alibaba/MNN.git --depth=1 # 2. build MNN cd MNN @@ -11,7 +11,10 @@ cd ../.. # 3. copy headers and libs cp -r MNN/include/MNN include -cp MNN/build/libMNN.dylib MNN/build/express/libMNN_Express.dylib libs +# linux +cp MNN/build/libMNN.so MNN/build/express/libMNN_Express.so libs 2> /dev/null || : +# macos +cp MNN/build/libMNN.dylib MNN/build/express/libMNN_Express.dylib libs 2> /dev/null || : # 4. build mnn-llm mkdir build diff --git a/script/download_model.sh b/script/download_model.sh index ee7bdf15..b9acd63e 100755 --- a/script/download_model.sh +++ b/script/download_model.sh @@ -2,10 +2,10 @@ model=$1 mkdir $model cd $model # download models -wget -c https://github.com/wangzhaode/mnn-llm/releases/download/$model-mnn/embedding.mnn -wget -c https://github.com/wangzhaode/mnn-llm/releases/download/$model-mnn/lm.mnn +wget -c -nv https://github.com/wangzhaode/mnn-llm/releases/download/$model-mnn/embedding.mnn +wget -c -nv https://github.com/wangzhaode/mnn-llm/releases/download/$model-mnn/lm.mnn for i in `seq 0 27` do - wget -c https://github.com/wangzhaode/mnn-llm/releases/download/$model-mnn/block_$i.mnn + wget -c -nv https://github.com/wangzhaode/mnn-llm/releases/download/$model-mnn/block_$i.mnn done cd .. \ No newline at end of file diff --git a/script/linux_build.sh b/script/linux_build.sh deleted file mode 100755 index 369f56f8..00000000 --- a/script/linux_build.sh +++ /dev/null @@ -1,21 +0,0 @@ -# 1. clone MNN -git clone https://github.com/alibaba/MNN.git -b 2.7.1 --depth=1 - -# 2. build MNN -cd MNN -mkdir build -cd build -cmake -DMNN_LOW_MEMORY=ON .. -make -j4 -cd ../.. - -# 3. copy headers and libs -cp -r MNN/include/MNN include -cp MNN/build/libMNN.so MNN/build/express/libMNN_Express.so libs - -# 4. build mnn-llm -mkdir build -cd build -cmake .. -make -j4 -cd .. \ No newline at end of file diff --git a/script/model_test.ps1 b/script/model_test.ps1 new file mode 100644 index 00000000..dd88a587 --- /dev/null +++ b/script/model_test.ps1 @@ -0,0 +1,26 @@ +Param( + [String]$model +) + +function model_test($model) { + Write-Output "test model : ${model}" + powershell .\script\download_model.ps1 ${model} + cd build + .\Release\cli_demo -m ..\${model} + cd .. +} + +function test_all() { + model_test chatglm-6b + model_test chatglm2-6b + model_test codegeex2-6b + model_test qwen-7b-chat + model_test baichuan2-7b-chat + model_test llama2-7b-chat +} + +if ($model -eq "all") { + test_all +} else { + model_test $model +} \ No newline at end of file diff --git a/script/model_test.sh b/script/model_test.sh new file mode 100755 index 00000000..a67a37b1 --- /dev/null +++ b/script/model_test.sh @@ -0,0 +1,22 @@ +model_test() { + echo 'test model: ' $1 + ./script/download_model.sh $1 + cd build + ./cli_demo -m ../$1 + cd .. +} + +test_all() { + model_test chatglm-6b + model_test chatglm2-6b + model_test codegeex2-6b + model_test qwen-7b-chat + model_test baichuan2-7b-chat + model_test llama2-7b-chat +} + +if [ $1 -eq 'all' ]; then + test_all +else + model_test $1 +fi \ No newline at end of file diff --git a/script/package.ps1 b/script/package.ps1 new file mode 100755 index 00000000..4042c549 --- /dev/null +++ b/script/package.ps1 @@ -0,0 +1,17 @@ +Param( + [String]$package_path +) + +mkdir $package_path +cd $package_path +mkdir resource +mkdir build +cd .. + +# copy file +echo 'copy files ...' +cp -r resource\tokenizer $package_path\resource +cp -r script $package_path +cp -r build\Release $package_path\build +cp libs\*.dll $package_path\build\Release +cp libs\*.lib $package_path\build\Release \ No newline at end of file diff --git a/script/package.sh b/script/package.sh new file mode 100755 index 00000000..cb3985ed --- /dev/null +++ b/script/package.sh @@ -0,0 +1,23 @@ +if [ $# -lt 1 ]; then + echo 'Usage: ./package.sh $package_path' + exit 1 +fi + +package_path=$1 + +# make dir +mkdir -p $package_path +cd $package_path +mkdir resource +mkdir build +cd .. + +# copy file +echo 'copy files ...' +cp -r resource/tokenizer $package_path/resource +cp -r script $package_path +cp build/*_demo $package_path/build +# linux +cp libs/*.so build/*.so $package_path/build 2> /dev/null || : +# macos +cp libs/*.dylib build/*.dylib $package_path/build 2> /dev/null || : \ No newline at end of file