Support ARM64 (#82)

* use lower version

* use latest

* Revert "use latest"

This reverts commit d0e3974.

* add arm64

* fix error

* Add disable sqlite3 flag

* fix error

* disable static

* if not work

* update

* fix release issue

.github/workflows/main.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: windows-latest 
     needs: Linux
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         include:
         - arch: x86
@@ -126,8 +126,87 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 
+  # copy from https://github.com/Cyan4973/xxHash/blob/dev/.github/workflows/ci.yml
+  # Linux ARM64
+  # All tests are using QEMU and gcc cross compiler.
+
+  qemu-consistency:
+    name: QEMU ${{ matrix.name }}
+    needs: Linux
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true  # 'false' means Don't stop matrix workflows even if some matrix failed.
+      matrix:
+        include: [
+          { name: 'ARM64',           xcc_pkg: gcc-aarch64-linux-gnu,        xcc: aarch64-linux-gnu-gcc,        xemu_pkg: qemu-system-arm,   xemu: qemu-aarch64-static, os: ubuntu-latest, },
+
+          { name: 'ARM64, gcc-7',    xcc_pkg: gcc-7-aarch64-linux-gnu,      xcc: aarch64-linux-gnu-gcc-7,      xemu_pkg: qemu-system-arm,   xemu: qemu-aarch64-static, os: ubuntu-18.04, },
+        ]
+    env:                        # Set environment variables
+      CC: ${{ matrix.xcc }}
+      XEMU: ${{ matrix.xemu }}
+      # LDFLAGS: -static
+    steps:
+    - uses: actions/checkout@v2 # https://github.com/actions/checkout
+    - name: apt update & install (1)
+      run: |
+        sudo apt-get update
+        sudo apt-get install gcc-multilib g++-multilib qemu-utils qemu-user-static
+    - name: Environment info (1)
+      run: |
+        echo && apt-cache search "^gcc-" | grep "linux" | sort
+    - name: apt update & install (2)
+      run: |
+        sudo apt-get install ${{ matrix.xcc_pkg }} ${{ matrix.xemu_pkg }}
+    - name: Environment info (2)
+      run: |
+        echo && which $CC
+        echo && $CC --version
+        echo && $CC -v  # Show built-in specs
+        echo && which $XEMU
+        echo && $XEMU --version
+    - name: ARM64
+      uses: lukka/run-cmake@v3
+      with:
+        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
+        cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
+        cmakeBuildType: 'Release'
+        buildDirectory: "${{ github.workspace }}/build"
+        buildWithCMake: true
+        buildWithCMakeArgs: --config Release
+        # cmakeAppendedArgs: " -DCODE_COVERAGE=OFF -DBUILD_SHELL=OFF -DBUILD_TEST_EXAMPLE=OFF -DBUILD_STATIC"
+        cmakeAppendedArgs: " -DCODE_COVERAGE=OFF -DBUILD_SHELL=OFF -DBUILD_TEST_EXAMPLE=OFF"
+    - name: "Check file existence"
+      uses: andstor/file-existence-action@v1
+      with:
+        allow_failure: true
+        # files: "${{ github.workspace }}/build/src/libsimple.a, ${{ github.workspace }}/build/src/libPINYIN_TEXT.a, ${{ github.workspace }}/build/test/dict/jieba.dict.utf8"
+        files: "${{ github.workspace }}/build/src/libsimple.so, ${{ github.workspace }}/build/test/dict/jieba.dict.utf8"
+    - name: Package
+      if: startsWith(github.ref, 'refs/tags/')
+      run: |
+        mkdir libsimple-${{ matrix.xcc }}
+        # cp -r src/libsimple.a src/libPINYIN_TEXT.a test/dict/ libsimple-linux-${{ matrix.xcc }}/
+        cp -r src/libsimple.so test/dict/ libsimple-${{ matrix.xcc }}/
+        zip -r libsimple-${{ matrix.xcc }}.zip libsimple-${{ matrix.xcc }}
+      working-directory: "${{ github.workspace }}/build"
+    - name: Release
+      if: startsWith(github.ref, 'refs/tags/')
+      uses: softprops/action-gh-release@v1
+      with:
+        draft: true
+        files: ${{ github.workspace }}/build/libsimple-${{ matrix.xcc }}.zip
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   Linux:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+        - os: ubuntu-18.04
+        - os: ubuntu-latest
     steps:
     - name: "Release Build Type"
       if: startsWith(github.ref, 'refs/tags/')
@@ -142,9 +221,11 @@ jobs:
       run: sudo apt-get update
 
     - name: Install lcov
+      if: startsWith(github.ref, 'refs/tags/') != true
       run: sudo apt-get install lcov
 
     - name: Install codecov
+      if: startsWith(github.ref, 'refs/tags/') != true
       shell: bash
       run: sudo pip install codecov
 
@@ -170,25 +251,25 @@ jobs:
         buildDirectory: "${{ github.workspace }}/build"
         buildWithCMake: true
         buildWithCMakeArgs: --config ${{ env.BUILD_TYPE }} 
-        cmakeAppendedArgs: " -DCODE_COVERAGE=OFF "
+        cmakeAppendedArgs: " -DCODE_COVERAGE=OFF -DBUILD_SHELL=OFF -DBUILD_TEST_EXAMPLE=OFF"
 
     - name: 'Run CTest'
-      if: startsWith(github.ref, 'refs/tags/') != true
+      if: ${{ startsWith(github.ref, 'refs/tags/') != true && matrix.os == 'ubuntu-latest' }}
       run: ctest -V -C ${{ env.BUILD_TYPE }}
       working-directory: "${{ github.workspace }}/build"
 
     # CODE COVERAGE
     - name: Code coverage - Capture coverage info
-      if: startsWith(github.ref, 'refs/tags/') != true
+      if: ${{ startsWith(github.ref, 'refs/tags/') != true && matrix.os == 'ubuntu-latest' }}
       run: lcov --directory . --capture --output-file coverage.info 
     - name: Code coverage - Filter out system, external, and unit test source files
-      if: startsWith(github.ref, 'refs/tags/') != true
+      if: ${{ startsWith(github.ref, 'refs/tags/') != true && matrix.os == 'ubuntu-latest' }}
       run: lcov --remove coverage.info --output-file coverage_filter.info '/Library/*' '/usr/*' '*/test/*' '*/cmrc/*' '*/entry.cc' '*/simple_highlight.*'
     - name: Code coverage - Output coverage data for debugging
-      if: startsWith(github.ref, 'refs/tags/') != true
+      if: ${{ startsWith(github.ref, 'refs/tags/') != true && matrix.os == 'ubuntu-latest' }}
       run: lcov --list coverage_filter.info
     - name: Code coverage - Upload to CodeCov
-      if: startsWith(github.ref, 'refs/tags/') != true
+      if: ${{ startsWith(github.ref, 'refs/tags/') != true && matrix.os == 'ubuntu-latest' }}
       run:  bash <(curl -s https://codecov.io/bash) -f coverage_filter.info || echo "Codecov did not collect coverage reports"
 
     - name: "Check file existence"
@@ -199,9 +280,11 @@ jobs:
 
     # npm run
     - uses: actions/setup-node@v2
+      if: startsWith(github.ref, 'refs/tags/') != true
       with:
         node-version: '16'
     - name: run node example
+      if: startsWith(github.ref, 'refs/tags/') != true
       working-directory: ./examples/node/
       run: |
         npm install
@@ -210,32 +293,42 @@ jobs:
 
     # python run
     - uses: actions/setup-python@v2
+      if: startsWith(github.ref, 'refs/tags/') != true
       with:
         python-version: '3.x' # Version range or exact version of a Python version to use, using SemVer's version range syntax
     - name: run python example
+      if: startsWith(github.ref, 'refs/tags/') != true
       working-directory: ./examples/python3/
       run: python db_connector.py "${{ github.workspace }}/build/src/libsimple"
 
     # create release
     - name: "Build Changelog"
       id: build_changelog
-      if: startsWith(github.ref, 'refs/tags/')
+      if: ${{ startsWith(github.ref, 'refs/tags/') && matrix.os == 'ubuntu-latest' }}
       uses: mikepenz/release-changelog-builder-action@v1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Package
       if: startsWith(github.ref, 'refs/tags/')
       run: |
-        mkdir libsimple-linux-x64
-        cp -r src/libsimple.so test/dict/ libsimple-linux-x64/
-        zip -r libsimple-linux-x64.zip libsimple-linux-x64
+        mkdir libsimple-linux-${{ matrix.os }}
+        cp -r src/libsimple.so test/dict/ libsimple-linux-${{ matrix.os }}/
+        zip -r libsimple-linux-${{ matrix.os }}.zip libsimple-linux-${{ matrix.os }}
       working-directory: "${{ github.workspace }}/build"
+    - name: Release without changelog
+      if: ${{ startsWith(github.ref, 'refs/tags/') && matrix.os != 'ubuntu-latest' }}
+      uses: softprops/action-gh-release@v1
+      with:
+        draft: true
+        files: ${{ github.workspace }}/build/libsimple-linux-${{ matrix.os }}.zip
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Release
-      if: startsWith(github.ref, 'refs/tags/')
+      if: ${{ startsWith(github.ref, 'refs/tags/') && matrix.os == 'ubuntu-latest' }}
       uses: softprops/action-gh-release@v1
       with:
         draft: true
-        files: ${{ github.workspace }}/build/libsimple-linux-x64.zip
+        files: ${{ github.workspace }}/build/libsimple-linux-${{ matrix.os }}.zip
         body: ${{steps.build_changelog.outputs.changelog}}
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.2)
+cmake_minimum_required(VERSION 3.19)
 project(simple-tokenizer)
 
 set(CMAKE_CXX_STANDARD 14)
@@ -11,6 +11,11 @@ if (MSVC)
    add_compile_options("$<$<C_COMPILER_ID:MSVC>:/utf-8>")
 endif()
 
+if (APPLE)
+   set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64" CACHE STRING "")
+   set(CMAKE_OSX_DEPLOYMENT_TARGET "10.11" CACHE STRING "Minimum OS X deployment version")
+endif()
+
 if (IOS)
   list(APPEND CMAKE_ARGS
     "-DPLATFORM=${PLATFORM}"
@@ -21,14 +26,17 @@ set(SQLITE3_HEADERS_DIR ${PROJECT_SOURCE_DIR}/contrib/sqlite3)
 set(SIMPLE_HEADERS_DIR ${PROJECT_SOURCE_DIR}/src)
 
 # setup sqlite
-set(SQLITE_ROOT ${PROJECT_SOURCE_DIR}/build/sqlite)
-set(SQLITE_INSTALL_DIR ${SQLITE_ROOT})
-set(SQLITE_INCLUDE_DIR ${SQLITE_INSTALL_DIR}/include)
-set(SQLITE_LIB_DIR ${SQLITE_INSTALL_DIR}/lib)
-if (NOT IOS)
-  option(SQLITE_ENABLE_FTS5         "enables full text searches version 5"         ON)
-  option(BUILD_SHELL "build SQLite3 shell application" ON)
-  add_subdirectory(contrib/sqlite3)
+option(BUILD_SQLITE3 "Enable build SQLite3" ON)
+if(BUILD_SQLITE3)
+  set(SQLITE_ROOT ${PROJECT_SOURCE_DIR}/build/sqlite)
+  set(SQLITE_INSTALL_DIR ${SQLITE_ROOT})
+  set(SQLITE_INCLUDE_DIR ${SQLITE_INSTALL_DIR}/include)
+  set(SQLITE_LIB_DIR ${SQLITE_INSTALL_DIR}/lib)
+  if (NOT IOS)
+    option(SQLITE_ENABLE_FTS5         "enables full text searches version 5"         ON)
+    option(BUILD_SHELL "build SQLite3 shell application" ON)
+    add_subdirectory(contrib/sqlite3)
+  endif()
 endif()
 
 include(contrib/CMakeRC.cmake)
@@ -65,7 +73,8 @@ endif()
 
 add_subdirectory(src)
 
-if (NOT IOS)
+OPTION(BUILD_TEST_EXAMPLE "Option to build tests and examples" ON)
+if (NOT IOS AND BUILD_TEST_EXAMPLE)
   add_subdirectory(examples/cpp)
   enable_testing()
   add_subdirectory(test)

src/CMakeLists.txt

@@ -25,7 +25,8 @@ set(SOURCE_FILES
     entry.cc
 )
 
-if (IOS)
+OPTION(BUILD_STATIC "Option to build static lib" OFF)
+if (IOS OR BUILD_STATIC)
 # iOS only support static library.
 add_library(simple STATIC ${SOURCE_FILES})
 else()
@@ -50,7 +51,7 @@ target_link_libraries(simple PUBLIC coverage_config PRIVATE PINYIN_TEXT SQLite3)
 
 install(TARGETS simple DESTINATION bin)
 
-if (IOS)
+if (IOS OR BUILD_STATIC)
 # iOS build as static library. so we need install PINYIN_TEXT too.
 install(TARGETS PINYIN_TEXT DESTINATION bin)
 endif()