[Build] Switch to scikit-build as the build backend (#4624)

* switch to skbuild

* Switch the build system to scikit-build

* include bc and libmolten

* find llvm runtime bc

* fix bc files installation

* install bc after compile

* Add more message

* Auto Format

* fix findpython

* Kickstart CI

* add empty line

* add missing dependency

* fix python args

* start CI

* Fix clang tidy run

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: Taichi Gardener <taichigardener@gmail.com>
Co-authored-by: Ailing <ailzhang@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/scripts/check_clang_tidy.sh

@@ -5,8 +5,5 @@ CI_SETUP_CMAKE_ARGS=$1
 cd taichi
 python3 -m pip install -r requirements_dev.txt
 
-rm -rf build && mkdir build && cd build
-cmake $CI_SETUP_CMAKE_ARGS ..
-
-cd ..
-python3 ./scripts/run_clang_tidy.py $PWD/taichi -clang-tidy-binary clang-tidy-10 -checks=-*,performance-inefficient-string-concatenation,readability-identifier-naming -header-filter=$PWD/taichi -p $PWD/build -j2
+export CI_SETUP_CMAKE_ARGS
+python3 ./scripts/run_clang_tidy.py $PWD/taichi -clang-tidy-binary clang-tidy-10 -checks=-*,performance-inefficient-string-concatenation,readability-identifier-naming -header-filter=$PWD/taichi -j2

.github/workflows/scripts/win_build.ps1

@@ -76,7 +76,6 @@ python -m venv venv
 . venv\Scripts\activate.ps1
 python -m pip install wheel
 python -m pip install -r requirements_dev.txt
-python -m pip install -r requirements_test.txt
 if (-not $?) { exit 1 }
 WriteInfo("Building Taichi")
 $env:TAICHI_CMAKE_ARGS += " -DCLANG_EXECUTABLE=$libsDir\\taichi_clang\\bin\\clang++.exe"

.gitignore

@@ -85,3 +85,4 @@ _build
 !docs/**/*.json
 imgui.ini
 /venv/
+/_skbuild/

CMakeLists.txt

@@ -49,7 +49,6 @@ else ()
 endif ()
 
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build")
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build")
 
 find_program(CCACHE_PROGRAM ccache)
 if(CCACHE_PROGRAM)
@@ -157,6 +156,7 @@ foreach(arch IN LISTS HOST_ARCH CUDA_ARCH)
       WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/taichi/runtime/llvm"
   )
   add_dependencies(${CORE_LIBRARY_NAME} "generate_llvm_runtime_${arch}")
+  install(FILES "${PROJECT_SOURCE_DIR}/taichi/runtime/llvm/runtime_${arch}.bc" DESTINATION ${CMAKE_INSTALL_PREFIX}/python/taichi/_lib/runtime)
 endforeach()
 
 configure_file(taichi/common/version.h.in ${CMAKE_SOURCE_DIR}/taichi/common/version.h)

MANIFEST.in

@@ -1,8 +1,3 @@
-include MANIFEST.in
-include version.txt
-include python/*.txt
-include python/*.py
-include *.cfg
 include python/taichi/*.md
 recursive-include python/taichi/assets *
 recursive-include python/taichi/examples *.py

cmake/PythonNumpyPybind11.cmake

@@ -1,93 +1,16 @@
 # Python, numpy, and pybind11
+execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pybind11 --cmake
+                OUTPUT_VARIABLE pybind11_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import numpy;print(numpy.get_include())"
+                OUTPUT_VARIABLE NUMPY_INCLUDE_DIR OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-if (PYTHON_EXECUTABLE)
-    message("Using ${PYTHON_EXECUTABLE} as python executable.")
-else ()
-    if (WIN32)
-        message("Using 'python' as python interpreter.")
-        set(PYTHON_EXECUTABLE python)
-    else ()
-        message("Using 'python3' as python interpreter.")
-        set(PYTHON_EXECUTABLE python3)
-    endif()
-endif ()
+message("-- Python: Using ${PYTHON_EXECUTABLE} as the interpreter")
+message("    version: ${PYTHON_VERSION_STRING}")
+message("    include: ${PYTHON_INCLUDE_DIR}")
+message("    library: ${PYTHON_LIBRARY}")
+message("    numpy include: ${NUMPY_INCLUDE_DIR}")
 
-if (WIN32)
-    execute_process(COMMAND where ${PYTHON_EXECUTABLE}
-        OUTPUT_VARIABLE PYTHON_EXECUTABLE_PATHS)
-    if (${PYTHON_EXECUTABLE_PATHS})
-        string(FIND ${PYTHON_EXECUTABLE_PATHS} "\n" _LINE_BREAK_LOC)
-        string(SUBSTRING ${PYTHON_EXECUTABLE_PATHS} 0 ${_LINE_BREAK_LOC} PYTHON_EXECUTABLE_PATH)
-    else ()
-        set(PYTHON_EXECUTABLE_PATH ${PYTHON_EXECUTABLE})
-    endif ()
-else ()
-    execute_process(COMMAND which ${PYTHON_EXECUTABLE}
-            OUTPUT_VARIABLE PYTHON_EXECUTABLE_PATH)
-endif()
-execute_process(COMMAND ${PYTHON_EXECUTABLE} -c
-        "import sys;\
-        from distutils import sysconfig;\
-        sys.stdout.write(sysconfig.get_python_version())"
-        OUTPUT_VARIABLE PYTHON_VERSION)
-execute_process(COMMAND ${PYTHON_EXECUTABLE} --version)
-execute_process(COMMAND ${PYTHON_EXECUTABLE} -c
-        "import sys;\
-        from distutils import sysconfig;\
-        sys.stdout.write(\
-        (sysconfig.get_config_var('INCLUDEPY')\
-        if sysconfig.get_config_var('INCLUDEDIR') is not None else None)\
-        or sysconfig.get_python_inc())"
-        OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS)
-execute_process(COMMAND ${PYTHON_EXECUTABLE} -c
-        "import sys;\
-        from distutils import sysconfig;\
-        sys.stdout.write((sysconfig.get_config_var('LIBDIR') or sysconfig.get_python_lib()).replace('\\\\','/'))"
-        OUTPUT_VARIABLE PYTHON_LIBRARY_DIR)
 
+include_directories(${NUMPY_INCLUDE_DIR})
 
-execute_process(COMMAND ${PYTHON_EXECUTABLE} -c
-        "import sys;\
-        sys.stdout.write(str(sys.version_info[1]))"
-        OUTPUT_VARIABLE PYTHON_MINOR_VERSION)
-
-
-if (WIN32)
-  execute_process(COMMAND ${PYTHON_EXECUTABLE} -c
-          "import sys;sys.stdout.write(sys.base_prefix.replace('\\\\', '/'))"
-          OUTPUT_VARIABLE PYTHON_BASE_PREFIX)
-  link_directories(${PYTHON_BASE_PREFIX}/libs)
-  set(PYTHON_LIBRARIES ${PYTHON_BASE_PREFIX}/libs/python3.lib)
-  set(PYTHON_LIBRARIES ${PYTHON_BASE_PREFIX}/libs/python3${PYTHON_MINOR_VERSION}.lib)
-else()
-  find_library(PYTHON_LIBRARY NAMES python${PYTHON_VERSION} python${PYTHON_VERSION}m PATHS ${PYTHON_LIBRARY_DIR}
-          NO_DEFAULT_PATH NO_SYSTEM_ENVIRONMENT_PATH PATH_SUFFIXES x86_64-linux-gnu)
-  set(PYTHON_LIBRARIES ${PYTHON_LIBRARY})
-endif()
-
-
-include_directories(${PYTHON_INCLUDE_DIRS})
-message("    version: ${PYTHON_VERSION}")
-message("    include: ${PYTHON_INCLUDE_DIRS}")
-message("    library: ${PYTHON_LIBRARIES}")
-
-execute_process(COMMAND ${PYTHON_EXECUTABLE} -c
-        "import numpy.distutils, sys;\
-        sys.stdout.write(':'.join(numpy.distutils.misc_util.get_numpy_include_dirs()))"
-        OUTPUT_VARIABLE PYTHON_NUMPY_INCLUDE_DIR)
-
-message("    numpy include: ${PYTHON_NUMPY_INCLUDE_DIR}")
-include_directories(${PYTHON_NUMPY_INCLUDE_DIR})
-
-execute_process(COMMAND ${PYTHON_EXECUTABLE} -c
-        "import sys; import pybind11; sys.stdout.write(pybind11.get_include() + ';' + pybind11.get_include(True))"
-        OUTPUT_VARIABLE PYBIND11_INCLUDE_DIR
-        RESULT_VARIABLE PYBIND11_IMPORT_RET)
-if (NOT PYBIND11_IMPORT_RET)
-    # returns zero if success
-    message("    pybind11 include: ${PYBIND11_INCLUDE_DIR}")
-else ()
-    message(FATAL_ERROR "Cannot import pybind11. Please install. ([sudo] pip3 install --user pybind11)")
-endif ()
-
-include_directories(${PYBIND11_INCLUDE_DIR})
+find_package(pybind11 CONFIG REQUIRED)

cmake/TaichiCore.cmake

@@ -16,6 +16,7 @@ option(TI_EMSCRIPTENED "Build using emscripten" OFF)
 # projects.
 set(CMAKE_CXX_VISIBILITY_PRESET hidden)
 set(CMAKE_VISIBILITY_INLINES_HIDDEN ON)
+set(INSTALL_LIB_DIR ${CMAKE_INSTALL_PREFIX}/python/taichi/_lib)
 
 if(ANDROID)
     set(TI_WITH_VULKAN ON)
@@ -384,6 +385,9 @@ if (TI_WITH_VULKAN)
         find_library(MOLTEN_VK libMoltenVK.dylib PATHS $HOMEBREW_CELLAR/molten-vk $VULKAN_SDK REQUIRED)
         configure_file(${MOLTEN_VK} ${CMAKE_BINARY_DIR}/libMoltenVK.dylib COPYONLY)
         message(STATUS "MoltenVK library ${MOLTEN_VK}")
+        if (EXISTS ${CMAKE_BINARY_DIR}/libMoltenVK.dylib)
+            install(FILES ${CMAKE_BINARY_DIR}/libMoltenVK.dylib DESTINATION ${INSTALL_LIB_DIR}/runtime)
+        endif()
     endif()
 endif ()
 
@@ -437,7 +441,7 @@ if(NOT TI_EMSCRIPTENED)
     # Cannot compile Python source code with Android, but TI_EXPORT_CORE should be set and
     # Android should only use the isolated library ignoring those source code.
     if (NOT ANDROID)
-        add_library(${CORE_WITH_PYBIND_LIBRARY_NAME} SHARED ${TAICHI_PYBIND_SOURCE})
+        pybind11_add_module(${CORE_WITH_PYBIND_LIBRARY_NAME} ${TAICHI_PYBIND_SOURCE})
     else()
         add_library(${CORE_WITH_PYBIND_LIBRARY_NAME} SHARED)
     endif ()
@@ -459,6 +463,10 @@ if(NOT TI_EMSCRIPTENED)
         set_target_properties(${CORE_WITH_PYBIND_LIBRARY_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY
                 "${CMAKE_CURRENT_SOURCE_DIR}/runtimes")
     endif ()
+
+    install(TARGETS ${CORE_WITH_PYBIND_LIBRARY_NAME}
+            RUNTIME DESTINATION ${INSTALL_LIB_DIR}/core
+            LIBRARY DESTINATION ${INSTALL_LIB_DIR}/core)
 endif()
 
 if(TI_EMSCRIPTENED)
@@ -487,3 +495,8 @@ endif()
     target_link_libraries(${CORE_LIBRARY_NAME} imgui)
 
 endif()
+
+if (NOT APPLE)
+    install(FILES ${CMAKE_SOURCE_DIR}/external/cuda_libdevice/slim_libdevice.10.bc
+            DESTINATION ${INSTALL_LIB_DIR}/runtime)
+endif()

pyproject.toml

@@ -1,5 +1,9 @@
 [build-system]
-requires = ["setuptools", "wheel", "numpy", "pybind11", "cmake"]
+requires = [
+    "setuptools", "wheel",
+    "numpy", "pybind11", "cmake",
+    "scikit-build", "ninja; platform_system != 'Windows'",
+]
 build-backend = "setuptools.build_meta"
 
 [tool.pytest.ini_options]

requirements_dev.txt

@@ -14,3 +14,6 @@ twine
 wheel
 astunparse
 pre-commit
+scikit-build
+numpy
+ninja; platform_system != 'Windows'

scripts/run_clang_tidy.py

@@ -77,6 +77,18 @@ def make_absolute(f, directory):
     return os.path.normpath(os.path.join(directory, f))
 
 
+def cmake_configure(source_path='.'):
+    import shlex
+
+    from skbuild.cmaker import CMaker
+    from skbuild.constants import CMAKE_BUILD_DIR
+
+    cmaker = CMaker()
+    cmake_args = shlex.split(os.getenv('CI_SETUP_CMAKE_ARGS', ''))
+    cmaker.configure(cmake_args)
+    return CMAKE_BUILD_DIR()
+
+
 def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path,
                         header_filter, extra_arg, extra_arg_before, quiet,
                         config):
@@ -265,8 +277,7 @@ def main():
     if args.build_path is not None:
         build_path = args.build_path
     else:
-        # Find our database
-        build_path = find_compilation_database(db_path)
+        build_path = cmake_configure('.')
 
     try:
         invocation = [args.clang_tidy_binary, '-list-checks']