[refactor] Create taichi_isolated_core that is free from pybind11

cmake/TaichiCore.cmake

@@ -1,5 +1,3 @@
-set(CORE_LIBRARY_NAME taichi_core)
-
 option(USE_STDCPP "Use -stdlib=libc++" OFF)
 option(TI_WITH_CUDA "Build with the CUDA backend" ON)
 option(TI_WITH_OPENGL "Build with the OpenGL backend" ON)
@@ -78,41 +76,33 @@ endif()
 # library into a shared lib.
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
-# The short-term goal is to have a sub-library that is mostly Taichi-focused,
-# free from the "application" layer such as pybind11 or GUI. At a minimum, we
-# must decouple from pybind11/python-environment. This sub-lib will then be
-# unit testtable.
+# The short-term goal is to have a sub-library, "taichi_isolated_core", that is
+# mostly Taichi-focused, free from the "application" layer such as pybind11 or
+# GUI. At a minimum, we must decouple from pybind11/python-environment. Then we
+# can 1) unit test a major part of Taichi, and 2) integrate a new frontend lang
+# with "taichi_isolated_core".
+#
 # TODO(#2198): Long-term speaking, we should create a separate library for each
 # sub-module. This way we can guarantee that the lib dependencies form a DAG.
-file(GLOB TAICHI_TESTABLE_SRC
-      "taichi/common/*.cpp"
-      "taichi/common/*.h"
-      "taichi/ir/ir_builder.*"
-      "taichi/ir/ir.*"
-      "taichi/ir/offloaded_task_type.*"
-      "taichi/ir/snode_types.*"
-      "taichi/ir/snode.*"
-      "taichi/ir/statements.*"
-      "taichi/ir/type_factory.*"
-      "taichi/ir/type_utils.*"
-      "taichi/ir/type.*"
-      "taichi/transforms/statement_usage_replace.cpp"
-      "taichi/program/arch.*"
-      "taichi/program/compile_config.*"
-      "taichi/system/timer.*"
-      "taichi/system/profiler.*"
+file(GLOB TAICHI_PYBIND_SOURCE
+      "taichi/python/*.cpp"
+      "taichi/python/*.h"
 )
-
-# TODO(#2196): Maybe we can do the following renaming in the end?
+list(REMOVE_ITEM TAICHI_CORE_SOURCE ${TAICHI_PYBIND_SOURCE})
+
+# TODO(#2196): Rename these CMAKE variables:
+# CORE_LIBRARY_NAME --> TAICHI_ISOLATED_CORE_LIB_NAME
+# CORE_WITH_PYBIND_LIBRARY_NAME --> TAICHI_CORE_LIB_NAME
+#
+# However, the better strategy is probably to rename the actual library:
+#
 # taichi_core --> taichi_pylib (this requires python-side refactoring...)
-# taichi_testable_lib --> taichi_core
-set(TAICHI_TESTABLE_LIB taichi_testable_lib)
-add_library(${TAICHI_TESTABLE_LIB} STATIC ${TAICHI_TESTABLE_SRC})
-
-list(REMOVE_ITEM TAICHI_CORE_SOURCE ${TAICHI_TESTABLE_SRC})
-
-add_library(${CORE_LIBRARY_NAME} SHARED ${TAICHI_CORE_SOURCE})
-target_link_libraries(${CORE_LIBRARY_NAME} ${TAICHI_TESTABLE_LIB})
+# taichi_isolated_core --> taichi_core
+#
+# But this requires more efforts, because taichi_core is already referenced
+# everywhere in python.
+set(CORE_LIBRARY_NAME taichi_isolated_core)
+add_library(${CORE_LIBRARY_NAME} OBJECT ${TAICHI_CORE_SOURCE})
 
 if (APPLE)
     # Ask OS X to minic Linux dynamic linking behavior
@@ -203,7 +193,6 @@ else()
     # windows
     target_link_libraries(${CORE_LIBRARY_NAME} Winmm)
 endif ()
-message("PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES})
 
 foreach (source IN LISTS TAICHI_CORE_SOURCE)
     file(RELATIVE_PATH source_rel ${CMAKE_CURRENT_LIST_DIR} ${source})
@@ -220,3 +209,11 @@ if (WIN32)
     set_target_properties(${CORE_LIBRARY_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY
             "${CMAKE_CURRENT_SOURCE_DIR}/runtimes")
 endif ()
+
+message("PYTHON_LIBRARIES: " ${PYTHON_LIBRARIES})
+
+set(CORE_WITH_PYBIND_LIBRARY_NAME taichi_core)
+add_library(${CORE_WITH_PYBIND_LIBRARY_NAME} SHARED ${TAICHI_PYBIND_SOURCE})
+# It is actually possible to link with an OBJECT library
+# https://cmake.org/cmake/help/v3.13/command/target_link_libraries.html?highlight=target_link_libraries#linking-object-libraries
+target_link_libraries(${CORE_WITH_PYBIND_LIBRARY_NAME} PUBLIC ${CORE_LIBRARY_NAME})

cmake/TaichiTests.cmake

@@ -16,7 +16,7 @@ include_directories(
 )
 
 add_executable(${TESTS_NAME} ${TAICHI_TESTS_SOURCE})
-target_link_libraries(${TESTS_NAME} taichi_testable_lib)
+target_link_libraries(${TESTS_NAME} taichi_isolated_core)
 target_link_libraries(${TESTS_NAME} gtest_main)
 
 add_test(NAME ${TESTS_NAME} COMMAND ${TESTS_NAME})

taichi/backends/cuda/detect_cuda.cpp

@@ -0,0 +1,17 @@
+#include "taichi/backends/cuda/detect_cuda.h"
+
+#if defined(TI_WITH_CUDA)
+#include "taichi/backends/cuda/cuda_driver.h"
+#endif
+
+namespace taichi {
+
+bool is_cuda_api_available() {
+#if defined(TI_WITH_CUDA)
+  return lang::CUDADriver::get_instance_without_context().detected();
+#else
+  return false;
+#endif
+}
+
+}  // namespace taichi

taichi/backends/cuda/detect_cuda.h

@@ -0,0 +1,5 @@
+#pragma once
+
+namespace taichi {
+bool is_cuda_api_available();
+}  // namespace taichi

taichi/backends/metal/kernel_manager.cpp

@@ -10,7 +10,7 @@
 #include "taichi/backends/metal/constants.h"
 #include "taichi/inc/constants.h"
 #include "taichi/math/arithmetic.h"
-#include "taichi/python/print_buffer.h"
+#include "taichi/program/py_print_buffer.h"
 #include "taichi/util/action_recorder.h"
 #include "taichi/util/file_sequence_writer.h"
 #include "taichi/util/str.h"

taichi/backends/opengl/opengl_api.cpp

@@ -4,8 +4,8 @@
 #include "taichi/backends/opengl/opengl_kernel_util.h"
 #include "taichi/program/kernel.h"
 #include "taichi/program/program.h"
+#include "taichi/program/py_print_buffer.h"
 #include "taichi/util/environ_config.h"
-#include "taichi/python/print_buffer.h"
 #include "taichi/backends/opengl/shaders/runtime.h"
 #include "taichi/backends/opengl/shaders/listman.h"
 

taichi/ir/ir.cpp

@@ -8,7 +8,8 @@
 #include "taichi/ir/statements.h"
 #include "taichi/ir/transforms.h"
 
-TLANG_NAMESPACE_BEGIN
+namespace taichi {
+namespace lang {
 
 std::string snode_access_flag_name(SNodeAccessFlag type) {
   if (type == SNodeAccessFlag::block_local) {
@@ -514,4 +515,5 @@ LocalAddress::LocalAddress(Stmt *var, int offset) : var(var), offset(offset) {
   TI_ASSERT(var->is<AllocaStmt>());
 }
 
-TLANG_NAMESPACE_END
+}  // namespace lang
+}  // namespace taichi

taichi/ir/ir.h

@@ -14,7 +14,8 @@
 #include "taichi/ir/type_factory.h"
 #include "taichi/util/short_name.h"
 
-TLANG_NAMESPACE_BEGIN
+namespace taichi {
+namespace lang {
 
 class IRNode;
 class Block;
@@ -595,7 +596,7 @@ class Stmt : public IRNode {
     TI_NOT_IMPLEMENTED
   }
 
-  virtual ~Stmt() override = default;
+  virtual ~Stmt() = default;
 };
 
 class Block : public IRNode {
@@ -776,4 +777,5 @@ inline void StmtFieldManager::operator()(const char *key, T &&value) {
   }
 }
 
-TLANG_NAMESPACE_END
+}  // namespace lang
+}  // namespace taichi

taichi/ir/statements.h

@@ -5,7 +5,8 @@
 #include "taichi/ir/stmt_op_types.h"
 #include "taichi/program/arch.h"
 
-TLANG_NAMESPACE_BEGIN
+namespace taichi {
+namespace lang {
 
 /**
  * Allocate a local variable with initial value 0.
@@ -1211,4 +1212,5 @@ class BitStructStoreStmt : public Stmt {
   TI_DEFINE_ACCEPT_AND_CLONE;
 };
 
-TLANG_NAMESPACE_END
+}  // namespace lang
+}  // namespace taichi

taichi/lang_util.cpp

@@ -13,6 +13,8 @@ TI_NAMESPACE_BEGIN
 namespace lang {
 
 CompileConfig default_compile_config;
+std::string compiled_lib_dir;
+std::string runtime_tmp_dir;
 
 real get_cpu_frequency() {
   static real cpu_frequency = 0;

taichi/program/program.cpp

@@ -11,6 +11,7 @@
 #include "taichi/backends/cuda/codegen_cuda.h"
 #include "taichi/backends/cuda/cuda_context.h"
 #endif
+#include "taichi/backends/cuda/detect_cuda.h"
 #include "taichi/backends/metal/codegen_metal.h"
 #include "taichi/backends/opengl/codegen_opengl.h"
 #include "taichi/backends/cpu/codegen_cpu.h"
@@ -37,12 +38,6 @@
 #include <xmmintrin.h>
 #endif
 
-TI_NAMESPACE_BEGIN
-
-bool is_cuda_api_available();
-
-TI_NAMESPACE_END
-
 TLANG_NAMESPACE_BEGIN
 
 namespace {

taichi/program/py_print_buffer.cpp

@@ -0,0 +1,7 @@
+#include "taichi/program/py_print_buffer.h"
+
+namespace taichi {
+
+PythonPrintBuffer py_cout;
+
+}  // namespace taichi

taichi/python/print_buffer.h → taichi/program/py_print_buffer.h

@@ -3,10 +3,12 @@
 #include <sstream>
 #include <iostream>
 
-TI_NAMESPACE_BEGIN
+namespace taichi {
 
+// PythonPrintBuffer holds the logs printed from kernel before sending them back
+// to python. The name could be a bit misleading, as it is really just a string
+// buffer, and can be used without Python.
 struct PythonPrintBuffer {
-  /* holds kernel print result before switching back to python */
   std::stringstream ss;
   bool enabled{false};
 
@@ -27,4 +29,4 @@ struct PythonPrintBuffer {
 
 extern PythonPrintBuffer py_cout;
 
-TI_NAMESPACE_END
+}  // namespace taichi

taichi/python/export_lang.cpp

@@ -40,9 +40,6 @@ void async_print_sfg();
 std::string async_dump_dot(std::optional<std::string> rankdir,
                            int embed_states_threshold);
 
-std::string compiled_lib_dir;
-std::string runtime_tmp_dir;
-
 Expr expr_index(const Expr &expr, const Expr &index) {
   return expr[index];
 }

taichi/python/export_misc.cpp

@@ -3,18 +3,19 @@
     The use of this software is governed by the LICENSE file.
 *******************************************************************************/
 
+#include "taichi/backends/cuda/detect_cuda.h"
 #include "taichi/backends/metal/api.h"
 #include "taichi/backends/opengl/opengl_api.h"
 #include "taichi/common/core.h"
 #include "taichi/common/interface.h"
 #include "taichi/common/task.h"
 #include "taichi/math/math.h"
+#include "taichi/program/py_print_buffer.h"
 #include "taichi/python/exception.h"
 #include "taichi/python/export.h"
-#include "taichi/python/print_buffer.h"
+#include "taichi/python/memory_usage_monitor.h"
 #include "taichi/system/benchmark.h"
 #include "taichi/system/dynamic_loader.h"
-#include "taichi/system/memory_usage_monitor.h"
 #include "taichi/system/profiler.h"
 #include "taichi/util/statistics.h"
 #if defined(TI_WITH_CUDA)
@@ -29,8 +30,6 @@ extern bool is_c_backend_available();
 
 TI_NAMESPACE_BEGIN
 
-PythonPrintBuffer py_cout;
-
 Config config_from_py_dict(py::dict &c) {
   Config config;
   for (auto item : c) {
@@ -86,14 +85,6 @@ void stop_duplicating_stdout_to_file(const std::string &fn) {
   TI_NOT_IMPLEMENTED;
 }
 
-bool is_cuda_api_available() {
-#if defined(TI_WITH_CUDA)
-  return lang::CUDADriver::get_instance_without_context().detected();
-#else
-  return false;
-#endif
-}
-
 void export_misc(py::module &m) {
   py::class_<Config>(m, "Config");
   py::register_exception_translator([](std::exception_ptr p) {

taichi/python/interfaces_registry.cpp

@@ -8,7 +8,7 @@
 namespace taichi {
 
 #define TI_INTERFACE_DEF_WITH_PYBIND11(class_name, base_alias)                \
-  TI_INTERFACE_DEF(class_name, base_alias)                                    \
+                                                                              \
   class InterfaceInjector_##class_name {                                      \
    public:                                                                    \
     InterfaceInjector_##class_name(const std::string &name) {                 \

taichi/system/memory_usage_monitor.cpp → taichi/python/memory_usage_monitor.cpp

@@ -1,12 +1,12 @@
-#include "taichi/system/memory_usage_monitor.h"
-#include "taichi/system/timer.h"
+#include "taichi/python/memory_usage_monitor.h"
+
+#include "pybind11/embed.h"
+#include "pybind11/pybind11.h"
 #include "taichi/common/core.h"
 #include "taichi/common/task.h"
+#include "taichi/math/scalar.h"
 #include "taichi/system/threading.h"
 #include "taichi/system/timer.h"
-#include "taichi/math/scalar.h"
-#include "pybind11/pybind11.h"
-#include "pybind11/embed.h"
 
 TI_NAMESPACE_BEGIN
 

taichi/system/memory_usage_monitor.h → taichi/python/memory_usage_monitor.h

@@ -1,4 +1,6 @@
-#include "virtual_memory.h"
+#pragma once
+
+#include "taichi/system/virtual_memory.h"
 
 TI_NAMESPACE_BEGIN
 

taichi/python/py_exception_translator.cpp

@@ -0,0 +1,31 @@
+#include <csignal>
+
+#include "pybind11/pybind11.h"
+
+namespace taichi {
+
+namespace {
+
+// Translates and propagates a CPP exception to Python.
+// TODO(#2198): How can we glob all these global variables used as initializaers
+// into a single function?
+class ExceptionTranslationImpl {
+ public:
+  explicit ExceptionTranslationImpl() {
+    pybind11::register_exception_translator([](std::exception_ptr p) {
+      try {
+        if (p)
+          std::rethrow_exception(p);
+      } catch (const std::string &e) {
+        PyErr_SetString(PyExc_RuntimeError, e.c_str());
+      } catch (const std::exception &e) {
+        PyErr_SetString(PyExc_RuntimeError, e.what());
+      }
+    });
+  }
+};
+
+ExceptionTranslationImpl _;
+
+}  // namespace
+}  // namespace taichi