CoreML support on MacOS

.github/workflows/build.yml

@@ -44,11 +44,11 @@ jobs:
         run: ./scripts/install_deps_mac.sh
       - name: build
         run: ./scripts/build_mac.sh
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: mac-archive
-          path: build/obs-virtualbg-*.zip
+      # - name: Upload artifacts
+      #   uses: actions/upload-artifact@v2
+      #   with:
+      #     name: mac-archive
+      #     path: build/obs-virtualbg-*.zip
   build_linux:
     name: build on Linux
     runs-on: ubuntu-latest

CMakeLists.txt

@@ -30,7 +30,7 @@ set(SOURCES
 
 add_executable(blur_generator src/blur_gen.cpp)
 target_link_libraries(blur_generator PRIVATE Halide::Generator)
-add_halide_library(blur FROM blur_generator GENERATOR blur_generator TARGETS cmake AUTOSCHEDULER Halide::Adams2019 SCHEDULE blur.schedule STMT_HTML blur_schedule.html)
+add_halide_library(blur FROM blur_generator GENERATOR blur_generator TARGETS host AUTOSCHEDULER Halide::Adams2019 SCHEDULE blur.schedule STMT_HTML blur_schedule.html)
 
 add_library(${CMAKE_PROJECT_NAME} MODULE
     ${SOURCES})
@@ -179,6 +179,8 @@ add_custom_command(TARGET ${CMAKE_PROJECT_NAME} POST_BUILD
   endif()
   include(CPack)
 elseif(APPLE)
+  target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE USE_COREML=1 OSX=1)
+  target_link_libraries(${CMAKE_PROJECT_NAME} "-framework CoreML")
   # set(CMAKE_OSX_DEPLOYMENT_TARGET 10.13)
   set(CMAKE_OSX_DEPLOYMENT_TARGET 11.0)
   # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libstdc++ -fvisibility=default -framework Foundation")

cmake/FindOnnxRuntime.cmake

@@ -1,14 +1,13 @@
 set(OnnxRuntime_DEPS_DIR ${CMAKE_SOURCE_DIR}/deps/onnxruntime)
 if(NOT WIN32 AND EXISTS ${OnnxRuntime_DEPS_DIR})
-    file(GLOB OnnxRuntime_LIBRARIES_EX ${OnnxRuntime_DEPS_DIR}/lib/lib*.a ${OnnxRuntime_DEPS_DIR}/lib/lib*.so)
+    file(GLOB OnnxRuntime_LIBRARIES_EX ${OnnxRuntime_DEPS_DIR}/lib/lib*)
     set(OnnxRuntime_LIBRARIES ${OnnxRuntime_LIBRARIES_EX})
     set(OnnxRuntime_INCLUDE_DIR ${OnnxRuntime_DEPS_DIR}/include)
     message("OnnxRuntime_DEPS_DIR Found!!!!!! ${OnnxRuntime_DEPS_DIR} ${OnnxRuntime_LIBRARIES}")
 else()
   message("kotti [${OnnxRuntimePath}]")
   find_path(OnnxRuntime_INCLUDE_DIR
       NAMES "onnxruntime_c_api.h"
-      "onnxruntime/core/session/onnxruntime_c_api.h"
       PATHS
           ENV OnnxRuntimePath
           ${OnnxRuntimePath}
@@ -19,7 +18,7 @@ else()
   )
 
   find_library(OnnxRuntime_LIBRARIES
-      NAMES onnxruntime.lib onnxruntime_providers_shared.lib onnxruntime_providers_cuda.lib libonnxruntime.dylib
+      NAMES onnxruntime.lib onnxruntime_providers_shared.lib onnxruntime_providers_cuda.lib libonnxruntime.dylib *.a
       PATHS
           ENV OnnxRuntimePath
           ${OnnxRuntimePath}

scripts/build_mac.sh

@@ -16,6 +16,8 @@ mkdir build
 pushd build
   # cmake .. -DobsPath=../deps/obs-studio -DLLVM_DIR=/usr/local/Cellar/llvm/12.0.1/lib/cmake/llvm
   cmake .. \
+    -DCMAKE_OSX_ARCHITECTURES="x86_64" \
+    -DCMAKE_APPLE_SILICON_PROCESSOR=x86_64 \
     -DobsLibPath=/Applications/OBS.app/Contents/Frameworks \
     -DobsIncludePath=$(cd ../deps/obs-studio/libobs; pwd) \
     -DOnnxRuntimePath=$(cd ../deps/onnxruntime; pwd) \

scripts/build_onnxruntime_mac.sh

@@ -3,18 +3,42 @@
 set -xe
 cd $(dirname $0)/..
 
-ORT_VERSION=1.10.0
+ORT_VERSION=1.11.0
 
 [ -d deps/onnxruntime ] && exit 0
 mkdir -p deps/onnxruntime
 
-git -C deps clone --single-branch --depth 1 -b v${ORT_VERSION} https://github.com/microsoft/onnxruntime.git
+# git -C deps clone --single-branch --depth 1 -b v${ORT_VERSION} https://github.com/microsoft/onnxruntime.git
+git -C deps clone --recursive --single-branch --depth 1 -b master https://github.com/microsoft/onnxruntime.git
 pushd deps/onnxruntime
-  ./build.sh --config RelWithDebInfo --parallel --skip_tests
+  ./build.sh --config RelWithDebInfo --parallel \
+    --use_coreml \
+    --skip_tests \
+    --cmake_extra_defines=onnxruntime_BUILD_UNIT_TESTS=OFF \
+    --osx_arch=x86_64 \
+    --cmake_extra_defines=CMAKE_APPLE_SILICON_PROCESSOR=x86_64 \
+    --cmake_extra_defines=CMAKE_OSX_DEPLOYMENT_TARGET=11.0
   pushd build/MacOS/RelWithDebInfo
     cmake ../../../cmake -DCMAKE_INSTALL_PREFIX=$(cd ../../../..; pwd)/onnxruntime
     cmake --build . --target install
     find . -name '*.a' -exec cp {} $(cd ../../../..; pwd)/onnxruntime/lib/ \;
     # cmake --install . --prefix=$(cd ../../../..; pwd)/onnxruntime --config RelWithDebInfo
   popd
+  pushd cmake/external/re2
+    mkdir b
+    pushd b
+      cmake -DCMAKE_APPLE_SILICON_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 ..
+      cmake --build . --target re2
+      cp libre2.a ../../../../lib/
+    popd
+  popd
+  pushd cmake/external/nsync
+    mkdir b
+    pushd b
+      cmake -DCMAKE_APPLE_SILICON_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 ..
+      cmake --build . --target nsync_cpp
+      cp libnsync_cpp.a ../../../../lib/
+    popd
+  popd
+  cp -f include/onnxruntime/core/session/*.h include/onnxruntime/core/providers/coreml/*.h include/
 popd

src/obs-virtualbg-detector.cpp

@@ -14,9 +14,11 @@
 #endif
 #endif
 
-#ifdef __APPLE__
-#include <onnxruntime/core/session/onnxruntime_cxx_api.h>
-// #include <onnxruntime/core/providers/cpu/cpu_provider_factory.h>
+#ifdef OSX
+#include <onnxruntime_c_api.h>
+
+#include <onnxruntime_cxx_api.h>
+#include <coreml_provider_factory.h>
 #else
 #include <onnxruntime_c_api.h>
 #include <onnxruntime_cxx_api.h>
@@ -100,8 +102,13 @@ void detector_setup_ort_session_gpu(Ort::SessionOptions &sessionOptions) {
          "error: %s",
          ex.what());
   }
-#else
-#if _WIN32
+#elif USE_COREML
+  try {
+    Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_CoreML(sessionOptions, 0));
+  } catch (const std::exception &ex) {
+    blog(LOG_ERROR,"[Virtual BG detector] Can't append Execution Provider CoreML. error: %s", ex.what());
+  }
+#elif _WIN32
   try {
     sessionOptions.DisableMemPattern();
     Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_DML(sessionOptions, 0));
@@ -113,7 +120,6 @@ void detector_setup_ort_session_gpu(Ort::SessionOptions &sessionOptions) {
          ex.what());
   }
 #endif
-#endif
 }
 
 void detector_setup_ort_session_load_model(virtual_bg_filter_data *filter_data,
@@ -219,7 +225,7 @@ void detector_update(void *data, obs_data_t *settings) {
   filter_data->threshold = (float)obs_data_get_double(settings, THRESHOLD_VALUE);
   filter_data->use_mask_blur = obs_data_get_bool(settings, USE_MASK_BLUR);
 
-#if defined(_WIN32) || USE_CUDA
+#if defined(_WIN32) || USE_CUDA || USE_COREML
   filter_data->use_gpu = obs_data_get_bool(settings, USE_GPU);
 #endif
 
@@ -262,7 +268,7 @@ void detector_defaults(obs_data_t *settings) {
   obs_data_set_default_bool(settings, USE_THRESHOLD, true);
   obs_data_set_default_double(settings, THRESHOLD_VALUE, 0.5);
   obs_data_set_default_bool(settings, USE_MASK_BLUR, true);
-#if defined(_WIN32) || USE_CUDA
+#if defined(_WIN32) || USE_CUDA || USE_COREML
   obs_data_set_default_bool(settings, USE_GPU, false);
 #endif
 }
@@ -273,7 +279,7 @@ obs_properties_t *detector_properties(void *data) {
   obs_properties_add_bool(ppts, USE_THRESHOLD, obs_module_text(USE_THRESHOLD));
   obs_properties_add_float_slider(ppts, THRESHOLD_VALUE, obs_module_text(THRESHOLD_VALUE), 0.0, 1.0, 0.05);
   obs_properties_add_bool(ppts, USE_MASK_BLUR, obs_module_text(USE_MASK_BLUR));
-#if defined(_WIN32) || USE_CUDA
+#if defined(_WIN32) || USE_CUDA || USE_COREML
   obs_properties_add_bool(ppts, USE_GPU, obs_module_text(USE_GPU));
 #endif