Fix OpenMP on macOS (#6114)

* Fix OpenMP on macOS

* Remove hardcode

* Adopt the way from LightGBM

CMakeLists.txt

@@ -289,8 +289,28 @@ endif()
 
 # OpenMP (optional)
 option(WITH_OPENMP "Build with parallelization using OpenMP" TRUE)
+option(USE_HOMEBREW_FALLBACK "(macOS-only) also look in 'brew --prefix' for libraries (e.g. OpenMP)" TRUE)
 if(WITH_OPENMP)
-  find_package(OpenMP COMPONENTS C CXX)
+  if(APPLE)
+    find_package(OpenMP COMPONENTS C CXX)
+    if(NOT OpenMP_FOUND)
+      if(USE_HOMEBREW_FALLBACK)
+        # libomp 15.0+ from brew is keg-only, so have to search in other locations.
+        # See https://github.com/Homebrew/homebrew-core/issues/112107#issuecomment-1278042927.
+        execute_process(COMMAND brew --prefix libomp
+                        OUTPUT_VARIABLE HOMEBREW_LIBOMP_PREFIX
+                        OUTPUT_STRIP_TRAILING_WHITESPACE)
+        set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_LIBOMP_PREFIX}/include")
+        set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I${HOMEBREW_LIBOMP_PREFIX}/include")
+        set(OpenMP_C_LIB_NAMES omp)
+        set(OpenMP_CXX_LIB_NAMES omp)
+        set(OpenMP_omp_LIBRARY ${HOMEBREW_LIBOMP_PREFIX}/lib/libomp.dylib)
+      endif()
+      find_package(OpenMP COMPONENTS C CXX)
+    endif()
+  else()
+    find_package(OpenMP COMPONENTS C CXX)
+  endif()
 endif()
 if(OpenMP_FOUND)
   string(APPEND CMAKE_C_FLAGS " ${OpenMP_C_FLAGS}")