From 26b475ef3f73009808f0fba04ced81f8824053f4 Mon Sep 17 00:00:00 2001 From: yangchuan Date: Tue, 10 Jan 2023 11:31:53 +0800 Subject: [PATCH] Enable compiling in macOS. Add some no-error compile options. --- cpp/CMake/ConfigArrow.cmake | 18 ++++++++++++++---- cpp/CMakeLists.txt | 12 +++++++++--- cpp/core/benchmarks/CompressionBenchmark.cc | 6 ++++++ cpp/core/benchmarks/ShuffleSplitBenchmark.cc | 10 +++++++--- cpp/core/jni/JniWrapper.cc | 15 +-------------- cpp/velox/benchmarks/BenchmarkUtils.cc | 5 +++++ cpp/velox/compute/DwrfDatasource.h | 1 - cpp/velox/compute/VeloxBackend.h | 1 - cpp/velox/tests/ShuffleSplitTest.cc | 6 +++--- 9 files changed, 45 insertions(+), 29 deletions(-) diff --git a/cpp/CMake/ConfigArrow.cmake b/cpp/CMake/ConfigArrow.cmake index def19463ea88..8644b4da919f 100644 --- a/cpp/CMake/ConfigArrow.cmake +++ b/cpp/CMake/ConfigArrow.cmake @@ -21,11 +21,21 @@ if (${BUILD_VELOX_BACKEND} STREQUAL "ON" AND ${BUILD_GAZELLE_CPP_BACKEND} STREQU endif() if(${BUILD_VELOX_BACKEND} STREQUAL "ON") - set(ARROW_SHARED_LIBRARY_SUFFIX ".so.1000") - set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".so.1000.0.0") + if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(ARROW_SHARED_LIBRARY_SUFFIX ".1000.dylib") + set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".1000.0.0.dylib") + else() + set(ARROW_SHARED_LIBRARY_SUFFIX ".so.1000") + set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".so.1000.0.0") + endif () else() - set(ARROW_SHARED_LIBRARY_SUFFIX ".so.800") - set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".so.800.0.0") + if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(ARROW_SHARED_LIBRARY_SUFFIX ".800.dylib") + set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".800.0.0.dylib") + else() + set(ARROW_SHARED_LIBRARY_SUFFIX ".so.800") + set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".so.800.0.0") + endif () endif() set(ARROW_LIB_NAME "arrow") diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 93745fa3716c..ca3cd07120d0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -77,6 +77,11 @@ add_compile_options(-Wno-error=parentheses) add_compile_options(-Wno-error=unused-function) add_compile_options(-Wno-error=unused-variable) add_compile_options(-Wno-error=unused-but-set-variable) +add_compile_options(-Wno-error=unused-result) +add_compile_options(-Wno-error=nullability-completeness) +add_compile_options(-Wno-error=potentially-evaluated-expression) +add_compile_options(-Wno-error=mismatched-tags) +add_compile_options(-Wno-error=string-plus-int) # # Dependencies @@ -95,8 +100,7 @@ find_package(Threads REQUIRED) find_package(JNI REQUIRED) if(BUILD_TESTS) - include(GoogleTest) - enable_testing() + find_package(GTest REQUIRED) endif() function(ADD_TEST_CASE TEST_NAME) @@ -120,8 +124,9 @@ function(ADD_TEST_CASE TEST_NAME) message(FATAL_ERROR "No sources specified for test ${TEST_NAME}") endif() + enable_testing() add_executable(${TEST_NAME} ${SOURCES}) - target_link_libraries(${TEST_NAME} spark_columnar_jni gtest gtest_main Threads::Threads) + target_link_libraries(${TEST_NAME} spark_columnar_jni GTest::gtest GTest::gtest_main Threads::Threads) target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/core) if(ARG_EXTRA_LINK_LIBS) @@ -136,6 +141,7 @@ function(ADD_TEST_CASE TEST_NAME) add_dependencies(${TEST_NAME} ${ARG_EXTRA_DEPENDENCIES}) endif() + include(GoogleTest) gtest_discover_tests(${TEST_NAME}) endfunction() diff --git a/cpp/core/benchmarks/CompressionBenchmark.cc b/cpp/core/benchmarks/CompressionBenchmark.cc index 2093e9863e9b..606cbffb3cc1 100644 --- a/cpp/core/benchmarks/CompressionBenchmark.cc +++ b/cpp/core/benchmarks/CompressionBenchmark.cc @@ -24,7 +24,9 @@ #include #include #include +#if !defined(__APPLE__) #include +#endif #include #include @@ -256,10 +258,14 @@ class BenchmarkCompression { protected: long SetCPU(uint32_t cpuindex) { +#if !defined(__APPLE__) cpu_set_t cs; CPU_ZERO(&cs); CPU_SET(cpuindex, &cs); return sched_setaffinity(0, sizeof(cs), &cs); +#else + return 0; +#endif } virtual void DoCompress( diff --git a/cpp/core/benchmarks/ShuffleSplitBenchmark.cc b/cpp/core/benchmarks/ShuffleSplitBenchmark.cc index 50b314cca39a..1879f75a1a80 100644 --- a/cpp/core/benchmarks/ShuffleSplitBenchmark.cc +++ b/cpp/core/benchmarks/ShuffleSplitBenchmark.cc @@ -25,7 +25,9 @@ #include #include #include +#if !defined(__APPLE__) #include +#endif #include #include @@ -290,10 +292,14 @@ class BenchmarkShuffleSplit { protected: long SetCPU(uint32_t cpuindex) { +#if !defined(__APPLE__) cpu_set_t cs; CPU_ZERO(&cs); CPU_SET(cpuindex, &cs); return sched_setaffinity(0, sizeof(cs), &cs); +#else + return 0; +#endif } virtual void Do_Split( @@ -383,9 +389,7 @@ class BenchmarkShuffleSplit_CacheScan_Benchmark : public BenchmarkShuffleSplit { for (auto _ : state) { for_each( - batches.begin(), - batches.end(), - [&splitter, &split_time, &options](std::shared_ptr& record_batch) { + batches.begin(), batches.end(), [&splitter, &split_time](std::shared_ptr& record_batch) { TIME_NANO_OR_THROW(split_time, splitter->Split(*record_batch)); }); // std::cout << " split done memory allocated = " << diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index fe4387d673d2..0ccfabe43fdb 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -16,7 +16,6 @@ */ #include -#include #include #include "compute/Backend.h" @@ -736,7 +735,7 @@ JNIEXPORT jlong JNICALL Java_io_glutenproject_vectorized_ShuffleSplitterJniWrapp std::string error_message = "Invalid splitter id " + std::to_string(splitter_id); gluten::JniThrow(error_message); } - jlong spilled_size; + int64_t spilled_size; gluten::JniAssertOkOrThrow(splitter->SpillFixedSize(size, &spilled_size), "(shuffle) nativeSpill: spill failed"); return spilled_size; JNI_METHOD_END(-1L) @@ -908,18 +907,6 @@ Java_io_glutenproject_memory_alloc_NativeMemoryAllocator_bytesAllocated(JNIEnv* JNI_METHOD_END(-1L) } -JNIEXPORT void JNICALL Java_io_glutenproject_tpc_MallocUtils_mallocTrim(JNIEnv* env, jobject obj) { - // malloc_stats_print(statsPrint, nullptr, nullptr); - std::cout << "Calling malloc_trim... " << std::endl; - malloc_trim(0); -} - -JNIEXPORT void JNICALL Java_io_glutenproject_tpc_MallocUtils_mallocStats(JNIEnv* env, jobject obj) { - // malloc_stats_print(statsPrint, nullptr, nullptr); - std::cout << "Calling malloc_stats... " << std::endl; - malloc_stats(); -} - #ifdef __cplusplus } #endif diff --git a/cpp/velox/benchmarks/BenchmarkUtils.cc b/cpp/velox/benchmarks/BenchmarkUtils.cc index 665a53c51658..9ace495089fa 100644 --- a/cpp/velox/benchmarks/BenchmarkUtils.cc +++ b/cpp/velox/benchmarks/BenchmarkUtils.cc @@ -25,6 +25,9 @@ #include #include #include +#if !defined(__APPLE__) +#include +#endif #include #include "compute/VeloxBackend.h" @@ -157,6 +160,7 @@ std::shared_ptr getInputFromBatchStream(const std::strin void setCpu(uint32_t cpuindex) { static const auto total_cores = std::thread::hardware_concurrency(); cpuindex = cpuindex % total_cores; +#if !defined(__APPLE__) cpu_set_t cs; CPU_ZERO(&cs); CPU_SET(cpuindex, &cs); @@ -164,4 +168,5 @@ void setCpu(uint32_t cpuindex) { std::cerr << "Error binding CPU " << std::to_string(cpuindex) << std::endl; exit(EXIT_FAILURE); } +#endif } diff --git a/cpp/velox/compute/DwrfDatasource.h b/cpp/velox/compute/DwrfDatasource.h index 04db72cfbdf6..b9a5f76f31e6 100644 --- a/cpp/velox/compute/DwrfDatasource.h +++ b/cpp/velox/compute/DwrfDatasource.h @@ -53,7 +53,6 @@ class DwrfDatasource { private: std::string file_path_; std::string final_path_; - int32_t count_ = 0; int64_t num_rbs_ = 0; std::shared_ptr schema_; std::vector row_vecs_; diff --git a/cpp/velox/compute/VeloxBackend.h b/cpp/velox/compute/VeloxBackend.h index 7a40ec7b8683..49a6bb9a7480 100644 --- a/cpp/velox/compute/VeloxBackend.h +++ b/cpp/velox/compute/VeloxBackend.h @@ -97,7 +97,6 @@ class WholeStageResIter { std::shared_ptr pool_; std::shared_ptr metrics_ = nullptr; - int64_t metricVeloxToArrowNanos_ = 0; /// All the children plan node ids with postorder traversal. std::vector orderedNodeIds_; diff --git a/cpp/velox/tests/ShuffleSplitTest.cc b/cpp/velox/tests/ShuffleSplitTest.cc index 004ed6e3f867..3482ddb727c3 100644 --- a/cpp/velox/tests/ShuffleSplitTest.cc +++ b/cpp/velox/tests/ShuffleSplitTest.cc @@ -106,7 +106,7 @@ class MyMemoryPool final : public arrow::MemoryPool { class SplitterTest : public ::testing::Test { protected: - void SetUp() { + void SetUp() override { auto hash_partition_key = field("hash_partition_key", arrow::int32()); auto f_na = field("f_na", arrow::null()); auto f_int8_a = field("f_int8_a", arrow::int8()); @@ -119,8 +119,8 @@ class SplitterTest : public ::testing::Test { auto f_nullable_string = field("f_nullable_string", arrow::utf8()); auto f_decimal = field("f_decimal128", arrow::decimal(10, 2)); - ARROW_ASSIGN_OR_THROW(tmp_dir_1_, std::move(arrow::internal::TemporaryDir::Make(tmp_dir_prefix))) - ARROW_ASSIGN_OR_THROW(tmp_dir_2_, std::move(arrow::internal::TemporaryDir::Make(tmp_dir_prefix))) + ARROW_ASSIGN_OR_THROW(tmp_dir_1_, arrow::internal::TemporaryDir::Make(tmp_dir_prefix)) + ARROW_ASSIGN_OR_THROW(tmp_dir_2_, arrow::internal::TemporaryDir::Make(tmp_dir_prefix)) auto config_dirs = tmp_dir_1_->path().ToString() + "," + tmp_dir_2_->path().ToString(); setenv("NATIVESQL_SPARK_LOCAL_DIRS", config_dirs.c_str(), 1);