Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Gluten-825]Enable compiling in macOS. #827

Closed
wants to merge 1 commit into from
Closed

[Gluten-825]Enable compiling in macOS. #827

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions cpp/CMake/ConfigArrow.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,21 @@ if (${BUILD_VELOX_BACKEND} STREQUAL "ON" AND ${BUILD_GAZELLE_CPP_BACKEND} STREQU
endif()

if(${BUILD_VELOX_BACKEND} STREQUAL "ON")
set(ARROW_SHARED_LIBRARY_SUFFIX ".so.1000")
set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".so.1000.0.0")
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(ARROW_SHARED_LIBRARY_SUFFIX ".1000.dylib")
set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".1000.0.0.dylib")
else()
set(ARROW_SHARED_LIBRARY_SUFFIX ".so.1000")
set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".so.1000.0.0")
endif ()
else()
set(ARROW_SHARED_LIBRARY_SUFFIX ".so.800")
set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".so.800.0.0")
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(ARROW_SHARED_LIBRARY_SUFFIX ".800.dylib")
set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".800.0.0.dylib")
else()
set(ARROW_SHARED_LIBRARY_SUFFIX ".so.800")
set(ARROW_SHARED_LIBRARY_PARENT_SUFFIX ".so.800.0.0")
endif ()
endif()

set(ARROW_LIB_NAME "arrow")
Expand Down
12 changes: 9 additions & 3 deletions cpp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ add_compile_options(-Wno-error=parentheses)
add_compile_options(-Wno-error=unused-function)
add_compile_options(-Wno-error=unused-variable)
add_compile_options(-Wno-error=unused-but-set-variable)
add_compile_options(-Wno-error=unused-result)
add_compile_options(-Wno-error=nullability-completeness)
add_compile_options(-Wno-error=potentially-evaluated-expression)
add_compile_options(-Wno-error=mismatched-tags)
add_compile_options(-Wno-error=string-plus-int)

#
# Dependencies
Expand All @@ -95,8 +100,7 @@ find_package(Threads REQUIRED)
find_package(JNI REQUIRED)

if(BUILD_TESTS)
include(GoogleTest)
enable_testing()
find_package(GTest REQUIRED)
endif()

function(ADD_TEST_CASE TEST_NAME)
Expand All @@ -120,8 +124,9 @@ function(ADD_TEST_CASE TEST_NAME)
message(FATAL_ERROR "No sources specified for test ${TEST_NAME}")
endif()

enable_testing()
add_executable(${TEST_NAME} ${SOURCES})
target_link_libraries(${TEST_NAME} spark_columnar_jni gtest gtest_main Threads::Threads)
target_link_libraries(${TEST_NAME} spark_columnar_jni GTest::gtest GTest::gtest_main Threads::Threads)
target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/core)

if(ARG_EXTRA_LINK_LIBS)
Expand All @@ -136,6 +141,7 @@ function(ADD_TEST_CASE TEST_NAME)
add_dependencies(${TEST_NAME} ${ARG_EXTRA_DEPENDENCIES})
endif()

include(GoogleTest)
gtest_discover_tests(${TEST_NAME})
endfunction()

Expand Down
6 changes: 6 additions & 0 deletions cpp/core/benchmarks/CompressionBenchmark.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@
#include <execinfo.h>
#include <parquet/arrow/reader.h>
#include <parquet/file_reader.h>
#if !defined(__APPLE__)
#include <sched.h>
#endif
#include <sys/mman.h>

#include <chrono>
Expand Down Expand Up @@ -256,10 +258,14 @@ class BenchmarkCompression {

protected:
long SetCPU(uint32_t cpuindex) {
#if !defined(__APPLE__)
cpu_set_t cs;
CPU_ZERO(&cs);
CPU_SET(cpuindex, &cs);
return sched_setaffinity(0, sizeof(cs), &cs);
#else
return 0;
#endif
}

virtual void DoCompress(
Expand Down
10 changes: 7 additions & 3 deletions cpp/core/benchmarks/ShuffleSplitBenchmark.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@
#include <execinfo.h>
#include <parquet/arrow/reader.h>
#include <parquet/file_reader.h>
#if !defined(__APPLE__)
#include <sched.h>
#endif
#include <sys/mman.h>

#include <chrono>
Expand Down Expand Up @@ -290,10 +292,14 @@ class BenchmarkShuffleSplit {

protected:
long SetCPU(uint32_t cpuindex) {
#if !defined(__APPLE__)
cpu_set_t cs;
CPU_ZERO(&cs);
CPU_SET(cpuindex, &cs);
return sched_setaffinity(0, sizeof(cs), &cs);
#else
return 0;
#endif
}

virtual void Do_Split(
Expand Down Expand Up @@ -383,9 +389,7 @@ class BenchmarkShuffleSplit_CacheScan_Benchmark : public BenchmarkShuffleSplit {

for (auto _ : state) {
for_each(
batches.begin(),
batches.end(),
[&splitter, &split_time, &options](std::shared_ptr<arrow::RecordBatch>& record_batch) {
batches.begin(), batches.end(), [&splitter, &split_time](std::shared_ptr<arrow::RecordBatch>& record_batch) {
TIME_NANO_OR_THROW(split_time, splitter->Split(*record_batch));
});
// std::cout << " split done memory allocated = " <<
Expand Down
15 changes: 1 addition & 14 deletions cpp/core/jni/JniWrapper.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
*/

#include <jni.h>
#include <malloc.h>
#include <filesystem>

#include "compute/Backend.h"
Expand Down Expand Up @@ -736,7 +735,7 @@ JNIEXPORT jlong JNICALL Java_io_glutenproject_vectorized_ShuffleSplitterJniWrapp
std::string error_message = "Invalid splitter id " + std::to_string(splitter_id);
gluten::JniThrow(error_message);
}
jlong spilled_size;
int64_t spilled_size;
gluten::JniAssertOkOrThrow(splitter->SpillFixedSize(size, &spilled_size), "(shuffle) nativeSpill: spill failed");
return spilled_size;
JNI_METHOD_END(-1L)
Expand Down Expand Up @@ -908,18 +907,6 @@ Java_io_glutenproject_memory_alloc_NativeMemoryAllocator_bytesAllocated(JNIEnv*
JNI_METHOD_END(-1L)
}

JNIEXPORT void JNICALL Java_io_glutenproject_tpc_MallocUtils_mallocTrim(JNIEnv* env, jobject obj) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After search all project about this interface, I can not find any usage.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@zhztheplayer Can we remove the API?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1, it should be removed.

// malloc_stats_print(statsPrint, nullptr, nullptr);
std::cout << "Calling malloc_trim... " << std::endl;
malloc_trim(0);
}

JNIEXPORT void JNICALL Java_io_glutenproject_tpc_MallocUtils_mallocStats(JNIEnv* env, jobject obj) {
// malloc_stats_print(statsPrint, nullptr, nullptr);
std::cout << "Calling malloc_stats... " << std::endl;
malloc_stats();
}

#ifdef __cplusplus
}
#endif
5 changes: 5 additions & 0 deletions cpp/velox/benchmarks/BenchmarkUtils.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
#include <filesystem>
#include <fstream>
#include <sstream>
#if !defined(__APPLE__)
#include <sched.h>
#endif
#include <thread>

#include "compute/VeloxBackend.h"
Expand Down Expand Up @@ -157,11 +160,13 @@ std::shared_ptr<gluten::ResultIterator> getInputFromBatchStream(const std::strin
void setCpu(uint32_t cpuindex) {
static const auto total_cores = std::thread::hardware_concurrency();
cpuindex = cpuindex % total_cores;
#if !defined(__APPLE__)
cpu_set_t cs;
CPU_ZERO(&cs);
CPU_SET(cpuindex, &cs);
if (sched_setaffinity(0, sizeof(cs), &cs) == -1) {
std::cerr << "Error binding CPU " << std::to_string(cpuindex) << std::endl;
exit(EXIT_FAILURE);
}
#endif
}
1 change: 0 additions & 1 deletion cpp/velox/compute/DwrfDatasource.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ class DwrfDatasource {
private:
std::string file_path_;
std::string final_path_;
int32_t count_ = 0;
int64_t num_rbs_ = 0;
std::shared_ptr<arrow::Schema> schema_;
std::vector<facebook::velox::RowVectorPtr> row_vecs_;
Expand Down
1 change: 0 additions & 1 deletion cpp/velox/compute/VeloxBackend.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,6 @@ class WholeStageResIter {
std::shared_ptr<facebook::velox::memory::MemoryPool> pool_;

std::shared_ptr<Metrics> metrics_ = nullptr;
int64_t metricVeloxToArrowNanos_ = 0;

/// All the children plan node ids with postorder traversal.
std::vector<facebook::velox::core::PlanNodeId> orderedNodeIds_;
Expand Down
6 changes: 3 additions & 3 deletions cpp/velox/tests/ShuffleSplitTest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ class MyMemoryPool final : public arrow::MemoryPool {

class SplitterTest : public ::testing::Test {
protected:
void SetUp() {
void SetUp() override {
auto hash_partition_key = field("hash_partition_key", arrow::int32());
auto f_na = field("f_na", arrow::null());
auto f_int8_a = field("f_int8_a", arrow::int8());
Expand All @@ -119,8 +119,8 @@ class SplitterTest : public ::testing::Test {
auto f_nullable_string = field("f_nullable_string", arrow::utf8());
auto f_decimal = field("f_decimal128", arrow::decimal(10, 2));

ARROW_ASSIGN_OR_THROW(tmp_dir_1_, std::move(arrow::internal::TemporaryDir::Make(tmp_dir_prefix)))
ARROW_ASSIGN_OR_THROW(tmp_dir_2_, std::move(arrow::internal::TemporaryDir::Make(tmp_dir_prefix)))
ARROW_ASSIGN_OR_THROW(tmp_dir_1_, arrow::internal::TemporaryDir::Make(tmp_dir_prefix))
ARROW_ASSIGN_OR_THROW(tmp_dir_2_, arrow::internal::TemporaryDir::Make(tmp_dir_prefix))
auto config_dirs = tmp_dir_1_->path().ToString() + "," + tmp_dir_2_->path().ToString();

setenv("NATIVESQL_SPARK_LOCAL_DIRS", config_dirs.c_str(), 1);
Expand Down