Build for GPU on CircleCI

.circleci/dist_compile.yml

@@ -287,6 +287,11 @@ jobs:
           command: |
             make benchmarks-build NUM_THREADS=8 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=4
           no_output_timeout: 1h
+      - run:
+          name: "Build Velox With GPU support"
+          command: |
+            make gpu NUM_THREADS=8 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=4 CUDA_ARCHITECTURES=90 CUDA_COMPILER=/usr/local/cuda-12.3/bin/nvcc
+          no_output_timeout: 1h
       - post-steps
 
   linux-adapters:

.github/workflows/docker.yml

@@ -66,6 +66,16 @@ jobs:
             tags: "ghcr.io/facebookincubator/velox-dev:presto-java"
 
     steps:
+      - name: "Checkout Repo"
+        uses: actions/checkout@v3
+        with:
+          sparse-checkout: |
+            scripts
+
+      - name: Free up disk space
+        run: |
+          ./scripts/gha-free-space.sh
+
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:

CMake/ResolveDependency.cmake

@@ -102,6 +102,10 @@ function(set_with_default var_name envvar_name default)
     set(${var_name}
         $ENV{${envvar_name}}
         PARENT_SCOPE)
+  elseif(DEFINED ${envvar_name})
+    set(${var_name}
+        ${${envvar_name}}
+        PARENT_SCOPE)
   else()
     set(${var_name}
         ${default}

CMake/resolve_dependency_modules/folly/CMakeLists.txt

@@ -28,13 +28,18 @@ message(STATUS "Building Folly from source")
 if(gflags_SOURCE STREQUAL "BUNDLED")
   set(glog_patch && git apply ${CMAKE_CURRENT_LIST_DIR}/folly-gflags-glog.patch)
 endif()
+if(VELOX_ENABLE_GPU)
+  set(cudacc_patch && git apply ${CMAKE_CURRENT_LIST_DIR}/folly-cudacc.patch)
+endif()
+
+set(VELOX_FOLLY_PATCH_COMMAND git apply ${CMAKE_CURRENT_LIST_DIR}/folly-no-export.patch
+                                ${glog_patch} ${cudacc_patch})
 
 FetchContent_Declare(
   folly
   URL ${VELOX_FOLLY_SOURCE_URL}
   URL_HASH ${VELOX_FOLLY_BUILD_SHA256_CHECKSUM}
-  PATCH_COMMAND git apply ${CMAKE_CURRENT_LIST_DIR}/folly-no-export.patch
-                ${glog_patch})
+  PATCH_COMMAND ${VELOX_FOLLY_PATCH_COMMAND})
 
 if(ON_APPLE_M1)
   # folly will wrongly assume x86_64 if this is not set

CMake/resolve_dependency_modules/folly/folly-cudacc.patch

@@ -0,0 +1,68 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# Copyright (c) Rivos, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+--- a/folly/Conv.h
++++ b/folly/Conv.h
+@@ -286,8 +286,8 @@ namespace detail {
+ template <class... T>
+ using LastElement = type_pack_element_t<sizeof...(T) - 1, T...>;
+
+-#ifdef _MSC_VER
+-// MSVC can't quite figure out the LastElementImpl::call() stuff
++#if defined(_MSC_VER) || defined(__CUDACC__)
++// MSVC and NVCC can't quite figure out the LastElementImpl::call() stuff
+ // in the base implementation, so we have to use tuples instead,
+ // which result in significantly more templates being compiled,
+ // though the runtime performance is the same.
+--- a/folly/synchronization/RelaxedAtomic.h
++++ b/folly/synchronization/RelaxedAtomic.h
+@@ -98,7 +98,7 @@ struct relaxed_atomic_base : protected std::atomic<T> {
+ };
+
+ template <typename T>
+-struct relaxed_atomic_integral_base : private relaxed_atomic_base<T> {
++struct relaxed_atomic_integral_base : protected relaxed_atomic_base<T> {
+  private:
+   using atomic = std::atomic<T>;
+   using base = relaxed_atomic_base<T>;
+@@ -108,7 +108,9 @@ struct relaxed_atomic_integral_base : private relaxed_atomic_base<T> {
+
+   using base::relaxed_atomic_base;
+   using base::operator=;
++#ifndef __CUDACC__
+   using base::operator T;
++#endif
+   using base::compare_exchange_strong;
+   using base::compare_exchange_weak;
+   using base::exchange;
+@@ -206,7 +208,9 @@ struct relaxed_atomic : detail::relaxed_atomic_base<T> {
+
+   using base::relaxed_atomic_base;
+   using base::operator=;
++#ifndef __CUDACC__
+   using base::operator T;
++#endif
+ };
+
+ template <typename T>
+@@ -220,7 +224,9 @@ struct relaxed_atomic<T*> : detail::relaxed_atomic_base<T*> {
+
+   using detail::relaxed_atomic_base<T*>::relaxed_atomic_base;
+   using base::operator=;
++#ifndef __CUDACC__
+   using base::operator T*;
++#endif
+
+   T* fetch_add(std::ptrdiff_t arg) noexcept {
+     return atomic::fetch_add(arg, std::memory_order_relaxed);

Makefile

@@ -50,6 +50,14 @@ ifdef AZURESDK_ROOT_DIR
 CMAKE_FLAGS += -DAZURESDK_ROOT_DIR=$(AZURESDK_ROOT_DIR)
 endif
 
+ifdef CUDA_ARCHITECTURES
+CMAKE_FLAGS += -DCMAKE_CUDA_ARCHITECTURES="$(CUDA_ARCHITECTURES)"
+endif
+
+ifdef CUDA_COMPILER
+CMAKE_FLAGS += -DCMAKE_CUDA_COMPILER="$(CUDA_COMPILER)"
+endif
+
 # Use Ninja if available. If Ninja is used, pass through parallelism control flags.
 USE_NINJA ?= 1
 ifeq ($(USE_NINJA), 1)
@@ -108,6 +116,14 @@ minimal:				 #: Minimal build
 	$(MAKE) cmake BUILD_DIR=release BUILD_TYPE=release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DVELOX_BUILD_MINIMAL=ON"
 	$(MAKE) build BUILD_DIR=release
 
+gpu:						 #: Build with GPU support
+	$(MAKE) cmake BUILD_DIR=release BUILD_TYPE=release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DVELOX_ENABLE_GPU=ON -Dfolly_SOURCE=BUNDLED"
+	$(MAKE) build BUILD_DIR=release
+
+gpu_debug:			 #: Build with debugging symbols and GPU support
+	$(MAKE) cmake BUILD_DIR=debug BUILD_TYPE=debug EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} -DVELOX_ENABLE_GPU=ON -Dfolly_SOURCE=BUNDLED"
+	$(MAKE) build BUILD_DIR=debug
+
 dwio:						#: Minimal build with dwio enabled.
 	$(MAKE) cmake BUILD_DIR=release BUILD_TYPE=release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS} \
 																										    							  -DVELOX_BUILD_MINIMAL_WITH_DWIO=ON"

scripts/circleci-container.dockfile

@@ -16,6 +16,7 @@
 FROM quay.io/centos/centos:stream8
 ARG cpu_target
 ENV CPU_TARGET=$cpu_target
+ENV AGGRESSIVE_CLEANUP=true
 
 COPY scripts/setup-helper-functions.sh /
 COPY scripts/setup-centos8.sh /

scripts/gha-free-space.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eux
+
+df -h
+# Remove Web browser packages
+sudo apt purge -y \
+  firefox \
+  google-chrome-stable \
+  microsoft-edge-stable
+# Remove things in /usr/local
+echo "::group::/usr/local/*"
+du -msc /usr/local/* | sort -n
+# ~1GB
+sudo rm -rf \
+  /usr/local/aws-cil \
+  /usr/local/aws-sam-cil \
+  /usr/local/julia* || :
+echo "::endgroup::"
+echo "::group::/usr/local/bin/*"
+du -msc /usr/local/bin/* | sort -n
+# ~1GB (From 1.2GB to 214MB)
+sudo rm -rf \
+  /usr/local/bin/aliyun \
+  /usr/local/bin/azcopy \
+  /usr/local/bin/bicep \
+  /usr/local/bin/cmake-gui \
+  /usr/local/bin/cpack \
+  /usr/local/bin/helm \
+  /usr/local/bin/hub \
+  /usr/local/bin/kubectl \
+  /usr/local/bin/minikube \
+  /usr/local/bin/node \
+  /usr/local/bin/packer \
+  /usr/local/bin/pulumi* \
+  /usr/local/bin/sam \
+  /usr/local/bin/stack \
+  /usr/local/bin/terraform || :
+# 142M
+sudo rm -rf /usr/local/bin/oc || :
+echo "::endgroup::"
+echo "::group::/usr/local/share/*"
+du -msc /usr/local/share/* | sort -n
+# 506MB
+sudo rm -rf /usr/local/share/chromium || :
+# 1.3GB
+sudo rm -rf /usr/local/share/powershell || :
+echo "::endgroup::"
+echo "::group::/usr/local/lib/*"
+du -msc /usr/local/lib/* | sort -n
+# 15GB
+sudo rm -rf /usr/local/lib/android || :
+# 341MB
+sudo rm -rf /usr/local/lib/heroku || :
+# 1.2GB
+sudo rm -rf /usr/local/lib/node_modules || :
+echo "::endgroup::"
+echo "::group::/opt/*"
+du -msc /opt/* | sort -n
+# 679MB
+sudo rm -rf /opt/az || :
+echo "::endgroup::"
+echo "::group::/opt/microsoft/*"
+du -msc /opt/microsoft/* | sort -n
+# 197MB
+sudo rm -rf /opt/microsoft/powershell || :
+echo "::endgroup::"
+echo "::group::/opt/hostedtoolcache/*"
+du -msc /opt/hostedtoolcache/* | sort -n
+# 5.3GB
+sudo rm -rf /opt/hostedtoolcache/CodeQL || :
+# 1.4GB
+sudo rm -rf /opt/hostedtoolcache/go || :
+# 489MB
+sudo rm -rf /opt/hostedtoolcache/PyPy || :
+# 376MB
+sudo rm -rf /opt/hostedtoolcache/node || :
+echo "::endgroup::"
+df -h

scripts/setup-adapters.sh

@@ -24,13 +24,18 @@ source $SCRIPTDIR/setup-helper-functions.sh
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
 MACHINE=$(uname -m)
+AGGRESSIVE_CLEANUP="${AGGRESSIVE_CLEANUP:-false}"
 
 function install_aws_deps {
   local AWS_REPO_NAME="aws/aws-sdk-cpp"
   local AWS_SDK_VERSION="1.11.169"
 
   github_checkout $AWS_REPO_NAME $AWS_SDK_VERSION --depth 1 --recurse-submodules
   cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS:BOOL=OFF -DMINIMIZE_SIZE:BOOL=ON -DENABLE_TESTING:BOOL=OFF -DBUILD_ONLY:STRING="s3;identity-management"
+  if [ $AGGRESSIVE_CLEANUP = "true" ]; then
+    rm -rf $(basename ${AWS_REPO_NAME})
+  fi
+
   # Dependencies for S3 testing
   # We need this specific version of Minio for testing.
   if [[ "$OSTYPE" == linux-gnu* ]]; then
@@ -62,25 +67,37 @@ function install_gcs-sdk-cpp {
   sed -i 's/^#define ABSL_OPTION_USE_\(.*\) 2/#define ABSL_OPTION_USE_\1 0/' "absl/base/options.h"
   cmake_install -DBUILD_SHARED_LIBS=OFF \
     -DABSL_BUILD_TESTING=OFF
+  if [ $AGGRESSIVE_CLEANUP = "true" ]; then
+    rm -rf abseil-cpp
+  fi
 
   # crc32
   github_checkout google/crc32c 1.1.2 --depth 1
   cmake_install -DBUILD_SHARED_LIBS=OFF \
     -DCRC32C_BUILD_TESTS=OFF \
     -DCRC32C_BUILD_BENCHMARKS=OFF \
     -DCRC32C_USE_GLOG=OFF
+  if [ $AGGRESSIVE_CLEANUP = "true" ]; then
+    rm -rf crc32c
+  fi
 
   # nlohmann json
   github_checkout nlohmann/json v3.11.2 --depth 1
   cmake_install -DBUILD_SHARED_LIBS=OFF \
     -DJSON_BuildTests=OFF
+  if [ $AGGRESSIVE_CLEANUP = "true" ]; then
+    rm -rf json
+  fi
 
   # google-cloud-cpp
   github_checkout googleapis/google-cloud-cpp v2.10.1 --depth 1
   cmake_install -DBUILD_SHARED_LIBS=OFF \
     -DCMAKE_INSTALL_MESSAGE=NEVER \
     -DGOOGLE_CLOUD_CPP_ENABLE_EXAMPLES=OFF \
     -DGOOGLE_CLOUD_CPP_ENABLE=storage
+  if [ $AGGRESSIVE_CLEANUP = "true" ]; then
+    rm -rf google-cloud-cpp
+  fi
 }
 
 function install_azure-storage-sdk-cpp {
@@ -117,6 +134,10 @@ function install_azure-storage-sdk-cpp {
   # install azure-storage-files-datalake
   cd sdk/storage/azure-storage-files-datalake
   cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF
+
+  if [ $AGGRESSIVE_CLEANUP = "true" ]; then
+    rm -rf azure-sdk-for-cpp
+  fi
 }
 
 function install_hdfs_deps {
@@ -132,9 +153,13 @@ function install_hdfs_deps {
     sed -i "s/dumpversion/dumpfullversion/" ./CMake/Platform.cmake
     # Dependencies for Hadoop testing
     wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop
-    cp -a hadoop /usr/local/
+    mv hadoop /usr/local/
   fi
   cmake_install
+
+  if [ $AGGRESSIVE_CLEANUP = "true" ]; then
+    rm -rf hawq
+  fi
 }
 
 cd "${DEPENDENCY_DIR}" || exit