ARROW-17427: [Java] Add Windows build script that produces DLLs (#14203)

Authored-by: Sutou Kouhei <kou@clear-code.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

ci/scripts/java_build.sh

@@ -87,7 +87,7 @@ if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
   ${mvn} -Darrow.c.jni.dist.dir=${java_jni_dist_dir} -Parrow-c-data install
 fi
 
-if [ "${ARROW_JNI}" = "ON" ]; then
+if [ "${ARROW_JAVA_JNI}" = "ON" ]; then
   ${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} -Parrow-jni install
 fi
 

ci/scripts/java_jni_build.sh

@@ -50,10 +50,10 @@ esac
 : ${ARROW_JAVA_BUILD_TESTS:=${ARROW_BUILD_TESTS:-OFF}}
 : ${CMAKE_BUILD_TYPE:=release}
 cmake \
-  -DARROW_JAVA_JNI_ENABLE_DATASET=${ARROW_DATASET:-ON} \
-  -DARROW_JAVA_JNI_ENABLE_GANDIVA=${ARROW_GANDIVA:-ON} \
-  -DARROW_JAVA_JNI_ENABLE_ORC=${ARROW_ORC:-ON} \
-  -DARROW_JAVA_JNI_ENABLE_PLASMA=${ARROW_PLASMA:-ON} \
+  -DARROW_JAVA_JNI_ENABLE_DATASET=${ARROW_DATASET:-OFF} \
+  -DARROW_JAVA_JNI_ENABLE_GANDIVA=${ARROW_GANDIVA:-OFF} \
+  -DARROW_JAVA_JNI_ENABLE_ORC=${ARROW_ORC:-OFF} \
+  -DARROW_JAVA_JNI_ENABLE_PLASMA=${ARROW_PLASMA:-OFF} \
   -DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \
   -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
   -DCMAKE_PREFIX_PATH=${arrow_install_dir} \
@@ -75,4 +75,9 @@ cmake --build . --config ${CMAKE_BUILD_TYPE} --target install
 popd
 
 mkdir -p ${dist_dir}
-mv ${prefix_dir}/lib/* ${dist_dir}/
+# For Windows. *.dll are installed into bin/ on Windows.
+if [ -d "${prefix_dir}/bin" ]; then
+  mv ${prefix_dir}/bin/* ${dist_dir}/
+else
+  mv ${prefix_dir}/lib/* ${dist_dir}/
+fi

ci/scripts/java_jni_macos_build.sh

@@ -32,12 +32,14 @@ echo "=== Building Arrow C++ libraries ==="
 install_dir=${build_dir}/cpp-install
 : ${ARROW_BUILD_TESTS:=ON}
 : ${ARROW_DATASET:=ON}
-: ${ARROW_FILESYSTEM:=ON}
+export ARROW_DATASET
 : ${ARROW_GANDIVA:=ON}
+export ARROW_GANDIVA
 : ${ARROW_ORC:=ON}
+export ARROW_ORC
 : ${ARROW_PARQUET:=ON}
-: ${ARROW_PLASMA_JAVA_CLIENT:=ON}
 : ${ARROW_PLASMA:=ON}
+export ARROW_PLASMA
 : ${ARROW_S3:=ON}
 : ${ARROW_USE_CCACHE:=OFF}
 : ${CMAKE_BUILD_TYPE:=Release}
@@ -61,7 +63,6 @@ cmake \
   -DARROW_CSV=${ARROW_DATASET} \
   -DARROW_DATASET=${ARROW_DATASET} \
   -DARROW_DEPENDENCY_USE_SHARED=OFF \
-  -DARROW_FILESYSTEM=${ARROW_FILESYSTEM} \
   -DARROW_GANDIVA=${ARROW_GANDIVA} \
   -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON \
   -DARROW_ORC=${ARROW_ORC} \

ci/scripts/java_jni_manylinux_build.sh

@@ -34,14 +34,16 @@ devtoolset_version=$(rpm -qa "devtoolset-*-gcc" --queryformat %{VERSION} | \
 devtoolset_include_cpp="/opt/rh/devtoolset-${devtoolset_version}/root/usr/include/c++/${devtoolset_version}"
 : ${ARROW_BUILD_TESTS:=ON}
 : ${ARROW_DATASET:=ON}
+export ARROW_DATASET
 : ${ARROW_GANDIVA:=ON}
-: ${ARROW_FILESYSTEM:=ON}
+export ARROW_GANDIVA
 : ${ARROW_JEMALLOC:=ON}
 : ${ARROW_RPATH_ORIGIN:=ON}
 : ${ARROW_ORC:=ON}
+export ARROW_ORC
 : ${ARROW_PARQUET:=ON}
 : ${ARROW_PLASMA:=ON}
-: ${ARROW_PLASMA_JAVA_CLIENT:=ON}
+export ARROW_PLASMA
 : ${ARROW_S3:=ON}
 : ${ARROW_USE_CCACHE:=OFF}
 : ${CMAKE_BUILD_TYPE:=release}
@@ -70,7 +72,6 @@ cmake \
   -DARROW_DATASET=${ARROW_DATASET} \
   -DARROW_DEPENDENCY_SOURCE="VCPKG" \
   -DARROW_DEPENDENCY_USE_SHARED=OFF \
-  -DARROW_FILESYSTEM=${ARROW_FILESYSTEM} \
   -DARROW_GANDIVA_PC_CXX_FLAGS=${GANDIVA_CXX_FLAGS} \
   -DARROW_GANDIVA=${ARROW_GANDIVA} \
   -DARROW_JEMALLOC=${ARROW_JEMALLOC} \

ci/scripts/java_jni_windows_build.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+arrow_dir=${1}
+build_dir=${2}
+# The directory where the final binaries will be stored when scripts finish
+dist_dir=${3}
+
+echo "=== Clear output directories and leftovers ==="
+# Clear output directories and leftovers
+rm -rf ${build_dir}
+
+echo "=== Building Arrow C++ libraries ==="
+install_dir=${build_dir}/cpp-install
+: ${ARROW_BUILD_TESTS:=ON}
+: ${ARROW_DATASET:=ON}
+export ARROW_DATASET
+# We can enable this after ARROW-17817 is resolved.
+: ${ARROW_ORC:=OFF}
+export ARROW_ORC
+: ${ARROW_PARQUET:=ON}
+: ${ARROW_S3:=ON}
+: ${ARROW_USE_CCACHE:=OFF}
+: ${CMAKE_BUILD_TYPE:=release}
+: ${CMAKE_UNITY_BUILD:=ON}
+
+if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
+  echo "=== ccache statistics before build ==="
+  ccache -s
+fi
+
+export ARROW_TEST_DATA="${arrow_dir}/testing/data"
+export PARQUET_TEST_DATA="${arrow_dir}/cpp/submodules/parquet-testing/data"
+export AWS_EC2_METADATA_DISABLED=TRUE
+
+mkdir -p "${build_dir}/cpp"
+pushd "${build_dir}/cpp"
+
+cmake \
+  -DARROW_BUILD_SHARED=OFF \
+  -DARROW_BUILD_TESTS=ON \
+  -DARROW_CSV=${ARROW_DATASET} \
+  -DARROW_DATASET=${ARROW_DATASET} \
+  -DARROW_DEPENDENCY_USE_SHARED=OFF \
+  -DARROW_ORC=${ARROW_ORC} \
+  -DARROW_PARQUET=${ARROW_PARQUET} \
+  -DARROW_S3=${ARROW_S3} \
+  -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \
+  -DARROW_WITH_BROTLI=ON \
+  -DARROW_WITH_LZ4=ON \
+  -DARROW_WITH_SNAPPY=ON \
+  -DARROW_WITH_ZSTD=ON \
+  -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
+  -DCMAKE_INSTALL_LIBDIR=lib \
+  -DCMAKE_INSTALL_PREFIX=${install_dir} \
+  -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
+  -GNinja \
+  ${arrow_dir}/cpp
+ninja install
+
+if [ "${ARROW_BUILD_TESTS}" = "ON" ]; then
+  # MinIO is required
+  exclude_tests="arrow-s3fs-test"
+  # unstable
+  exclude_tests="${exclude_tests}|arrow-compute-hash-join-node-test"
+  exclude_tests="${exclude_tests}|arrow-dataset-scanner-test"
+  # strptime
+  exclude_tests="${exclude_tests}|arrow-utility-test"
+  ctest \
+    --exclude-regex "${exclude_tests}" \
+    --label-regex unittest \
+    --output-on-failure \
+    --parallel $(nproc) \
+    --timeout 300
+fi
+
+popd
+
+
+${arrow_dir}/ci/scripts/java_jni_build.sh \
+  ${arrow_dir} \
+  ${install_dir} \
+  ${build_dir} \
+  ${dist_dir}
+
+if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
+  echo "=== ccache statistics after build ==="
+  ccache -s
+fi
+
+
+echo "=== Checking shared dependencies for libraries ==="
+pushd ${dist_dir}
+# TODO
+# archery linking check-dependencies \
+#   --allow libm \
+#   --allow librt \
+#   --allow libz \
+#   libarrow_cdata_jni.dll \
+#   libarrow_dataset_jni.dll \
+popd

cpp/src/arrow/ArrowConfig.cmake.in

@@ -107,9 +107,19 @@ if(TARGET Arrow::arrow_static AND NOT TARGET Arrow::arrow_bundled_dependencies)
   # https://cmake.org/cmake/help/latest/policy/CMP0057.html
   cmake_policy(PUSH)
   cmake_policy(SET CMP0057 NEW)
-  if(APPLE AND "AWS::aws-c-common" IN_LIST ARROW_BUNDLED_STATIC_LIBS)
-    find_library(CORE_FOUNDATION CoreFoundation)
-    target_link_libraries(Arrow::arrow_bundled_dependencies INTERFACE ${CORE_FOUNDATION})
+  if("AWS::aws-c-common" IN_LIST ARROW_BUNDLED_STATIC_LIBS)
+    if(APPLE)
+      find_library(CORE_FOUNDATION CoreFoundation)
+      target_link_libraries(Arrow::arrow_bundled_dependencies
+                            INTERFACE ${CORE_FOUNDATION})
+    elseif(WIN32)
+      target_link_libraries(Arrow::arrow_bundled_dependencies
+                            INTERFACE "winhttp.lib"
+                                      "bcrypt.lib"
+                                      "wininet.lib"
+                                      "userenv.lib"
+                                      "version.lib")
+    endif()
   endif()
   cmake_policy(POP)
 endif()

cpp/src/arrow/dataset/CMakeLists.txt

@@ -82,6 +82,10 @@ add_arrow_lib(arrow_dataset
               STATIC_INSTALL_INTERFACE_LIBS
               ${ARROW_DATASET_STATIC_INSTALL_INTERFACE_LIBS})
 
+if(ARROW_BUILD_STATIC AND WIN32)
+  target_compile_definitions(arrow_dataset_static PUBLIC ARROW_DS_STATIC)
+endif()
+
 if(ARROW_TEST_LINKAGE STREQUAL "static")
   set(ARROW_DATASET_TEST_LINK_LIBS arrow_dataset_static ${ARROW_TEST_STATIC_LINK_LIBS})
 else()

cpp/src/arrow/flight/CMakeLists.txt

@@ -246,6 +246,10 @@ add_arrow_lib(arrow_flight
               STATIC_INSTALL_INTERFACE_LIBS
               Arrow::arrow_static)
 
+if(ARROW_BUILD_STATIC AND WIN32)
+  target_compile_definitions(arrow_flight_static PUBLIC ARROW_FLIGHT_STATIC)
+endif()
+
 foreach(LIB_TARGET ${ARROW_FLIGHT_LIBRARIES})
   target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_FLIGHT_EXPORTING)
 endforeach()

cpp/src/arrow/flight/sql/CMakeLists.txt

@@ -63,6 +63,10 @@ add_arrow_lib(arrow_flight_sql
               PRIVATE_INCLUDES
               "${Protobuf_INCLUDE_DIRS}")
 
+if(ARROW_BUILD_STATIC AND WIN32)
+  target_compile_definitions(arrow_flight_sql_static PUBLIC ARROW_FLIGHT_SQL_STATIC)
+endif()
+
 if(MSVC)
   # Suppress warnings caused by Protobuf (casts)
   set_source_files_properties(protocol_internal.cc PROPERTIES COMPILE_FLAGS "/wd4267")

cpp/src/parquet/CMakeLists.txt

@@ -273,7 +273,7 @@ endif()
 
 if(ARROW_BUILD_STATIC AND WIN32)
   # ARROW-4848: Static Parquet lib needs to import static symbols on Windows
-  target_compile_definitions(parquet_static PUBLIC ARROW_STATIC)
+  target_compile_definitions(parquet_static PUBLIC PARQUET_STATIC)
 endif()
 
 add_definitions(-DPARQUET_THRIFT_VERSION_MAJOR=${Thrift_VERSION_MAJOR})

dev/tasks/java-jars/github.yml

@@ -83,10 +83,41 @@ jobs:
           name: macos-shared-lib
           path: arrow-shared-libs-macos.tar.gz
 
+  build-cpp-windows:
+    name: Build C++ libraries Windows
+    runs-on: windows-2019
+    steps:
+      {{ macros.github_checkout_arrow()|indent }}
+      {{ macros.github_install_archery()|indent }}
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+      - name: Download Timezone Database
+        shell: bash
+        run: arrow/ci/scripts/download_tz_database.sh
+      - name: Build C++ libraries
+        shell: cmd
+        run: |
+          call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
+          bash -c "arrow/ci/scripts/java_jni_windows_build.sh $(pwd)/arrow $(pwd)/arrow/cpp-build $(pwd)/arrow/java-dist"
+      - name: Compress into single artifact to keep directory structure
+        shell: bash
+        run: tar -cvzf arrow-shared-libs-windows.tar.gz arrow/java-dist/
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: windows-shared-lib
+          path: arrow-shared-libs-windows.tar.gz
+
   package-jars:
     name: Build jar files
     runs-on: macos-latest
-    needs: [build-cpp-macos, build-cpp-ubuntu]
+    needs:
+      - build-cpp-ubuntu
+      - build-cpp-macos
+      - build-cpp-windows
     steps:
       {{ macros.github_checkout_arrow(fetch_depth=0)|indent }}
       - name: Download Linux C++ Libraries
@@ -97,22 +128,31 @@ jobs:
         uses: actions/download-artifact@v2
         with:
           name: macos-shared-lib
+      - name: Download Windows C++ libraries
+        uses: actions/download-artifact@v2
+        with:
+          name: windows-shared-lib
       - name: Descompress artifacts
         run: |
           tar -xvzf arrow-shared-libs-linux.tar.gz
           tar -xvzf arrow-shared-libs-macos.tar.gz
+          tar -xvzf arrow-shared-libs-windows.tar.gz
       - name: Test that shared libraries exist
         run: |
           set -x
           test -f arrow/java-dist/libarrow_cdata_jni.dylib
-          test -f arrow/java-dist/libarrow_dataset_jni.dylib
-          test -f arrow/java-dist/libgandiva_jni.dylib
-          test -f arrow/java-dist/libplasma_java.dylib
-          test -f arrow/java-dist/libarrow_orc_jni.dylib
           test -f arrow/java-dist/libarrow_cdata_jni.so
+          test -f arrow/java-dist/arrow_cdata_jni.dll
+          test -f arrow/java-dist/libarrow_dataset_jni.dylib
           test -f arrow/java-dist/libarrow_dataset_jni.so
+          test -f arrow/java-dist/arrow_dataset_jni.dll
+          test -f arrow/java-dist/libarrow_orc_jni.dylib
           test -f arrow/java-dist/libarrow_orc_jni.so
+          # We can enable this after ARROW-17817 is resolved.
+          # test -f arrow/java-dist/arrow_orc_jni.dll
+          test -f arrow/java-dist/libgandiva_jni.dylib
           test -f arrow/java-dist/libgandiva_jni.so
+          test -f arrow/java-dist/libplasma_java.dylib
           test -f arrow/java-dist/libplasma_java.so
       - name: Build bundled jar
         run: |

java/adapter/orc/CMakeLists.txt

@@ -37,4 +37,6 @@ set_property(TARGET arrow_java_jni_orc PROPERTY OUTPUT_NAME "arrow_orc_jni")
 target_link_libraries(arrow_java_jni_orc arrow_java_jni_orc_headers jni
                       Arrow::arrow_static)
 
-install(TARGETS arrow_java_jni_orc DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(TARGETS arrow_java_jni_orc
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})

java/adapter/orc/pom.xml

@@ -32,7 +32,7 @@
             <scope>compile</scope>
             <classifier>${arrow.vector.classifier}</classifier>
         </dependency>
-	<dependency>
+        <dependency>
             <groupId>org.apache.orc</groupId>
             <artifactId>orc-core</artifactId>
             <version>1.7.6</version>
@@ -122,7 +122,7 @@
             <resource>
                 <directory>${arrow.cpp.build.dir}</directory>
                 <includes>
-                    <include>**/libarrow_orc_jni.*</include>
+                    <include>**/*arrow_orc_jni.*</include>
                 </includes>
             </resource>
         </resources>

java/c/CMakeLists.txt

@@ -30,4 +30,6 @@ add_library(arrow_java_jni_cdata SHARED src/main/cpp/jni_wrapper.cc)
 set_property(TARGET arrow_java_jni_cdata PROPERTY OUTPUT_NAME "arrow_cdata_jni")
 target_link_libraries(arrow_java_jni_cdata arrow_java_jni_cdata_headers jni)
 
-install(TARGETS arrow_java_jni_cdata DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(TARGETS arrow_java_jni_cdata
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})

java/dataset/CMakeLists.txt

@@ -42,4 +42,6 @@ if(BUILD_TESTING)
   add_test(NAME arrow-java-jni-dataset-test COMMAND arrow-java-jni-dataset-test)
 endif()
 
-install(TARGETS arrow_java_jni_dataset DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(TARGETS arrow_java_jni_dataset
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})