Add Dependencies for Arrow's Azure Filesystem (#1431)

automerged PR by conda-forge/automerge-action
conda-forge · Jun 21, 2024 · 5016d21 · 5016d21
2 parents 86c9a18 + 5415d77
commit 5016d21
Show file tree

Hide file tree

Showing 16 changed files with 107 additions and 10 deletions.
diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml
diff --git a/...ompiler_version11c_stdlib_version2.17cuda_compiler_version11.8cxx_compiler_version11.yaml b/...ompiler_version11c_stdlib_version2.17cuda_compiler_version11.8cxx_compiler_version11.yaml
@@ -2,6 +2,14 @@ aws_crt_cpp:
 - 0.26.12
 aws_sdk_cpp:
 - 1.11.329
+azure_core_cpp:
+- 1.12.0
+azure_identity_cpp:
+- 1.8.0
+azure_storage_blobs_cpp:
+- 12.11.0
+azure_storage_files_datalake_cpp:
+- 12.10.0
 bzip2:
 - '1'
 c_compiler:

diff --git a/...ompiler_version12c_stdlib_version2.12cuda_compiler_versionNonecxx_compiler_version12.yaml b/...ompiler_version12c_stdlib_version2.12cuda_compiler_versionNonecxx_compiler_version12.yaml
@@ -2,6 +2,14 @@ aws_crt_cpp:
 - 0.26.12
 aws_sdk_cpp:
 - 1.11.329
+azure_core_cpp:
+- 1.12.0
+azure_identity_cpp:
+- 1.8.0
+azure_storage_blobs_cpp:
+- 12.11.0
+azure_storage_files_datalake_cpp:
+- 12.10.0
 bzip2:
 - '1'
 c_compiler:

diff --git a/...rt/linux_aarch64_c_compiler_version10cuda_compiler_version11.8cxx_compiler_version10.yaml b/...rt/linux_aarch64_c_compiler_version10cuda_compiler_version11.8cxx_compiler_version10.yaml
@@ -4,6 +4,14 @@ aws_crt_cpp:
 - 0.26.12
 aws_sdk_cpp:
 - 1.11.329
+azure_core_cpp:
+- 1.12.0
+azure_identity_cpp:
+- 1.8.0
+azure_storage_blobs_cpp:
+- 12.11.0
+azure_storage_files_datalake_cpp:
+- 12.10.0
 bzip2:
 - '1'
 c_compiler:

diff --git a/...rt/linux_aarch64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml b/...rt/linux_aarch64_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml
@@ -4,6 +4,14 @@ aws_crt_cpp:
 - 0.26.12
 aws_sdk_cpp:
 - 1.11.329
+azure_core_cpp:
+- 1.12.0
+azure_identity_cpp:
+- 1.8.0
+azure_storage_blobs_cpp:
+- 12.11.0
+azure_storage_files_datalake_cpp:
+- 12.10.0
 bzip2:
 - '1'
 c_compiler:

diff --git a/...rt/linux_ppc64le_c_compiler_version10cuda_compiler_version11.8cxx_compiler_version10.yaml b/...rt/linux_ppc64le_c_compiler_version10cuda_compiler_version11.8cxx_compiler_version10.yaml
@@ -2,6 +2,14 @@ aws_crt_cpp:
 - 0.26.12
 aws_sdk_cpp:
 - 1.11.329
+azure_core_cpp:
+- 1.12.0
+azure_identity_cpp:
+- 1.8.0
+azure_storage_blobs_cpp:
+- 12.11.0
+azure_storage_files_datalake_cpp:
+- 12.10.0
 bzip2:
 - '1'
 c_compiler:

diff --git a/...rt/linux_ppc64le_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml b/...rt/linux_ppc64le_c_compiler_version13cuda_compiler_versionNonecxx_compiler_version13.yaml
@@ -2,6 +2,14 @@ aws_crt_cpp:
 - 0.26.12
 aws_sdk_cpp:
 - 1.11.329
+azure_core_cpp:
+- 1.12.0
+azure_identity_cpp:
+- 1.8.0
+azure_storage_blobs_cpp:
+- 12.11.0
+azure_storage_files_datalake_cpp:
+- 12.10.0
 bzip2:
 - '1'
 c_compiler:

diff --git a/.ci_support/migrations/azure_core_cpp1120.yaml b/.ci_support/migrations/azure_core_cpp1120.yaml
@@ -0,0 +1,21 @@
+migrator_ts: 1718824596
+__migrator:
+  kind: version
+  migration_number: 1
+  bump_number: 1
+  commit_message: Pin azure-core-cpp 1.12.0
+
+azure_core_cpp:
+  - 1.12.0
+azure_identity_cpp:
+  - 1.8.0
+azure_storage_blobs_cpp:
+  - 12.11.0
+azure_storage_common_cpp:
+  - 12.6.0
+azure_storage_files_datalake_cpp:
+  - 12.10.0
+azure_storage_files_shares_cpp:
+  - 12.9.0
+azure_storage_queues_cpp:
+  - 12.2.0
diff --git a/.ci_support/osx_64_.yaml b/.ci_support/osx_64_.yaml
@@ -6,6 +6,14 @@ aws_crt_cpp:
 - 0.26.12
 aws_sdk_cpp:
 - 1.11.329
+azure_core_cpp:
+- 1.12.0
+azure_identity_cpp:
+- 1.8.0
+azure_storage_blobs_cpp:
+- 12.11.0
+azure_storage_files_datalake_cpp:
+- 12.10.0
 bzip2:
 - '1'
 c_compiler:

diff --git a/.ci_support/osx_arm64_.yaml b/.ci_support/osx_arm64_.yaml
@@ -6,6 +6,14 @@ aws_crt_cpp:
 - 0.26.12
 aws_sdk_cpp:
 - 1.11.329
+azure_core_cpp:
+- 1.12.0
+azure_identity_cpp:
+- 1.8.0
+azure_storage_blobs_cpp:
+- 12.11.0
+azure_storage_files_datalake_cpp:
+- 12.10.0
 bzip2:
 - '1'
 c_compiler:

diff --git a/...ort/win_64_cuda_compiler_versionNone.yaml → ...ompilerNonecuda_compiler_versionNone.yaml b/...ort/win_64_cuda_compiler_versionNone.yaml → ...ompilerNonecuda_compiler_versionNone.yaml
diff --git a/...ort/win_64_cuda_compiler_version11.8.yaml → ...ompilernvcccuda_compiler_version11.8.yaml b/...ort/win_64_cuda_compiler_version11.8.yaml → ...ompilernvcccuda_compiler_version11.8.yaml
diff --git a/README.md b/README.md
diff --git a/recipe/bld.bat b/recipe/bld.bat
@@ -10,13 +10,14 @@ if "%cuda_compiler_version%"=="None" (
     set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=ON"
 )
 
-:: # reusable variable for dependencies we cannot yet unvendor
+:: # reusable variable for dependencies we cannot yet enable
 set "READ_RECIPE_META_YAML_WHY_NOT=OFF"
 
 :: for available switches see
 :: https://github.com/apache/arrow/blame/apache-arrow-12.0.0/cpp/cmake_modules/DefineOptions.cmake
 cmake -G "Ninja" ^
       -DARROW_ACERO=ON ^
+      -DARROW_AZURE=%READ_RECIPE_META_YAML_WHY_NOT% ^
       -DARROW_BOOST_USE_SHARED:BOOL=ON ^
       -DARROW_BUILD_STATIC:BOOL=OFF ^
       -DARROW_BUILD_TESTS:BOOL=OFF ^

diff --git a/recipe/build.sh b/recipe/build.sh
@@ -58,6 +58,7 @@ export READ_RECIPE_META_YAML_WHY_NOT=OFF
 # placeholder in ARROW_GDB_INSTALL_DIR must match _la_placeholder in activate.sh
 cmake -GNinja \
     -DARROW_ACERO=ON \
+    -DARROW_AZURE=ON \
     -DARROW_BOOST_USE_SHARED=ON \
     -DARROW_BUILD_BENCHMARKS=OFF \
     -DARROW_BUILD_STATIC=OFF \

diff --git a/recipe/meta.yaml b/recipe/meta.yaml
@@ -27,7 +27,7 @@ source:
     folder: testing
 
 build:
-  number: 9
+  number: 10
   # for cuda support, building with one version is enough to be compatible with
   # all later versions, since arrow is only using libcuda, and not libcudart.
   skip: true  # [cuda_compiler_version not in ("None", cuda_compiler_version_min)]
@@ -58,6 +58,12 @@ requirements:
     - llvmdev {{ llvm_version }}
     - aws-crt-cpp
     - aws-sdk-cpp
+    # azure filesystem dependencies, currently broken on windows, see
+    # https://github.com/apache/arrow/issues/41990
+    - azure-core-cpp                    # [unix]
+    - azure-identity-cpp                # [unix]
+    - azure-storage-blobs-cpp           # [unix]
+    - azure-storage-files-datalake-cpp  # [unix]
     - brotli
     - bzip2
     # not yet: https://github.com/conda-forge/cpp-opentelemetry-sdk-feedstock/issues/38
@@ -186,6 +192,10 @@ outputs:
       host:
         - aws-crt-cpp
         - aws-sdk-cpp
+        - azure-core-cpp                    # [unix]
+        - azure-identity-cpp                # [unix]
+        - azure-storage-blobs-cpp           # [unix]
+        - azure-storage-files-datalake-cpp  # [unix]
         - brotli
         - bzip2
         # not yet: https://github.com/conda-forge/cpp-opentelemetry-sdk-feedstock/issues/38