fix pyarrow hang

dmlc · Jan 19, 2022 · 7b503ca · 7b503ca
1 parent be9df4c
commit 7b503ca
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 4 deletions.
diff --git a/python-package/xgboost/data.py b/python-package/xgboost/data.py
@@ -488,7 +488,8 @@ def _from_arrow(
     if enable_categorical:
         raise ValueError("categorical data in datatable is not supported yet.")
 
-    rb_iter = iter(data.to_batches())
+    # use_async=True to workaround pyarrow 6.0.1 hang, see Modin-3982 and ARROW-15362
+    rb_iter = iter(data.to_batches(use_async=True))
     it = RecordBatchDataIter(rb_iter)
     next_callback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p)(it.next)
     handle = ctypes.c_void_p()

diff --git a/tests/ci_build/conda_env/cpu_test.yml b/tests/ci_build/conda_env/cpu_test.yml
@@ -34,10 +34,10 @@ dependencies:
 - llvmlite
 - py-ubjson
 - cffi
+- pyarrow
 - pip:
   - shap
   - ipython                     # required by shap at import time.
   - sphinx_rtd_theme
   - datatable
-  - pyarrow==4.0.0
   - modin[all]
diff --git a/tests/ci_build/conda_env/win64_test.yml b/tests/ci_build/conda_env/win64_test.yml
@@ -18,5 +18,4 @@ dependencies:
 - pip
 - py-ubjson
 - cffi
-- pip:
-  - pyarrow==4.0.0
+- pyarrow
-Original file line number
+Diff line change
@@ Expand Up / @@ -18,5 +18,4 @@ dependencies: @@
     - pip
     - py-ubjson
     - cffi
-    - pip:
-      - pyarrow==4.0.0
+    - pyarrow