Skip to content

Commit

Permalink
GH-38431: [Python][CI] Update fs.type_name checks for s3fs tests (#38455
Browse files Browse the repository at this point in the history
)

### Rationale for this change

Appveyor CI is failing https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/48347810. It seems the reason for the test failures is a change in the `type_name` (from `"py::fsspec+s3"` to `"py::fsspec+('s3', 's3a')"`) and due to it tests are not being skipped.

### What changes are included in this PR?

Update the check for `type_name` in case of `PyFileSystem(FSSpecHandler(s3fs.S3FileSystem()))`.
* Closes: #38431

Authored-by: AlenkaF <frim.alenka@gmail.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>
  • Loading branch information
AlenkaF authored and raulcd committed Nov 6, 2023
1 parent 2dcee3f commit 5a37e74
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 7 deletions.
2 changes: 1 addition & 1 deletion ci/conda_env_python.txt
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,6 @@ numpy>=1.16.6
pytest
pytest-faulthandler
pytest-lazy-fixture
s3fs>=2021.8.0
s3fs>=2023.10.0
setuptools
setuptools_scm<8.0.0
12 changes: 6 additions & 6 deletions python/pyarrow/tests/test_fs.py
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ def check_mtime_or_absent(file_info):


def skip_fsspec_s3fs(fs):
if fs.type_name == "py::fsspec+s3":
if fs.type_name == "py::fsspec+('s3', 's3a')":
pytest.xfail(reason="Not working with fsspec's s3fs")


Expand Down Expand Up @@ -631,7 +631,7 @@ def test_get_file_info(fs, pathfn):
assert aaa_info.path == aaa
assert 'aaa' in repr(aaa_info)
assert aaa_info.extension == ''
if fs.type_name == "py::fsspec+s3":
if fs.type_name == "py::fsspec+('s3', 's3a')":
# s3fs doesn't create empty directories
assert aaa_info.type == FileType.NotFound
else:
Expand All @@ -646,7 +646,7 @@ def test_get_file_info(fs, pathfn):
assert bb_info.type == FileType.File
assert 'FileType.File' in repr(bb_info)
assert bb_info.size == 0
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+s3"]:
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+('s3', 's3a')"]:
check_mtime(bb_info)

assert c_info.path == str(c)
Expand All @@ -655,7 +655,7 @@ def test_get_file_info(fs, pathfn):
assert c_info.type == FileType.File
assert 'FileType.File' in repr(c_info)
assert c_info.size == 4
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+s3"]:
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+('s3', 's3a')"]:
check_mtime(c_info)

assert zzz_info.path == str(zzz)
Expand Down Expand Up @@ -698,7 +698,7 @@ def test_get_file_info_with_selector(fs, pathfn):
assert selector.base_dir == base_dir

infos = fs.get_file_info(selector)
if fs.type_name == "py::fsspec+s3":
if fs.type_name == "py::fsspec+('s3', 's3a')":
# s3fs only lists directories if they are not empty
len(infos) == 4
else:
Expand All @@ -719,7 +719,7 @@ def test_get_file_info_with_selector(fs, pathfn):
selector = FileSelector(base_dir, recursive=False)

infos = fs.get_file_info(selector)
if fs.type_name == "py::fsspec+s3":
if fs.type_name == "py::fsspec+('s3', 's3a')":
# s3fs only lists directories if they are not empty
assert len(infos) == 3
else:
Expand Down

0 comments on commit 5a37e74

Please sign in to comment.