Skip to content

Commit

Permalink
port get_directory fix (#14432) (#14537)
Browse files Browse the repository at this point in the history
  • Loading branch information
zzstoatzz authored Jul 10, 2024
1 parent 92e1187 commit 0169663
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 8 deletions.
3 changes: 2 additions & 1 deletion src/integrations/prefect-gcp/prefect_gcp/cloud_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -667,7 +667,8 @@ async def get_directory(
if blob_path[-1] == "/":
# object is a folder and will be created if it contains any objects
continue
local_file_path = os.path.join(local_path, blob_path)
relative_blob_path = os.path.relpath(blob_path, from_path)
local_file_path = os.path.join(local_path, relative_blob_path)
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)

with disable_run_logger():
Expand Down
27 changes: 20 additions & 7 deletions src/integrations/prefect-gcp/tests/test_cloud_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,8 @@ def test_get_directory(self, gcs_bucket, tmp_path, from_path, local_path):
prefix = os.path.join(gcs_bucket.bucket_folder, from_path or "")
if local_path is None:
local_path = os.path.abspath(".")
else:
local_path = os.path.abspath(os.path.expanduser(local_path))

if from_path is None:
from_path = gcs_bucket.bucket_folder
Expand All @@ -166,17 +168,28 @@ def test_get_directory(self, gcs_bucket, tmp_path, from_path, local_path):
for file_path in actual:
assert os.path.isfile(file_path)

# blob.txt, base_folder/nested_blob.txt, base_folder/sub_folder/nested_blob.txt
# check that returned file paths are correct
if not prefix:
assert len(actual) == 4
# base_folder/sub_folder/nested_blob.txt
expected = [
os.path.join(local_path, "blob.txt"),
os.path.join(local_path, "base_folder/nested_blob.txt"),
os.path.join(local_path, "base_folder/sub_folder/nested_blob.txt"),
os.path.join(local_path, "dotted.folder/nested_blob.txt"),
]
elif prefix == "base_folder/sub_folder":
assert len(actual) == 1
# base_folder/nested_blob.txt, base_folder/sub_folder/nested_blob.txt
# base_folder/sub_folder/nested_blob.txt
expected = [
os.path.join(local_path, "nested_blob.txt"),
]
elif prefix == "base_folder" or prefix == "base_folder/":
assert len(actual) == 2
# base_folder/nested_blob.txt, base_folder/sub_folder/nested_blob.txt
expected = [
os.path.join(local_path, "nested_blob.txt"),
os.path.join(local_path, "sub_folder/nested_blob.txt"),
]
else:
assert len(actual) == 0
expected = []
assert actual == expected

@pytest.mark.parametrize("to_path", [None, "to_path"])
@pytest.mark.parametrize("ignore", [True, False])
Expand Down

0 comments on commit 0169663

Please sign in to comment.