Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add matchGlob parameter to list_blobs #1055

Merged
merged 5 commits into from
Jun 14, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions google/cloud/storage/bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -1290,6 +1290,7 @@ def list_blobs(
client=None,
timeout=_DEFAULT_TIMEOUT,
retry=DEFAULT_RETRY,
match_glob=None,
):
"""Return an iterator used to find blobs in the bucket.

Expand Down Expand Up @@ -1365,6 +1366,12 @@ def list_blobs(
:param retry:
(Optional) How to retry the RPC. See: :ref:`configuring_retries`

:type match_glob: str
:param match_glob:
(Optional) A glob pattern used to filter results (for example, foo*bar).
The string value must be UTF-8 encoded. See:
https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob

:rtype: :class:`~google.api_core.page_iterator.Iterator`
:returns: Iterator of all :class:`~google.cloud.storage.blob.Blob`
in this bucket matching the arguments.
Expand All @@ -1384,6 +1391,7 @@ def list_blobs(
fields=fields,
timeout=timeout,
retry=retry,
match_glob=match_glob,
)

def list_notifications(
Expand Down
9 changes: 9 additions & 0 deletions google/cloud/storage/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1127,6 +1127,7 @@ def list_blobs(
page_size=None,
timeout=_DEFAULT_TIMEOUT,
retry=DEFAULT_RETRY,
match_glob=None,
):
"""Return an iterator used to find blobs in the bucket.

Expand Down Expand Up @@ -1220,6 +1221,11 @@ def list_blobs(
See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
information on retry types and how to configure them.

match_glob (str):
(Optional) A glob pattern used to filter results (for example, foo*bar).
The string value must be UTF-8 encoded. See:
https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob

Returns:
Iterator of all :class:`~google.cloud.storage.blob.Blob`
in this bucket matching the arguments. The RPC call
Expand All @@ -1238,6 +1244,9 @@ def list_blobs(
if delimiter is not None:
extra_params["delimiter"] = delimiter

if match_glob is not None:
extra_params["matchGlob"] = match_glob

if start_offset is not None:
extra_params["startOffset"] = start_offset

Expand Down
32 changes: 32 additions & 0 deletions tests/system/test_bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -621,6 +621,38 @@ def test_bucket_list_blobs_hierarchy_w_include_trailing_delimiter(
assert iterator.prefixes == expected_prefixes


@_helpers.retry_failures
def test_bucket_list_blobs_w_match_glob(
storage_client,
buckets_to_delete,
blobs_to_delete,
):
bucket_name = _helpers.unique_name("w-matchglob")
bucket = _helpers.retry_429_503(storage_client.create_bucket)(bucket_name)
buckets_to_delete.append(bucket)

payload = b"helloworld"
blob_names = ["foo/bar", "foo/baz", "foo/foobar", "foobar"]
for name in blob_names:
blob = bucket.blob(name)
blob.upload_from_string(payload)
blobs_to_delete.append(blob)

match_glob_results = {
"foo*bar": ["foobar"],
"foo**bar": ["foo/bar", "foo/foobar", "foobar"],
"**/foobar": ["foo/foobar", "foobar"],
"*/ba[rz]": ["foo/bar", "foo/baz"],
"*/ba[!a-y]": ["foo/baz"],
"**/{foobar,baz}": ["foo/baz", "foo/foobar", "foobar"],
"foo/{foo*,*baz}": ["foo/baz", "foo/foobar"],
}
for match_glob, expected_names in match_glob_results.items():
blob_iter = bucket.list_blobs(match_glob=match_glob)
blobs = list(blob_iter)
assert [blob.name for blob in blobs] == expected_names


def test_bucket_w_retention_period(
storage_client,
buckets_to_delete,
Expand Down
6 changes: 6 additions & 0 deletions tests/unit/test_bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -1143,6 +1143,7 @@ def test_list_blobs_w_defaults(self):
expected_max_results = None
expected_prefix = None
expected_delimiter = None
expected_match_glob = None
expected_start_offset = None
expected_end_offset = None
expected_include_trailing_delimiter = None
Expand All @@ -1163,6 +1164,7 @@ def test_list_blobs_w_defaults(self):
fields=expected_fields,
timeout=self._get_default_timeout(),
retry=DEFAULT_RETRY,
match_glob=expected_match_glob,
)

def test_list_blobs_w_explicit(self):
Expand All @@ -1171,6 +1173,7 @@ def test_list_blobs_w_explicit(self):
page_token = "ABCD"
prefix = "subfolder"
delimiter = "/"
match_glob = "**txt"
start_offset = "c"
end_offset = "g"
include_trailing_delimiter = True
Expand All @@ -1197,6 +1200,7 @@ def test_list_blobs_w_explicit(self):
client=other_client,
timeout=timeout,
retry=retry,
match_glob=match_glob,
)

self.assertIs(iterator, other_client.list_blobs.return_value)
Expand All @@ -1205,6 +1209,7 @@ def test_list_blobs_w_explicit(self):
expected_max_results = max_results
expected_prefix = prefix
expected_delimiter = delimiter
expected_match_glob = match_glob
expected_start_offset = start_offset
expected_end_offset = end_offset
expected_include_trailing_delimiter = include_trailing_delimiter
Expand All @@ -1225,6 +1230,7 @@ def test_list_blobs_w_explicit(self):
fields=expected_fields,
timeout=timeout,
retry=retry,
match_glob=expected_match_glob,
)

def test_list_notifications_w_defaults(self):
Expand Down
3 changes: 3 additions & 0 deletions tests/unit/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1928,6 +1928,7 @@ def test_list_blobs_w_explicit_w_user_project(self):
page_token = "ABCD"
prefix = "subfolder"
delimiter = "/"
match_glob = "**txt"
start_offset = "c"
end_offset = "g"
include_trailing_delimiter = True
Expand Down Expand Up @@ -1962,6 +1963,7 @@ def test_list_blobs_w_explicit_w_user_project(self):
page_size=page_size,
timeout=timeout,
retry=retry,
match_glob=match_glob,
)

self.assertIs(iterator, client._list_resource.return_value)
Expand All @@ -1976,6 +1978,7 @@ def test_list_blobs_w_explicit_w_user_project(self):
"projection": projection,
"prefix": prefix,
"delimiter": delimiter,
"matchGlob": match_glob,
"startOffset": start_offset,
"endOffset": end_offset,
"includeTrailingDelimiter": include_trailing_delimiter,
Expand Down