From 42337e12c43103f66cb0ca578ad04f4dd43d714a Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Fri, 10 May 2019 13:21:04 -0700 Subject: [PATCH 1/4] New method for client to download blob object or uri to a file. --- storage/google/cloud/storage/client.py | 12 ++++++++++++ storage/tests/unit/test_client.py | 27 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/storage/google/cloud/storage/client.py b/storage/google/cloud/storage/client.py index cbd82bdb909d..f9e7e8a50135 100644 --- a/storage/google/cloud/storage/client.py +++ b/storage/google/cloud/storage/client.py @@ -14,6 +14,7 @@ """Client for interacting with the Google Cloud Storage API.""" +from six.moves.urllib.parse import urlsplit from google.auth.credentials import AnonymousCredentials @@ -24,6 +25,7 @@ from google.cloud.storage._http import Connection from google.cloud.storage.batch import Batch from google.cloud.storage.bucket import Bucket +from google.cloud.storage.blob import Blob _marker = object() @@ -309,6 +311,16 @@ def create_bucket(self, bucket_or_name, requester_pays=None, project=None): bucket.create(client=self, project=project) return bucket + def download_blob_to_file(self, blob_or_uri, file_obj, start=None, end=None): + try: + blob_or_uri.download_to_file(file_obj, client=self, start=start, end=end) + except AttributeError: + scheme, netloc, path, query, frag = urlsplit(blob_or_uri) + bucket = Bucket(self, name=netloc) + blob_or_uri = Blob(path, bucket) + + blob_or_uri.download_to_file(file_obj, client=self, start=start, end=end) + def list_buckets( self, max_results=None, diff --git a/storage/tests/unit/test_client.py b/storage/tests/unit/test_client.py index c136b9089cb6..c25ee794900a 100644 --- a/storage/tests/unit/test_client.py +++ b/storage/tests/unit/test_client.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import json import unittest @@ -516,6 +517,32 @@ def test_create_bucket_with_object_success(self): json_sent = http.request.call_args_list[0][1]["data"] self.assertEqual(json_expected, json.loads(json_sent)) + def test_download_blob_to_file_with_blob(self): + project = "PROJECT" + credentials = _make_credentials() + client = self._make_one(project=project, credentials=credentials) + blob = mock.Mock() + file_obj = io.BytesIO() + + client.download_blob_to_file(blob, file_obj) + blob.download_to_file.assert_called_once_with( + file_obj, client=client, start=None, end=None + ) + + def test_download_blob_to_file_with_uri(self): + project = "PROJECT" + credentials = _make_credentials() + client = self._make_one(project=project, credentials=credentials) + blob = mock.Mock() + file_obj = io.BytesIO() + + with mock.patch("google.cloud.storage.client.Blob", return_value=blob): + client.download_blob_to_file("gs://bucket_name/path/to/object", file_obj) + + blob.download_to_file.assert_called_once_with( + file_obj, client=client, start=None, end=None + ) + def test_list_buckets_wo_project(self): CREDENTIALS = _make_credentials() client = self._make_one(project=None, credentials=CREDENTIALS) From 62c99d505791791f800109e97b06773d2c24fb96 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Fri, 10 May 2019 13:29:02 -0700 Subject: [PATCH 2/4] Adding docstring. --- storage/google/cloud/storage/client.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/storage/google/cloud/storage/client.py b/storage/google/cloud/storage/client.py index f9e7e8a50135..dd79a8423383 100644 --- a/storage/google/cloud/storage/client.py +++ b/storage/google/cloud/storage/client.py @@ -312,6 +312,21 @@ def create_bucket(self, bucket_or_name, requester_pays=None, project=None): return bucket def download_blob_to_file(self, blob_or_uri, file_obj, start=None, end=None): + """Download the contents of a blob object or blob URI into a file-like object. + + Args: + blob_or_uri (Union[ \ + :class:`~google.cloud.storage.blob.Blob`, \ + str, \ + ]): + The blob resource to pass or URI to download. + file_obj (file): + A file handle to which to write the blob's data. + start (int): + Optional, the first byte in a range to be downloaded. + end (int): + Optional, The last byte in a range to be downloaded. + """ try: blob_or_uri.download_to_file(file_obj, client=self, start=start, end=end) except AttributeError: From edcb4e2f2f86f2b89a29d483d299c4737653de3a Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Tue, 14 May 2019 11:10:28 -0700 Subject: [PATCH 3/4] Made changes based on feedback and updated docstring to reflect examples. --- storage/google/cloud/storage/client.py | 34 +++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/storage/google/cloud/storage/client.py b/storage/google/cloud/storage/client.py index dd79a8423383..7bd63d6ca27a 100644 --- a/storage/google/cloud/storage/client.py +++ b/storage/google/cloud/storage/client.py @@ -316,21 +316,47 @@ def download_blob_to_file(self, blob_or_uri, file_obj, start=None, end=None): Args: blob_or_uri (Union[ \ - :class:`~google.cloud.storage.blob.Blob`, \ - str, \ + :class:`~google.cloud.storage.blob.Blob`, \ + str, \ ]): The blob resource to pass or URI to download. file_obj (file): A file handle to which to write the blob's data. start (int): - Optional, the first byte in a range to be downloaded. + Optional. The first byte in a range to be downloaded. end (int): - Optional, The last byte in a range to be downloaded. + Optional. The last byte in a range to be downloaded. + + Examples: + Download a blob using using a blob resource. + + >>> from google.cloud import storage + >>> client = storage.Client() + + >>> bucket = client.get_bucket('my-bucket-name') + >>> blob = storage.Blob('path/to/blob', bucket) + + >>> with open('file-to-download-to') as file_obj: + >>> client.download_blob_to_file(blob, file) # API request. + + + Download a blob using a URI. + + >>> from google.cloud import storage + >>> client = storage.Client() + + >>> with open('file-to-download-to') as file_obj: + >>> client.download_blob_to_file( + >>> 'gs::/bucket_name/path/to/blob', file) + + """ try: blob_or_uri.download_to_file(file_obj, client=self, start=start, end=end) except AttributeError: scheme, netloc, path, query, frag = urlsplit(blob_or_uri) + if scheme != "gs": + raise ValueError("URI scheme must be gs") bucket = Bucket(self, name=netloc) blob_or_uri = Blob(path, bucket) From 52d3ed88eb37d540ac6786c7a5364defc5bf0199 Mon Sep 17 00:00:00 2001 From: Layla Bristol Date: Tue, 14 May 2019 22:00:43 -0700 Subject: [PATCH 4/4] Corrected typo and added additional test for uri coverage. --- storage/google/cloud/storage/client.py | 2 +- storage/tests/unit/test_client.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/storage/google/cloud/storage/client.py b/storage/google/cloud/storage/client.py index 7bd63d6ca27a..0a107f7149bb 100644 --- a/storage/google/cloud/storage/client.py +++ b/storage/google/cloud/storage/client.py @@ -347,7 +347,7 @@ def download_blob_to_file(self, blob_or_uri, file_obj, start=None, end=None): >>> with open('file-to-download-to') as file_obj: >>> client.download_blob_to_file( - >>> 'gs::/bucket_name/path/to/blob', file) + >>> 'gs://bucket_name/path/to/blob', file) """ diff --git a/storage/tests/unit/test_client.py b/storage/tests/unit/test_client.py index c25ee794900a..00fa9a7cc469 100644 --- a/storage/tests/unit/test_client.py +++ b/storage/tests/unit/test_client.py @@ -17,6 +17,7 @@ import unittest import mock +import pytest import requests from six.moves import http_client @@ -543,6 +544,19 @@ def test_download_blob_to_file_with_uri(self): file_obj, client=client, start=None, end=None ) + def test_download_blob_to_file_with_invalid_uri(self): + project = "PROJECT" + credentials = _make_credentials() + client = self._make_one(project=project, credentials=credentials) + blob = mock.Mock() + file_obj = io.BytesIO() + + with mock.patch("google.cloud.storage.client.Blob", return_value=blob): + with pytest.raises(ValueError, match="URI scheme must be gs"): + client.download_blob_to_file( + "http://bucket_name/path/to/object", file_obj + ) + def test_list_buckets_wo_project(self): CREDENTIALS = _make_credentials() client = self._make_one(project=None, credentials=CREDENTIALS)