diff --git a/gcloud/_helpers.py b/gcloud/_helpers.py index 5853785adfaf..f24aa57b8e71 100644 --- a/gcloud/_helpers.py +++ b/gcloud/_helpers.py @@ -394,6 +394,27 @@ def _to_bytes(value, encoding='ascii'): raise TypeError('%r could not be converted to bytes' % (value,)) +def _bytes_to_unicode(value): + """Converts bytes to a unicode value, if necessary. + + :type value: bytes + :param value: bytes value to attempt string conversion on. + + :rtype: str + :returns: The original value converted to unicode (if bytes) or as passed + in if it started out as unicode. + + :raises: :class:`ValueError` if the value could not be converted to + unicode. + """ + result = (value.decode('utf-8') + if isinstance(value, six.binary_type) else value) + if isinstance(result, six.text_type): + return result + else: + raise ValueError('%r could not be converted to unicode' % (value,)) + + def _pb_timestamp_to_datetime(timestamp): """Convert a Timestamp protobuf to a datetime object. diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py index 92c11e412320..6b1280a31289 100644 --- a/gcloud/storage/blob.py +++ b/gcloud/storage/blob.py @@ -14,7 +14,9 @@ """Create / interact with Google Cloud Storage blobs.""" +import base64 import copy +import hashlib from io import BytesIO import json import mimetypes @@ -26,6 +28,8 @@ from six.moves.urllib.parse import quote from gcloud._helpers import _rfc3339_to_datetime +from gcloud._helpers import _to_bytes +from gcloud._helpers import _bytes_to_unicode from gcloud.credentials import generate_signed_url from gcloud.exceptions import NotFound from gcloud.exceptions import make_exception @@ -276,7 +280,7 @@ def delete(self, client=None): """ return self.bucket.delete_blob(self.name, client=client) - def download_to_file(self, file_obj, client=None): + def download_to_file(self, file_obj, encryption_key=None, client=None): """Download the contents of this blob into a file-like object. .. note:: @@ -284,9 +288,33 @@ def download_to_file(self, file_obj, client=None): If the server-set property, :attr:`media_link`, is not yet initialized, makes an additional API request to load it. + Downloading a file that has been encrypted with a `customer-supplied`_ + encryption key:: + + >>> from gcloud import storage + >>> from gcloud.storage import Blob + + >>> client = storage.Client(project='my-project') + >>> bucket = client.get_bucket('my-bucket') + >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19' + >>> blob = Blob('secure-data', bucket) + >>> with open('/tmp/my-secure-file', 'wb') as file_obj: + >>> blob.download_to_file(file_obj, + ... encryption_key=encryption_key) + + The ``encryption_key`` should be a str or bytes with a length of at + least 32. + + .. _customer-supplied: https://cloud.google.com/storage/docs/\ + encryption#customer-supplied + :type file_obj: file :param file_obj: A file handle to which to write the blob's data. + :type encryption_key: str or bytes + :param encryption_key: Optional 32 byte encryption key for + customer-supplied encryption. + :type client: :class:`gcloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. @@ -305,7 +333,11 @@ def download_to_file(self, file_obj, client=None): if self.chunk_size is not None: download.chunksize = self.chunk_size - request = Request(download_url, 'GET') + headers = {} + if encryption_key: + _set_encryption_headers(encryption_key, headers) + + request = Request(download_url, 'GET', headers) # Use the private ``_connection`` rather than the public # ``.connection``, since the public connection may be a batch. A @@ -315,12 +347,16 @@ def download_to_file(self, file_obj, client=None): # it has all three (http, API_BASE_URL and build_api_url). download.initialize_download(request, client._connection.http) - def download_to_filename(self, filename, client=None): + def download_to_filename(self, filename, encryption_key=None, client=None): """Download the contents of this blob into a named file. :type filename: string :param filename: A filename to be passed to ``open``. + :type encryption_key: str or bytes + :param encryption_key: Optional 32 byte encryption key for + customer-supplied encryption. + :type client: :class:`gcloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. @@ -328,14 +364,19 @@ def download_to_filename(self, filename, client=None): :raises: :class:`gcloud.exceptions.NotFound` """ with open(filename, 'wb') as file_obj: - self.download_to_file(file_obj, client=client) + self.download_to_file(file_obj, encryption_key=encryption_key, + client=client) mtime = time.mktime(self.updated.timetuple()) os.utime(file_obj.name, (mtime, mtime)) - def download_as_string(self, client=None): + def download_as_string(self, encryption_key=None, client=None): """Download the contents of this blob as a string. + :type encryption_key: str or bytes + :param encryption_key: Optional 32 byte encryption key for + customer-supplied encryption. + :type client: :class:`gcloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. @@ -345,7 +386,8 @@ def download_as_string(self, client=None): :raises: :class:`gcloud.exceptions.NotFound` """ string_buffer = BytesIO() - self.download_to_file(string_buffer, client=client) + self.download_to_file(string_buffer, encryption_key=encryption_key, + client=client) return string_buffer.getvalue() @staticmethod @@ -358,8 +400,10 @@ def _check_response_error(request, http_response): raise make_exception(faux_response, http_response.content, error_info=request.url) + # pylint: disable=too-many-locals def upload_from_file(self, file_obj, rewind=False, size=None, - content_type=None, num_retries=6, client=None): + encryption_key=None, content_type=None, num_retries=6, + client=None): """Upload the contents of this blob from a file-like object. The content type of the upload will either be @@ -378,6 +422,25 @@ def upload_from_file(self, file_obj, rewind=False, size=None, `lifecycle `_ API documents for details. + Uploading a file with a `customer-supplied`_ encryption key:: + + >>> from gcloud import storage + >>> from gcloud.storage import Blob + + >>> client = storage.Client(project='my-project') + >>> bucket = client.get_bucket('my-bucket') + >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19' + >>> blob = Blob('secure-data', bucket) + >>> with open('my-file', 'rb') as my_file: + >>> blob.upload_from_file(my_file, + ... encryption_key=encryption_key) + + The ``encryption_key`` should be a str or bytes with a length of at + least 32. + + .. _customer-supplied: https://cloud.google.com/storage/docs/\ + encryption#customer-supplied + :type file_obj: file :param file_obj: A file handle open for reading. @@ -391,6 +454,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None, :func:`os.fstat`. (If the file handle is not from the filesystem this won't be possible.) + :type encryption_key: str or bytes + :param encryption_key: Optional 32 byte encryption key for + customer-supplied encryption. + :type content_type: string or ``NoneType`` :param content_type: Optional type of content being uploaded. @@ -434,6 +501,9 @@ def upload_from_file(self, file_obj, rewind=False, size=None, 'User-Agent': connection.USER_AGENT, } + if encryption_key: + _set_encryption_headers(encryption_key, headers) + upload = Upload(file_obj, content_type, total_bytes, auto_transfer=False) @@ -473,9 +543,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None, six.string_types): # pragma: NO COVER Python3 response_content = response_content.decode('utf-8') self._set_properties(json.loads(response_content)) + # pylint: enable=too-many-locals def upload_from_filename(self, filename, content_type=None, - client=None): + encryption_key=None, client=None): """Upload this blob's contents from the content of a named file. The content type of the upload will either be @@ -500,6 +571,10 @@ def upload_from_filename(self, filename, content_type=None, :type content_type: string or ``NoneType`` :param content_type: Optional type of content being uploaded. + :type encryption_key: str or bytes + :param encryption_key: Optional 32 byte encryption key for + customer-supplied encryption. + :type client: :class:`gcloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. @@ -510,10 +585,10 @@ def upload_from_filename(self, filename, content_type=None, with open(filename, 'rb') as file_obj: self.upload_from_file(file_obj, content_type=content_type, - client=client) + encryption_key=encryption_key, client=client) def upload_from_string(self, data, content_type='text/plain', - client=None): + encryption_key=None, client=None): """Upload contents of this blob from the provided string. .. note:: @@ -535,6 +610,10 @@ def upload_from_string(self, data, content_type='text/plain', :param content_type: Optional type of content being uploaded. Defaults to ``'text/plain'``. + :type encryption_key: str or bytes + :param encryption_key: Optional 32 byte encryption key for + customer-supplied encryption. + :type client: :class:`gcloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. @@ -545,7 +624,7 @@ def upload_from_string(self, data, content_type='text/plain', string_buffer.write(data) self.upload_from_file(file_obj=string_buffer, rewind=True, size=len(data), content_type=content_type, - client=client) + encryption_key=encryption_key, client=client) def make_public(self, client=None): """Make this blob public giving all users read access. @@ -838,3 +917,21 @@ def __init__(self, bucket_name, object_name): self.query_params = {'name': object_name} self._bucket_name = bucket_name self._relative_path = '' + + +def _set_encryption_headers(key, headers): + """Builds customer encyrption key headers + + :type key: str or bytes + :param key: 32 byte key to build request key and hash. + + :type headers: dict + :param headers: dict of HTTP headers being sent in request. + """ + key = _to_bytes(key) + sha256_key = hashlib.sha256(key).digest() + key_hash = base64.b64encode(sha256_key).rstrip() + encoded_key = base64.b64encode(key).rstrip() + headers['X-Goog-Encryption-Algorithm'] = 'AES256' + headers['X-Goog-Encryption-Key'] = _bytes_to_unicode(encoded_key) + headers['X-Goog-Encryption-Key-Sha256'] = _bytes_to_unicode(key_hash) diff --git a/gcloud/storage/test_blob.py b/gcloud/storage/test_blob.py index a116114843ed..3ae92ed76a3d 100644 --- a/gcloud/storage/test_blob.py +++ b/gcloud/storage/test_blob.py @@ -383,6 +383,51 @@ def test_download_to_filename(self): self.assertEqual(wrote, b'abcdef') self.assertEqual(mtime, updatedTime) + def test_download_to_filename_w_key(self): + import os + import time + from six.moves.http_client import OK + from six.moves.http_client import PARTIAL_CONTENT + from gcloud._testing import _NamedTemporaryFile + + BLOB_NAME = 'blob-name' + KEY = 'aa426195405adee2c8081bb9e7e74b19' + HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk=' + HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0=' + chunk1_response = {'status': PARTIAL_CONTENT, + 'content-range': 'bytes 0-2/6'} + chunk2_response = {'status': OK, + 'content-range': 'bytes 3-5/6'} + connection = _Connection( + (chunk1_response, b'abc'), + (chunk2_response, b'def'), + ) + client = _Client(connection) + bucket = _Bucket(client) + MEDIA_LINK = 'http://example.com/media/' + properties = {'mediaLink': MEDIA_LINK, + 'updated': '2014-12-06T13:13:50.690Z'} + blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties) + blob._CHUNK_SIZE_MULTIPLE = 1 + blob.chunk_size = 3 + + with _NamedTemporaryFile() as temp: + blob.download_to_filename(temp.name, encryption_key=KEY) + with open(temp.name, 'rb') as file_obj: + wrote = file_obj.read() + mtime = os.path.getmtime(temp.name) + updatedTime = time.mktime(blob.updated.timetuple()) + + rq = connection.http._requested + headers = dict( + [(x.title(), str(y)) for x, y in rq[0]['headers'].items()]) + self.assertEqual(headers['X-Goog-Encryption-Algorithm'], 'AES256') + self.assertEqual(headers['X-Goog-Encryption-Key'], HEADER_KEY_VALUE) + self.assertEqual(headers['X-Goog-Encryption-Key-Sha256'], + HEADER_KEY_HASH_VALUE) + self.assertEqual(wrote, b'abcdef') + self.assertEqual(mtime, updatedTime) + def test_download_as_string(self): from six.moves.http_client import OK from six.moves.http_client import PARTIAL_CONTENT @@ -685,6 +730,63 @@ def test_upload_from_file_w_slash_in_name(self): self.assertEqual(headers['Content-Length'], '6') self.assertEqual(headers['Content-Type'], 'application/octet-stream') + def test_upload_from_filename_w_key(self): + from six.moves.http_client import OK + from six.moves.urllib.parse import parse_qsl + from six.moves.urllib.parse import urlsplit + from gcloud._testing import _NamedTemporaryFile + from gcloud.streaming import http_wrapper + + BLOB_NAME = 'blob-name' + UPLOAD_URL = 'http://example.com/upload/name/key' + DATA = b'ABCDEF' + KEY = 'aa426195405adee2c8081bb9e7e74b19' + HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk=' + HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0=' + EXPECTED_CONTENT_TYPE = 'foo/bar' + properties = {'contentType': EXPECTED_CONTENT_TYPE} + loc_response = {'status': OK, 'location': UPLOAD_URL} + chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE, + 'range': 'bytes 0-4'} + chunk2_response = {'status': OK} + connection = _Connection( + (loc_response, '{}'), + (chunk1_response, ''), + (chunk2_response, ''), + ) + client = _Client(connection) + bucket = _Bucket(client) + blob = self._makeOne(BLOB_NAME, bucket=bucket, + properties=properties) + blob._CHUNK_SIZE_MULTIPLE = 1 + blob.chunk_size = 5 + + with _NamedTemporaryFile(suffix='.jpeg') as temp: + with open(temp.name, 'wb') as file_obj: + file_obj.write(DATA) + blob.upload_from_filename(temp.name, + content_type=EXPECTED_CONTENT_TYPE, + encryption_key=KEY) + + rq = connection.http._requested + self.assertEqual(len(rq), 1) + self.assertEqual(rq[0]['method'], 'POST') + uri = rq[0]['uri'] + scheme, netloc, path, qs, _ = urlsplit(uri) + self.assertEqual(scheme, 'http') + self.assertEqual(netloc, 'example.com') + self.assertEqual(path, '/b/name/o') + self.assertEqual(dict(parse_qsl(qs)), + {'uploadType': 'media', 'name': BLOB_NAME}) + headers = dict( + [(x.title(), str(y)) for x, y in rq[0]['headers'].items()]) + self.assertEqual(headers['X-Goog-Encryption-Algorithm'], 'AES256') + self.assertEqual(headers['X-Goog-Encryption-Key'], HEADER_KEY_VALUE) + self.assertEqual(headers['X-Goog-Encryption-Key-Sha256'], + HEADER_KEY_HASH_VALUE) + self.assertEqual(headers['Content-Length'], '6') + self.assertEqual(headers['Content-Type'], 'foo/bar') + def _upload_from_filename_test_helper(self, properties=None, content_type_arg=None, expected_content_type=None): @@ -837,6 +939,54 @@ def test_upload_from_string_w_text(self): self.assertEqual(headers['Content-Type'], 'text/plain') self.assertEqual(rq[0]['body'], ENCODED) + def test_upload_from_string_text_w_key(self): + from six.moves.http_client import OK + from six.moves.urllib.parse import parse_qsl + from six.moves.urllib.parse import urlsplit + from gcloud.streaming import http_wrapper + BLOB_NAME = 'blob-name' + KEY = 'aa426195405adee2c8081bb9e7e74b19' + HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk=' + HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0=' + UPLOAD_URL = 'http://example.com/upload/name/key' + DATA = u'ABCDEF\u1234' + ENCODED = DATA.encode('utf-8') + loc_response = {'status': OK, 'location': UPLOAD_URL} + chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE, + 'range': 'bytes 0-4'} + chunk2_response = {'status': OK} + connection = _Connection( + (loc_response, '{}'), + (chunk1_response, ''), + (chunk2_response, ''), + ) + client = _Client(connection) + bucket = _Bucket(client=client) + blob = self._makeOne(BLOB_NAME, bucket=bucket) + blob._CHUNK_SIZE_MULTIPLE = 1 + blob.chunk_size = 5 + blob.upload_from_string(DATA, encryption_key=KEY) + rq = connection.http._requested + self.assertEqual(len(rq), 1) + self.assertEqual(rq[0]['method'], 'POST') + uri = rq[0]['uri'] + scheme, netloc, path, qs, _ = urlsplit(uri) + self.assertEqual(scheme, 'http') + self.assertEqual(netloc, 'example.com') + self.assertEqual(path, '/b/name/o') + self.assertEqual(dict(parse_qsl(qs)), + {'uploadType': 'media', 'name': BLOB_NAME}) + headers = dict( + [(x.title(), str(y)) for x, y in rq[0]['headers'].items()]) + + self.assertEqual(headers['X-Goog-Encryption-Algorithm'], 'AES256') + self.assertEqual(headers['X-Goog-Encryption-Key'], HEADER_KEY_VALUE) + self.assertEqual(headers['X-Goog-Encryption-Key-Sha256'], + HEADER_KEY_HASH_VALUE) + self.assertEqual(headers['Content-Length'], str(len(ENCODED))) + self.assertEqual(headers['Content-Type'], 'text/plain') + self.assertEqual(rq[0]['body'], ENCODED) + def test_make_public(self): from six.moves.http_client import OK from gcloud.storage.acl import _ACLEntity diff --git a/gcloud/test__helpers.py b/gcloud/test__helpers.py index dffd8f0cc38c..77c2108ffabc 100644 --- a/gcloud/test__helpers.py +++ b/gcloud/test__helpers.py @@ -586,6 +586,27 @@ def test_with_nonstring_type(self): self.assertRaises(TypeError, self._callFUT, value) +class Test__bytes_to_unicode(unittest2.TestCase): + + def _callFUT(self, *args, **kwargs): + from gcloud._helpers import _bytes_to_unicode + return _bytes_to_unicode(*args, **kwargs) + + def test_with_bytes(self): + value = b'bytes-val' + encoded_value = 'bytes-val' + self.assertEqual(self._callFUT(value), encoded_value) + + def test_with_unicode(self): + value = u'string-val' + encoded_value = 'string-val' + self.assertEqual(self._callFUT(value), encoded_value) + + def test_with_nonstring_type(self): + value = object() + self.assertRaises(ValueError, self._callFUT, value) + + class Test__pb_timestamp_to_datetime(unittest2.TestCase): def _callFUT(self, timestamp):