Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Streaming: strip unused features #1209

Merged
merged 17 commits into from
Nov 19, 2015
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 20 additions & 19 deletions gcloud/storage/blob.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,18 @@
import six
from six.moves.urllib.parse import quote # pylint: disable=F0401

from gcloud.streaming import http_wrapper
from gcloud.streaming import transfer

from gcloud._helpers import _RFC3339_MICROS
from gcloud._helpers import UTC
from gcloud.credentials import generate_signed_url
from gcloud.exceptions import NotFound
from gcloud.storage._helpers import _PropertyMixin
from gcloud.storage._helpers import _scalar_property
from gcloud.storage.acl import ObjectACL
from gcloud.streaming.http_wrapper import Request
from gcloud.streaming.http_wrapper import make_api_request
from gcloud.streaming.transfer import Download
from gcloud.streaming.transfer import RESUMABLE_UPLOAD
from gcloud.streaming.transfer import Upload


_API_ACCESS_ENDPOINT = 'https://storage.googleapis.com'
Expand Down Expand Up @@ -258,22 +260,22 @@ def download_to_file(self, file_obj, client=None):
download_url = self.media_link

# Use apitools 'Download' facility.
download = transfer.Download.FromStream(file_obj, auto_transfer=False)
download = Download.from_stream(file_obj, auto_transfer=False)
headers = {}
if self.chunk_size is not None:
download.chunksize = self.chunk_size
headers['Range'] = 'bytes=0-%d' % (self.chunk_size - 1,)
request = http_wrapper.Request(download_url, 'GET', headers)
request = Request(download_url, 'GET', headers)

# Use the private ``_connection`` rather than the public
# ``.connection``, since the public connection may be a batch. A
# batch wraps a client's connection, but does not store the `http`
# object. The rest (API_BASE_URL and build_api_url) are also defined
# on the Batch class, but we just use the wrapped connection since
# it has all three (http, API_BASE_URL and build_api_url).
download.InitializeDownload(request, client._connection.http)
download.initialize_download(request, client._connection.http)

download.StreamInChunks()
download.stream_file(use_chunks=True)

This comment was marked as spam.


def download_to_filename(self, filename, client=None):
"""Download the contents of this blob into a named file.
Expand Down Expand Up @@ -383,9 +385,8 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
'User-Agent': connection.USER_AGENT,
}

upload = transfer.Upload(file_obj, content_type, total_bytes,
auto_transfer=False,
chunksize=self.chunk_size)
upload = Upload(file_obj, content_type, total_bytes,
auto_transfer=False, chunksize=self.chunk_size)

url_builder = _UrlBuilder(bucket_name=self.bucket.name,
object_name=self.name)
Expand All @@ -397,21 +398,21 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
path=self.bucket.path + '/o')

# Use apitools 'Upload' facility.
request = http_wrapper.Request(upload_url, 'POST', headers)
request = Request(upload_url, 'POST', headers)

upload.ConfigureRequest(upload_config, request, url_builder)
upload.configure_request(upload_config, request, url_builder)
query_params = url_builder.query_params
base_url = connection.API_BASE_URL + '/upload'
request.url = connection.build_api_url(api_base_url=base_url,
path=self.bucket.path + '/o',
query_params=query_params)
upload.InitializeUpload(request, connection.http)
upload.initialize_upload(request, connection.http)

if upload.strategy == transfer.RESUMABLE_UPLOAD:
http_response = upload.StreamInChunks()
if upload.strategy == RESUMABLE_UPLOAD:
http_response = upload.stream_file(use_chunks=True)
else:
http_response = http_wrapper.MakeRequest(connection.http, request,
retries=num_retries)
http_response = make_api_request(connection.http, request,
retries=num_retries)
response_content = http_response.content
if not isinstance(response_content,
six.string_types): # pragma: NO COVER Python3
Expand Down Expand Up @@ -765,7 +766,7 @@ def updated(self):


class _UploadConfig(object):
"""Faux message FBO apitools' 'ConfigureRequest'.
"""Faux message FBO apitools' 'configure_request'.

Values extracted from apitools
'samples/storage_sample/storage/storage_v1_client.py'
Expand All @@ -779,7 +780,7 @@ class _UploadConfig(object):


class _UrlBuilder(object):
"""Faux builder FBO apitools' 'ConfigureRequest'"""
"""Faux builder FBO apitools' 'configure_request'"""
def __init__(self, bucket_name, object_name):
self.query_params = {'name': object_name}
self._bucket_name = bucket_name
Expand Down
2 changes: 1 addition & 1 deletion gcloud/storage/test_blob.py
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,7 @@ def test_upload_from_file_resumable(self):
blob.chunk_size = 5

# Set the threshhold low enough that we force a resumable uploada.
with _Monkey(transfer, _RESUMABLE_UPLOAD_THRESHOLD=5):
with _Monkey(transfer, RESUMABLE_UPLOAD_THRESHOLD=5):
with _NamedTemporaryFile() as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
Expand Down
32 changes: 1 addition & 31 deletions gcloud/streaming/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,41 +64,11 @@ class InvalidUserInputError(InvalidDataError):
"""User-provided input is invalid."""


class InvalidDataFromServerError(InvalidDataError, CommunicationError):

"""Data received from the server is malformed."""


class BatchError(Error):

"""Error generated while constructing a batch request."""


class ConfigurationError(Error):

"""Base class for configuration errors."""


class GeneratedClientError(Error):

"""The generated client configuration is invalid."""


class ConfigurationValueError(UserError):

"""Some part of the user-specified client configuration is invalid."""


class ResourceUnavailableError(Error):

"""User requested an unavailable resource."""


class CredentialsError(Error):

"""Errors related to invalid credentials."""


class TransferError(CommunicationError):

"""Errors related to transfers."""
Expand Down Expand Up @@ -138,7 +108,7 @@ class BadStatusCodeError(HttpError):
"""The request completed but returned a bad status code."""


class NotYetImplementedError(GeneratedClientError):
class NotYetImplementedError(Error):

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.

This comment was marked as spam.


"""This functionality is not yet implemented."""

Expand Down
Loading