Skip to content

Commit

Permalink
Converting _datetime_from_millis to handle microseconds.
Browse files Browse the repository at this point in the history
Also making it require a float (or int) instead of bailing
out for NoneType. Changing the use of the method in
bigquery to not pass potentially null values and to multiply
by 1000.0 (convert from millis to micros). Also updating
micros -> datetime conversion in datastore to use the newly
converted method.
  • Loading branch information
dhermes committed Aug 11, 2015
1 parent 41e8dc5 commit 41e6439
Show file tree
Hide file tree
Showing 5 changed files with 40 additions and 33 deletions.
19 changes: 8 additions & 11 deletions gcloud/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,19 +222,16 @@ def _millis(when):
return millis


def _datetime_from_millis(value):
"""Convert non-none timestamp to datetime, assuming UTC.
def _datetime_from_microseconds(value):
"""Convert timestamp to datetime, assuming UTC.
:rtype: :class:`datetime.datetime`, or ``NoneType``
:type value: float
:param value: The timestamp to convert
:rtype: :class:`datetime.datetime`
:returns: The datetime object created from the value.
"""
if value is not None:
# back-end returns timestamps as milliseconds since the epoch
seconds = int(value / 1000.0)
microseconds = 1000.0 * (value - 1000 * seconds)
return (
_EPOCH +
datetime.timedelta(seconds=seconds, microseconds=microseconds)
)
return _EPOCH + datetime.timedelta(microseconds=value)


def _microseconds_from_datetime(value):
Expand Down
12 changes: 9 additions & 3 deletions gcloud/bigquery/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"""Define API Datasets."""
import six

from gcloud._helpers import _datetime_from_millis
from gcloud._helpers import _datetime_from_microseconds
from gcloud.exceptions import NotFound
from gcloud.bigquery.table import Table

Expand Down Expand Up @@ -114,7 +114,10 @@ def created(self):
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the creation time (None until set from the server).
"""
return _datetime_from_millis(self._properties.get('creationTime'))
creation_time = self._properties.get('creationTime')
if creation_time is not None:
# creation_time will be in milliseconds.
return _datetime_from_microseconds(1000.0 * creation_time)

@property
def dataset_id(self):
Expand All @@ -141,7 +144,10 @@ def modified(self):
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the modification time (None until set from the server).
"""
return _datetime_from_millis(self._properties.get('lastModifiedTime'))
modified_time = self._properties.get('lastModifiedTime')
if modified_time is not None:
# modified_time will be in milliseconds.
return _datetime_from_microseconds(1000.0 * modified_time)

@property
def self_link(self):
Expand Down
20 changes: 15 additions & 5 deletions gcloud/bigquery/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import six

from gcloud._helpers import _datetime_from_millis
from gcloud._helpers import _datetime_from_microseconds
from gcloud._helpers import _millis_from_datetime
from gcloud.exceptions import NotFound

Expand Down Expand Up @@ -116,7 +116,10 @@ def created(self):
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the creation time (None until set from the server).
"""
return _datetime_from_millis(self._properties.get('creationTime'))
creation_time = self._properties.get('creationTime')
if creation_time is not None:
# creation_time will be in milliseconds.
return _datetime_from_microseconds(1000.0 * creation_time)

@property
def etag(self):
Expand All @@ -134,7 +137,10 @@ def modified(self):
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the modification time (None until set from the server).
"""
return _datetime_from_millis(self._properties.get('lastModifiedTime'))
modified_time = self._properties.get('lastModifiedTime')
if modified_time is not None:
# modified_time will be in milliseconds.
return _datetime_from_microseconds(1000.0 * modified_time)

@property
def num_bytes(self):
Expand Down Expand Up @@ -212,7 +218,10 @@ def expires(self):
:rtype: ``datetime.datetime``, or ``NoneType``
:returns: the expiration time, or None
"""
return _datetime_from_millis(self._properties.get('expirationTime'))
expiration_time = self._properties.get('expirationTime')
if expiration_time is not None:
# expiration_time will be in milliseconds.
return _datetime_from_microseconds(1000.0 * expiration_time)

@expires.setter
def expires(self, value):
Expand Down Expand Up @@ -727,7 +736,8 @@ def _bool_from_json(value, field):

def _datetime_from_json(value, field):
if _not_null(value, field):
return _datetime_from_millis(float(value))
# Field value will be in milliseconds.

This comment has been minimized.

Copy link
@arterrey

arterrey Sep 8, 2015

@dhermes I'm getting values out of BigQuery in seconds. So * 1000.0 is not enough - it needs to be * 1000000.0 to work. However, I'm not sure if that is a bug in the BigQuery API or a bug in this client library.

This comment has been minimized.

Copy link
@dhermes

dhermes Sep 8, 2015

Author Contributor

Thanks a lot for noticing! I filed #1125 for you and asked a follow up question there.

This comment has been minimized.

Copy link
@arterrey

arterrey Sep 8, 2015

oops, I also notices and filed #1126

This comment has been minimized.

Copy link
@dhermes

dhermes Sep 8, 2015

Author Contributor

No worries. Let's follow up on #1125.

return _datetime_from_microseconds(1000.0 * float(value))


def _record_from_json(value, field):
Expand Down
8 changes: 3 additions & 5 deletions gcloud/datastore/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from google.protobuf.internal.type_checkers import Int64ValueChecker
import six

from gcloud._helpers import UTC
from gcloud._helpers import _datetime_from_microseconds
from gcloud._helpers import _microseconds_from_datetime
from gcloud.datastore import _datastore_v1_pb2 as datastore_pb
from gcloud.datastore.entity import Entity
Expand Down Expand Up @@ -224,9 +224,7 @@ def _get_value_from_value_pb(value_pb):
result = None
if value_pb.HasField('timestamp_microseconds_value'):
microseconds = value_pb.timestamp_microseconds_value
naive = (datetime.datetime.utcfromtimestamp(0) +
datetime.timedelta(microseconds=microseconds))
result = naive.replace(tzinfo=UTC)
result = _datetime_from_microseconds(microseconds)

elif value_pb.HasField('key_value'):
result = key_from_protobuf(value_pb.key_value)
Expand Down Expand Up @@ -285,7 +283,7 @@ def _set_protobuf_value(value_pb, val):
:type value_pb: :class:`gcloud.datastore._datastore_v1_pb2.Value`
:param value_pb: The value protobuf to which the value is being assigned.
:type val: `datetime.datetime`, boolean, float, integer, string,
:type val: :class:`datetime.datetime`, boolean, float, integer, string,
:class:`gcloud.datastore.key.Key`,
:class:`gcloud.datastore.entity.Entity`,
:param val: The value to be assigned.
Expand Down
14 changes: 5 additions & 9 deletions gcloud/test__helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,25 +334,21 @@ def test_w_naive_datetime(self):
self.assertEqual(result, MILLIS)


class Test__datetime_from_millis(unittest2.TestCase):
class Test__datetime_from_microseconds(unittest2.TestCase):

def _callFUT(self, value):
from gcloud._helpers import _datetime_from_millis
return _datetime_from_millis(value)
from gcloud._helpers import _datetime_from_microseconds
return _datetime_from_microseconds(value)

def test_w_none(self):
self.assertTrue(self._callFUT(None) is None)

def test_w_millis(self):
def test_it(self):
import datetime
from gcloud._helpers import UTC
from gcloud._helpers import _microseconds_from_datetime

NOW = datetime.datetime(2015, 7, 29, 17, 45, 21, 123456,
tzinfo=UTC)
NOW_MICROS = _microseconds_from_datetime(NOW)
MILLIS = NOW_MICROS / 1000.0
self.assertEqual(self._callFUT(MILLIS), NOW)
self.assertEqual(self._callFUT(NOW_MICROS), NOW)


class _AppIdentity(object):
Expand Down

0 comments on commit 41e6439

Please sign in to comment.