From 44cda35e0de151c856ab50f89421fe36da3bd984 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Dec 2016 11:43:26 -0500 Subject: [PATCH 1/8] Reorder to match docs. --- bigquery/google/cloud/bigquery/_helpers.py | 12 ++++---- bigquery/unit_tests/test__helpers.py | 34 +++++++++++----------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 8cebe9fbec01..7d2fbc504c4f 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -47,6 +47,11 @@ def _bool_from_json(value, field): return value.lower() in ['t', 'true', '1'] +def _string_from_json(value, _): + """NOOP string -> string coercion""" + return value + + def _timestamp_from_json(value, field): """Coerce 'value' to a datetime, if set or not nullable.""" if _not_null(value, field): @@ -82,11 +87,6 @@ def _record_from_json(value, field): return record -def _string_from_json(value, _): - """NOOP string -> string coercion""" - return value - - _CELLDATA_FROM_JSON = { 'INTEGER': _int_from_json, 'INT64': _int_from_json, @@ -94,11 +94,11 @@ def _string_from_json(value, _): 'FLOAT64': _float_from_json, 'BOOLEAN': _bool_from_json, 'BOOL': _bool_from_json, + 'STRING': _string_from_json, 'TIMESTAMP': _timestamp_from_json, 'DATETIME': _datetime_from_json, 'DATE': _date_from_json, 'RECORD': _record_from_json, - 'STRING': _string_from_json, } diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index b3ccb1d715f5..60720dfce56c 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -105,6 +105,23 @@ def test_w_value_other(self): self.assertFalse(coerced) +class Test_string_from_json(unittest.TestCase): + + def _call_fut(self, value, field): + from google.cloud.bigquery._helpers import _string_from_json + return _string_from_json(value, field) + + def test_w_none_nullable(self): + self.assertIsNone(self._call_fut(None, _Field('NULLABLE'))) + + def test_w_none_required(self): + self.assertIsNone(self._call_fut(None, _Field('RECORD'))) + + def test_w_string_value(self): + coerced = self._call_fut('Wonderful!', object()) + self.assertEqual(coerced, 'Wonderful!') + + class Test_timestamp_from_json(unittest.TestCase): def _call_fut(self, value, field): @@ -238,23 +255,6 @@ def test_w_record_subfield(self): self.assertEqual(coerced, expected) -class Test_string_from_json(unittest.TestCase): - - def _call_fut(self, value, field): - from google.cloud.bigquery._helpers import _string_from_json - return _string_from_json(value, field) - - def test_w_none_nullable(self): - self.assertIsNone(self._call_fut(None, _Field('NULLABLE'))) - - def test_w_none_required(self): - self.assertIsNone(self._call_fut(None, _Field('RECORD'))) - - def test_w_string_value(self): - coerced = self._call_fut('Wonderful!', object()) - self.assertEqual(coerced, 'Wonderful!') - - class Test_row_from_json(unittest.TestCase): def _call_fut(self, row, schema): From 9f6237569e1a9a07c803bfe5e746e41f7c6ce314 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Dec 2016 11:51:11 -0500 Subject: [PATCH 2/8] Fix field mode for test. --- bigquery/unit_tests/test__helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index 60720dfce56c..f1cd7b6deb32 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -115,7 +115,7 @@ def test_w_none_nullable(self): self.assertIsNone(self._call_fut(None, _Field('NULLABLE'))) def test_w_none_required(self): - self.assertIsNone(self._call_fut(None, _Field('RECORD'))) + self.assertIsNone(self._call_fut(None, _Field('REQUIRED'))) def test_w_string_value(self): coerced = self._call_fut('Wonderful!', object()) From a8b534cb519f80cca4d823f45d5c72ff7db1bee3 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Dec 2016 11:59:03 -0500 Subject: [PATCH 3/8] Add support for 'BYTES' columns / parameters. --- bigquery/google/cloud/bigquery/_helpers.py | 16 +++++++++ bigquery/unit_tests/test__helpers.py | 38 ++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index 7d2fbc504c4f..b162b4ddad28 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -14,6 +14,7 @@ """Shared helper functions for BigQuery API classes.""" +import base64 from collections import OrderedDict import datetime @@ -52,6 +53,12 @@ def _string_from_json(value, _): return value +def _bytes_from_json(value, field): + """Base64-decode value""" + if _not_null(value, field): + return base64.decodestring(value) + + def _timestamp_from_json(value, field): """Coerce 'value' to a datetime, if set or not nullable.""" if _not_null(value, field): @@ -95,6 +102,7 @@ def _record_from_json(value, field): 'BOOLEAN': _bool_from_json, 'BOOL': _bool_from_json, 'STRING': _string_from_json, + 'BYTES': _bytes_from_json, 'TIMESTAMP': _timestamp_from_json, 'DATETIME': _datetime_from_json, 'DATE': _date_from_json, @@ -121,6 +129,13 @@ def _bool_to_json(value): return value +def _bytes_to_json(value): + """Coerce 'value' to an JSON-compatible representation.""" + if isinstance(value, bytes): + value = base64.encodestring(value) + return value + + def _timestamp_to_json(value): """Coerce 'value' to an JSON-compatible representation.""" if isinstance(value, datetime.datetime): @@ -149,6 +164,7 @@ def _date_to_json(value): 'FLOAT64': _float_to_json, 'BOOLEAN': _bool_to_json, 'BOOL': _bool_to_json, + 'BYTES': _bytes_to_json, 'TIMESTAMP': _timestamp_to_json, 'DATETIME': _datetime_to_json, 'DATE': _date_to_json, diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index f1cd7b6deb32..c60ea7a41d27 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -122,6 +122,27 @@ def test_w_string_value(self): self.assertEqual(coerced, 'Wonderful!') +class Test_bytes_from_json(unittest.TestCase): + + def _call_fut(self, value, field): + from google.cloud.bigquery._helpers import _bytes_from_json + return _bytes_from_json(value, field) + + def test_w_none_nullable(self): + self.assertIsNone(self._call_fut(None, _Field('NULLABLE'))) + + def test_w_none_required(self): + with self.assertRaises(TypeError): + self._call_fut(None, _Field('REQUIRED')) + + def test_w_base64_encoded_value(self): + import base64 + expected = 'Wonderful!' + encoded = base64.encodestring(expected) + coerced = self._call_fut(encoded, object()) + self.assertEqual(coerced, expected) + + class Test_timestamp_from_json(unittest.TestCase): def _call_fut(self, value, field): @@ -471,6 +492,23 @@ def test_w_string(self): self.assertEqual(self._call_fut('false'), 'false') +class Test_bytes_to_json(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud.bigquery._helpers import _bytes_to_json + return _bytes_to_json(value) + + def test_w_non_bytes(self): + non_bytes = object() + self.assertIs(self._call_fut(non_bytes), non_bytes) + + def test_w_bytes(self): + import base64 + source = b'source' + expected = base64.encodestring(source) + self.assertEqual(self._call_fut(source), expected) + + class Test_timestamp_to_json(unittest.TestCase): def _call_fut(self, value): From 20cdfc78bebb710e3bb2bf92c866acdb51af2991 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Dec 2016 12:11:21 -0500 Subject: [PATCH 4/8] Add helper for parsing zoneless time strings. --- core/google/cloud/_helpers.py | 13 +++++++++++++ core/unit_tests/test__helpers.py | 12 ++++++++++++ 2 files changed, 25 insertions(+) diff --git a/core/google/cloud/_helpers.py b/core/google/cloud/_helpers.py index 9b4ec5736cb0..50936915fa33 100644 --- a/core/google/cloud/_helpers.py +++ b/core/google/cloud/_helpers.py @@ -251,6 +251,19 @@ def _date_from_iso8601_date(value): return datetime.datetime.strptime(value, '%Y-%m-%d').date() +def _time_from_iso8601_time_naive(value): + """Convert a zoneless ISO8601 time string to naive datetime time + + :type value: str + :param value: The time string to convert + + :rtype: :class:`datetime.time` + :returns: A datetime time object created from the string + + """ + return datetime.datetime.strptime(value, '%H:%M:%S').time() + + def _rfc3339_to_datetime(dt_str): """Convert a microsecond-precision timetamp to a native datetime. diff --git a/core/unit_tests/test__helpers.py b/core/unit_tests/test__helpers.py index 78391e56ef42..08c27ae556e2 100644 --- a/core/unit_tests/test__helpers.py +++ b/core/unit_tests/test__helpers.py @@ -265,6 +265,18 @@ def test_todays_date(self): self.assertEqual(self._call_fut(TODAY.strftime("%Y-%m-%d")), TODAY) +class Test___time_from_iso8601_time_naive(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud._helpers import _time_from_iso8601_time_naive + return _time_from_iso8601_time_naive(value) + + def test_todays_date(self): + import datetime + WHEN = datetime.time(12, 9, 42) + self.assertEqual(self._call_fut(("12:09:42")), WHEN) + + class Test__rfc3339_to_datetime(unittest.TestCase): def _call_fut(self, dt_str): From 175e02771062bd933e11e44669765a56217b252a Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Dec 2016 12:24:06 -0500 Subject: [PATCH 5/8] Add support for 'TIME' columns / parameters. --- bigquery/google/cloud/bigquery/_helpers.py | 18 +++++++++++ bigquery/unit_tests/test__helpers.py | 37 ++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/bigquery/google/cloud/bigquery/_helpers.py b/bigquery/google/cloud/bigquery/_helpers.py index b162b4ddad28..44183a3a3f2b 100644 --- a/bigquery/google/cloud/bigquery/_helpers.py +++ b/bigquery/google/cloud/bigquery/_helpers.py @@ -23,6 +23,7 @@ from google.cloud._helpers import _datetime_to_rfc3339 from google.cloud._helpers import _microseconds_from_datetime from google.cloud._helpers import _RFC3339_NO_FRACTION +from google.cloud._helpers import _time_from_iso8601_time_naive def _not_null(value, field): @@ -76,9 +77,17 @@ def _datetime_from_json(value, field): def _date_from_json(value, field): """Coerce 'value' to a datetime date, if set or not nullable""" if _not_null(value, field): + # value will be a string, in YYYY-MM-DD form. return _date_from_iso8601_date(value) +def _time_from_json(value, field): + """Coerce 'value' to a datetime date, if set or not nullable""" + if _not_null(value, field): + # value will be a string, in HH:MM:SS form. + return _time_from_iso8601_time_naive(value) + + def _record_from_json(value, field): """Coerce 'value' to a mapping, if set or not nullable.""" if _not_null(value, field): @@ -106,6 +115,7 @@ def _record_from_json(value, field): 'TIMESTAMP': _timestamp_from_json, 'DATETIME': _datetime_from_json, 'DATE': _date_from_json, + 'TIME': _time_from_json, 'RECORD': _record_from_json, } @@ -157,6 +167,13 @@ def _date_to_json(value): return value +def _time_to_json(value): + """Coerce 'value' to an JSON-compatible representation.""" + if isinstance(value, datetime.time): + value = value.isoformat() + return value + + _SCALAR_VALUE_TO_JSON = { 'INTEGER': _int_to_json, 'INT64': _int_to_json, @@ -168,6 +185,7 @@ def _date_to_json(value): 'TIMESTAMP': _timestamp_to_json, 'DATETIME': _datetime_to_json, 'DATE': _date_to_json, + 'TIME': _time_to_json, } diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index c60ea7a41d27..5dc27c1c06ac 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -215,6 +215,27 @@ def test_w_string_value(self): datetime.date(1987, 9, 22)) +class Test_time_from_json(unittest.TestCase): + + def _call_fut(self, value, field): + from google.cloud.bigquery._helpers import _time_from_json + return _time_from_json(value, field) + + def test_w_none_nullable(self): + self.assertIsNone(self._call_fut(None, _Field('NULLABLE'))) + + def test_w_none_required(self): + with self.assertRaises(TypeError): + self._call_fut(None, _Field('REQUIRED')) + + def test_w_string_value(self): + import datetime + coerced = self._call_fut('12:12:27', object()) + self.assertEqual( + coerced, + datetime.time(12, 12, 27)) + + class Test_record_from_json(unittest.TestCase): def _call_fut(self, value, field): @@ -560,6 +581,22 @@ def test_w_datetime(self): self.assertEqual(self._call_fut(when), '2016-12-03') +class Test_time_to_json(unittest.TestCase): + + def _call_fut(self, value): + from google.cloud.bigquery._helpers import _time_to_json + return _time_to_json(value) + + def test_w_string(self): + RFC3339 = '12:13:41' + self.assertEqual(self._call_fut(RFC3339), RFC3339) + + def test_w_datetime(self): + import datetime + when = datetime.time(12, 13, 41) + self.assertEqual(self._call_fut(when), '12:13:41') + + class Test_ConfigurationProperty(unittest.TestCase): @staticmethod From 240d44aa71a7ab0090fc9dfb9a286c8e1501fb7c Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Dec 2016 12:44:15 -0500 Subject: [PATCH 6/8] Add system tests for standard SQL scalar column types. --- system_tests/bigquery.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py index bf913d499afb..635d9a6fb5a8 100644 --- a/system_tests/bigquery.py +++ b/system_tests/bigquery.py @@ -479,12 +479,49 @@ def _job_done(instance): # raise an error, and that the job completed (in the `retry()` # above). - def test_sync_query_w_nested_arrays_and_structs(self): + def test_sync_query_w_standard_sql_types(self): + import datetime + from google.cloud._helpers import UTC + naive = datetime.datetime(2016, 12, 5, 12, 41, 9) + stamp = "%s %s" %(naive.date().isoformat(), naive.time().isoformat()) + zoned = naive.replace(tzinfo=UTC) EXAMPLES = [ { 'sql': 'SELECT 1', 'expected': 1, }, + { + 'sql': 'SELECT 1.3', + 'expected': 1.3, + }, + { + 'sql': 'SELECT TRUE', + 'expected': True, + }, + { + 'sql': 'SELECT "ABC"', + 'expected': 'ABC', + }, + { + 'sql': 'SELECT CAST("foo" AS BYTES)', + 'expected': b'foo', + }, + { + 'sql': 'SELECT TIMESTAMP "%s"' % (stamp,), + 'expected': zoned, + }, + { + 'sql': 'SELECT DATETIME(TIMESTAMP "%s")' % (stamp,), + 'expected': naive, + }, + { + 'sql': 'SELECT DATE(TIMESTAMP "%s")' % (stamp,), + 'expected': naive.date(), + }, + { + 'sql': 'SELECT TIME(TIMESTAMP "%s")' % (stamp,), + 'expected': naive.time(), + }, { 'sql': 'SELECT (1, 2)', 'expected': {'_field_1': 1, '_field_2': 2}, From 2c1ccd2ea1a5753bc9977db0b254b7d9e192f64a Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Dec 2016 13:23:55 -0500 Subject: [PATCH 7/8] Fix base64 encodeing of test value on Py3k. --- bigquery/unit_tests/test__helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/unit_tests/test__helpers.py b/bigquery/unit_tests/test__helpers.py index 5dc27c1c06ac..76eacc9a04a8 100644 --- a/bigquery/unit_tests/test__helpers.py +++ b/bigquery/unit_tests/test__helpers.py @@ -137,7 +137,7 @@ def test_w_none_required(self): def test_w_base64_encoded_value(self): import base64 - expected = 'Wonderful!' + expected = b'Wonderful!' encoded = base64.encodestring(expected) coerced = self._call_fut(encoded, object()) self.assertEqual(coerced, expected) From 2518708970a81f841453a7744231666a12506c39 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 5 Dec 2016 13:49:42 -0500 Subject: [PATCH 8/8] pylint --- system_tests/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/bigquery.py b/system_tests/bigquery.py index 635d9a6fb5a8..ff6d3ceff7d9 100644 --- a/system_tests/bigquery.py +++ b/system_tests/bigquery.py @@ -483,7 +483,7 @@ def test_sync_query_w_standard_sql_types(self): import datetime from google.cloud._helpers import UTC naive = datetime.datetime(2016, 12, 5, 12, 41, 9) - stamp = "%s %s" %(naive.date().isoformat(), naive.time().isoformat()) + stamp = "%s %s" % (naive.date().isoformat(), naive.time().isoformat()) zoned = naive.replace(tzinfo=UTC) EXAMPLES = [ {