From 260672ca3c0702b0b21dd6673c6dd2bba9ebc8c6 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Wed, 7 Apr 2021 16:47:24 -0600 Subject: [PATCH 01/20] split out and pytestify list_tables tests. Also, exercise dataset polymorphism in some of the tests. --- tests/unit/conftest.py | 18 ++++ tests/unit/helpers.py | 42 +++++++++ tests/unit/test_client.py | 155 ------------------------------- tests/unit/test_list_tables.py | 161 +++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 155 deletions(-) create mode 100644 tests/unit/conftest.py create mode 100644 tests/unit/test_list_tables.py diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 000000000..9a9602cc0 --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,18 @@ +import pytest + +from .helpers import make_client + + +@pytest.fixture +def client(): + yield make_client() + + +@pytest.fixture +def PROJECT(): + yield "PROJECT" + + +@pytest.fixture +def DS_ID(): + yield "DATASET_ID" diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py index b51b0bbb7..19f6136bf 100644 --- a/tests/unit/helpers.py +++ b/tests/unit/helpers.py @@ -12,6 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import google.cloud.bigquery.client +import google.cloud.bigquery.dataset +import mock +import pytest + def make_connection(*responses): import google.cloud.bigquery._http @@ -31,3 +36,40 @@ def _to_pyarrow(value): import pyarrow return pyarrow.array([value])[0] + + +def make_client(project="PROJECT", credentials=None, **kw): + if credentials is None: + credentials = mock.Mock(spec=google.auth.credentials.Credentials) + + return google.cloud.bigquery.client.Client(project, credentials, **kw) + + +def make_dataset(project, ds_id): + return google.cloud.bigquery.dataset.Dataset( + google.cloud.bigquery.dataset.DatasetReference(project, ds_id) + ) + + +def make_dataset_list_item(project, ds_id): + return google.cloud.bigquery.dataset.DatasetListItem( + dict(datasetReference=dict(projectId=project, datasetId=ds_id)) + ) + + +def identity(x): + return x + + +def get_reference(x): + return x.reference + + +dataset_like = [ + (google.cloud.bigquery.dataset.DatasetReference, identity), + (make_dataset, identity), +] + +dataset_polymorphic = pytest.mark.parametrize( + "make_dataset,get_reference", dataset_like +) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 26ef340de..34365176d 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -2926,29 +2926,6 @@ def test_update_table_delete_property(self): self.assertEqual(req[1]["data"], sent) self.assertIsNone(table3.description) - def test_list_tables_empty_w_timeout(self): - path = "/projects/{}/datasets/{}/tables".format(self.PROJECT, self.DS_ID) - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = make_connection({}) - - dataset = DatasetReference(self.PROJECT, self.DS_ID) - iterator = client.list_tables(dataset, timeout=7.5) - self.assertIs(iterator.dataset, dataset) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - page = next(iterator.pages) - - final_attributes.assert_called_once_with({"path": path}, client, None) - tables = list(page) - token = iterator.next_page_token - - self.assertEqual(tables, []) - self.assertIsNone(token) - conn.api_request.assert_called_once_with( - method="GET", path=path, query_params={}, timeout=7.5 - ) def test_list_models_empty_w_timeout(self): path = "/projects/{}/datasets/{}/models".format(self.PROJECT, self.DS_ID) @@ -3125,138 +3102,6 @@ def test_list_routines_wrong_type(self): DatasetReference(self.PROJECT, self.DS_ID).table("foo") ) - def test_list_tables_defaults(self): - from google.cloud.bigquery.table import TableListItem - - TABLE_1 = "table_one" - TABLE_2 = "table_two" - PATH = "projects/%s/datasets/%s/tables" % (self.PROJECT, self.DS_ID) - TOKEN = "TOKEN" - DATA = { - "nextPageToken": TOKEN, - "tables": [ - { - "kind": "bigquery#table", - "id": "%s:%s.%s" % (self.PROJECT, self.DS_ID, TABLE_1), - "tableReference": { - "tableId": TABLE_1, - "datasetId": self.DS_ID, - "projectId": self.PROJECT, - }, - "type": "TABLE", - }, - { - "kind": "bigquery#table", - "id": "%s:%s.%s" % (self.PROJECT, self.DS_ID, TABLE_2), - "tableReference": { - "tableId": TABLE_2, - "datasetId": self.DS_ID, - "projectId": self.PROJECT, - }, - "type": "TABLE", - }, - ], - } - - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = make_connection(DATA) - dataset = DatasetReference(self.PROJECT, self.DS_ID) - - iterator = client.list_tables(dataset) - self.assertIs(iterator.dataset, dataset) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - page = next(iterator.pages) - - final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) - tables = list(page) - token = iterator.next_page_token - - self.assertEqual(len(tables), len(DATA["tables"])) - for found, expected in zip(tables, DATA["tables"]): - self.assertIsInstance(found, TableListItem) - self.assertEqual(found.full_table_id, expected["id"]) - self.assertEqual(found.table_type, expected["type"]) - self.assertEqual(token, TOKEN) - - conn.api_request.assert_called_once_with( - method="GET", path="/%s" % PATH, query_params={}, timeout=None - ) - - def test_list_tables_explicit(self): - from google.cloud.bigquery.table import TableListItem - - TABLE_1 = "table_one" - TABLE_2 = "table_two" - PATH = "projects/%s/datasets/%s/tables" % (self.PROJECT, self.DS_ID) - TOKEN = "TOKEN" - DATA = { - "tables": [ - { - "kind": "bigquery#dataset", - "id": "%s:%s.%s" % (self.PROJECT, self.DS_ID, TABLE_1), - "tableReference": { - "tableId": TABLE_1, - "datasetId": self.DS_ID, - "projectId": self.PROJECT, - }, - "type": "TABLE", - }, - { - "kind": "bigquery#dataset", - "id": "%s:%s.%s" % (self.PROJECT, self.DS_ID, TABLE_2), - "tableReference": { - "tableId": TABLE_2, - "datasetId": self.DS_ID, - "projectId": self.PROJECT, - }, - "type": "TABLE", - }, - ] - } - - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = make_connection(DATA) - dataset = DatasetReference(self.PROJECT, self.DS_ID) - - iterator = client.list_tables( - # Test with string for dataset ID. - self.DS_ID, - max_results=3, - page_token=TOKEN, - ) - self.assertEqual(iterator.dataset, dataset) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - page = next(iterator.pages) - - final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) - tables = list(page) - token = iterator.next_page_token - - self.assertEqual(len(tables), len(DATA["tables"])) - for found, expected in zip(tables, DATA["tables"]): - self.assertIsInstance(found, TableListItem) - self.assertEqual(found.full_table_id, expected["id"]) - self.assertEqual(found.table_type, expected["type"]) - self.assertIsNone(token) - - conn.api_request.assert_called_once_with( - method="GET", - path="/%s" % PATH, - query_params={"maxResults": 3, "pageToken": TOKEN}, - timeout=None, - ) - - def test_list_tables_wrong_type(self): - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - with self.assertRaises(TypeError): - client.list_tables(DatasetReference(self.PROJECT, self.DS_ID).table("foo")) def test_delete_dataset(self): from google.cloud.bigquery.dataset import Dataset diff --git a/tests/unit/test_list_tables.py b/tests/unit/test_list_tables.py new file mode 100644 index 000000000..8f0566132 --- /dev/null +++ b/tests/unit/test_list_tables.py @@ -0,0 +1,161 @@ +from .helpers import make_connection, dataset_polymorphic +import google.cloud.bigquery.dataset +import mock +import pytest + + +@dataset_polymorphic +def test_list_tables_empty_w_timeout( + make_dataset, get_reference, client, PROJECT, DS_ID +): + path = "/projects/{}/datasets/{}/tables".format(PROJECT, DS_ID) + conn = client._connection = make_connection({}) + + dataset = make_dataset(PROJECT, DS_ID) + iterator = client.list_tables(dataset, timeout=7.5) + assert iterator.dataset is get_reference(dataset) + with mock.patch( + "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" + ) as final_attributes: + page = next(iterator.pages) + + final_attributes.assert_called_once_with({"path": path}, client, None) + tables = list(page) + token = iterator.next_page_token + + assert tables == [] + assert token is None + conn.api_request.assert_called_once_with( + method="GET", path=path, query_params={}, timeout=7.5 + ) + + +@dataset_polymorphic +def test_list_tables_defaults(make_dataset, get_reference, client, PROJECT, DS_ID): + from google.cloud.bigquery.table import TableListItem + + TABLE_1 = "table_one" + TABLE_2 = "table_two" + PATH = "projects/%s/datasets/%s/tables" % (PROJECT, DS_ID) + TOKEN = "TOKEN" + DATA = { + "nextPageToken": TOKEN, + "tables": [ + { + "kind": "bigquery#table", + "id": "%s:%s.%s" % (PROJECT, DS_ID, TABLE_1), + "tableReference": { + "tableId": TABLE_1, + "datasetId": DS_ID, + "projectId": PROJECT, + }, + "type": "TABLE", + }, + { + "kind": "bigquery#table", + "id": "%s:%s.%s" % (PROJECT, DS_ID, TABLE_2), + "tableReference": { + "tableId": TABLE_2, + "datasetId": DS_ID, + "projectId": PROJECT, + }, + "type": "TABLE", + }, + ], + } + + conn = client._connection = make_connection(DATA) + dataset = make_dataset(PROJECT, DS_ID) + + iterator = client.list_tables(dataset) + assert iterator.dataset is dataset + with mock.patch( + "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" + ) as final_attributes: + page = next(iterator.pages) + + final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) + tables = list(page) + token = iterator.next_page_token + + assert len(tables) == len(DATA["tables"]) + for found, expected in zip(tables, DATA["tables"]): + assert isinstance(found, TableListItem) + assert found.full_table_id == expected["id"] + assert found.table_type == expected["type"] + assert token == TOKEN + + conn.api_request.assert_called_once_with( + method="GET", path="/%s" % PATH, query_params={}, timeout=None + ) + + +def test_list_tables_explicit(client, PROJECT, DS_ID): + from google.cloud.bigquery.table import TableListItem + + TABLE_1 = "table_one" + TABLE_2 = "table_two" + PATH = "projects/%s/datasets/%s/tables" % (PROJECT, DS_ID) + TOKEN = "TOKEN" + DATA = { + "tables": [ + { + "kind": "bigquery#dataset", + "id": "%s:%s.%s" % (PROJECT, DS_ID, TABLE_1), + "tableReference": { + "tableId": TABLE_1, + "datasetId": DS_ID, + "projectId": PROJECT, + }, + "type": "TABLE", + }, + { + "kind": "bigquery#dataset", + "id": "%s:%s.%s" % (PROJECT, DS_ID, TABLE_2), + "tableReference": { + "tableId": TABLE_2, + "datasetId": DS_ID, + "projectId": PROJECT, + }, + "type": "TABLE", + }, + ] + } + + conn = client._connection = make_connection(DATA) + dataset = google.cloud.bigquery.dataset.DatasetReference(PROJECT, DS_ID) + + iterator = client.list_tables( + # Test with string for dataset ID. + DS_ID, + max_results=3, + page_token=TOKEN, + ) + assert iterator.dataset == dataset + with mock.patch( + "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" + ) as final_attributes: + page = next(iterator.pages) + + final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) + tables = list(page) + token = iterator.next_page_token + + assert len(tables) == len(DATA["tables"]) + for found, expected in zip(tables, DATA["tables"]): + assert isinstance(found, TableListItem) + assert found.full_table_id == expected["id"] + assert found.table_type == expected["type"] + assert token is None + + conn.api_request.assert_called_once_with( + method="GET", + path="/%s" % PATH, + query_params={"maxResults": 3, "pageToken": TOKEN}, + timeout=None, + ) + + +def test_list_tables_wrong_type(client): + with pytest.raises(TypeError): + client.list_tables(42) From 3a122d7f2e7528e7beee4b98d738feff5c344252 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Wed, 7 Apr 2021 16:59:50 -0600 Subject: [PATCH 02/20] list_tables now accepts DatasetListItem objects --- google/cloud/bigquery/client.py | 5 ++++- tests/unit/helpers.py | 3 ++- tests/unit/test_list_tables.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 305d60d3b..e9dc0b28c 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -1321,7 +1321,10 @@ def list_tables( ) if not isinstance(dataset, (Dataset, DatasetReference)): - raise TypeError("dataset must be a Dataset, DatasetReference, or string") + if isinstance(dataset, DatasetListItem): + dataset = dataset.reference + else: + raise TypeError("dataset must be a Dataset, DatasetReference, or string") path = "%s/tables" % dataset.path span_attributes = {"path": path} diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py index 19f6136bf..6e15dfd2f 100644 --- a/tests/unit/helpers.py +++ b/tests/unit/helpers.py @@ -68,7 +68,8 @@ def get_reference(x): dataset_like = [ (google.cloud.bigquery.dataset.DatasetReference, identity), (make_dataset, identity), -] + (make_dataset_list_item, get_reference), + ] dataset_polymorphic = pytest.mark.parametrize( "make_dataset,get_reference", dataset_like diff --git a/tests/unit/test_list_tables.py b/tests/unit/test_list_tables.py index 8f0566132..6c9f9e558 100644 --- a/tests/unit/test_list_tables.py +++ b/tests/unit/test_list_tables.py @@ -13,7 +13,7 @@ def test_list_tables_empty_w_timeout( dataset = make_dataset(PROJECT, DS_ID) iterator = client.list_tables(dataset, timeout=7.5) - assert iterator.dataset is get_reference(dataset) + assert iterator.dataset == get_reference(dataset) with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: @@ -68,7 +68,7 @@ def test_list_tables_defaults(make_dataset, get_reference, client, PROJECT, DS_I dataset = make_dataset(PROJECT, DS_ID) iterator = client.list_tables(dataset) - assert iterator.dataset is dataset + assert iterator.dataset == get_reference(dataset) with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: From d453031fc588c221f90d2616b9e3e0f3dc5438cc Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Wed, 7 Apr 2021 17:21:19 -0600 Subject: [PATCH 03/20] Get coverage to 100% But why do we run coverage on test code? --- tests/unit/helpers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py index 6e15dfd2f..049bfc513 100644 --- a/tests/unit/helpers.py +++ b/tests/unit/helpers.py @@ -38,10 +38,8 @@ def _to_pyarrow(value): return pyarrow.array([value])[0] -def make_client(project="PROJECT", credentials=None, **kw): - if credentials is None: - credentials = mock.Mock(spec=google.auth.credentials.Credentials) - +def make_client(project="PROJECT", **kw): + credentials = mock.Mock(spec=google.auth.credentials.Credentials) return google.cloud.bigquery.client.Client(project, credentials, **kw) From 61b53bcaeee3f0d1a5c701d868a5688c7533b33e Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Wed, 7 Apr 2021 17:41:18 -0600 Subject: [PATCH 04/20] lint --- tests/unit/helpers.py | 2 +- tests/unit/test_client.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py index 049bfc513..69cf557f0 100644 --- a/tests/unit/helpers.py +++ b/tests/unit/helpers.py @@ -67,7 +67,7 @@ def get_reference(x): (google.cloud.bigquery.dataset.DatasetReference, identity), (make_dataset, identity), (make_dataset_list_item, get_reference), - ] +] dataset_polymorphic = pytest.mark.parametrize( "make_dataset,get_reference", dataset_like diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 34365176d..cf2dac1dd 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -2926,7 +2926,6 @@ def test_update_table_delete_property(self): self.assertEqual(req[1]["data"], sent) self.assertIsNone(table3.description) - def test_list_models_empty_w_timeout(self): path = "/projects/{}/datasets/{}/models".format(self.PROJECT, self.DS_ID) creds = _make_credentials() @@ -3102,7 +3101,6 @@ def test_list_routines_wrong_type(self): DatasetReference(self.PROJECT, self.DS_ID).table("foo") ) - def test_delete_dataset(self): from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.dataset import DatasetReference From 0b1b7c856b4ac312a1b597cf1233e18986b69869 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Wed, 7 Apr 2021 17:41:58 -0600 Subject: [PATCH 05/20] Update exception text for DatasetListItem --- google/cloud/bigquery/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index e9dc0b28c..338be6b50 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -1324,7 +1324,10 @@ def list_tables( if isinstance(dataset, DatasetListItem): dataset = dataset.reference else: - raise TypeError("dataset must be a Dataset, DatasetReference, or string") + raise TypeError( + "dataset must be a Dataset, DatasetReference, DatasetListItem," + " or string" + ) path = "%s/tables" % dataset.path span_attributes = {"path": path} From 1349d4b1419717364484e68cf98227b0dd14bd86 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 09:43:39 -0600 Subject: [PATCH 06/20] Bypass opentelemetry tracing in unit tests. --- tests/unit/conftest.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 9a9602cc0..c6e6e8801 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,3 +1,6 @@ +import contextlib +import google.cloud.bigquery.opentelemetry_tracing +import mock import pytest from .helpers import make_client @@ -16,3 +19,18 @@ def PROJECT(): @pytest.fixture def DS_ID(): yield "DATASET_ID" + + +@pytest.fixture(autouse=True, scope="session") +def bypass_opentelemetry_tracing(): + @contextlib.contextmanager + def create_span(name, attributes=None, client=None, job_ref=None): + # Make existing test code that checks _get_final_span_attributes work: + google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes( + attributes, client, job_ref + ) + yield + + # Note that we have to mock in client, because it uses a from import. :/ + with mock.patch("google.cloud.bigquery.client.create_span", create_span): + yield From a93d6dc20f84150d11a2d875976737a8310af6c7 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 09:46:59 -0600 Subject: [PATCH 07/20] Got rid of opentelemetry tracing checks. They aren't needed. --- tests/unit/test_list_tables.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/tests/unit/test_list_tables.py b/tests/unit/test_list_tables.py index 6c9f9e558..177bc9a94 100644 --- a/tests/unit/test_list_tables.py +++ b/tests/unit/test_list_tables.py @@ -14,12 +14,8 @@ def test_list_tables_empty_w_timeout( dataset = make_dataset(PROJECT, DS_ID) iterator = client.list_tables(dataset, timeout=7.5) assert iterator.dataset == get_reference(dataset) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - page = next(iterator.pages) + page = next(iterator.pages) - final_attributes.assert_called_once_with({"path": path}, client, None) tables = list(page) token = iterator.next_page_token @@ -69,12 +65,8 @@ def test_list_tables_defaults(make_dataset, get_reference, client, PROJECT, DS_I iterator = client.list_tables(dataset) assert iterator.dataset == get_reference(dataset) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - page = next(iterator.pages) + page = next(iterator.pages) - final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) tables = list(page) token = iterator.next_page_token @@ -132,12 +124,8 @@ def test_list_tables_explicit(client, PROJECT, DS_ID): page_token=TOKEN, ) assert iterator.dataset == dataset - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - page = next(iterator.pages) + page = next(iterator.pages) - final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) tables = list(page) token = iterator.next_page_token From b4ea2a86aaeff64c7dd3eb43cdecfc79f1f93820 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 10:20:08 -0600 Subject: [PATCH 08/20] abstracted dataset-argument handling And applied it to `list_tables` and `list_models`. --- google/cloud/bigquery/client.py | 41 ++++++++++++++++----------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 338be6b50..47bc54e4e 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -449,6 +449,24 @@ def _create_bqstorage_client(self): return bigquery_storage.BigQueryReadClient(credentials=self._credentials) + + def _dataset_from_arg(self, dataset): + if isinstance(dataset, str): + dataset = DatasetReference.from_string( + dataset, default_project=self.project + ) + + if not isinstance(dataset, (Dataset, DatasetReference)): + if isinstance(dataset, DatasetListItem): + dataset = dataset.reference + else: + raise TypeError( + "dataset must be a Dataset, DatasetReference, DatasetListItem," + " or string" + ) + return dataset + + def create_dataset( self, dataset, exists_ok=False, retry=DEFAULT_RETRY, timeout=None ): @@ -1160,13 +1178,7 @@ def list_models( :class:`~google.cloud.bigquery.model.Model` contained within the requested dataset. """ - if isinstance(dataset, str): - dataset = DatasetReference.from_string( - dataset, default_project=self.project - ) - - if not isinstance(dataset, (Dataset, DatasetReference)): - raise TypeError("dataset must be a Dataset, DatasetReference, or string") + dataset = self._dataset_from_arg(dataset) path = "%s/models" % dataset.path span_attributes = {"path": path} @@ -1315,20 +1327,7 @@ def list_tables( :class:`~google.cloud.bigquery.table.TableListItem` contained within the requested dataset. """ - if isinstance(dataset, str): - dataset = DatasetReference.from_string( - dataset, default_project=self.project - ) - - if not isinstance(dataset, (Dataset, DatasetReference)): - if isinstance(dataset, DatasetListItem): - dataset = dataset.reference - else: - raise TypeError( - "dataset must be a Dataset, DatasetReference, DatasetListItem," - " or string" - ) - + dataset = self._dataset_from_arg(dataset) path = "%s/tables" % dataset.path span_attributes = {"path": path} From 712878982c61b31080797de52c74f71bb6bed855 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 10:22:04 -0600 Subject: [PATCH 09/20] Converted list_model tests to pytest and included check for dataset polymorphism --- tests/unit/test_client.py | 82 ---------------------------------- tests/unit/test_list_models.py | 72 +++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 82 deletions(-) create mode 100644 tests/unit/test_list_models.py diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index cf2dac1dd..9b0ed038d 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -2926,88 +2926,6 @@ def test_update_table_delete_property(self): self.assertEqual(req[1]["data"], sent) self.assertIsNone(table3.description) - def test_list_models_empty_w_timeout(self): - path = "/projects/{}/datasets/{}/models".format(self.PROJECT, self.DS_ID) - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = make_connection({}) - - dataset_id = "{}.{}".format(self.PROJECT, self.DS_ID) - iterator = client.list_models(dataset_id, timeout=7.5) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - page = next(iterator.pages) - - final_attributes.assert_called_once_with({"path": path}, client, None) - models = list(page) - token = iterator.next_page_token - - self.assertEqual(models, []) - self.assertIsNone(token) - conn.api_request.assert_called_once_with( - method="GET", path=path, query_params={}, timeout=7.5 - ) - - def test_list_models_defaults(self): - from google.cloud.bigquery.model import Model - - MODEL_1 = "model_one" - MODEL_2 = "model_two" - PATH = "projects/%s/datasets/%s/models" % (self.PROJECT, self.DS_ID) - TOKEN = "TOKEN" - DATA = { - "nextPageToken": TOKEN, - "models": [ - { - "modelReference": { - "modelId": MODEL_1, - "datasetId": self.DS_ID, - "projectId": self.PROJECT, - } - }, - { - "modelReference": { - "modelId": MODEL_2, - "datasetId": self.DS_ID, - "projectId": self.PROJECT, - } - }, - ], - } - - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = make_connection(DATA) - dataset = DatasetReference(self.PROJECT, self.DS_ID) - - iterator = client.list_models(dataset) - self.assertIs(iterator.dataset, dataset) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - page = next(iterator.pages) - - final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) - models = list(page) - token = iterator.next_page_token - - self.assertEqual(len(models), len(DATA["models"])) - for found, expected in zip(models, DATA["models"]): - self.assertIsInstance(found, Model) - self.assertEqual(found.model_id, expected["modelReference"]["modelId"]) - self.assertEqual(token, TOKEN) - - conn.api_request.assert_called_once_with( - method="GET", path="/%s" % PATH, query_params={}, timeout=None - ) - - def test_list_models_wrong_type(self): - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - with self.assertRaises(TypeError): - client.list_models(DatasetReference(self.PROJECT, self.DS_ID).model("foo")) - def test_list_routines_empty_w_timeout(self): creds = _make_credentials() client = self._make_one(project=self.PROJECT, credentials=creds) diff --git a/tests/unit/test_list_models.py b/tests/unit/test_list_models.py new file mode 100644 index 000000000..d98b39b94 --- /dev/null +++ b/tests/unit/test_list_models.py @@ -0,0 +1,72 @@ +from .helpers import make_connection, dataset_polymorphic +import google.cloud.bigquery.dataset +import mock +import pytest + + +def test_list_models_empty_w_timeout(client, PROJECT, DS_ID): + path = "/projects/{}/datasets/{}/models".format(PROJECT, DS_ID) + conn = client._connection = make_connection({}) + + dataset_id = "{}.{}".format(PROJECT, DS_ID) + iterator = client.list_models(dataset_id, timeout=7.5) + page = next(iterator.pages) + models = list(page) + token = iterator.next_page_token + + assert models == [] + assert token is None + conn.api_request.assert_called_once_with( + method="GET", path=path, query_params={}, timeout=7.5 + ) + +@dataset_polymorphic +def test_list_models_defaults(make_dataset, get_reference, client, PROJECT, DS_ID): + from google.cloud.bigquery.model import Model + + MODEL_1 = "model_one" + MODEL_2 = "model_two" + PATH = "projects/%s/datasets/%s/models" % (PROJECT, DS_ID) + TOKEN = "TOKEN" + DATA = { + "nextPageToken": TOKEN, + "models": [ + { + "modelReference": { + "modelId": MODEL_1, + "datasetId": DS_ID, + "projectId": PROJECT, + } + }, + { + "modelReference": { + "modelId": MODEL_2, + "datasetId": DS_ID, + "projectId": PROJECT, + } + }, + ], + } + + conn = client._connection = make_connection(DATA) + dataset = make_dataset(PROJECT, DS_ID) + + iterator = client.list_models(dataset) + assert iterator.dataset == get_reference(dataset) + page = next(iterator.pages) + models = list(page) + token = iterator.next_page_token + + assert len(models) == len(DATA["models"]) + for found, expected in zip(models, DATA["models"]): + assert isinstance(found, Model) + assert found.model_id == expected["modelReference"]["modelId"] + assert token == TOKEN + + conn.api_request.assert_called_once_with( + method="GET", path="/%s" % PATH, query_params={}, timeout=None + ) + +def test_list_models_wrong_type(client): + with pytest.raises(TypeError): + client.list_models(42) From 0ee721bf171f74af0424e7a828ab40ebd6c79322 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 10:22:46 -0600 Subject: [PATCH 10/20] removed unneeded blanl lines. --- tests/unit/test_list_tables.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/unit/test_list_tables.py b/tests/unit/test_list_tables.py index 177bc9a94..c294bee86 100644 --- a/tests/unit/test_list_tables.py +++ b/tests/unit/test_list_tables.py @@ -15,7 +15,6 @@ def test_list_tables_empty_w_timeout( iterator = client.list_tables(dataset, timeout=7.5) assert iterator.dataset == get_reference(dataset) page = next(iterator.pages) - tables = list(page) token = iterator.next_page_token @@ -66,7 +65,6 @@ def test_list_tables_defaults(make_dataset, get_reference, client, PROJECT, DS_I iterator = client.list_tables(dataset) assert iterator.dataset == get_reference(dataset) page = next(iterator.pages) - tables = list(page) token = iterator.next_page_token @@ -125,7 +123,6 @@ def test_list_tables_explicit(client, PROJECT, DS_ID): ) assert iterator.dataset == dataset page = next(iterator.pages) - tables = list(page) token = iterator.next_page_token From c778e9bd76ddb2d6811fe9f5a50bef5897bfe0a9 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 10:46:14 -0600 Subject: [PATCH 11/20] Made list_routines accept DatasetListItem and conveted list_routines tests to pytest. --- google/cloud/bigquery/client.py | 9 +--- tests/unit/test_client.py | 93 --------------------------------- 2 files changed, 1 insertion(+), 101 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 47bc54e4e..d9ee9b1b8 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -1249,14 +1249,7 @@ def list_routines( :class:`~google.cloud.bigquery.routine.Routine`s contained within the requested dataset, limited by ``max_results``. """ - if isinstance(dataset, str): - dataset = DatasetReference.from_string( - dataset, default_project=self.project - ) - - if not isinstance(dataset, (Dataset, DatasetReference)): - raise TypeError("dataset must be a Dataset, DatasetReference, or string") - + dataset = self._dataset_from_arg(dataset) path = "{}/routines".format(dataset.path) span_attributes = {"path": path} diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 9b0ed038d..6ec87482c 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -2926,99 +2926,6 @@ def test_update_table_delete_property(self): self.assertEqual(req[1]["data"], sent) self.assertIsNone(table3.description) - def test_list_routines_empty_w_timeout(self): - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = make_connection({}) - - iterator = client.list_routines("test-routines.test_routines", timeout=7.5) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - page = next(iterator.pages) - - final_attributes.assert_called_once_with( - {"path": "/projects/test-routines/datasets/test_routines/routines"}, - client, - None, - ) - routines = list(page) - token = iterator.next_page_token - - self.assertEqual(routines, []) - self.assertIsNone(token) - conn.api_request.assert_called_once_with( - method="GET", - path="/projects/test-routines/datasets/test_routines/routines", - query_params={}, - timeout=7.5, - ) - - def test_list_routines_defaults(self): - from google.cloud.bigquery.routine import Routine - - project_id = "test-routines" - dataset_id = "test_routines" - path = "/projects/test-routines/datasets/test_routines/routines" - routine_1 = "routine_one" - routine_2 = "routine_two" - token = "TOKEN" - resource = { - "nextPageToken": token, - "routines": [ - { - "routineReference": { - "routineId": routine_1, - "datasetId": dataset_id, - "projectId": project_id, - } - }, - { - "routineReference": { - "routineId": routine_2, - "datasetId": dataset_id, - "projectId": project_id, - } - }, - ], - } - - creds = _make_credentials() - client = self._make_one(project=project_id, credentials=creds) - conn = client._connection = make_connection(resource) - dataset = DatasetReference(client.project, dataset_id) - - iterator = client.list_routines(dataset) - self.assertIs(iterator.dataset, dataset) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - page = next(iterator.pages) - - final_attributes.assert_called_once_with({"path": path}, client, None) - routines = list(page) - actual_token = iterator.next_page_token - - self.assertEqual(len(routines), len(resource["routines"])) - for found, expected in zip(routines, resource["routines"]): - self.assertIsInstance(found, Routine) - self.assertEqual( - found.routine_id, expected["routineReference"]["routineId"] - ) - self.assertEqual(actual_token, token) - - conn.api_request.assert_called_once_with( - method="GET", path=path, query_params={}, timeout=None - ) - - def test_list_routines_wrong_type(self): - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - with self.assertRaises(TypeError): - client.list_routines( - DatasetReference(self.PROJECT, self.DS_ID).table("foo") - ) - def test_delete_dataset(self): from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.dataset import DatasetReference From 4fb971b4865f2c65dbbffb66b43aa6ea4afb4005 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 12:56:07 -0600 Subject: [PATCH 12/20] create_dataset accepts DatasetListItem Also converted create_dataset tests to pytest. (And fixed some long lines.) --- google/cloud/bigquery/client.py | 47 ++- tests/unit/conftest.py | 5 + tests/unit/test_client.py | 588 +++++++----------------------- tests/unit/test_create_dataset.py | 379 +++++++++++++++++++ 4 files changed, 533 insertions(+), 486 deletions(-) create mode 100644 tests/unit/test_create_dataset.py diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index d9ee9b1b8..40b8ebf1a 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -449,7 +449,6 @@ def _create_bqstorage_client(self): return bigquery_storage.BigQueryReadClient(credentials=self._credentials) - def _dataset_from_arg(self, dataset): if isinstance(dataset, str): dataset = DatasetReference.from_string( @@ -466,7 +465,6 @@ def _dataset_from_arg(self, dataset): ) return dataset - def create_dataset( self, dataset, exists_ok=False, retry=DEFAULT_RETRY, timeout=None ): @@ -509,10 +507,7 @@ def create_dataset( >>> dataset = client.create_dataset(dataset) """ - if isinstance(dataset, str): - dataset = DatasetReference.from_string( - dataset, default_project=self.project - ) + dataset = self._dataset_from_arg(dataset) if isinstance(dataset, DatasetReference): dataset = Dataset(dataset) @@ -697,7 +692,11 @@ def get_dataset(self, dataset_ref, retry=DEFAULT_RETRY, timeout=None): return Dataset.from_api_repr(api_response) def get_iam_policy( - self, table, requested_policy_version=1, retry=DEFAULT_RETRY, timeout=None, + self, + table, + requested_policy_version=1, + retry=DEFAULT_RETRY, + timeout=None, ): if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -722,7 +721,12 @@ def get_iam_policy( return Policy.from_api_repr(response) def set_iam_policy( - self, table, policy, updateMask=None, retry=DEFAULT_RETRY, timeout=None, + self, + table, + policy, + updateMask=None, + retry=DEFAULT_RETRY, + timeout=None, ): if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -751,7 +755,11 @@ def set_iam_policy( return Policy.from_api_repr(response) def test_iam_permissions( - self, table, permissions, retry=DEFAULT_RETRY, timeout=None, + self, + table, + permissions, + retry=DEFAULT_RETRY, + timeout=None, ): if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -1382,14 +1390,7 @@ def delete_dataset( Defaults to ``False``. If ``True``, ignore "not found" errors when deleting the dataset. """ - if isinstance(dataset, str): - dataset = DatasetReference.from_string( - dataset, default_project=self.project - ) - - if not isinstance(dataset, (Dataset, DatasetReference)): - raise TypeError("dataset must be a Dataset or a DatasetReference") - + dataset = self._dataset_from_arg(dataset) params = {} path = dataset.path if delete_contents: @@ -1557,7 +1558,13 @@ def delete_table( raise def _get_query_results( - self, job_id, retry, project=None, timeout_ms=None, location=None, timeout=None, + self, + job_id, + retry, + project=None, + timeout_ms=None, + location=None, + timeout=None, ): """Get the query results object for a query job. @@ -1699,8 +1706,8 @@ def create_job(self, job_config, retry=DEFAULT_RETRY, timeout=None): timeout=timeout, ) elif "extract" in job_config: - extract_job_config = google.cloud.bigquery.job.ExtractJobConfig.from_api_repr( - job_config + extract_job_config = ( + google.cloud.bigquery.job.ExtractJobConfig.from_api_repr(job_config) ) source = _get_sub_prop(job_config, ["extract", "sourceTable"]) if source: diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index c6e6e8801..60ae06304 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -21,6 +21,11 @@ def DS_ID(): yield "DATASET_ID" +@pytest.fixture +def LOCATION(): + yield "us-central" + + @pytest.fixture(autouse=True, scope="session") def bypass_opentelemetry_tracing(): @contextlib.contextmanager diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 6ec87482c..bb03e9ab5 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -231,7 +231,9 @@ def test__call_api_applying_custom_retry_on_timeout(self): client = self._make_one(project=self.PROJECT, credentials=creds) api_request_patcher = mock.patch.object( - client._connection, "api_request", side_effect=[TimeoutError, "result"], + client._connection, + "api_request", + side_effect=[TimeoutError, "result"], ) retry = DEFAULT_RETRY.with_deadline(1).with_predicate( lambda exc: isinstance(exc, TimeoutError) @@ -254,7 +256,9 @@ def test__call_api_span_creator_not_called(self): client = self._make_one(project=self.PROJECT, credentials=creds) api_request_patcher = mock.patch.object( - client._connection, "api_request", side_effect=[TimeoutError, "result"], + client._connection, + "api_request", + side_effect=[TimeoutError, "result"], ) retry = DEFAULT_RETRY.with_deadline(1).with_predicate( lambda exc: isinstance(exc, TimeoutError) @@ -276,7 +280,9 @@ def test__call_api_span_creator_called(self): client = self._make_one(project=self.PROJECT, credentials=creds) api_request_patcher = mock.patch.object( - client._connection, "api_request", side_effect=[TimeoutError, "result"], + client._connection, + "api_request", + side_effect=[TimeoutError, "result"], ) retry = DEFAULT_RETRY.with_deadline(1).with_predicate( lambda exc: isinstance(exc, TimeoutError) @@ -439,7 +445,9 @@ def test_get_service_account_email_w_custom_retry(self): "email": "bq-123@bigquery-encryption.iam.gserviceaccount.com", } api_request_patcher = mock.patch.object( - client._connection, "api_request", side_effect=[ValueError, resource], + client._connection, + "api_request", + side_effect=[ValueError, resource], ) retry = DEFAULT_RETRY.with_deadline(1).with_predicate( @@ -856,439 +864,6 @@ def fail_bqstorage_import(name, globals, locals, fromlist, level): ] assert matching_warnings, "Missing dependency warning not raised." - def test_create_dataset_minimal(self): - from google.cloud.bigquery.dataset import Dataset - - PATH = "projects/%s/datasets" % self.PROJECT - RESOURCE = { - "datasetReference": {"projectId": self.PROJECT, "datasetId": self.DS_ID}, - "etag": "etag", - "id": "%s:%s" % (self.PROJECT, self.DS_ID), - } - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = make_connection(RESOURCE) - - ds_ref = DatasetReference(self.PROJECT, self.DS_ID) - before = Dataset(ds_ref) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - after = client.create_dataset(before, timeout=7.5) - - final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) - - self.assertEqual(after.dataset_id, self.DS_ID) - self.assertEqual(after.project, self.PROJECT) - self.assertEqual(after.etag, RESOURCE["etag"]) - self.assertEqual(after.full_dataset_id, RESOURCE["id"]) - - conn.api_request.assert_called_once_with( - method="POST", - path="/%s" % PATH, - data={ - "datasetReference": { - "projectId": self.PROJECT, - "datasetId": self.DS_ID, - }, - "labels": {}, - }, - timeout=7.5, - ) - - def test_create_dataset_w_attrs(self): - from google.cloud.bigquery.dataset import Dataset, AccessEntry - - PATH = "projects/%s/datasets" % self.PROJECT - DESCRIPTION = "DESC" - FRIENDLY_NAME = "FN" - LOCATION = "US" - USER_EMAIL = "phred@example.com" - LABELS = {"color": "red"} - VIEW = { - "projectId": "my-proj", - "datasetId": "starry-skies", - "tableId": "northern-hemisphere", - } - RESOURCE = { - "datasetReference": {"projectId": self.PROJECT, "datasetId": self.DS_ID}, - "etag": "etag", - "id": "%s:%s" % (self.PROJECT, self.DS_ID), - "description": DESCRIPTION, - "friendlyName": FRIENDLY_NAME, - "location": LOCATION, - "defaultTableExpirationMs": "3600", - "labels": LABELS, - "access": [{"role": "OWNER", "userByEmail": USER_EMAIL}, {"view": VIEW}], - } - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = make_connection(RESOURCE) - entries = [ - AccessEntry("OWNER", "userByEmail", USER_EMAIL), - AccessEntry(None, "view", VIEW), - ] - - ds_ref = DatasetReference(self.PROJECT, self.DS_ID) - before = Dataset(ds_ref) - before.access_entries = entries - before.description = DESCRIPTION - before.friendly_name = FRIENDLY_NAME - before.default_table_expiration_ms = 3600 - before.location = LOCATION - before.labels = LABELS - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - after = client.create_dataset(before) - - final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) - - self.assertEqual(after.dataset_id, self.DS_ID) - self.assertEqual(after.project, self.PROJECT) - self.assertEqual(after.etag, RESOURCE["etag"]) - self.assertEqual(after.full_dataset_id, RESOURCE["id"]) - self.assertEqual(after.description, DESCRIPTION) - self.assertEqual(after.friendly_name, FRIENDLY_NAME) - self.assertEqual(after.location, LOCATION) - self.assertEqual(after.default_table_expiration_ms, 3600) - self.assertEqual(after.labels, LABELS) - - conn.api_request.assert_called_once_with( - method="POST", - path="/%s" % PATH, - data={ - "datasetReference": { - "projectId": self.PROJECT, - "datasetId": self.DS_ID, - }, - "description": DESCRIPTION, - "friendlyName": FRIENDLY_NAME, - "location": LOCATION, - "defaultTableExpirationMs": "3600", - "access": [ - {"role": "OWNER", "userByEmail": USER_EMAIL}, - {"view": VIEW}, - ], - "labels": LABELS, - }, - timeout=None, - ) - - def test_create_dataset_w_custom_property(self): - # The library should handle sending properties to the API that are not - # yet part of the library - from google.cloud.bigquery.dataset import Dataset - - path = "/projects/%s/datasets" % self.PROJECT - resource = { - "datasetReference": {"projectId": self.PROJECT, "datasetId": self.DS_ID}, - "newAlphaProperty": "unreleased property", - } - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = make_connection(resource) - - ds_ref = DatasetReference(self.PROJECT, self.DS_ID) - before = Dataset(ds_ref) - before._properties["newAlphaProperty"] = "unreleased property" - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - after = client.create_dataset(before) - - final_attributes.assert_called_once_with({"path": path}, client, None) - - self.assertEqual(after.dataset_id, self.DS_ID) - self.assertEqual(after.project, self.PROJECT) - self.assertEqual(after._properties["newAlphaProperty"], "unreleased property") - - conn.api_request.assert_called_once_with( - method="POST", - path=path, - data={ - "datasetReference": { - "projectId": self.PROJECT, - "datasetId": self.DS_ID, - }, - "newAlphaProperty": "unreleased property", - "labels": {}, - }, - timeout=None, - ) - - def test_create_dataset_w_client_location_wo_dataset_location(self): - from google.cloud.bigquery.dataset import Dataset - - PATH = "projects/%s/datasets" % self.PROJECT - RESOURCE = { - "datasetReference": {"projectId": self.PROJECT, "datasetId": self.DS_ID}, - "etag": "etag", - "id": "%s:%s" % (self.PROJECT, self.DS_ID), - "location": self.LOCATION, - } - creds = _make_credentials() - client = self._make_one( - project=self.PROJECT, credentials=creds, location=self.LOCATION - ) - conn = client._connection = make_connection(RESOURCE) - - ds_ref = DatasetReference(self.PROJECT, self.DS_ID) - before = Dataset(ds_ref) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - after = client.create_dataset(before) - - final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) - - self.assertEqual(after.dataset_id, self.DS_ID) - self.assertEqual(after.project, self.PROJECT) - self.assertEqual(after.etag, RESOURCE["etag"]) - self.assertEqual(after.full_dataset_id, RESOURCE["id"]) - self.assertEqual(after.location, self.LOCATION) - - conn.api_request.assert_called_once_with( - method="POST", - path="/%s" % PATH, - data={ - "datasetReference": { - "projectId": self.PROJECT, - "datasetId": self.DS_ID, - }, - "labels": {}, - "location": self.LOCATION, - }, - timeout=None, - ) - - def test_create_dataset_w_client_location_w_dataset_location(self): - from google.cloud.bigquery.dataset import Dataset - - PATH = "projects/%s/datasets" % self.PROJECT - OTHER_LOCATION = "EU" - RESOURCE = { - "datasetReference": {"projectId": self.PROJECT, "datasetId": self.DS_ID}, - "etag": "etag", - "id": "%s:%s" % (self.PROJECT, self.DS_ID), - "location": OTHER_LOCATION, - } - creds = _make_credentials() - client = self._make_one( - project=self.PROJECT, credentials=creds, location=self.LOCATION - ) - conn = client._connection = make_connection(RESOURCE) - - ds_ref = DatasetReference(self.PROJECT, self.DS_ID) - before = Dataset(ds_ref) - before.location = OTHER_LOCATION - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - after = client.create_dataset(before) - - final_attributes.assert_called_once_with({"path": "/%s" % PATH}, client, None) - - self.assertEqual(after.dataset_id, self.DS_ID) - self.assertEqual(after.project, self.PROJECT) - self.assertEqual(after.etag, RESOURCE["etag"]) - self.assertEqual(after.full_dataset_id, RESOURCE["id"]) - self.assertEqual(after.location, OTHER_LOCATION) - - conn.api_request.assert_called_once_with( - method="POST", - path="/%s" % PATH, - data={ - "datasetReference": { - "projectId": self.PROJECT, - "datasetId": self.DS_ID, - }, - "labels": {}, - "location": OTHER_LOCATION, - }, - timeout=None, - ) - - def test_create_dataset_w_reference(self): - path = "/projects/%s/datasets" % self.PROJECT - resource = { - "datasetReference": {"projectId": self.PROJECT, "datasetId": self.DS_ID}, - "etag": "etag", - "id": "%s:%s" % (self.PROJECT, self.DS_ID), - "location": self.LOCATION, - } - creds = _make_credentials() - client = self._make_one( - project=self.PROJECT, credentials=creds, location=self.LOCATION - ) - conn = client._connection = make_connection(resource) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - dataset = client.create_dataset(DatasetReference(self.PROJECT, self.DS_ID)) - - final_attributes.assert_called_once_with({"path": path}, client, None) - - self.assertEqual(dataset.dataset_id, self.DS_ID) - self.assertEqual(dataset.project, self.PROJECT) - self.assertEqual(dataset.etag, resource["etag"]) - self.assertEqual(dataset.full_dataset_id, resource["id"]) - self.assertEqual(dataset.location, self.LOCATION) - - conn.api_request.assert_called_once_with( - method="POST", - path=path, - data={ - "datasetReference": { - "projectId": self.PROJECT, - "datasetId": self.DS_ID, - }, - "labels": {}, - "location": self.LOCATION, - }, - timeout=None, - ) - - def test_create_dataset_w_fully_qualified_string(self): - path = "/projects/%s/datasets" % self.PROJECT - resource = { - "datasetReference": {"projectId": self.PROJECT, "datasetId": self.DS_ID}, - "etag": "etag", - "id": "%s:%s" % (self.PROJECT, self.DS_ID), - "location": self.LOCATION, - } - creds = _make_credentials() - client = self._make_one( - project=self.PROJECT, credentials=creds, location=self.LOCATION - ) - conn = client._connection = make_connection(resource) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - dataset = client.create_dataset("{}.{}".format(self.PROJECT, self.DS_ID)) - - final_attributes.assert_called_once_with({"path": path}, client, None) - - self.assertEqual(dataset.dataset_id, self.DS_ID) - self.assertEqual(dataset.project, self.PROJECT) - self.assertEqual(dataset.etag, resource["etag"]) - self.assertEqual(dataset.full_dataset_id, resource["id"]) - self.assertEqual(dataset.location, self.LOCATION) - - conn.api_request.assert_called_once_with( - method="POST", - path=path, - data={ - "datasetReference": { - "projectId": self.PROJECT, - "datasetId": self.DS_ID, - }, - "labels": {}, - "location": self.LOCATION, - }, - timeout=None, - ) - - def test_create_dataset_w_string(self): - path = "/projects/%s/datasets" % self.PROJECT - resource = { - "datasetReference": {"projectId": self.PROJECT, "datasetId": self.DS_ID}, - "etag": "etag", - "id": "%s:%s" % (self.PROJECT, self.DS_ID), - "location": self.LOCATION, - } - creds = _make_credentials() - client = self._make_one( - project=self.PROJECT, credentials=creds, location=self.LOCATION - ) - conn = client._connection = make_connection(resource) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - dataset = client.create_dataset(self.DS_ID) - - final_attributes.assert_called_once_with({"path": path}, client, None) - - self.assertEqual(dataset.dataset_id, self.DS_ID) - self.assertEqual(dataset.project, self.PROJECT) - self.assertEqual(dataset.etag, resource["etag"]) - self.assertEqual(dataset.full_dataset_id, resource["id"]) - self.assertEqual(dataset.location, self.LOCATION) - - conn.api_request.assert_called_once_with( - method="POST", - path=path, - data={ - "datasetReference": { - "projectId": self.PROJECT, - "datasetId": self.DS_ID, - }, - "labels": {}, - "location": self.LOCATION, - }, - timeout=None, - ) - - def test_create_dataset_alreadyexists_w_exists_ok_false(self): - creds = _make_credentials() - client = self._make_one( - project=self.PROJECT, credentials=creds, location=self.LOCATION - ) - client._connection = make_connection( - google.api_core.exceptions.AlreadyExists("dataset already exists") - ) - - with pytest.raises(google.api_core.exceptions.AlreadyExists): - client.create_dataset(self.DS_ID) - - def test_create_dataset_alreadyexists_w_exists_ok_true(self): - post_path = "/projects/{}/datasets".format(self.PROJECT) - get_path = "/projects/{}/datasets/{}".format(self.PROJECT, self.DS_ID) - resource = { - "datasetReference": {"projectId": self.PROJECT, "datasetId": self.DS_ID}, - "etag": "etag", - "id": "{}:{}".format(self.PROJECT, self.DS_ID), - "location": self.LOCATION, - } - creds = _make_credentials() - client = self._make_one( - project=self.PROJECT, credentials=creds, location=self.LOCATION - ) - conn = client._connection = make_connection( - google.api_core.exceptions.AlreadyExists("dataset already exists"), resource - ) - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - dataset = client.create_dataset(self.DS_ID, exists_ok=True) - - final_attributes.assert_called_with({"path": get_path}, client, None) - - self.assertEqual(dataset.dataset_id, self.DS_ID) - self.assertEqual(dataset.project, self.PROJECT) - self.assertEqual(dataset.etag, resource["etag"]) - self.assertEqual(dataset.full_dataset_id, resource["id"]) - self.assertEqual(dataset.location, self.LOCATION) - - conn.api_request.assert_has_calls( - [ - mock.call( - method="POST", - path=post_path, - data={ - "datasetReference": { - "projectId": self.PROJECT, - "datasetId": self.DS_ID, - }, - "labels": {}, - "location": self.LOCATION, - }, - timeout=None, - ), - mock.call(method="GET", path=get_path, timeout=None), - ] - ) - def test_create_routine_w_minimal_resource(self): from google.cloud.bigquery.routine import Routine from google.cloud.bigquery.routine import RoutineReference @@ -1314,7 +889,10 @@ def test_create_routine_w_minimal_resource(self): final_attributes.assert_called_once_with({"path": path}, client, None) conn.api_request.assert_called_once_with( - method="POST", path=path, data=resource, timeout=7.5, + method="POST", + path=path, + data=resource, + timeout=7.5, ) self.assertEqual( actual_routine.reference, RoutineReference.from_string(full_routine_id) @@ -1348,7 +926,10 @@ def test_create_routine_w_conflict(self): } } conn.api_request.assert_called_once_with( - method="POST", path=path, data=resource, timeout=None, + method="POST", + path=path, + data=resource, + timeout=None, ) @unittest.skipIf(opentelemetry is None, "Requires `opentelemetry`") @@ -1384,7 +965,10 @@ def test_span_status_is_set(self): } } conn.api_request.assert_called_once_with( - method="POST", path=path, data=resource, timeout=None, + method="POST", + path=path, + data=resource, + timeout=None, ) def test_create_routine_w_conflict_exists_ok(self): @@ -1420,7 +1004,12 @@ def test_create_routine_w_conflict_exists_ok(self): self.assertEqual(actual_routine.routine_id, "minimal_routine") conn.api_request.assert_has_calls( [ - mock.call(method="POST", path=path, data=resource, timeout=None,), + mock.call( + method="POST", + path=path, + data=resource, + timeout=None, + ), mock.call( method="GET", path="/projects/test-routine-project/datasets/test_routines/routines/minimal_routine", @@ -1819,7 +1408,9 @@ def test_create_table_alreadyexists_w_exists_ok_false(self): client.create_table("{}.{}".format(self.DS_ID, self.TABLE_ID)) final_attributes.assert_called_with( - {"path": post_path, "dataset_id": self.TABLE_REF.dataset_id}, client, None, + {"path": post_path, "dataset_id": self.TABLE_REF.dataset_id}, + client, + None, ) conn.api_request.assert_called_once_with( @@ -1989,7 +1580,9 @@ def test_get_routine(self): final_attributes.assert_called_once_with({"path": path}, client, None) conn.api_request.assert_called_once_with( - method="GET", path=path, timeout=7.5, + method="GET", + path=path, + timeout=7.5, ) self.assertEqual( actual_routine.reference, @@ -2071,7 +1664,9 @@ def test_get_iam_policy(self): from google.api_core.iam import Policy PATH = "/projects/{}/datasets/{}/tables/{}:getIamPolicy".format( - self.PROJECT, self.DS_ID, self.TABLE_ID, + self.PROJECT, + self.DS_ID, + self.TABLE_ID, ) BODY = {"options": {"requestedPolicyVersion": 1}} ETAG = "CARDI" @@ -2122,7 +1717,9 @@ def test_get_iam_policy_w_invalid_table(self): client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) table_resource_string = "projects/{}/datasets/{}/tables/{}".format( - self.PROJECT, self.DS_ID, self.TABLE_ID, + self.PROJECT, + self.DS_ID, + self.TABLE_ID, ) with self.assertRaises(TypeError): @@ -2342,7 +1939,11 @@ def test_update_dataset(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - ds2 = client.update_dataset(ds, fields=fields, timeout=7.5,) + ds2 = client.update_dataset( + ds, + fields=fields, + timeout=7.5, + ) final_attributes.assert_called_once_with( {"path": "/%s" % PATH, "fields": fields}, client, None @@ -2521,7 +2122,11 @@ def test_update_routine(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - actual_routine = client.update_routine(routine, fields, timeout=7.5,) + actual_routine = client.update_routine( + routine, + fields, + timeout=7.5, + ) final_attributes.assert_called_once_with( {"path": routine.path, "fields": fields}, client, None @@ -2681,7 +2286,9 @@ def test_update_table_w_custom_property(self): updated_table = client.update_table(table, ["newAlphaProperty"]) final_attributes.assert_called_once_with( - {"path": "/%s" % path, "fields": ["newAlphaProperty"]}, client, None, + {"path": "/%s" % path, "fields": ["newAlphaProperty"]}, + client, + None, ) conn.api_request.assert_called_once_with( @@ -2716,7 +2323,9 @@ def test_update_table_only_use_legacy_sql(self): updated_table = client.update_table(table, ["view_use_legacy_sql"]) final_attributes.assert_called_once_with( - {"path": "/%s" % path, "fields": ["view_use_legacy_sql"]}, client, None, + {"path": "/%s" % path, "fields": ["view_use_legacy_sql"]}, + client, + None, ) conn.api_request.assert_called_once_with( @@ -2797,7 +2406,9 @@ def test_update_table_w_query(self): updated_table = client.update_table(table, updated_properties) final_attributes.assert_called_once_with( - {"path": "/%s" % path, "fields": updated_properties}, client, None, + {"path": "/%s" % path, "fields": updated_properties}, + client, + None, ) self.assertEqual(updated_table.schema, table.schema) @@ -3123,7 +2734,9 @@ def test_delete_routine(self): final_attributes.assert_called_once_with({"path": path}, client, None) conn.api_request.assert_called_with( - method="DELETE", path=path, timeout=7.5, + method="DELETE", + path=path, + timeout=7.5, ) def test_delete_routine_w_wrong_type(self): @@ -3150,7 +2763,9 @@ def test_delete_routine_w_not_found_ok_false(self): final_attributes.assert_called_once_with({"path": path}, client, None) conn.api_request.assert_called_with( - method="DELETE", path=path, timeout=None, + method="DELETE", + path=path, + timeout=None, ) def test_delete_routine_w_not_found_ok_true(self): @@ -3172,7 +2787,9 @@ def test_delete_routine_w_not_found_ok_true(self): final_attributes.assert_called_once_with({"path": path}, client, None) conn.api_request.assert_called_with( - method="DELETE", path=path, timeout=None, + method="DELETE", + path=path, + timeout=None, ) def test_delete_table(self): @@ -4577,7 +4194,10 @@ def test_extract_table(self): # Check that extract_table actually starts the job. conn.api_request.assert_called_once_with( - method="POST", path="/projects/PROJECT/jobs", data=RESOURCE, timeout=7.5, + method="POST", + path="/projects/PROJECT/jobs", + data=RESOURCE, + timeout=7.5, ) # Check the job resource. @@ -4819,7 +4439,10 @@ def test_extract_table_for_source_type_model(self): # Check that extract_table actually starts the job. conn.api_request.assert_called_once_with( - method="POST", path="/projects/PROJECT/jobs", data=RESOURCE, timeout=7.5, + method="POST", + path="/projects/PROJECT/jobs", + data=RESOURCE, + timeout=7.5, ) # Check the job resource. @@ -4862,7 +4485,10 @@ def test_extract_table_for_source_type_model_w_string_model_id(self): # Check that extract_table actually starts the job. conn.api_request.assert_called_once_with( - method="POST", path="/projects/PROJECT/jobs", data=RESOURCE, timeout=7.5, + method="POST", + path="/projects/PROJECT/jobs", + data=RESOURCE, + timeout=7.5, ) def test_extract_table_for_source_type_model_w_model_object(self): @@ -4901,7 +4527,10 @@ def test_extract_table_for_source_type_model_w_model_object(self): # Check that extract_table actually starts the job. conn.api_request.assert_called_once_with( - method="POST", path="/projects/PROJECT/jobs", data=RESOURCE, timeout=7.5, + method="POST", + path="/projects/PROJECT/jobs", + data=RESOURCE, + timeout=7.5, ) def test_extract_table_for_invalid_source_type_model(self): @@ -5087,7 +4716,11 @@ def test_query_preserving_explicit_job_config(self): from google.cloud.bigquery import QueryJobConfig - client = self._make_one(project=self.PROJECT, credentials=creds, _http=http,) + client = self._make_one( + project=self.PROJECT, + credentials=creds, + _http=http, + ) conn = client._connection = make_connection(resource) job_config = QueryJobConfig() @@ -5851,7 +5484,10 @@ def test_insert_rows_w_repeated_fields(self): self.assertEqual(len(errors), 0) conn.api_request.assert_called_once_with( - method="POST", path="/%s" % PATH, data=SENT, timeout=None, + method="POST", + path="/%s" % PATH, + data=SENT, + timeout=None, ) def test_insert_rows_w_record_schema(self): @@ -5925,7 +5561,9 @@ def test_insert_rows_w_explicit_none_insert_ids(self): from google.cloud.bigquery.table import Table PATH = "projects/{}/datasets/{}/tables/{}/insertAll".format( - self.PROJECT, self.DS_ID, self.TABLE_ID, + self.PROJECT, + self.DS_ID, + self.TABLE_ID, ) creds = _make_credentials() http = object() @@ -5951,7 +5589,10 @@ def _row_data(row): self.assertEqual(len(errors), 0) conn.api_request.assert_called_once_with( - method="POST", path="/{}".format(PATH), data=SENT, timeout=None, + method="POST", + path="/{}".format(PATH), + data=SENT, + timeout=None, ) def test_insert_rows_errors(self): @@ -6327,7 +5968,10 @@ def test_insert_rows_json(self): self.assertEqual(len(errors), 0) conn.api_request.assert_called_once_with( - method="POST", path="/%s" % PATH, data=SENT, timeout=7.5, + method="POST", + path="/%s" % PATH, + data=SENT, + timeout=7.5, ) def test_insert_rows_json_with_string_id(self): @@ -6363,7 +6007,9 @@ def test_insert_rows_json_w_explicit_none_insert_ids(self): conn = client._connection = make_connection({}) errors = client.insert_rows_json( - "proj.dset.tbl", rows, row_ids=[None] * len(rows), + "proj.dset.tbl", + rows, + row_ids=[None] * len(rows), ) self.assertEqual(len(errors), 0) @@ -7029,7 +6675,10 @@ def test_load_table_from_file_resumable(self): ) with do_upload_patch as do_upload: client.load_table_from_file( - file_obj, self.TABLE_REF, job_id="job_id", job_config=job_config, + file_obj, + self.TABLE_REF, + job_id="job_id", + job_config=job_config, ) do_upload.assert_called_once_with( @@ -7347,7 +6996,10 @@ def test_load_table_from_dataframe(self): policy_tags=PolicyTagList(names=("baz",)), ), "accounts": SchemaField( - "accounts", "INTEGER", mode="REPEATED", description="array column", + "accounts", + "INTEGER", + mode="REPEATED", + description="array column", ), } get_table_schema = [ @@ -8442,7 +8094,11 @@ def test__do_resumable_upload_custom_project(self): client = self._make_client(transport) result = client._do_resumable_upload( - file_obj, self.EXPECTED_CONFIGURATION, None, None, project="custom-project", + file_obj, + self.EXPECTED_CONFIGURATION, + None, + None, + project="custom-project", ) content = result.content.decode("utf-8") diff --git a/tests/unit/test_create_dataset.py b/tests/unit/test_create_dataset.py new file mode 100644 index 000000000..77a298beb --- /dev/null +++ b/tests/unit/test_create_dataset.py @@ -0,0 +1,379 @@ +from google.cloud.bigquery.dataset import Dataset, DatasetReference +from .helpers import make_connection, dataset_polymorphic, make_client +import google.cloud.bigquery.dataset +import mock +import pytest + + +@dataset_polymorphic +def test_create_dataset_minimal(make_dataset, get_reference, client, PROJECT, DS_ID): + PATH = "projects/%s/datasets" % PROJECT + RESOURCE = { + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, + "etag": "etag", + "id": "%s:%s" % (PROJECT, DS_ID), + } + conn = client._connection = make_connection(RESOURCE) + + dataset = make_dataset(PROJECT, DS_ID) + after = client.create_dataset(dataset, timeout=7.5) + + assert after.dataset_id == DS_ID + assert after.project == PROJECT + assert after.etag == RESOURCE["etag"] + assert after.full_dataset_id == RESOURCE["id"] + + conn.api_request.assert_called_once_with( + method="POST", + path="/%s" % PATH, + data={ + "datasetReference": { + "projectId": PROJECT, + "datasetId": DS_ID, + }, + "labels": {}, + }, + timeout=7.5, + ) + + +def test_create_dataset_w_attrs(client, PROJECT, DS_ID): + from google.cloud.bigquery.dataset import AccessEntry + + PATH = "projects/%s/datasets" % PROJECT + DESCRIPTION = "DESC" + FRIENDLY_NAME = "FN" + LOCATION = "US" + USER_EMAIL = "phred@example.com" + LABELS = {"color": "red"} + VIEW = { + "projectId": "my-proj", + "datasetId": "starry-skies", + "tableId": "northern-hemisphere", + } + RESOURCE = { + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, + "etag": "etag", + "id": "%s:%s" % (PROJECT, DS_ID), + "description": DESCRIPTION, + "friendlyName": FRIENDLY_NAME, + "location": LOCATION, + "defaultTableExpirationMs": "3600", + "labels": LABELS, + "access": [{"role": "OWNER", "userByEmail": USER_EMAIL}, {"view": VIEW}], + } + conn = client._connection = make_connection(RESOURCE) + entries = [ + AccessEntry("OWNER", "userByEmail", USER_EMAIL), + AccessEntry(None, "view", VIEW), + ] + + ds_ref = DatasetReference(PROJECT, DS_ID) + before = Dataset(ds_ref) + before.access_entries = entries + before.description = DESCRIPTION + before.friendly_name = FRIENDLY_NAME + before.default_table_expiration_ms = 3600 + before.location = LOCATION + before.labels = LABELS + after = client.create_dataset(before) + + assert after.dataset_id == DS_ID + assert after.project == PROJECT + assert after.etag == RESOURCE["etag"] + assert after.full_dataset_id == RESOURCE["id"] + assert after.description == DESCRIPTION + assert after.friendly_name == FRIENDLY_NAME + assert after.location == LOCATION + assert after.default_table_expiration_ms == 3600 + assert after.labels == LABELS + + conn.api_request.assert_called_once_with( + method="POST", + path="/%s" % PATH, + data={ + "datasetReference": { + "projectId": PROJECT, + "datasetId": DS_ID, + }, + "description": DESCRIPTION, + "friendlyName": FRIENDLY_NAME, + "location": LOCATION, + "defaultTableExpirationMs": "3600", + "access": [ + {"role": "OWNER", "userByEmail": USER_EMAIL}, + {"view": VIEW}, + ], + "labels": LABELS, + }, + timeout=None, + ) + + +def test_create_dataset_w_custom_property(client, PROJECT, DS_ID): + # The library should handle sending properties to the API that are not + # yet part of the library + + path = "/projects/%s/datasets" % PROJECT + resource = { + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, + "newAlphaProperty": "unreleased property", + } + conn = client._connection = make_connection(resource) + + ds_ref = DatasetReference(PROJECT, DS_ID) + before = Dataset(ds_ref) + before._properties["newAlphaProperty"] = "unreleased property" + after = client.create_dataset(before) + + assert after.dataset_id == DS_ID + assert after.project == PROJECT + assert after._properties["newAlphaProperty"] == "unreleased property" + + conn.api_request.assert_called_once_with( + method="POST", + path=path, + data={ + "datasetReference": { + "projectId": PROJECT, + "datasetId": DS_ID, + }, + "newAlphaProperty": "unreleased property", + "labels": {}, + }, + timeout=None, + ) + + +def test_create_dataset_w_client_location_wo_dataset_location(PROJECT, DS_ID, LOCATION): + PATH = "projects/%s/datasets" % PROJECT + RESOURCE = { + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, + "etag": "etag", + "id": "%s:%s" % (PROJECT, DS_ID), + "location": LOCATION, + } + client = make_client(location=LOCATION) + conn = client._connection = make_connection(RESOURCE) + + ds_ref = DatasetReference(PROJECT, DS_ID) + before = Dataset(ds_ref) + after = client.create_dataset(before) + + assert after.dataset_id == DS_ID + assert after.project == PROJECT + assert after.etag == RESOURCE["etag"] + assert after.full_dataset_id == RESOURCE["id"] + assert after.location == LOCATION + + conn.api_request.assert_called_once_with( + method="POST", + path="/%s" % PATH, + data={ + "datasetReference": { + "projectId": PROJECT, + "datasetId": DS_ID, + }, + "labels": {}, + "location": LOCATION, + }, + timeout=None, + ) + + +def test_create_dataset_w_client_location_w_dataset_location(PROJECT, DS_ID, LOCATION): + PATH = "projects/%s/datasets" % PROJECT + OTHER_LOCATION = "EU" + RESOURCE = { + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, + "etag": "etag", + "id": "%s:%s" % (PROJECT, DS_ID), + "location": OTHER_LOCATION, + } + client = make_client(location=LOCATION) + conn = client._connection = make_connection(RESOURCE) + + ds_ref = DatasetReference(PROJECT, DS_ID) + before = Dataset(ds_ref) + before.location = OTHER_LOCATION + after = client.create_dataset(before) + + assert after.dataset_id == DS_ID + assert after.project == PROJECT + assert after.etag == RESOURCE["etag"] + assert after.full_dataset_id == RESOURCE["id"] + assert after.location == OTHER_LOCATION + + conn.api_request.assert_called_once_with( + method="POST", + path="/%s" % PATH, + data={ + "datasetReference": { + "projectId": PROJECT, + "datasetId": DS_ID, + }, + "labels": {}, + "location": OTHER_LOCATION, + }, + timeout=None, + ) + + +def test_create_dataset_w_reference(PROJECT, DS_ID, LOCATION): + path = "/projects/%s/datasets" % PROJECT + resource = { + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, + "etag": "etag", + "id": "%s:%s" % (PROJECT, DS_ID), + "location": LOCATION, + } + client = make_client(location=LOCATION) + conn = client._connection = make_connection(resource) + dataset = client.create_dataset(DatasetReference(PROJECT, DS_ID)) + + assert dataset.dataset_id == DS_ID + assert dataset.project == PROJECT + assert dataset.etag == resource["etag"] + assert dataset.full_dataset_id == resource["id"] + assert dataset.location == LOCATION + + conn.api_request.assert_called_once_with( + method="POST", + path=path, + data={ + "datasetReference": { + "projectId": PROJECT, + "datasetId": DS_ID, + }, + "labels": {}, + "location": LOCATION, + }, + timeout=None, + ) + + +def test_create_dataset_w_fully_qualified_string(PROJECT, DS_ID, LOCATION): + path = "/projects/%s/datasets" % PROJECT + resource = { + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, + "etag": "etag", + "id": "%s:%s" % (PROJECT, DS_ID), + "location": LOCATION, + } + client = make_client(location=LOCATION) + conn = client._connection = make_connection(resource) + dataset = client.create_dataset("{}.{}".format(PROJECT, DS_ID)) + + assert dataset.dataset_id == DS_ID + assert dataset.project == PROJECT + assert dataset.etag == resource["etag"] + assert dataset.full_dataset_id == resource["id"] + assert dataset.location == LOCATION + + conn.api_request.assert_called_once_with( + method="POST", + path=path, + data={ + "datasetReference": { + "projectId": PROJECT, + "datasetId": DS_ID, + }, + "labels": {}, + "location": LOCATION, + }, + timeout=None, + ) + + +def test_create_dataset_w_string(PROJECT, DS_ID, LOCATION): + path = "/projects/%s/datasets" % PROJECT + resource = { + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, + "etag": "etag", + "id": "%s:%s" % (PROJECT, DS_ID), + "location": LOCATION, + } + client = make_client(location=LOCATION) + conn = client._connection = make_connection(resource) + with mock.patch( + "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" + ) as final_attributes: + dataset = client.create_dataset(DS_ID) + + final_attributes.assert_called_once_with({"path": path}, client, None) + + assert dataset.dataset_id == DS_ID + assert dataset.project == PROJECT + assert dataset.etag == resource["etag"] + assert dataset.full_dataset_id == resource["id"] + assert dataset.location == LOCATION + + conn.api_request.assert_called_once_with( + method="POST", + path=path, + data={ + "datasetReference": { + "projectId": PROJECT, + "datasetId": DS_ID, + }, + "labels": {}, + "location": LOCATION, + }, + timeout=None, + ) + + +def test_create_dataset_alreadyexists_w_exists_ok_false(PROJECT, DS_ID, LOCATION): + client = make_client(location=LOCATION) + client._connection = make_connection( + google.api_core.exceptions.AlreadyExists("dataset already exists") + ) + + with pytest.raises(google.api_core.exceptions.AlreadyExists): + client.create_dataset(DS_ID) + + +def test_create_dataset_alreadyexists_w_exists_ok_true(PROJECT, DS_ID, LOCATION): + post_path = "/projects/{}/datasets".format(PROJECT) + get_path = "/projects/{}/datasets/{}".format(PROJECT, DS_ID) + resource = { + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, + "etag": "etag", + "id": "{}:{}".format(PROJECT, DS_ID), + "location": LOCATION, + } + client = make_client(location=LOCATION) + conn = client._connection = make_connection( + google.api_core.exceptions.AlreadyExists("dataset already exists"), resource + ) + with mock.patch( + "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" + ) as final_attributes: + dataset = client.create_dataset(DS_ID, exists_ok=True) + + final_attributes.assert_called_with({"path": get_path}, client, None) + + assert dataset.dataset_id == DS_ID + assert dataset.project == PROJECT + assert dataset.etag == resource["etag"] + assert dataset.full_dataset_id == resource["id"] + assert dataset.location == LOCATION + + conn.api_request.assert_has_calls( + [ + mock.call( + method="POST", + path=post_path, + data={ + "datasetReference": { + "projectId": PROJECT, + "datasetId": DS_ID, + }, + "labels": {}, + "location": LOCATION, + }, + timeout=None, + ), + mock.call(method="GET", path=get_path, timeout=None), + ] + ) From 4154204190c8b599861d965c5277df8b109b414d Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 13:31:58 -0600 Subject: [PATCH 13/20] Converted list_routine tests to pytest --- tests/unit/test_list_routines.py | 77 ++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/unit/test_list_routines.py diff --git a/tests/unit/test_list_routines.py b/tests/unit/test_list_routines.py new file mode 100644 index 000000000..7fa0da33e --- /dev/null +++ b/tests/unit/test_list_routines.py @@ -0,0 +1,77 @@ +from .helpers import make_connection, dataset_polymorphic +import google.cloud.bigquery.dataset +import mock +import pytest + + +def test_list_routines_empty_w_timeout(client): + conn = client._connection = make_connection({}) + + iterator = client.list_routines("test-routines.test_routines", timeout=7.5) + page = next(iterator.pages) + routines = list(page) + token = iterator.next_page_token + + assert routines == [] + assert token is None + conn.api_request.assert_called_once_with( + method="GET", + path="/projects/test-routines/datasets/test_routines/routines", + query_params={}, + timeout=7.5, + ) + + +@dataset_polymorphic +def test_list_routines_defaults(make_dataset, get_reference, client, PROJECT): + from google.cloud.bigquery.routine import Routine + + project_id = PROJECT + dataset_id = "test_routines" + path = f"/projects/{PROJECT}/datasets/test_routines/routines" + routine_1 = "routine_one" + routine_2 = "routine_two" + token = "TOKEN" + resource = { + "nextPageToken": token, + "routines": [ + { + "routineReference": { + "routineId": routine_1, + "datasetId": dataset_id, + "projectId": project_id, + } + }, + { + "routineReference": { + "routineId": routine_2, + "datasetId": dataset_id, + "projectId": project_id, + } + }, + ], + } + + conn = client._connection = make_connection(resource) + dataset = make_dataset(client.project, dataset_id) + + iterator = client.list_routines(dataset) + assert iterator.dataset == get_reference(dataset) + page = next(iterator.pages) + routines = list(page) + actual_token = iterator.next_page_token + + assert len(routines) == len(resource["routines"]) + for found, expected in zip(routines, resource["routines"]): + assert isinstance(found, Routine) + assert found.routine_id == expected["routineReference"]["routineId"] + assert actual_token == token + + conn.api_request.assert_called_once_with( + method="GET", path=path, query_params={}, timeout=None + ) + + +def test_list_routines_wrong_type(client): + with pytest.raises(TypeError): + client.list_routines(42) From be900635483aad65b28dc19029d74baa60b180cc Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 13:32:45 -0600 Subject: [PATCH 14/20] include string dataset representation in dataset polymorphism. --- tests/unit/helpers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py index 69cf557f0..32f1512f5 100644 --- a/tests/unit/helpers.py +++ b/tests/unit/helpers.py @@ -43,6 +43,10 @@ def make_client(project="PROJECT", **kw): return google.cloud.bigquery.client.Client(project, credentials, **kw) +def make_dataset_reference_string(project, ds_id): + return f"{project}.{ds_id}" + + def make_dataset(project, ds_id): return google.cloud.bigquery.dataset.Dataset( google.cloud.bigquery.dataset.DatasetReference(project, ds_id) @@ -67,6 +71,8 @@ def get_reference(x): (google.cloud.bigquery.dataset.DatasetReference, identity), (make_dataset, identity), (make_dataset_list_item, get_reference), + (make_dataset_reference_string, + google.cloud.bigquery.dataset.DatasetReference.from_string), ] dataset_polymorphic = pytest.mark.parametrize( From bbe1fd25e036287d63e2ece29fd154474d322e77 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 13:34:02 -0600 Subject: [PATCH 15/20] removed some unused imports --- tests/unit/test_list_tables.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/test_list_tables.py b/tests/unit/test_list_tables.py index c294bee86..fdd3aa857 100644 --- a/tests/unit/test_list_tables.py +++ b/tests/unit/test_list_tables.py @@ -1,6 +1,5 @@ from .helpers import make_connection, dataset_polymorphic import google.cloud.bigquery.dataset -import mock import pytest From 2aa4954895663ddf99227f99c0181dee25b43872 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 13:35:06 -0600 Subject: [PATCH 16/20] Updated delete_dataset tests - Polymorphoc on dataset - pytest --- tests/unit/test_client.py | 97 ------------------------------- tests/unit/test_delete_dataset.py | 63 ++++++++++++++++++++ 2 files changed, 63 insertions(+), 97 deletions(-) create mode 100644 tests/unit/test_delete_dataset.py diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index bb03e9ab5..8f95008c3 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -2537,103 +2537,6 @@ def test_update_table_delete_property(self): self.assertEqual(req[1]["data"], sent) self.assertIsNone(table3.description) - def test_delete_dataset(self): - from google.cloud.bigquery.dataset import Dataset - from google.cloud.bigquery.dataset import DatasetReference - - ds_ref = DatasetReference(self.PROJECT, self.DS_ID) - datasets = (ds_ref, Dataset(ds_ref), "{}.{}".format(self.PROJECT, self.DS_ID)) - PATH = "projects/%s/datasets/%s" % (self.PROJECT, self.DS_ID) - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = make_connection(*([{}] * len(datasets))) - for arg in datasets: - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - client.delete_dataset(arg, timeout=7.5) - - final_attributes.assert_called_once_with( - {"path": "/%s" % PATH}, client, None - ) - - conn.api_request.assert_called_with( - method="DELETE", path="/%s" % PATH, query_params={}, timeout=7.5 - ) - - def test_delete_dataset_delete_contents(self): - from google.cloud.bigquery.dataset import Dataset - - PATH = "projects/%s/datasets/%s" % (self.PROJECT, self.DS_ID) - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - conn = client._connection = make_connection({}, {}) - ds_ref = DatasetReference(self.PROJECT, self.DS_ID) - for arg in (ds_ref, Dataset(ds_ref)): - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - client.delete_dataset(arg, delete_contents=True) - - final_attributes.assert_called_once_with( - {"path": "/%s" % PATH, "deleteContents": True}, client, None - ) - conn.api_request.assert_called_with( - method="DELETE", - path="/%s" % PATH, - query_params={"deleteContents": "true"}, - timeout=None, - ) - - def test_delete_dataset_wrong_type(self): - creds = _make_credentials() - client = self._make_one(project=self.PROJECT, credentials=creds) - with self.assertRaises(TypeError): - client.delete_dataset( - DatasetReference(self.PROJECT, self.DS_ID).table("foo") - ) - - def test_delete_dataset_w_not_found_ok_false(self): - path = "/projects/{}/datasets/{}".format(self.PROJECT, self.DS_ID) - creds = _make_credentials() - http = object() - client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) - conn = client._connection = make_connection( - google.api_core.exceptions.NotFound("dataset not found") - ) - - with self.assertRaises(google.api_core.exceptions.NotFound): - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - client.delete_dataset(self.DS_ID) - - final_attributes.assert_called_once_with({"path": path}, client, None) - - conn.api_request.assert_called_with( - method="DELETE", path=path, query_params={}, timeout=None - ) - - def test_delete_dataset_w_not_found_ok_true(self): - path = "/projects/{}/datasets/{}".format(self.PROJECT, self.DS_ID) - creds = _make_credentials() - http = object() - client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) - conn = client._connection = make_connection( - google.api_core.exceptions.NotFound("dataset not found") - ) - - with mock.patch( - "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" - ) as final_attributes: - client.delete_dataset(self.DS_ID, not_found_ok=True) - - final_attributes.assert_called_once_with({"path": path}, client, None) - - conn.api_request.assert_called_with( - method="DELETE", path=path, query_params={}, timeout=None - ) - def test_delete_model(self): from google.cloud.bigquery.model import Model diff --git a/tests/unit/test_delete_dataset.py b/tests/unit/test_delete_dataset.py new file mode 100644 index 000000000..3dfec8b7d --- /dev/null +++ b/tests/unit/test_delete_dataset.py @@ -0,0 +1,63 @@ +from .helpers import make_connection, make_client, dataset_polymorphic +import google.api_core.exceptions +import pytest + + +@dataset_polymorphic +def test_delete_dataset(make_dataset, get_reference, client, PROJECT, DS_ID): + dataset = make_dataset(PROJECT, DS_ID) + PATH = "projects/%s/datasets/%s" % (PROJECT, DS_ID) + conn = client._connection = make_connection({}) + client.delete_dataset(dataset, timeout=7.5) + conn.api_request.assert_called_with( + method="DELETE", path="/%s" % PATH, query_params={}, timeout=7.5 + ) + + +@dataset_polymorphic +def test_delete_dataset_delete_contents( + make_dataset, get_reference, client, PROJECT, DS_ID +): + PATH = "projects/%s/datasets/%s" % (PROJECT, DS_ID) + conn = client._connection = make_connection({}) + dataset = make_dataset(PROJECT, DS_ID) + client.delete_dataset(dataset, delete_contents=True) + conn.api_request.assert_called_with( + method="DELETE", + path="/%s" % PATH, + query_params={"deleteContents": "true"}, + timeout=None, + ) + + +def test_delete_dataset_wrong_type(client): + with pytest.raises(TypeError): + client.delete_dataset(42) + + +def test_delete_dataset_w_not_found_ok_false(PROJECT, DS_ID): + path = "/projects/{}/datasets/{}".format(PROJECT, DS_ID) + http = object() + client = make_client(_http=http) + conn = client._connection = make_connection( + google.api_core.exceptions.NotFound("dataset not found") + ) + + with pytest.raises(google.api_core.exceptions.NotFound): + client.delete_dataset(DS_ID) + + conn.api_request.assert_called_with( + method="DELETE", path=path, query_params={}, timeout=None + ) + +def test_delete_dataset_w_not_found_ok_true(PROJECT, DS_ID): + path = "/projects/{}/datasets/{}".format(PROJECT, DS_ID) + http = object() + client = make_client(_http=http) + conn = client._connection = make_connection( + google.api_core.exceptions.NotFound("dataset not found") + ) + client.delete_dataset(DS_ID, not_found_ok=True) + conn.api_request.assert_called_with( + method="DELETE", path=path, query_params={}, timeout=None + ) From fd39b8b81a04c6d35426a3a1a19a45b560450e37 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 13:54:23 -0600 Subject: [PATCH 17/20] black --- tests/unit/helpers.py | 6 ++++-- tests/unit/test_delete_dataset.py | 1 + tests/unit/test_list_models.py | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py index 32f1512f5..67aeaca35 100644 --- a/tests/unit/helpers.py +++ b/tests/unit/helpers.py @@ -71,8 +71,10 @@ def get_reference(x): (google.cloud.bigquery.dataset.DatasetReference, identity), (make_dataset, identity), (make_dataset_list_item, get_reference), - (make_dataset_reference_string, - google.cloud.bigquery.dataset.DatasetReference.from_string), + ( + make_dataset_reference_string, + google.cloud.bigquery.dataset.DatasetReference.from_string, + ), ] dataset_polymorphic = pytest.mark.parametrize( diff --git a/tests/unit/test_delete_dataset.py b/tests/unit/test_delete_dataset.py index 3dfec8b7d..c57b517e0 100644 --- a/tests/unit/test_delete_dataset.py +++ b/tests/unit/test_delete_dataset.py @@ -50,6 +50,7 @@ def test_delete_dataset_w_not_found_ok_false(PROJECT, DS_ID): method="DELETE", path=path, query_params={}, timeout=None ) + def test_delete_dataset_w_not_found_ok_true(PROJECT, DS_ID): path = "/projects/{}/datasets/{}".format(PROJECT, DS_ID) http = object() diff --git a/tests/unit/test_list_models.py b/tests/unit/test_list_models.py index d98b39b94..85a5a6b80 100644 --- a/tests/unit/test_list_models.py +++ b/tests/unit/test_list_models.py @@ -20,6 +20,7 @@ def test_list_models_empty_w_timeout(client, PROJECT, DS_ID): method="GET", path=path, query_params={}, timeout=7.5 ) + @dataset_polymorphic def test_list_models_defaults(make_dataset, get_reference, client, PROJECT, DS_ID): from google.cloud.bigquery.model import Model @@ -67,6 +68,7 @@ def test_list_models_defaults(make_dataset, get_reference, client, PROJECT, DS_I method="GET", path="/%s" % PATH, query_params={}, timeout=None ) + def test_list_models_wrong_type(client): with pytest.raises(TypeError): client.list_models(42) From 05d89f8e2ff887735b70c412508bd644fbabdd37 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 14:11:12 -0600 Subject: [PATCH 18/20] lint --- google/cloud/bigquery/client.py | 31 ++---- tests/unit/test_client.py | 155 +++++++----------------------- tests/unit/test_create_dataset.py | 50 ++-------- tests/unit/test_list_models.py | 2 - tests/unit/test_list_routines.py | 2 - 5 files changed, 49 insertions(+), 191 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 40b8ebf1a..8ba3638f1 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -692,11 +692,7 @@ def get_dataset(self, dataset_ref, retry=DEFAULT_RETRY, timeout=None): return Dataset.from_api_repr(api_response) def get_iam_policy( - self, - table, - requested_policy_version=1, - retry=DEFAULT_RETRY, - timeout=None, + self, table, requested_policy_version=1, retry=DEFAULT_RETRY, timeout=None, ): if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -721,12 +717,7 @@ def get_iam_policy( return Policy.from_api_repr(response) def set_iam_policy( - self, - table, - policy, - updateMask=None, - retry=DEFAULT_RETRY, - timeout=None, + self, table, policy, updateMask=None, retry=DEFAULT_RETRY, timeout=None, ): if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -755,11 +746,7 @@ def set_iam_policy( return Policy.from_api_repr(response) def test_iam_permissions( - self, - table, - permissions, - retry=DEFAULT_RETRY, - timeout=None, + self, table, permissions, retry=DEFAULT_RETRY, timeout=None, ): if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -1558,13 +1545,7 @@ def delete_table( raise def _get_query_results( - self, - job_id, - retry, - project=None, - timeout_ms=None, - location=None, - timeout=None, + self, job_id, retry, project=None, timeout_ms=None, location=None, timeout=None, ): """Get the query results object for a query job. @@ -1706,8 +1687,8 @@ def create_job(self, job_config, retry=DEFAULT_RETRY, timeout=None): timeout=timeout, ) elif "extract" in job_config: - extract_job_config = ( - google.cloud.bigquery.job.ExtractJobConfig.from_api_repr(job_config) + extract_job_config = google.cloud.bigquery.job.ExtractJobConfig.from_api_repr( + job_config ) source = _get_sub_prop(job_config, ["extract", "sourceTable"]) if source: diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 8f95008c3..96e51678f 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -231,9 +231,7 @@ def test__call_api_applying_custom_retry_on_timeout(self): client = self._make_one(project=self.PROJECT, credentials=creds) api_request_patcher = mock.patch.object( - client._connection, - "api_request", - side_effect=[TimeoutError, "result"], + client._connection, "api_request", side_effect=[TimeoutError, "result"], ) retry = DEFAULT_RETRY.with_deadline(1).with_predicate( lambda exc: isinstance(exc, TimeoutError) @@ -256,9 +254,7 @@ def test__call_api_span_creator_not_called(self): client = self._make_one(project=self.PROJECT, credentials=creds) api_request_patcher = mock.patch.object( - client._connection, - "api_request", - side_effect=[TimeoutError, "result"], + client._connection, "api_request", side_effect=[TimeoutError, "result"], ) retry = DEFAULT_RETRY.with_deadline(1).with_predicate( lambda exc: isinstance(exc, TimeoutError) @@ -280,9 +276,7 @@ def test__call_api_span_creator_called(self): client = self._make_one(project=self.PROJECT, credentials=creds) api_request_patcher = mock.patch.object( - client._connection, - "api_request", - side_effect=[TimeoutError, "result"], + client._connection, "api_request", side_effect=[TimeoutError, "result"], ) retry = DEFAULT_RETRY.with_deadline(1).with_predicate( lambda exc: isinstance(exc, TimeoutError) @@ -445,9 +439,7 @@ def test_get_service_account_email_w_custom_retry(self): "email": "bq-123@bigquery-encryption.iam.gserviceaccount.com", } api_request_patcher = mock.patch.object( - client._connection, - "api_request", - side_effect=[ValueError, resource], + client._connection, "api_request", side_effect=[ValueError, resource], ) retry = DEFAULT_RETRY.with_deadline(1).with_predicate( @@ -889,10 +881,7 @@ def test_create_routine_w_minimal_resource(self): final_attributes.assert_called_once_with({"path": path}, client, None) conn.api_request.assert_called_once_with( - method="POST", - path=path, - data=resource, - timeout=7.5, + method="POST", path=path, data=resource, timeout=7.5, ) self.assertEqual( actual_routine.reference, RoutineReference.from_string(full_routine_id) @@ -926,10 +915,7 @@ def test_create_routine_w_conflict(self): } } conn.api_request.assert_called_once_with( - method="POST", - path=path, - data=resource, - timeout=None, + method="POST", path=path, data=resource, timeout=None, ) @unittest.skipIf(opentelemetry is None, "Requires `opentelemetry`") @@ -965,10 +951,7 @@ def test_span_status_is_set(self): } } conn.api_request.assert_called_once_with( - method="POST", - path=path, - data=resource, - timeout=None, + method="POST", path=path, data=resource, timeout=None, ) def test_create_routine_w_conflict_exists_ok(self): @@ -1004,12 +987,7 @@ def test_create_routine_w_conflict_exists_ok(self): self.assertEqual(actual_routine.routine_id, "minimal_routine") conn.api_request.assert_has_calls( [ - mock.call( - method="POST", - path=path, - data=resource, - timeout=None, - ), + mock.call(method="POST", path=path, data=resource, timeout=None,), mock.call( method="GET", path="/projects/test-routine-project/datasets/test_routines/routines/minimal_routine", @@ -1408,9 +1386,7 @@ def test_create_table_alreadyexists_w_exists_ok_false(self): client.create_table("{}.{}".format(self.DS_ID, self.TABLE_ID)) final_attributes.assert_called_with( - {"path": post_path, "dataset_id": self.TABLE_REF.dataset_id}, - client, - None, + {"path": post_path, "dataset_id": self.TABLE_REF.dataset_id}, client, None, ) conn.api_request.assert_called_once_with( @@ -1580,9 +1556,7 @@ def test_get_routine(self): final_attributes.assert_called_once_with({"path": path}, client, None) conn.api_request.assert_called_once_with( - method="GET", - path=path, - timeout=7.5, + method="GET", path=path, timeout=7.5, ) self.assertEqual( actual_routine.reference, @@ -1664,9 +1638,7 @@ def test_get_iam_policy(self): from google.api_core.iam import Policy PATH = "/projects/{}/datasets/{}/tables/{}:getIamPolicy".format( - self.PROJECT, - self.DS_ID, - self.TABLE_ID, + self.PROJECT, self.DS_ID, self.TABLE_ID, ) BODY = {"options": {"requestedPolicyVersion": 1}} ETAG = "CARDI" @@ -1717,9 +1689,7 @@ def test_get_iam_policy_w_invalid_table(self): client = self._make_one(project=self.PROJECT, credentials=creds, _http=http) table_resource_string = "projects/{}/datasets/{}/tables/{}".format( - self.PROJECT, - self.DS_ID, - self.TABLE_ID, + self.PROJECT, self.DS_ID, self.TABLE_ID, ) with self.assertRaises(TypeError): @@ -1939,11 +1909,7 @@ def test_update_dataset(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - ds2 = client.update_dataset( - ds, - fields=fields, - timeout=7.5, - ) + ds2 = client.update_dataset(ds, fields=fields, timeout=7.5,) final_attributes.assert_called_once_with( {"path": "/%s" % PATH, "fields": fields}, client, None @@ -2122,11 +2088,7 @@ def test_update_routine(self): with mock.patch( "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes" ) as final_attributes: - actual_routine = client.update_routine( - routine, - fields, - timeout=7.5, - ) + actual_routine = client.update_routine(routine, fields, timeout=7.5,) final_attributes.assert_called_once_with( {"path": routine.path, "fields": fields}, client, None @@ -2286,9 +2248,7 @@ def test_update_table_w_custom_property(self): updated_table = client.update_table(table, ["newAlphaProperty"]) final_attributes.assert_called_once_with( - {"path": "/%s" % path, "fields": ["newAlphaProperty"]}, - client, - None, + {"path": "/%s" % path, "fields": ["newAlphaProperty"]}, client, None, ) conn.api_request.assert_called_once_with( @@ -2323,9 +2283,7 @@ def test_update_table_only_use_legacy_sql(self): updated_table = client.update_table(table, ["view_use_legacy_sql"]) final_attributes.assert_called_once_with( - {"path": "/%s" % path, "fields": ["view_use_legacy_sql"]}, - client, - None, + {"path": "/%s" % path, "fields": ["view_use_legacy_sql"]}, client, None, ) conn.api_request.assert_called_once_with( @@ -2406,9 +2364,7 @@ def test_update_table_w_query(self): updated_table = client.update_table(table, updated_properties) final_attributes.assert_called_once_with( - {"path": "/%s" % path, "fields": updated_properties}, - client, - None, + {"path": "/%s" % path, "fields": updated_properties}, client, None, ) self.assertEqual(updated_table.schema, table.schema) @@ -2637,9 +2593,7 @@ def test_delete_routine(self): final_attributes.assert_called_once_with({"path": path}, client, None) conn.api_request.assert_called_with( - method="DELETE", - path=path, - timeout=7.5, + method="DELETE", path=path, timeout=7.5, ) def test_delete_routine_w_wrong_type(self): @@ -2666,9 +2620,7 @@ def test_delete_routine_w_not_found_ok_false(self): final_attributes.assert_called_once_with({"path": path}, client, None) conn.api_request.assert_called_with( - method="DELETE", - path=path, - timeout=None, + method="DELETE", path=path, timeout=None, ) def test_delete_routine_w_not_found_ok_true(self): @@ -2690,9 +2642,7 @@ def test_delete_routine_w_not_found_ok_true(self): final_attributes.assert_called_once_with({"path": path}, client, None) conn.api_request.assert_called_with( - method="DELETE", - path=path, - timeout=None, + method="DELETE", path=path, timeout=None, ) def test_delete_table(self): @@ -4097,10 +4047,7 @@ def test_extract_table(self): # Check that extract_table actually starts the job. conn.api_request.assert_called_once_with( - method="POST", - path="/projects/PROJECT/jobs", - data=RESOURCE, - timeout=7.5, + method="POST", path="/projects/PROJECT/jobs", data=RESOURCE, timeout=7.5, ) # Check the job resource. @@ -4342,10 +4289,7 @@ def test_extract_table_for_source_type_model(self): # Check that extract_table actually starts the job. conn.api_request.assert_called_once_with( - method="POST", - path="/projects/PROJECT/jobs", - data=RESOURCE, - timeout=7.5, + method="POST", path="/projects/PROJECT/jobs", data=RESOURCE, timeout=7.5, ) # Check the job resource. @@ -4388,10 +4332,7 @@ def test_extract_table_for_source_type_model_w_string_model_id(self): # Check that extract_table actually starts the job. conn.api_request.assert_called_once_with( - method="POST", - path="/projects/PROJECT/jobs", - data=RESOURCE, - timeout=7.5, + method="POST", path="/projects/PROJECT/jobs", data=RESOURCE, timeout=7.5, ) def test_extract_table_for_source_type_model_w_model_object(self): @@ -4430,10 +4371,7 @@ def test_extract_table_for_source_type_model_w_model_object(self): # Check that extract_table actually starts the job. conn.api_request.assert_called_once_with( - method="POST", - path="/projects/PROJECT/jobs", - data=RESOURCE, - timeout=7.5, + method="POST", path="/projects/PROJECT/jobs", data=RESOURCE, timeout=7.5, ) def test_extract_table_for_invalid_source_type_model(self): @@ -4619,11 +4557,7 @@ def test_query_preserving_explicit_job_config(self): from google.cloud.bigquery import QueryJobConfig - client = self._make_one( - project=self.PROJECT, - credentials=creds, - _http=http, - ) + client = self._make_one(project=self.PROJECT, credentials=creds, _http=http,) conn = client._connection = make_connection(resource) job_config = QueryJobConfig() @@ -5387,10 +5321,7 @@ def test_insert_rows_w_repeated_fields(self): self.assertEqual(len(errors), 0) conn.api_request.assert_called_once_with( - method="POST", - path="/%s" % PATH, - data=SENT, - timeout=None, + method="POST", path="/%s" % PATH, data=SENT, timeout=None, ) def test_insert_rows_w_record_schema(self): @@ -5464,9 +5395,7 @@ def test_insert_rows_w_explicit_none_insert_ids(self): from google.cloud.bigquery.table import Table PATH = "projects/{}/datasets/{}/tables/{}/insertAll".format( - self.PROJECT, - self.DS_ID, - self.TABLE_ID, + self.PROJECT, self.DS_ID, self.TABLE_ID, ) creds = _make_credentials() http = object() @@ -5492,10 +5421,7 @@ def _row_data(row): self.assertEqual(len(errors), 0) conn.api_request.assert_called_once_with( - method="POST", - path="/{}".format(PATH), - data=SENT, - timeout=None, + method="POST", path="/{}".format(PATH), data=SENT, timeout=None, ) def test_insert_rows_errors(self): @@ -5871,10 +5797,7 @@ def test_insert_rows_json(self): self.assertEqual(len(errors), 0) conn.api_request.assert_called_once_with( - method="POST", - path="/%s" % PATH, - data=SENT, - timeout=7.5, + method="POST", path="/%s" % PATH, data=SENT, timeout=7.5, ) def test_insert_rows_json_with_string_id(self): @@ -5910,9 +5833,7 @@ def test_insert_rows_json_w_explicit_none_insert_ids(self): conn = client._connection = make_connection({}) errors = client.insert_rows_json( - "proj.dset.tbl", - rows, - row_ids=[None] * len(rows), + "proj.dset.tbl", rows, row_ids=[None] * len(rows), ) self.assertEqual(len(errors), 0) @@ -6578,10 +6499,7 @@ def test_load_table_from_file_resumable(self): ) with do_upload_patch as do_upload: client.load_table_from_file( - file_obj, - self.TABLE_REF, - job_id="job_id", - job_config=job_config, + file_obj, self.TABLE_REF, job_id="job_id", job_config=job_config, ) do_upload.assert_called_once_with( @@ -6899,10 +6817,7 @@ def test_load_table_from_dataframe(self): policy_tags=PolicyTagList(names=("baz",)), ), "accounts": SchemaField( - "accounts", - "INTEGER", - mode="REPEATED", - description="array column", + "accounts", "INTEGER", mode="REPEATED", description="array column", ), } get_table_schema = [ @@ -7997,11 +7912,7 @@ def test__do_resumable_upload_custom_project(self): client = self._make_client(transport) result = client._do_resumable_upload( - file_obj, - self.EXPECTED_CONFIGURATION, - None, - None, - project="custom-project", + file_obj, self.EXPECTED_CONFIGURATION, None, None, project="custom-project", ) content = result.content.decode("utf-8") diff --git a/tests/unit/test_create_dataset.py b/tests/unit/test_create_dataset.py index 77a298beb..3eb8f1072 100644 --- a/tests/unit/test_create_dataset.py +++ b/tests/unit/test_create_dataset.py @@ -27,10 +27,7 @@ def test_create_dataset_minimal(make_dataset, get_reference, client, PROJECT, DS method="POST", path="/%s" % PATH, data={ - "datasetReference": { - "projectId": PROJECT, - "datasetId": DS_ID, - }, + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, "labels": {}, }, timeout=7.5, @@ -92,18 +89,12 @@ def test_create_dataset_w_attrs(client, PROJECT, DS_ID): method="POST", path="/%s" % PATH, data={ - "datasetReference": { - "projectId": PROJECT, - "datasetId": DS_ID, - }, + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, "description": DESCRIPTION, "friendlyName": FRIENDLY_NAME, "location": LOCATION, "defaultTableExpirationMs": "3600", - "access": [ - {"role": "OWNER", "userByEmail": USER_EMAIL}, - {"view": VIEW}, - ], + "access": [{"role": "OWNER", "userByEmail": USER_EMAIL}, {"view": VIEW}], "labels": LABELS, }, timeout=None, @@ -134,10 +125,7 @@ def test_create_dataset_w_custom_property(client, PROJECT, DS_ID): method="POST", path=path, data={ - "datasetReference": { - "projectId": PROJECT, - "datasetId": DS_ID, - }, + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, "newAlphaProperty": "unreleased property", "labels": {}, }, @@ -170,10 +158,7 @@ def test_create_dataset_w_client_location_wo_dataset_location(PROJECT, DS_ID, LO method="POST", path="/%s" % PATH, data={ - "datasetReference": { - "projectId": PROJECT, - "datasetId": DS_ID, - }, + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, "labels": {}, "location": LOCATION, }, @@ -208,10 +193,7 @@ def test_create_dataset_w_client_location_w_dataset_location(PROJECT, DS_ID, LOC method="POST", path="/%s" % PATH, data={ - "datasetReference": { - "projectId": PROJECT, - "datasetId": DS_ID, - }, + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, "labels": {}, "location": OTHER_LOCATION, }, @@ -241,10 +223,7 @@ def test_create_dataset_w_reference(PROJECT, DS_ID, LOCATION): method="POST", path=path, data={ - "datasetReference": { - "projectId": PROJECT, - "datasetId": DS_ID, - }, + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, "labels": {}, "location": LOCATION, }, @@ -274,10 +253,7 @@ def test_create_dataset_w_fully_qualified_string(PROJECT, DS_ID, LOCATION): method="POST", path=path, data={ - "datasetReference": { - "projectId": PROJECT, - "datasetId": DS_ID, - }, + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, "labels": {}, "location": LOCATION, }, @@ -312,10 +288,7 @@ def test_create_dataset_w_string(PROJECT, DS_ID, LOCATION): method="POST", path=path, data={ - "datasetReference": { - "projectId": PROJECT, - "datasetId": DS_ID, - }, + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, "labels": {}, "location": LOCATION, }, @@ -365,10 +338,7 @@ def test_create_dataset_alreadyexists_w_exists_ok_true(PROJECT, DS_ID, LOCATION) method="POST", path=post_path, data={ - "datasetReference": { - "projectId": PROJECT, - "datasetId": DS_ID, - }, + "datasetReference": {"projectId": PROJECT, "datasetId": DS_ID}, "labels": {}, "location": LOCATION, }, diff --git a/tests/unit/test_list_models.py b/tests/unit/test_list_models.py index 85a5a6b80..534a4b54c 100644 --- a/tests/unit/test_list_models.py +++ b/tests/unit/test_list_models.py @@ -1,6 +1,4 @@ from .helpers import make_connection, dataset_polymorphic -import google.cloud.bigquery.dataset -import mock import pytest diff --git a/tests/unit/test_list_routines.py b/tests/unit/test_list_routines.py index 7fa0da33e..82719fce6 100644 --- a/tests/unit/test_list_routines.py +++ b/tests/unit/test_list_routines.py @@ -1,6 +1,4 @@ from .helpers import make_connection, dataset_polymorphic -import google.cloud.bigquery.dataset -import mock import pytest From 440ac82d87f9b6c4adc9addf8867623e22de6778 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 14:14:47 -0600 Subject: [PATCH 19/20] We don't actually need to avoid opentelemetry And a 3.6 test dependened on it. --- tests/unit/conftest.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 60ae06304..07fc9b4ad 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,6 +1,3 @@ -import contextlib -import google.cloud.bigquery.opentelemetry_tracing -import mock import pytest from .helpers import make_client @@ -24,18 +21,3 @@ def DS_ID(): @pytest.fixture def LOCATION(): yield "us-central" - - -@pytest.fixture(autouse=True, scope="session") -def bypass_opentelemetry_tracing(): - @contextlib.contextmanager - def create_span(name, attributes=None, client=None, job_ref=None): - # Make existing test code that checks _get_final_span_attributes work: - google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes( - attributes, client, job_ref - ) - yield - - # Note that we have to mock in client, because it uses a from import. :/ - with mock.patch("google.cloud.bigquery.client.create_span", create_span): - yield From d4d329ea74a7729c7fb637a84b92b5f5b5f594ae Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 8 Apr 2021 14:18:10 -0600 Subject: [PATCH 20/20] fixed docstrings to include DatasetListItem in dataset polymorphic APIs. --- google/cloud/bigquery/client.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 8ba3638f1..10127e10d 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -477,6 +477,7 @@ def create_dataset( dataset (Union[ \ google.cloud.bigquery.dataset.Dataset, \ google.cloud.bigquery.dataset.DatasetReference, \ + google.cloud.bigquery.dataset.DatasetListItem, \ str, \ ]): A :class:`~google.cloud.bigquery.dataset.Dataset` to create. @@ -1146,6 +1147,7 @@ def list_models( dataset (Union[ \ google.cloud.bigquery.dataset.Dataset, \ google.cloud.bigquery.dataset.DatasetReference, \ + google.cloud.bigquery.dataset.DatasetListItem, \ str, \ ]): A reference to the dataset whose models to list from the @@ -1217,6 +1219,7 @@ def list_routines( dataset (Union[ \ google.cloud.bigquery.dataset.Dataset, \ google.cloud.bigquery.dataset.DatasetReference, \ + google.cloud.bigquery.dataset.DatasetListItem, \ str, \ ]): A reference to the dataset whose routines to list from the @@ -1288,6 +1291,7 @@ def list_tables( dataset (Union[ \ google.cloud.bigquery.dataset.Dataset, \ google.cloud.bigquery.dataset.DatasetReference, \ + google.cloud.bigquery.dataset.DatasetListItem, \ str, \ ]): A reference to the dataset whose tables to list from the @@ -1358,6 +1362,7 @@ def delete_dataset( dataset (Union[ \ google.cloud.bigquery.dataset.Dataset, \ google.cloud.bigquery.dataset.DatasetReference, \ + google.cloud.bigquery.dataset.DatasetListItem, \ str, \ ]): A reference to the dataset to delete. If a string is passed