From 42a4b989b92a4033b4516cfcc54a8021fc770056 Mon Sep 17 00:00:00 2001 From: steffnay Date: Fri, 28 Jan 2022 09:16:12 -0800 Subject: [PATCH 01/34] add view, dataset, routine properties --- google/cloud/bigquery/dataset.py | 351 +++++++++++++++++-------------- tests/unit/test_dataset.py | 143 +++++++++++++ 2 files changed, 340 insertions(+), 154 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 499072de2..cb0aeea30 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -23,9 +23,11 @@ from google.cloud.bigquery import _helpers from google.cloud.bigquery.model import ModelReference from google.cloud.bigquery.routine import RoutineReference -from google.cloud.bigquery.table import TableReference +from google.cloud.bigquery.table import TableReference, _table_arg_to_table_ref from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration +from typing import FrozenSet, Iterable, Optional, Union + def _get_table_reference(self, table_id): """Constructs a TableReference. @@ -73,6 +75,146 @@ def _get_routine_reference(self, routine_id): } ) +class DatasetReference(object): + """DatasetReferences are pointers to datasets. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#datasetreference + + Args: + project (str): The ID of the project + dataset_id (str): The ID of the dataset + + Raises: + ValueError: If either argument is not of type ``str``. + """ + + def __init__(self, project, dataset_id): + if not isinstance(project, str): + raise ValueError("Pass a string for project") + if not isinstance(dataset_id, str): + raise ValueError("Pass a string for dataset_id") + self._project = project + self._dataset_id = dataset_id + + @property + def project(self): + """str: Project ID of the dataset.""" + return self._project + + @property + def dataset_id(self): + """str: Dataset ID.""" + return self._dataset_id + + @property + def path(self): + """str: URL path for the dataset based on project and dataset ID.""" + return "/projects/%s/datasets/%s" % (self.project, self.dataset_id) + + table = _get_table_reference + + model = _get_model_reference + + routine = _get_routine_reference + + @classmethod + def from_api_repr(cls, resource: dict) -> "DatasetReference": + """Factory: construct a dataset reference given its API representation + + Args: + resource (Dict[str, str]): + Dataset reference resource representation returned from the API + + Returns: + google.cloud.bigquery.dataset.DatasetReference: + Dataset reference parsed from ``resource``. + """ + project = resource["projectId"] + dataset_id = resource["datasetId"] + return cls(project, dataset_id) + + @classmethod + def from_string( + cls, dataset_id: str, default_project: str = None + ) -> "DatasetReference": + """Construct a dataset reference from dataset ID string. + + Args: + dataset_id (str): + A dataset ID in standard SQL format. If ``default_project`` + is not specified, this must include both the project ID and + the dataset ID, separated by ``.``. + default_project (Optional[str]): + The project ID to use when ``dataset_id`` does not include a + project ID. + + Returns: + DatasetReference: + Dataset reference parsed from ``dataset_id``. + + Examples: + >>> DatasetReference.from_string('my-project-id.some_dataset') + DatasetReference('my-project-id', 'some_dataset') + + Raises: + ValueError: + If ``dataset_id`` is not a fully-qualified dataset ID in + standard SQL format. + """ + output_dataset_id = dataset_id + output_project_id = default_project + parts = _helpers._split_id(dataset_id) + + if len(parts) == 1 and not default_project: + raise ValueError( + "When default_project is not set, dataset_id must be a " + "fully-qualified dataset ID in standard SQL format, " + 'e.g., "project.dataset_id" got {}'.format(dataset_id) + ) + elif len(parts) == 2: + output_project_id, output_dataset_id = parts + elif len(parts) > 2: + raise ValueError( + "Too many parts in dataset_id. Expected a fully-qualified " + "dataset ID in standard SQL format. e.g. " + '"project.dataset_id", got {}'.format(dataset_id) + ) + + return cls(output_project_id, output_dataset_id) + + def to_api_repr(self) -> dict: + """Construct the API resource representation of this dataset reference + + Returns: + Dict[str, str]: dataset reference represented as an API resource + """ + return {"projectId": self._project, "datasetId": self._dataset_id} + + def _key(self): + """A tuple key that uniquely describes this field. + + Used to compute this instance's hashcode and evaluate equality. + + Returns: + Tuple[str]: The contents of this :class:`.DatasetReference`. + """ + return (self._project, self._dataset_id) + + def __eq__(self, other): + if not isinstance(other, DatasetReference): + return NotImplemented + return self._key() == other._key() + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self._key()) + + def __repr__(self): + return "DatasetReference{}".format(self._key()) + class AccessEntry(object): """Represents grant of an access role to an entity. @@ -145,17 +287,13 @@ class AccessEntry(object): def __init__(self, role=None, entity_type=None, entity_id=None): self._properties = {} - if entity_type in ("view", "routine", "dataset"): - if role is not None: - raise ValueError( - "Role must be None for a %r. Received " - "role: %r" % (entity_type, role) - ) - else: - if role is None: - raise ValueError( - "Role must be set for entity " "type %r" % (entity_type,) - ) + if entity_type is not None: + self._properties[entity_type] = entity_id + # else: + # if role is None: + # raise ValueError( + # "Role must be set for entity " "type %r" % (entity_type,) + # ) self._role = role self._entity_type = entity_type self._entity_id = entity_id @@ -165,6 +303,51 @@ def role(self): """str: The role of the entry.""" return self._role + @property + def view(self) -> Optional[TableReference]: + """some view representation (tableref or string?)""" + return self._properties["view"] + + @view.setter + def view(self, value): + # value = _table_arg_to_table_ref(value).to_api_repr() # maybe unneccessary + if self._role is not None: + raise ValueError( + "Role must be None for a view. Current " + "role: %r" % (self._role) + ) + self._properties["view"] = value + + @property + def dataset(self) -> Optional[DatasetReference]: + """some dataset representation (tableref or string?)""" + return self._properties["dataset"] + + @dataset.setter + def dataset(self, value): + if self._role is not None: + raise ValueError( + "Role must be None for a dataset. Current " + "role: %r" % (self._role) + ) + # value = _table_arg_to_table_ref(value).to_api_repr() # maybe unneccessary + self._properties["dataset"] = value + + @property + def routine(self) -> Optional[RoutineReference]: + """some routine representation (tableref or string?)""" + return self._properties["routine"] + + @routine.setter + def routine(self, value): + if self._role is not None: + raise ValueError( + "Role must be None for a routine. Current " + "role: %r" % (self._role) + ) + # value = _table_arg_to_table_ref(value).to_api_repr() # maybe unneccessary + self._properties["routine"] = value + @property def entity_type(self): """str: The entity_type of the entry.""" @@ -208,7 +391,8 @@ def to_api_repr(self): Dict[str, object]: Access entry represented as an API resource """ resource = copy.deepcopy(self._properties) - resource[self._entity_type] = self._entity_id + if self._entity_type is not None: + resource[self._entity_type] = self._entity_id if self._role is not None: resource["role"] = self._role return resource @@ -241,147 +425,6 @@ def from_api_repr(cls, resource: dict) -> "AccessEntry": return config -class DatasetReference(object): - """DatasetReferences are pointers to datasets. - - See - https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#datasetreference - - Args: - project (str): The ID of the project - dataset_id (str): The ID of the dataset - - Raises: - ValueError: If either argument is not of type ``str``. - """ - - def __init__(self, project, dataset_id): - if not isinstance(project, str): - raise ValueError("Pass a string for project") - if not isinstance(dataset_id, str): - raise ValueError("Pass a string for dataset_id") - self._project = project - self._dataset_id = dataset_id - - @property - def project(self): - """str: Project ID of the dataset.""" - return self._project - - @property - def dataset_id(self): - """str: Dataset ID.""" - return self._dataset_id - - @property - def path(self): - """str: URL path for the dataset based on project and dataset ID.""" - return "/projects/%s/datasets/%s" % (self.project, self.dataset_id) - - table = _get_table_reference - - model = _get_model_reference - - routine = _get_routine_reference - - @classmethod - def from_api_repr(cls, resource: dict) -> "DatasetReference": - """Factory: construct a dataset reference given its API representation - - Args: - resource (Dict[str, str]): - Dataset reference resource representation returned from the API - - Returns: - google.cloud.bigquery.dataset.DatasetReference: - Dataset reference parsed from ``resource``. - """ - project = resource["projectId"] - dataset_id = resource["datasetId"] - return cls(project, dataset_id) - - @classmethod - def from_string( - cls, dataset_id: str, default_project: str = None - ) -> "DatasetReference": - """Construct a dataset reference from dataset ID string. - - Args: - dataset_id (str): - A dataset ID in standard SQL format. If ``default_project`` - is not specified, this must include both the project ID and - the dataset ID, separated by ``.``. - default_project (Optional[str]): - The project ID to use when ``dataset_id`` does not include a - project ID. - - Returns: - DatasetReference: - Dataset reference parsed from ``dataset_id``. - - Examples: - >>> DatasetReference.from_string('my-project-id.some_dataset') - DatasetReference('my-project-id', 'some_dataset') - - Raises: - ValueError: - If ``dataset_id`` is not a fully-qualified dataset ID in - standard SQL format. - """ - output_dataset_id = dataset_id - output_project_id = default_project - parts = _helpers._split_id(dataset_id) - - if len(parts) == 1 and not default_project: - raise ValueError( - "When default_project is not set, dataset_id must be a " - "fully-qualified dataset ID in standard SQL format, " - 'e.g., "project.dataset_id" got {}'.format(dataset_id) - ) - elif len(parts) == 2: - output_project_id, output_dataset_id = parts - elif len(parts) > 2: - raise ValueError( - "Too many parts in dataset_id. Expected a fully-qualified " - "dataset ID in standard SQL format. e.g. " - '"project.dataset_id", got {}'.format(dataset_id) - ) - - return cls(output_project_id, output_dataset_id) - - def to_api_repr(self) -> dict: - """Construct the API resource representation of this dataset reference - - Returns: - Dict[str, str]: dataset reference represented as an API resource - """ - return {"projectId": self._project, "datasetId": self._dataset_id} - - def _key(self): - """A tuple key that uniquely describes this field. - - Used to compute this instance's hashcode and evaluate equality. - - Returns: - Tuple[str]: The contents of this :class:`.DatasetReference`. - """ - return (self._project, self._dataset_id) - - def __eq__(self, other): - if not isinstance(other, DatasetReference): - return NotImplemented - return self._key() == other._key() - - def __ne__(self, other): - return not self == other - - def __hash__(self): - return hash(self._key()) - - def __repr__(self): - return "DatasetReference{}".format(self._key()) - - class Dataset(object): """Datasets are containers for tables. diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index c554782bf..656bf36ad 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -198,7 +198,150 @@ def test_from_api_repr_entries_w_extra_keys(self): with self.assertRaises(ValueError): self._get_target_class().from_api_repr(resource) + # def test_properties(self): + # from google.cloud.bigquery import external_config + # from google.cloud.bigquery.dataset import AccessEntry + # from google.cloud.bigquery.external_config import AvroOptions + + # options = AvroOptions.from_api_repr({"useAvroLogicalTypes": True}) + # ec = external_config.ExternalConfig(external_config.ExternalSourceFormat.AVRO) + + # self.assertIsNone(ec.avro_options.use_avro_logical_types) + + # ec.avro_options = options + + # self.assertTrue(ec.avro_options.use_avro_logical_types) + # self.assertIs( + # ec.options._properties, ec._properties[AvroOptions._RESOURCE_NAME] + # ) + # self.assertIs( + # ec.avro_options._properties, ec._properties[AvroOptions._RESOURCE_NAME] + # ) + + # view = { + # "projectId": "my-project", + # "datasetId": "my_dataset", + # "tableId": "my_table", + # } + # table_resource_string = "abc.abc.abc" + # entry = self._make_one(None, "view", table_resource_string) + # # resource = entry.to_api_repr() + # # exp_resource = {"role": "READER"} + # print("******") + # print(repr(entry)) + # print("properties:") + # print(entry._properties) + # # print("_entity_type:") + # # print(entry._entity_type) + # # print("~~~~~~~") + # # table_resource_string = "abc.abc.abc" + # # entry.view = table_resource_string + # # print(repr(entry)) + # # print("properties:") + # # print(entry._properties) + + # # review = { + # # "projectId": "msdfafast", + # # "datasetId": "myasfadft", + # # "tableId": "myasdfadfale", + # # } + # # entry._properties["view"] = review + # # print(repr(entry)) + # # print("properties:") + # # print(entry._properties) + # # entry2 = AccessEntry("READER") + # # entry2.view = view + # # print("******2") + # # print(repr(entry2)) + # # print("properties2:") + # # print(entry2._properties) + # # entry._entity_type = "view" + # # print("set entity type") + # # print(entry2._properties) + + + # # print("~~~~~~~") + # # print(repr(resource)) + # # print('external config:') + # # print(ec) + # # print(ec._properties) + + # # print(resource.properties) + # self.assertEqual(resource, exp_resource) + + def test_view_getter_setter(self): + from google.cloud.bigquery import external_config + from google.cloud.bigquery.dataset import AccessEntry + from google.cloud.bigquery.external_config import AvroOptions + + view = { + "projectId": "my_project", + "datasetId": "my_dataset", + "tableId": "my_table", + } + entry = self._make_one(None) + entry.view = view + resource = entry.to_api_repr() + exp_resource = {"view": view} + self.assertEqual(resource, exp_resource) + + def test_view_getter_setter_incorrect_role(self): + view = { + "projectId": "my_project", + "datasetId": "my_dataset", + "tableId": "my_table", + } + entry = self._make_one("READER") + with self.assertRaises(ValueError): + entry.view = view + def test_dataset_getter_setter(self): + dataset = { + "dataset": { + "projectId": "my-project", + "datasetId": "my_dataset", + "tableId": "my_table", + } + } + entry = self._make_one(None) + entry.dataset = dataset + resource = entry.to_api_repr() + exp_resource = {"dataset": dataset} + self.assertEqual(resource, exp_resource) + + def test_dataset_getter_setter_incorrect_role(self): + dataset = { + "dataset": { + "projectId": "my-project", + "datasetId": "my_dataset", + "tableId": "my_table", + } + } + entry = self._make_one("READER") + with self.assertRaises(ValueError): + entry.dataset = dataset + + def test_routine_getter_setter(self): + routine = { + "projectId": "my-project", + "datasetId": "my_dataset", + "routineId": "my_routine", + } + entry = self._make_one(None) + entry.routine = routine + resource = entry.to_api_repr() + exp_resource = {"routine": routine} + self.assertEqual(resource, exp_resource) + + def test_routine_getter_setter_incorrect_role(self): + routine = { + "projectId": "my-project", + "datasetId": "my_dataset", + "routineId": "my_routine", + } + entry = self._make_one("READER") + with self.assertRaises(ValueError): + entry.routine = routine class TestDatasetReference(unittest.TestCase): @staticmethod def _get_target_class(): From 80c22ec931cd905ebf6d87c47308a5b3f78694c8 Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 31 Jan 2022 12:16:08 -0800 Subject: [PATCH 02/34] add properties and unit tests --- google/cloud/bigquery/dataset.py | 62 ++++++++--- tests/unit/test_dataset.py | 170 ++++++++----------------------- 2 files changed, 94 insertions(+), 138 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index cb0aeea30..f1bbe6ae8 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -75,6 +75,7 @@ def _get_routine_reference(self, routine_id): } ) + class DatasetReference(object): """DatasetReferences are pointers to datasets. @@ -294,14 +295,19 @@ def __init__(self, role=None, entity_type=None, entity_id=None): # raise ValueError( # "Role must be set for entity " "type %r" % (entity_type,) # ) - self._role = role + self._role = self._properties["role"] = role self._entity_type = entity_type self._entity_id = entity_id @property def role(self): """str: The role of the entry.""" - return self._role + if self._properties["role"]: + return self._properties["role"] + + @role.setter + def role(self, value): + self._properties["role"] = value @property def view(self) -> Optional[TableReference]: @@ -310,11 +316,9 @@ def view(self) -> Optional[TableReference]: @view.setter def view(self, value): - # value = _table_arg_to_table_ref(value).to_api_repr() # maybe unneccessary if self._role is not None: raise ValueError( - "Role must be None for a view. Current " - "role: %r" % (self._role) + "Role must be None for a view. Current " "role: %r" % (self._role) ) self._properties["view"] = value @@ -327,10 +331,8 @@ def dataset(self) -> Optional[DatasetReference]: def dataset(self, value): if self._role is not None: raise ValueError( - "Role must be None for a dataset. Current " - "role: %r" % (self._role) + "Role must be None for a dataset. Current " "role: %r" % (self._role) ) - # value = _table_arg_to_table_ref(value).to_api_repr() # maybe unneccessary self._properties["dataset"] = value @property @@ -342,12 +344,46 @@ def routine(self) -> Optional[RoutineReference]: def routine(self, value): if self._role is not None: raise ValueError( - "Role must be None for a routine. Current " - "role: %r" % (self._role) + "Role must be None for a routine. Current " "role: %r" % (self._role) ) - # value = _table_arg_to_table_ref(value).to_api_repr() # maybe unneccessary self._properties["routine"] = value + @property + def group_by_email(self) -> Optional[str]: + """groupbyEmail""" + return self._properties["groupByEmail"] + + @group_by_email.setter + def group_by_email(self, value): + self._properties["groupByEmail"] = value + + @property + def user_by_email(self) -> Optional[str]: + """userByEmail""" + return self._properties["userByEmail"] + + @user_by_email.setter + def user_by_email(self, value): + self._properties["userByEmail"] = value + + @property + def domain(self) -> Optional[str]: + """domain""" + return self._properties["domain"] + + @domain.setter + def domain(self, value): + self._properties["domain"] = value + + @property + def special_group(self) -> Optional[str]: + """specialGroup""" + return self._properties["specialGroup"] + + @special_group.setter + def special_group(self, value): + self._properties["specialGroup"] = value + @property def entity_type(self): """str: The entity_type of the entry.""" @@ -379,7 +415,9 @@ def _key(self): Returns: Tuple: The contents of this :class:`~google.cloud.bigquery.dataset.AccessEntry`. """ - return (self._role, self._entity_type, self._entity_id) + properties = self._properties.copy() + prop_tup = tuple(sorted(properties.items())) + return (self._role, self._entity_type, self._entity_id, prop_tup) def __hash__(self): return hash(self._key()) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 656bf36ad..bdcf06d91 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -34,16 +34,6 @@ def test_ctor_defaults(self): self.assertEqual(entry.entity_type, "userByEmail") self.assertEqual(entry.entity_id, "phred@example.com") - def test_ctor_bad_entity_type(self): - with self.assertRaises(ValueError): - self._make_one(None, "unknown", None) - - def test_ctor_view_with_role(self): - role = "READER" - entity_type = "view" - with self.assertRaises(ValueError): - self._make_one(role, entity_type, None) - def test_ctor_view_success(self): role = None entity_type = "view" @@ -53,12 +43,6 @@ def test_ctor_view_success(self): self.assertEqual(entry.entity_type, entity_type) self.assertEqual(entry.entity_id, entity_id) - def test_ctor_routine_with_role(self): - role = "READER" - entity_type = "routine" - with self.assertRaises(ValueError): - self._make_one(role, entity_type, None) - def test_ctor_routine_success(self): role = None entity_type = "routine" @@ -68,12 +52,6 @@ def test_ctor_routine_success(self): self.assertEqual(entry.entity_type, entity_type) self.assertEqual(entry.entity_id, entity_id) - def test_ctor_nonview_without_role(self): - role = None - entity_type = "userByEmail" - with self.assertRaises(ValueError): - self._make_one(role, entity_type, None) - def test___eq___role_mismatch(self): entry = self._make_one("OWNER", "userByEmail", "phred@example.com") other = self._make_one("WRITER", "userByEmail", "phred@example.com") @@ -127,7 +105,7 @@ def test_to_api_repr_view(self): } entry = self._make_one(None, "view", view) resource = entry.to_api_repr() - exp_resource = {"view": view} + exp_resource = {"view": view, "role": None} self.assertEqual(resource, exp_resource) def test_to_api_repr_routine(self): @@ -138,7 +116,7 @@ def test_to_api_repr_routine(self): } entry = self._make_one(None, "routine", routine) resource = entry.to_api_repr() - exp_resource = {"routine": routine} + exp_resource = {"routine": routine, "role": None} self.assertEqual(resource, exp_resource) def test_to_api_repr_dataset(self): @@ -148,21 +126,9 @@ def test_to_api_repr_dataset(self): } entry = self._make_one(None, "dataset", dataset) resource = entry.to_api_repr() - exp_resource = {"dataset": dataset} + exp_resource = {"dataset": dataset, "role": None} self.assertEqual(resource, exp_resource) - def test_to_api_w_incorrect_role(self): - dataset = { - "dataset": { - "projectId": "my-project", - "datasetId": "my_dataset", - "tableId": "my_table", - }, - "target_type": "VIEW", - } - with self.assertRaises(ValueError): - self._make_one("READER", "dataset", dataset) - def test_from_api_repr(self): resource = {"role": "OWNER", "userByEmail": "salmon@example.com"} entry = self._get_target_class().from_api_repr(resource) @@ -198,77 +164,6 @@ def test_from_api_repr_entries_w_extra_keys(self): with self.assertRaises(ValueError): self._get_target_class().from_api_repr(resource) - # def test_properties(self): - # from google.cloud.bigquery import external_config - # from google.cloud.bigquery.dataset import AccessEntry - # from google.cloud.bigquery.external_config import AvroOptions - - # options = AvroOptions.from_api_repr({"useAvroLogicalTypes": True}) - # ec = external_config.ExternalConfig(external_config.ExternalSourceFormat.AVRO) - - # self.assertIsNone(ec.avro_options.use_avro_logical_types) - - # ec.avro_options = options - - # self.assertTrue(ec.avro_options.use_avro_logical_types) - # self.assertIs( - # ec.options._properties, ec._properties[AvroOptions._RESOURCE_NAME] - # ) - # self.assertIs( - # ec.avro_options._properties, ec._properties[AvroOptions._RESOURCE_NAME] - # ) - - # view = { - # "projectId": "my-project", - # "datasetId": "my_dataset", - # "tableId": "my_table", - # } - # table_resource_string = "abc.abc.abc" - # entry = self._make_one(None, "view", table_resource_string) - # # resource = entry.to_api_repr() - # # exp_resource = {"role": "READER"} - # print("******") - # print(repr(entry)) - # print("properties:") - # print(entry._properties) - # # print("_entity_type:") - # # print(entry._entity_type) - # # print("~~~~~~~") - # # table_resource_string = "abc.abc.abc" - # # entry.view = table_resource_string - # # print(repr(entry)) - # # print("properties:") - # # print(entry._properties) - - # # review = { - # # "projectId": "msdfafast", - # # "datasetId": "myasfadft", - # # "tableId": "myasdfadfale", - # # } - # # entry._properties["view"] = review - # # print(repr(entry)) - # # print("properties:") - # # print(entry._properties) - # # entry2 = AccessEntry("READER") - # # entry2.view = view - # # print("******2") - # # print(repr(entry2)) - # # print("properties2:") - # # print(entry2._properties) - # # entry._entity_type = "view" - # # print("set entity type") - # # print(entry2._properties) - - - # # print("~~~~~~~") - # # print(repr(resource)) - # # print('external config:') - # # print(ec) - # # print(ec._properties) - - # # print(resource.properties) - # self.assertEqual(resource, exp_resource) - def test_view_getter_setter(self): from google.cloud.bigquery import external_config from google.cloud.bigquery.dataset import AccessEntry @@ -282,7 +177,7 @@ def test_view_getter_setter(self): entry = self._make_one(None) entry.view = view resource = entry.to_api_repr() - exp_resource = {"view": view} + exp_resource = {"view": view, "role": None} self.assertEqual(resource, exp_resource) def test_view_getter_setter_incorrect_role(self): @@ -296,27 +191,16 @@ def test_view_getter_setter_incorrect_role(self): entry.view = view def test_dataset_getter_setter(self): - dataset = { - "dataset": { - "projectId": "my-project", - "datasetId": "my_dataset", - "tableId": "my_table", - } - } + dataset = {"dataset": {"projectId": "my-project", "datasetId": "my_dataset",}} entry = self._make_one(None) entry.dataset = dataset resource = entry.to_api_repr() - exp_resource = {"dataset": dataset} + print("TO API REPR", resource) + exp_resource = {"dataset": dataset, "role": None} self.assertEqual(resource, exp_resource) def test_dataset_getter_setter_incorrect_role(self): - dataset = { - "dataset": { - "projectId": "my-project", - "datasetId": "my_dataset", - "tableId": "my_table", - } - } + dataset = {"dataset": {"projectId": "my-project", "datasetId": "my_dataset",}} entry = self._make_one("READER") with self.assertRaises(ValueError): entry.dataset = dataset @@ -330,9 +214,9 @@ def test_routine_getter_setter(self): entry = self._make_one(None) entry.routine = routine resource = entry.to_api_repr() - exp_resource = {"routine": routine} + exp_resource = {"routine": routine, "role": None} self.assertEqual(resource, exp_resource) - + def test_routine_getter_setter_incorrect_role(self): routine = { "projectId": "my-project", @@ -342,6 +226,40 @@ def test_routine_getter_setter_incorrect_role(self): entry = self._make_one("READER") with self.assertRaises(ValueError): entry.routine = routine + + def test_group_by_email_getter_setter(self): + email = "cloud-developer-relations@google.com" + entry = self._make_one(None) + entry.group_by_email = email + resource = entry.to_api_repr() + exp_resource = {"groupByEmail": email, "role": None} + self.assertEqual(resource, exp_resource) + + def test_user_by_email_getter_setter(self): + email = "cloud-developer-relations@google.com" + entry = self._make_one(None) + entry.user_by_email = email + resource = entry.to_api_repr() + exp_resource = {"userByEmail": email, "role": None} + self.assertEqual(resource, exp_resource) + + def test_domain_setter(self): + domain = "my_domain" + entry = self._make_one(None) + entry.domain = domain + resource = entry.to_api_repr() + exp_resource = {"domain": domain, "role": None} + self.assertEqual(resource, exp_resource) + + def test_special_group_getter_setter(self): + special_group = "my_special_group" + entry = self._make_one(None) + entry.special_group = special_group + resource = entry.to_api_repr() + exp_resource = {"specialGroup": special_group, "role": None} + self.assertEqual(resource, exp_resource) + + class TestDatasetReference(unittest.TestCase): @staticmethod def _get_target_class(): From 2f2a8cd31273a38e6b442cb065e30ca1da4428e3 Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 31 Jan 2022 12:51:44 -0800 Subject: [PATCH 03/34] lint --- google/cloud/bigquery/dataset.py | 5 ----- tests/unit/test_dataset.py | 1 - 2 files changed, 6 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index f1bbe6ae8..8befd7d87 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -290,11 +290,6 @@ def __init__(self, role=None, entity_type=None, entity_id=None): self._properties = {} if entity_type is not None: self._properties[entity_type] = entity_id - # else: - # if role is None: - # raise ValueError( - # "Role must be set for entity " "type %r" % (entity_type,) - # ) self._role = self._properties["role"] = role self._entity_type = entity_type self._entity_id = entity_id diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index bdcf06d91..e70e92678 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -195,7 +195,6 @@ def test_dataset_getter_setter(self): entry = self._make_one(None) entry.dataset = dataset resource = entry.to_api_repr() - print("TO API REPR", resource) exp_resource = {"dataset": dataset, "role": None} self.assertEqual(resource, exp_resource) From a8840ab51f5baf6acd1c28080063aa35aa69b385 Mon Sep 17 00:00:00 2001 From: steffnay Date: Sat, 5 Feb 2022 12:06:13 -0800 Subject: [PATCH 04/34] add properties and tests --- google/cloud/bigquery/dataset.py | 77 ++++++++++++++++------- tests/unit/test_dataset.py | 105 ++++++++++++++++++++++++++++--- 2 files changed, 154 insertions(+), 28 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 8befd7d87..4f5444e74 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -22,11 +22,11 @@ from google.cloud.bigquery import _helpers from google.cloud.bigquery.model import ModelReference -from google.cloud.bigquery.routine import RoutineReference +from google.cloud.bigquery.routine import Routine, RoutineReference from google.cloud.bigquery.table import TableReference, _table_arg_to_table_ref from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration -from typing import FrozenSet, Iterable, Optional, Union +from typing import FrozenSet, Iterable, Optional, Union, List def _get_table_reference(self, table_id): @@ -295,7 +295,7 @@ def __init__(self, role=None, entity_type=None, entity_id=None): self._entity_id = entity_id @property - def role(self): + def role(self) -> Optional[str]: """str: The role of the entry.""" if self._properties["role"]: return self._properties["role"] @@ -305,20 +305,7 @@ def role(self, value): self._properties["role"] = value @property - def view(self) -> Optional[TableReference]: - """some view representation (tableref or string?)""" - return self._properties["view"] - - @view.setter - def view(self, value): - if self._role is not None: - raise ValueError( - "Role must be None for a view. Current " "role: %r" % (self._role) - ) - self._properties["view"] = value - - @property - def dataset(self) -> Optional[DatasetReference]: + def dataset(self) -> Optional[dict]: """some dataset representation (tableref or string?)""" return self._properties["dataset"] @@ -328,10 +315,34 @@ def dataset(self, value): raise ValueError( "Role must be None for a dataset. Current " "role: %r" % (self._role) ) - self._properties["dataset"] = value + if not isinstance(value, dict): + if isinstance(value, str): + value = DatasetReference.from_string(value) + + if isinstance(value, (Dataset, DatasetListItem)): + value = value.reference + + value = value.to_api_repr() + + prop = { + "dataset": value, + "targetTypes": self._properties.get("targetTypes", None), + } + self._properties["dataset"] = prop + + @property + def target_types(self) -> Optional[List[str]]: + """target types""" + return self._properties["dataset"]["targetTypes"] + + @target_types.setter + def target_types(self, value): + if not "dataset" in self._properties: + self._properties["dataset"] = {} + self._properties["dataset"]["targetTypes"] = value @property - def routine(self) -> Optional[RoutineReference]: + def routine(self) -> Optional[dict]: """some routine representation (tableref or string?)""" return self._properties["routine"] @@ -341,8 +352,32 @@ def routine(self, value): raise ValueError( "Role must be None for a routine. Current " "role: %r" % (self._role) ) + if not isinstance(value, dict): + if isinstance(value, str): + value = RoutineReference.from_string(value) + + if isinstance(value, Routine): + value = value.reference + + value = value.to_api_repr() self._properties["routine"] = value + @property + def view(self) -> Optional[dict]: + """some view representation (tableref or string?)""" + return self._properties["view"] + + @view.setter + def view(self, value): + if self._role is not None: + raise ValueError( + "Role must be None for a view. Current " "role: %r" % (self._role) + ) + if not isinstance(value, dict): + value = _table_arg_to_table_ref(value) + value = value.to_api_repr() + self._properties["view"] = value + @property def group_by_email(self) -> Optional[str]: """groupbyEmail""" @@ -380,12 +415,12 @@ def special_group(self, value): self._properties["specialGroup"] = value @property - def entity_type(self): + def entity_type(self) -> Optional[str]: """str: The entity_type of the entry.""" return self._entity_type @property - def entity_id(self): + def entity_id(self) -> Optional[str]: """str: The entity_id of the entry.""" return self._entity_id diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index e70e92678..e4c16ed30 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -15,13 +15,15 @@ import unittest import mock +from google.cloud.bigquery.routine.routine import RoutineReference import pytest +from google.cloud.bigquery.dataset import AccessEntry, DatasetReference, TableReference class TestAccessEntry(unittest.TestCase): @staticmethod def _get_target_class(): - from google.cloud.bigquery.dataset import AccessEntry + # from google.cloud.bigquery.dataset import AccessEntry, DatasetReference return AccessEntry @@ -165,10 +167,6 @@ def test_from_api_repr_entries_w_extra_keys(self): self._get_target_class().from_api_repr(resource) def test_view_getter_setter(self): - from google.cloud.bigquery import external_config - from google.cloud.bigquery.dataset import AccessEntry - from google.cloud.bigquery.external_config import AvroOptions - view = { "projectId": "my_project", "datasetId": "my_dataset", @@ -180,6 +178,28 @@ def test_view_getter_setter(self): exp_resource = {"view": view, "role": None} self.assertEqual(resource, exp_resource) + def test_view_getter_setter_string(self): + project = "my_project" + dataset = "my_dataset" + table = "my_table" + entry = self._make_one(None) + entry.view = f"{project}.{dataset}.{table}" + resource = entry.to_api_repr() + exp_view = {"projectId": project, "datasetId": dataset, "tableId": table} + exp_resource = {"view": exp_view, "role": None} + self.assertEqual(resource, exp_resource) + + def test_view_getter_setter_table_ref(self): + project = "my_project" + dataset = "my_dataset" + table = "my_table" + entry = self._make_one(None) + entry.view = TableReference(DatasetReference(project, dataset), table) + resource = entry.to_api_repr() + exp_view = {"projectId": project, "datasetId": dataset, "tableId": table} + exp_resource = {"view": exp_view, "role": None} + self.assertEqual(resource, exp_resource) + def test_view_getter_setter_incorrect_role(self): view = { "projectId": "my_project", @@ -191,11 +211,46 @@ def test_view_getter_setter_incorrect_role(self): entry.view = view def test_dataset_getter_setter(self): - dataset = {"dataset": {"projectId": "my-project", "datasetId": "my_dataset",}} + dataset = {"dataset": {"projectId": "my-project", "datasetId": "my_dataset"}} entry = self._make_one(None) entry.dataset = dataset + print("ENTRY:", entry) resource = entry.to_api_repr() - exp_resource = {"dataset": dataset, "role": None} + print("RESOURCE:", resource) + exp_resource = { + "dataset": {"dataset": dataset, "targetTypes": None}, + "role": None, + } + self.assertEqual(resource, exp_resource) + + def test_dataset_getter_setter_string(self): + project = "my-project" + dataset_id = "my_dataset" + entry = self._make_one(None) + entry.dataset = f"{project}.{dataset_id}" + resource = entry.to_api_repr() + exp_resource = { + "dataset": { + "dataset": {"projectId": project, "datasetId": dataset_id}, + "targetTypes": None, + }, + "role": None, + } + self.assertEqual(resource, exp_resource) + + def test_dataset_getter_setter_dataset_ref(self): + project = "my-project" + dataset_id = "my_dataset" + entry = self._make_one(None) + entry.dataset = DatasetReference(project, dataset_id) + resource = entry.to_api_repr() + exp_resource = { + "dataset": { + "dataset": {"projectId": project, "datasetId": dataset_id}, + "targetTypes": None, + }, + "role": None, + } self.assertEqual(resource, exp_resource) def test_dataset_getter_setter_incorrect_role(self): @@ -216,6 +271,42 @@ def test_routine_getter_setter(self): exp_resource = {"routine": routine, "role": None} self.assertEqual(resource, exp_resource) + def test_routine_getter_setter_string(self): + project = "my-project" + dataset_id = "my_dataset" + routine_id = "my_routine" + entry = self._make_one(None) + entry.routine = f"{project}.{dataset_id}.{routine_id}" + resource = entry.to_api_repr() + exp_resource = { + "routine": { + "projectId": project, + "datasetId": dataset_id, + "routineId": routine_id, + }, + "role": None, + } + self.assertEqual(resource, exp_resource) + + def test_routine_getter_setter_routine_ref(self): + project = "my-project" + dataset_id = "my_dataset" + routine_id = "my_routine" + entry = self._make_one(None) + entry.routine = RoutineReference.from_string( + f"{project}.{dataset_id}.{routine_id}" + ) + resource = entry.to_api_repr() + exp_resource = { + "routine": { + "projectId": project, + "datasetId": dataset_id, + "routineId": routine_id, + }, + "role": None, + } + self.assertEqual(resource, exp_resource) + def test_routine_getter_setter_incorrect_role(self): routine = { "projectId": "my-project", From 4e37ddf7a8eb461bb195a33d7daaf9d746d35a00 Mon Sep 17 00:00:00 2001 From: steffnay Date: Sat, 5 Feb 2022 12:48:17 -0800 Subject: [PATCH 05/34] remove unused imports --- google/cloud/bigquery/dataset.py | 26 +++++++++++++------------- tests/unit/test_dataset.py | 6 +----- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 4f5444e74..3e4fa3080 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -26,7 +26,7 @@ from google.cloud.bigquery.table import TableReference, _table_arg_to_table_ref from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration -from typing import FrozenSet, Iterable, Optional, Union, List +from typing import Optional, List def _get_table_reference(self, table_id): @@ -296,7 +296,7 @@ def __init__(self, role=None, entity_type=None, entity_id=None): @property def role(self) -> Optional[str]: - """str: The role of the entry.""" + """The role of the entry.""" if self._properties["role"]: return self._properties["role"] @@ -306,7 +306,7 @@ def role(self, value): @property def dataset(self) -> Optional[dict]: - """some dataset representation (tableref or string?)""" + """API resource representation of a dataset reference.""" return self._properties["dataset"] @dataset.setter @@ -332,18 +332,18 @@ def dataset(self, value): @property def target_types(self) -> Optional[List[str]]: - """target types""" + """Which resources that the dataset in this entry applies to.""" return self._properties["dataset"]["targetTypes"] @target_types.setter def target_types(self, value): - if not "dataset" in self._properties: + if "dataset" not in self._properties: self._properties["dataset"] = {} self._properties["dataset"]["targetTypes"] = value @property def routine(self) -> Optional[dict]: - """some routine representation (tableref or string?)""" + """API resource representation of a routine reference.""" return self._properties["routine"] @routine.setter @@ -364,7 +364,7 @@ def routine(self, value): @property def view(self) -> Optional[dict]: - """some view representation (tableref or string?)""" + """API resource representation of a view reference.""" return self._properties["view"] @view.setter @@ -380,7 +380,7 @@ def view(self, value): @property def group_by_email(self) -> Optional[str]: - """groupbyEmail""" + """An email address of a Google Group to grant access to.""" return self._properties["groupByEmail"] @group_by_email.setter @@ -389,7 +389,7 @@ def group_by_email(self, value): @property def user_by_email(self) -> Optional[str]: - """userByEmail""" + """An email address of a user to grant access to.""" return self._properties["userByEmail"] @user_by_email.setter @@ -398,7 +398,7 @@ def user_by_email(self, value): @property def domain(self) -> Optional[str]: - """domain""" + """A domain to grant access to.""" return self._properties["domain"] @domain.setter @@ -407,7 +407,7 @@ def domain(self, value): @property def special_group(self) -> Optional[str]: - """specialGroup""" + """A special group to grant access to.""" return self._properties["specialGroup"] @special_group.setter @@ -416,12 +416,12 @@ def special_group(self, value): @property def entity_type(self) -> Optional[str]: - """str: The entity_type of the entry.""" + """The entity_type of the entry.""" return self._entity_type @property def entity_id(self) -> Optional[str]: - """str: The entity_id of the entry.""" + """The entity_id of the entry.""" return self._entity_id def __eq__(self, other): diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index e4c16ed30..dc800f321 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -23,8 +23,6 @@ class TestAccessEntry(unittest.TestCase): @staticmethod def _get_target_class(): - # from google.cloud.bigquery.dataset import AccessEntry, DatasetReference - return AccessEntry def _make_one(self, *args, **kw): @@ -214,9 +212,7 @@ def test_dataset_getter_setter(self): dataset = {"dataset": {"projectId": "my-project", "datasetId": "my_dataset"}} entry = self._make_one(None) entry.dataset = dataset - print("ENTRY:", entry) resource = entry.to_api_repr() - print("RESOURCE:", resource) exp_resource = { "dataset": {"dataset": dataset, "targetTypes": None}, "role": None, @@ -254,7 +250,7 @@ def test_dataset_getter_setter_dataset_ref(self): self.assertEqual(resource, exp_resource) def test_dataset_getter_setter_incorrect_role(self): - dataset = {"dataset": {"projectId": "my-project", "datasetId": "my_dataset",}} + dataset = {"dataset": {"projectId": "my-project", "datasetId": "my_dataset"}} entry = self._make_one("READER") with self.assertRaises(ValueError): entry.dataset = dataset From 6e2a7bc0c9acb4b6049fb224c3d4745ed239c04e Mon Sep 17 00:00:00 2001 From: steffnay Date: Sat, 5 Feb 2022 13:38:28 -0800 Subject: [PATCH 06/34] update test --- tests/unit/test_create_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_create_dataset.py b/tests/unit/test_create_dataset.py index 67b21225d..eb05df2a0 100644 --- a/tests/unit/test_create_dataset.py +++ b/tests/unit/test_create_dataset.py @@ -109,7 +109,7 @@ def test_create_dataset_w_attrs(client, PROJECT, DS_ID): "friendlyName": FRIENDLY_NAME, "location": LOCATION, "defaultTableExpirationMs": "3600", - "access": [{"role": "OWNER", "userByEmail": USER_EMAIL}, {"view": VIEW}], + "access": [{"role": "OWNER", "userByEmail": USER_EMAIL}, {"view": VIEW, "role": None}], "labels": LABELS, }, timeout=DEFAULT_TIMEOUT, From 38b1692cc28d08f920f4b1a55ccc910abe43d443 Mon Sep 17 00:00:00 2001 From: steffnay Date: Sat, 5 Feb 2022 14:53:10 -0800 Subject: [PATCH 07/34] add tests --- tests/unit/test_create_dataset.py | 5 ++- tests/unit/test_dataset.py | 59 ++++++++++++++++++++++++------- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/tests/unit/test_create_dataset.py b/tests/unit/test_create_dataset.py index eb05df2a0..81af52261 100644 --- a/tests/unit/test_create_dataset.py +++ b/tests/unit/test_create_dataset.py @@ -109,7 +109,10 @@ def test_create_dataset_w_attrs(client, PROJECT, DS_ID): "friendlyName": FRIENDLY_NAME, "location": LOCATION, "defaultTableExpirationMs": "3600", - "access": [{"role": "OWNER", "userByEmail": USER_EMAIL}, {"view": VIEW, "role": None}], + "access": [ + {"role": "OWNER", "userByEmail": USER_EMAIL}, + {"view": VIEW, "role": None}, + ], "labels": LABELS, }, timeout=DEFAULT_TIMEOUT, diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index dc800f321..95475dd4c 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -15,9 +15,14 @@ import unittest import mock -from google.cloud.bigquery.routine.routine import RoutineReference +from google.cloud.bigquery.routine.routine import Routine, RoutineReference import pytest -from google.cloud.bigquery.dataset import AccessEntry, DatasetReference, TableReference +from google.cloud.bigquery.dataset import ( + AccessEntry, + Dataset, + DatasetReference, + TableReference, +) class TestAccessEntry(unittest.TestCase): @@ -217,7 +222,9 @@ def test_dataset_getter_setter(self): "dataset": {"dataset": dataset, "targetTypes": None}, "role": None, } + prop = entry.dataset self.assertEqual(resource, exp_resource) + self.assertEqual(prop, exp_resource["dataset"]) def test_dataset_getter_setter_string(self): project = "my-project" @@ -249,6 +256,21 @@ def test_dataset_getter_setter_dataset_ref(self): } self.assertEqual(resource, exp_resource) + def test_dataset_getter_setter_dataset(self): + project = "my-project" + dataset_id = "my_dataset" + entry = self._make_one(None) + entry.dataset = Dataset(DatasetReference(project, dataset_id)) + resource = entry.to_api_repr() + exp_resource = { + "dataset": { + "dataset": {"projectId": project, "datasetId": dataset_id}, + "targetTypes": None, + }, + "role": None, + } + self.assertEqual(resource, exp_resource) + def test_dataset_getter_setter_incorrect_role(self): dataset = {"dataset": {"projectId": "my-project", "datasetId": "my_dataset"}} entry = self._make_one("READER") @@ -285,20 +307,31 @@ def test_routine_getter_setter_string(self): self.assertEqual(resource, exp_resource) def test_routine_getter_setter_routine_ref(self): - project = "my-project" - dataset_id = "my_dataset" - routine_id = "my_routine" + routine = { + "projectId": "my-project", + "datasetId": "my_dataset", + "routineId": "my_routine", + } entry = self._make_one(None) - entry.routine = RoutineReference.from_string( - f"{project}.{dataset_id}.{routine_id}" - ) + entry.routine = RoutineReference.from_api_repr(routine) resource = entry.to_api_repr() exp_resource = { - "routine": { - "projectId": project, - "datasetId": dataset_id, - "routineId": routine_id, - }, + "routine": routine, + "role": None, + } + self.assertEqual(resource, exp_resource) + + def test_routine_getter_setter_routine(self): + routine = { + "projectId": "my-project", + "datasetId": "my_dataset", + "routineId": "my_routine", + } + entry = self._make_one(None) + entry.routine = Routine(RoutineReference.from_api_repr(routine)) + resource = entry.to_api_repr() + exp_resource = { + "routine": routine, "role": None, } self.assertEqual(resource, exp_resource) From 4b99d788d58e845186a72326fc749f5940dc692f Mon Sep 17 00:00:00 2001 From: steffnay Date: Sun, 6 Feb 2022 10:23:43 -0800 Subject: [PATCH 08/34] add tests --- tests/unit/test_dataset.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 95475dd4c..45aa41a24 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -378,6 +378,21 @@ def test_special_group_getter_setter(self): exp_resource = {"specialGroup": special_group, "role": None} self.assertEqual(resource, exp_resource) + def test_role_getter_setter(self): + role = "READER" + entry = self._make_one(None) + entry.role = role + resource = entry.to_api_repr() + exp_resource = {"role": role} + self.assertEqual(resource, exp_resource) + + def test_target_types_getter_setter(self): + target_types = ["VIEWS"] + entry = self._make_one(None) + entry.target_types = target_types + resource = entry.to_api_repr() + self.assertEqual(resource["dataset"]["targetTypes"], target_types) + class TestDatasetReference(unittest.TestCase): @staticmethod From 7eff546e03123060a4b4498a215fbf51c1cdd347 Mon Sep 17 00:00:00 2001 From: steffnay Date: Sun, 6 Feb 2022 11:04:25 -0800 Subject: [PATCH 09/34] add tests --- tests/unit/test_dataset.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 45aa41a24..d0e3c8203 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -179,6 +179,7 @@ def test_view_getter_setter(self): entry.view = view resource = entry.to_api_repr() exp_resource = {"view": view, "role": None} + self.assertEqual(entry.view, view) self.assertEqual(resource, exp_resource) def test_view_getter_setter_string(self): @@ -334,6 +335,7 @@ def test_routine_getter_setter_routine(self): "routine": routine, "role": None, } + self.assertEqual(entry.routine, routine) self.assertEqual(resource, exp_resource) def test_routine_getter_setter_incorrect_role(self): @@ -360,6 +362,7 @@ def test_user_by_email_getter_setter(self): entry.user_by_email = email resource = entry.to_api_repr() exp_resource = {"userByEmail": email, "role": None} + self.assertEqual(entry.user_by_email, email) self.assertEqual(resource, exp_resource) def test_domain_setter(self): @@ -368,6 +371,7 @@ def test_domain_setter(self): entry.domain = domain resource = entry.to_api_repr() exp_resource = {"domain": domain, "role": None} + self.assertEqual(entry.domain, domain) self.assertEqual(resource, exp_resource) def test_special_group_getter_setter(self): @@ -376,6 +380,7 @@ def test_special_group_getter_setter(self): entry.special_group = special_group resource = entry.to_api_repr() exp_resource = {"specialGroup": special_group, "role": None} + self.assertEqual(entry.special_group, special_group) self.assertEqual(resource, exp_resource) def test_role_getter_setter(self): @@ -390,8 +395,8 @@ def test_target_types_getter_setter(self): target_types = ["VIEWS"] entry = self._make_one(None) entry.target_types = target_types - resource = entry.to_api_repr() - self.assertEqual(resource["dataset"]["targetTypes"], target_types) + self.assertEqual(entry.target_types, target_types) + self.assertEqual(entry.dataset["targetTypes"], target_types) class TestDatasetReference(unittest.TestCase): From d81076d494a4c74b05aa72bd64d56b0c0763deb8 Mon Sep 17 00:00:00 2001 From: steffnay Date: Sun, 6 Feb 2022 12:09:19 -0800 Subject: [PATCH 10/34] add more tests --- tests/unit/test_dataset.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index d0e3c8203..e96aaff68 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -354,6 +354,7 @@ def test_group_by_email_getter_setter(self): entry.group_by_email = email resource = entry.to_api_repr() exp_resource = {"groupByEmail": email, "role": None} + self.assertEqual(entry.group_by_email, email) self.assertEqual(resource, exp_resource) def test_user_by_email_getter_setter(self): @@ -398,6 +399,15 @@ def test_target_types_getter_setter(self): self.assertEqual(entry.target_types, target_types) self.assertEqual(entry.dataset["targetTypes"], target_types) + def test_target_types_getter_setter_w_dataset(self): + dataset = {"dataset": {"projectId": "my-project", "datasetId": "my_dataset"}} + target_types = ["VIEWS"] + entry = self._make_one(None) + entry.dataset = dataset + entry.target_types = target_types + self.assertEqual(entry.target_types, target_types) + self.assertEqual(entry.dataset["targetTypes"], target_types) + class TestDatasetReference(unittest.TestCase): @staticmethod From ee1427b9050dfa87ca67e783df5f421c3f8767c2 Mon Sep 17 00:00:00 2001 From: steffnay Date: Sun, 6 Feb 2022 12:53:17 -0800 Subject: [PATCH 11/34] update return type --- google/cloud/bigquery/dataset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 3e4fa3080..d623d5f11 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -297,8 +297,7 @@ def __init__(self, role=None, entity_type=None, entity_id=None): @property def role(self) -> Optional[str]: """The role of the entry.""" - if self._properties["role"]: - return self._properties["role"] + return self._properties["role"] @role.setter def role(self, value): From 7f68e892c0de7b2aeb35541bb4f6ccfcaf9901b7 Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Tue, 15 Feb 2022 23:31:03 -0800 Subject: [PATCH 12/34] Update google/cloud/bigquery/dataset.py Co-authored-by: Tim Swast --- google/cloud/bigquery/dataset.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index d623d5f11..08275bb7d 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -323,11 +323,7 @@ def dataset(self, value): value = value.to_api_repr() - prop = { - "dataset": value, - "targetTypes": self._properties.get("targetTypes", None), - } - self._properties["dataset"] = prop + _helpers._set_sub_prop(self._properties, ["dataset", "dataset"], prop) @property def target_types(self) -> Optional[List[str]]: From ec6dc26c348f2cdecdb55adbe00d706fdc975c25 Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Tue, 15 Feb 2022 23:31:19 -0800 Subject: [PATCH 13/34] Update google/cloud/bigquery/dataset.py Co-authored-by: Tim Swast --- google/cloud/bigquery/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 08275bb7d..33fad7598 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -297,7 +297,7 @@ def __init__(self, role=None, entity_type=None, entity_id=None): @property def role(self) -> Optional[str]: """The role of the entry.""" - return self._properties["role"] + return self._properties.get("role") @role.setter def role(self, value): From d797026488d0dc9029140ea6334ecc7069f06a88 Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Tue, 15 Feb 2022 23:31:28 -0800 Subject: [PATCH 14/34] Update google/cloud/bigquery/dataset.py Co-authored-by: Tim Swast --- google/cloud/bigquery/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 33fad7598..64d3c0340 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -328,7 +328,7 @@ def dataset(self, value): @property def target_types(self) -> Optional[List[str]]: """Which resources that the dataset in this entry applies to.""" - return self._properties["dataset"]["targetTypes"] + return _helpers._get_sub_prop(self._properties, ["dataset", "targetTypes"]) @target_types.setter def target_types(self, value): From 296d764ebd2247c78d2a7ba6b6c6083f2129a3dd Mon Sep 17 00:00:00 2001 From: steffnay Date: Tue, 15 Feb 2022 23:32:59 -0800 Subject: [PATCH 15/34] refactor to use get() and remove self._entity_id --- google/cloud/bigquery/dataset.py | 48 ++++++++++++++++---------------- tests/unit/test_feature.py | 36 ++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 24 deletions(-) create mode 100644 tests/unit/test_feature.py diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index d623d5f11..def557005 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -290,14 +290,14 @@ def __init__(self, role=None, entity_type=None, entity_id=None): self._properties = {} if entity_type is not None: self._properties[entity_type] = entity_id - self._role = self._properties["role"] = role + self._properties["role"] = role self._entity_type = entity_type - self._entity_id = entity_id + # self._entity_id = entity_id @property def role(self) -> Optional[str]: """The role of the entry.""" - return self._properties["role"] + return self._properties.get("role") @role.setter def role(self, value): @@ -306,13 +306,13 @@ def role(self, value): @property def dataset(self) -> Optional[dict]: """API resource representation of a dataset reference.""" - return self._properties["dataset"] + return self._properties.get("dataset") @dataset.setter def dataset(self, value): - if self._role is not None: + if self.role is not None: raise ValueError( - "Role must be None for a dataset. Current " "role: %r" % (self._role) + "Role must be None for a dataset. Current " "role: %r" % (self.role) ) if not isinstance(value, dict): if isinstance(value, str): @@ -332,7 +332,7 @@ def dataset(self, value): @property def target_types(self) -> Optional[List[str]]: """Which resources that the dataset in this entry applies to.""" - return self._properties["dataset"]["targetTypes"] + return self._properties.get("dataset").get("targetTypes") @target_types.setter def target_types(self, value): @@ -343,13 +343,13 @@ def target_types(self, value): @property def routine(self) -> Optional[dict]: """API resource representation of a routine reference.""" - return self._properties["routine"] + return self._properties.get("routine") @routine.setter def routine(self, value): - if self._role is not None: + if self.role is not None: raise ValueError( - "Role must be None for a routine. Current " "role: %r" % (self._role) + "Role must be None for a routine. Current " "role: %r" % (self.role) ) if not isinstance(value, dict): if isinstance(value, str): @@ -364,13 +364,13 @@ def routine(self, value): @property def view(self) -> Optional[dict]: """API resource representation of a view reference.""" - return self._properties["view"] + return self._properties.get("view") @view.setter def view(self, value): - if self._role is not None: + if self.role is not None: raise ValueError( - "Role must be None for a view. Current " "role: %r" % (self._role) + "Role must be None for a view. Current " "role: %r" % (self.role) ) if not isinstance(value, dict): value = _table_arg_to_table_ref(value) @@ -380,7 +380,7 @@ def view(self, value): @property def group_by_email(self) -> Optional[str]: """An email address of a Google Group to grant access to.""" - return self._properties["groupByEmail"] + return self._properties.get("groupByEmail") @group_by_email.setter def group_by_email(self, value): @@ -389,7 +389,7 @@ def group_by_email(self, value): @property def user_by_email(self) -> Optional[str]: """An email address of a user to grant access to.""" - return self._properties["userByEmail"] + return self._properties.get("userByEmail") @user_by_email.setter def user_by_email(self, value): @@ -398,7 +398,7 @@ def user_by_email(self, value): @property def domain(self) -> Optional[str]: """A domain to grant access to.""" - return self._properties["domain"] + return self._properties.get("domain") @domain.setter def domain(self, value): @@ -407,7 +407,7 @@ def domain(self, value): @property def special_group(self) -> Optional[str]: """A special group to grant access to.""" - return self._properties["specialGroup"] + return self._properties.get("specialGroup") @special_group.setter def special_group(self, value): @@ -421,7 +421,7 @@ def entity_type(self) -> Optional[str]: @property def entity_id(self) -> Optional[str]: """The entity_id of the entry.""" - return self._entity_id + return self._properties.get(self.entity_type) def __eq__(self, other): if not isinstance(other, AccessEntry): @@ -433,9 +433,9 @@ def __ne__(self, other): def __repr__(self): return "" % ( - self._role, + self.role, self._entity_type, - self._entity_id, + self.entity_id, ) def _key(self): @@ -446,7 +446,7 @@ def _key(self): """ properties = self._properties.copy() prop_tup = tuple(sorted(properties.items())) - return (self._role, self._entity_type, self._entity_id, prop_tup) + return (self.role, self._entity_type, self.entity_id, prop_tup) def __hash__(self): return hash(self._key()) @@ -459,9 +459,9 @@ def to_api_repr(self): """ resource = copy.deepcopy(self._properties) if self._entity_type is not None: - resource[self._entity_type] = self._entity_id - if self._role is not None: - resource["role"] = self._role + resource[self.entity_type] = self.entity_id + if self.role is not None: + resource["role"] = self.role return resource @classmethod diff --git a/tests/unit/test_feature.py b/tests/unit/test_feature.py new file mode 100644 index 000000000..9a6e154c4 --- /dev/null +++ b/tests/unit/test_feature.py @@ -0,0 +1,36 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# import datetime + +# import mock +# import pytest + +# from google.cloud.bigquery.retry import DEFAULT_TIMEOUT + +# # from google.cloud.bigquery.dataset import AccessEntry +# from .helpers import make_connection + + +# def test_list_jobs_w_timeout(client, PROJECT): +# from google.cloud.bigquery.dataset import AccessEntry +# entry = AccessEntry('OWNER', 'userByEmail', 'user@example.com') +# view = { +# 'projectId': 'my-project', +# 'datasetId': 'my_dataset', +# # 'tableId': 'my_table' +# } +# # entry = AccessEntry(None, 'view', view) +# entry = AccessEntry(None, 'dataset', view) +# assert (entry == {}) From fcd456144e2cacc32c8e02ea81e24b1c627f5745 Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Tue, 15 Feb 2022 23:59:49 -0800 Subject: [PATCH 16/34] delete unnecessary file --- tests/unit/test_feature.py | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 tests/unit/test_feature.py diff --git a/tests/unit/test_feature.py b/tests/unit/test_feature.py deleted file mode 100644 index 9a6e154c4..000000000 --- a/tests/unit/test_feature.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# import datetime - -# import mock -# import pytest - -# from google.cloud.bigquery.retry import DEFAULT_TIMEOUT - -# # from google.cloud.bigquery.dataset import AccessEntry -# from .helpers import make_connection - - -# def test_list_jobs_w_timeout(client, PROJECT): -# from google.cloud.bigquery.dataset import AccessEntry -# entry = AccessEntry('OWNER', 'userByEmail', 'user@example.com') -# view = { -# 'projectId': 'my-project', -# 'datasetId': 'my_dataset', -# # 'tableId': 'my_table' -# } -# # entry = AccessEntry(None, 'view', view) -# entry = AccessEntry(None, 'dataset', view) -# assert (entry == {}) From b11e914ea400a4d70b6259358d99a5eb648407db Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Thu, 17 Feb 2022 08:27:09 -0800 Subject: [PATCH 17/34] Update google/cloud/bigquery/dataset.py Co-authored-by: Tim Swast --- google/cloud/bigquery/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 56cfe41fa..2bd236dce 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -286,7 +286,7 @@ class AccessEntry(object): >>> entry = AccessEntry(None, 'view', view) """ - def __init__(self, role=None, entity_type=None, entity_id=None): + def __init__(self, role: Optional[str] = None, entity_type: Optional[str] = None, entity_id Optional[Dict[str, Any] = None): self._properties = {} if entity_type is not None: self._properties[entity_type] = entity_id From 9605acda54f830ba8181d4bcb17b63f19d1322e5 Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Thu, 17 Feb 2022 08:27:14 -0800 Subject: [PATCH 18/34] Update google/cloud/bigquery/dataset.py Co-authored-by: Tim Swast --- google/cloud/bigquery/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 2bd236dce..76828543c 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -362,7 +362,7 @@ def routine(self, value): self._properties["routine"] = value @property - def view(self) -> Optional[dict]: + def view(self) -> Optional[Dict[str, Any]]: """API resource representation of a view reference.""" return self._properties.get("view") From 5e93d912a07065b62ab0183f28d96e4b3dd840fa Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Thu, 17 Feb 2022 08:27:19 -0800 Subject: [PATCH 19/34] Update google/cloud/bigquery/dataset.py Co-authored-by: Tim Swast --- google/cloud/bigquery/dataset.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 76828543c..51171ec78 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -458,10 +458,6 @@ def to_api_repr(self): Dict[str, object]: Access entry represented as an API resource """ resource = copy.deepcopy(self._properties) - if self._entity_type is not None: - resource[self.entity_type] = self.entity_id - if self.role is not None: - resource["role"] = self.role return resource @classmethod From d40f5dbb8ad5c3f705c505f8fb07d8093d50c862 Mon Sep 17 00:00:00 2001 From: steffnay Date: Thu, 17 Feb 2022 10:20:51 -0800 Subject: [PATCH 20/34] add types, remove unnecessary checks --- google/cloud/bigquery/dataset.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 51171ec78..946ec1c67 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -26,7 +26,7 @@ from google.cloud.bigquery.table import TableReference, _table_arg_to_table_ref from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration -from typing import Optional, List +from typing import Optional, List, Dict, Any def _get_table_reference(self, table_id): @@ -286,7 +286,12 @@ class AccessEntry(object): >>> entry = AccessEntry(None, 'view', view) """ - def __init__(self, role: Optional[str] = None, entity_type: Optional[str] = None, entity_id Optional[Dict[str, Any] = None): + def __init__( + self, + role: Optional[str] = None, + entity_type: Optional[str] = None, + entity_id: Optional[Dict[str, Any]] = None, + ): self._properties = {} if entity_type is not None: self._properties[entity_type] = entity_id From 4d1ffa14e1141d1ad9b0c575ec6c570eddbf3618 Mon Sep 17 00:00:00 2001 From: steffnay Date: Thu, 17 Feb 2022 19:57:10 -0800 Subject: [PATCH 21/34] fix types --- google/cloud/bigquery/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 946ec1c67..06e21a39a 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -26,7 +26,7 @@ from google.cloud.bigquery.table import TableReference, _table_arg_to_table_ref from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration -from typing import Optional, List, Dict, Any +from typing import Optional, List, Dict, Any, Union def _get_table_reference(self, table_id): @@ -299,7 +299,7 @@ def __init__( self._entity_type = entity_type @property - def role(self) -> Optional[str]: + def role(self) -> Optional[Union[Dict[str, Any], str]]: """The role of the entry.""" return self._properties.get("role") @@ -424,7 +424,7 @@ def entity_type(self) -> Optional[str]: return self._entity_type @property - def entity_id(self) -> Optional[str]: + def entity_id(self) -> Optional[Union[Dict[str, Any], str]]: """The entity_id of the entry.""" return self._properties.get(self.entity_type) From 92c320935042fd6ca5abc84ac3e9e0e1e57c1f15 Mon Sep 17 00:00:00 2001 From: steffnay Date: Fri, 18 Feb 2022 12:03:32 -0800 Subject: [PATCH 22/34] types --- google/cloud/bigquery/dataset.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 06e21a39a..02e60b598 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -290,7 +290,7 @@ def __init__( self, role: Optional[str] = None, entity_type: Optional[str] = None, - entity_id: Optional[Dict[str, Any]] = None, + entity_id: Optional[Union[Dict[str, Any], str]] = None, ): self._properties = {} if entity_type is not None: @@ -299,7 +299,7 @@ def __init__( self._entity_type = entity_type @property - def role(self) -> Optional[Union[Dict[str, Any], str]]: + def role(self) -> Optional[str]: """The role of the entry.""" return self._properties.get("role") @@ -308,7 +308,7 @@ def role(self, value): self._properties["role"] = value @property - def dataset(self) -> Optional[dict]: + def dataset(self) -> Optional[Dict[str, Any]]: """API resource representation of a dataset reference.""" return self._properties.get("dataset") @@ -346,7 +346,7 @@ def target_types(self, value): _helpers._set_sub_prop(self._properties, ["dataset", "targetTypes"], value) @property - def routine(self) -> Optional[dict]: + def routine(self) -> Optional[Dict[str, Any]]: """API resource representation of a routine reference.""" return self._properties.get("routine") @@ -356,7 +356,7 @@ def routine(self, value): raise ValueError( "Role must be None for a routine. Current " "role: %r" % (self.role) ) - if not isinstance(value, dict): + if not isinstance(value, Dict): if isinstance(value, str): value = RoutineReference.from_string(value) @@ -424,9 +424,9 @@ def entity_type(self) -> Optional[str]: return self._entity_type @property - def entity_id(self) -> Optional[Union[Dict[str, Any], str]]: + def entity_id(self): """The entity_id of the entry.""" - return self._properties.get(self.entity_type) + return self._properties.get(self.entity_type) if self.entity_type else None def __eq__(self, other): if not isinstance(other, AccessEntry): @@ -437,10 +437,11 @@ def __ne__(self, other): return not self == other def __repr__(self): + return "" % ( self.role, self._entity_type, - self.entity_id, + str(self.entity_id), ) def _key(self): From f0521487f998a801b10481fa4388c144334ebc5b Mon Sep 17 00:00:00 2001 From: steffnay Date: Fri, 18 Feb 2022 14:01:12 -0800 Subject: [PATCH 23/34] add type casting --- google/cloud/bigquery/dataset.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 02e60b598..a667a6323 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -18,6 +18,8 @@ import copy +import typing + import google.cloud._helpers # type: ignore from google.cloud.bigquery import _helpers @@ -301,7 +303,7 @@ def __init__( @property def role(self) -> Optional[str]: """The role of the entry.""" - return self._properties.get("role") + return typing.cast(Optional[str], self._properties.get("role")) @role.setter def role(self, value): @@ -310,7 +312,7 @@ def role(self, value): @property def dataset(self) -> Optional[Dict[str, Any]]: """API resource representation of a dataset reference.""" - return self._properties.get("dataset") + return typing.cast(Optional[Dict[str, Any]], self._properties.get("dataset")) @dataset.setter def dataset(self, value): @@ -337,7 +339,10 @@ def dataset(self, value): @property def target_types(self) -> Optional[List[str]]: """Which resources that the dataset in this entry applies to.""" - return _helpers._get_sub_prop(self._properties, ["dataset", "targetTypes"]) + return typing.cast( + Optional[List[str]], + _helpers._get_sub_prop(self._properties, ["dataset", "targetTypes"]), + ) @target_types.setter def target_types(self, value): @@ -348,7 +353,7 @@ def target_types(self, value): @property def routine(self) -> Optional[Dict[str, Any]]: """API resource representation of a routine reference.""" - return self._properties.get("routine") + return typing.cast(Optional[Dict[str, Any]], self._properties.get("routine")) @routine.setter def routine(self, value): @@ -369,7 +374,7 @@ def routine(self, value): @property def view(self) -> Optional[Dict[str, Any]]: """API resource representation of a view reference.""" - return self._properties.get("view") + return typing.cast(Optional[Dict[str, Any]], self._properties.get("view")) @view.setter def view(self, value): @@ -385,7 +390,7 @@ def view(self, value): @property def group_by_email(self) -> Optional[str]: """An email address of a Google Group to grant access to.""" - return self._properties.get("groupByEmail") + return typing.cast(Optional[str], self._properties.get("groupByEmail")) @group_by_email.setter def group_by_email(self, value): @@ -394,7 +399,7 @@ def group_by_email(self, value): @property def user_by_email(self) -> Optional[str]: """An email address of a user to grant access to.""" - return self._properties.get("userByEmail") + return typing.cast(Optional[str], self._properties.get("userByEmail")) @user_by_email.setter def user_by_email(self, value): @@ -403,7 +408,7 @@ def user_by_email(self, value): @property def domain(self) -> Optional[str]: """A domain to grant access to.""" - return self._properties.get("domain") + return typing.cast(Optional[str], self._properties.get("domain")) @domain.setter def domain(self, value): @@ -412,7 +417,7 @@ def domain(self, value): @property def special_group(self) -> Optional[str]: """A special group to grant access to.""" - return self._properties.get("specialGroup") + return typing.cast(Optional[str], self._properties.get("specialGroup")) @special_group.setter def special_group(self, value): @@ -424,9 +429,9 @@ def entity_type(self) -> Optional[str]: return self._entity_type @property - def entity_id(self): + def entity_id(self) -> Optional[Union[Dict[str, Any], str]]: """The entity_id of the entry.""" - return self._properties.get(self.entity_type) if self.entity_type else None + return self._properties.get(self._entity_type) if self._entity_type else None def __eq__(self, other): if not isinstance(other, AccessEntry): @@ -439,9 +444,12 @@ def __ne__(self, other): def __repr__(self): return "" % ( + str, self.role, + str, self._entity_type, - str(self.entity_id), + str, + self.entity_id, ) def _key(self): From e0e68a4f0acf9929ce3e7c11d831c736ea34f3fc Mon Sep 17 00:00:00 2001 From: steffnay Date: Wed, 23 Feb 2022 11:06:17 -0800 Subject: [PATCH 24/34] refactor AccessEntry repr --- google/cloud/bigquery/dataset.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index a667a6323..f34e86074 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -443,14 +443,7 @@ def __ne__(self, other): def __repr__(self): - return "" % ( - str, - self.role, - str, - self._entity_type, - str, - self.entity_id, - ) + return f"" def _key(self): """ A tuple key that uniquely describes this field. From a33f86a630b8f187317433625e63ef184f2e1e67 Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 7 Mar 2022 15:02:04 -0800 Subject: [PATCH 25/34] update to return DatasetReference --- google/cloud/bigquery/dataset.py | 27 ++++++++++++--------- tests/unit/test_dataset.py | 41 ++++++++++++++++---------------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index f34e86074..2237937bb 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -230,16 +230,16 @@ class AccessEntry(object): See https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets. Args: - role (str): + role: Role granted to the entity. The following string values are supported: `'READER'`, `'WRITER'`, `'OWNER'`. It may also be :data:`None` if the ``entity_type`` is ``view``, ``routine``, or ``dataset``. - entity_type (str): + entity_type: Type of entity being granted the role. See :class:`google.cloud.bigquery.enums.EntityTypes` for supported types. - entity_id (Union[str, Dict[str, str]]): + entity_id: If the ``entity_type`` is not 'view', 'routine', or 'dataset', the ``entity_id`` is the ``str`` ID of the entity being granted the role. If the ``entity_type`` is 'view' or 'routine', the ``entity_id`` is a ``dict`` @@ -310,9 +310,12 @@ def role(self, value): self._properties["role"] = value @property - def dataset(self) -> Optional[Dict[str, Any]]: + def dataset(self) -> Optional[DatasetReference]: """API resource representation of a dataset reference.""" - return typing.cast(Optional[Dict[str, Any]], self._properties.get("dataset")) + return typing.cast( + Optional[DatasetReference], + _helpers._get_sub_prop(self._properties, ["dataset", "dataset"]), + ) @dataset.setter def dataset(self, value): @@ -320,14 +323,16 @@ def dataset(self, value): raise ValueError( "Role must be None for a dataset. Current " "role: %r" % (self.role) ) - if not isinstance(value, dict): - if isinstance(value, str): - value = DatasetReference.from_string(value) + if isinstance(value, dict): + value = DatasetReference.from_api_repr(value) - if isinstance(value, (Dataset, DatasetListItem)): - value = value.reference + if isinstance(value, str): + value = DatasetReference.from_string(value) - value = value.to_api_repr() + if isinstance(value, (Dataset, DatasetListItem)): + value = value.reference + + # value = value.to_api_repr() _helpers._set_sub_prop(self._properties, ["dataset", "dataset"], value) _helpers._set_sub_prop( diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 74a28ae23..d3ef40313 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -219,17 +219,19 @@ def test_view_getter_setter_incorrect_role(self): entry.view = view def test_dataset_getter_setter(self): - dataset = {"dataset": {"projectId": "my-project", "datasetId": "my_dataset"}} + dataset = {"projectId": "my-project", "datasetId": "my_dataset"} entry = self._make_one(None) entry.dataset = dataset + dataset_ref = DatasetReference.from_api_repr(dataset) resource = entry.to_api_repr() exp_resource = { - "dataset": {"dataset": dataset, "targetTypes": None}, + "dataset": {"dataset": dataset_ref, "targetTypes": None,}, "role": None, } prop = entry.dataset + print("prop", prop) self.assertEqual(resource, exp_resource) - self.assertEqual(prop, exp_resource["dataset"]) + self.assertEqual(prop, dataset_ref) def test_dataset_getter_setter_none(self): entry = self._make_one(None) @@ -239,13 +241,12 @@ def test_dataset_getter_setter_string(self): project = "my-project" dataset_id = "my_dataset" entry = self._make_one(None) - entry.dataset = f"{project}.{dataset_id}" + string_ref = f"{project}.{dataset_id}" + entry.dataset = string_ref + dataset_ref = DatasetReference.from_string(string_ref) resource = entry.to_api_repr() exp_resource = { - "dataset": { - "dataset": {"projectId": project, "datasetId": dataset_id}, - "targetTypes": None, - }, + "dataset": {"dataset": dataset_ref, "targetTypes": None,}, "role": None, } self.assertEqual(resource, exp_resource) @@ -253,14 +254,12 @@ def test_dataset_getter_setter_string(self): def test_dataset_getter_setter_dataset_ref(self): project = "my-project" dataset_id = "my_dataset" + dataset_ref = DatasetReference(project, dataset_id) entry = self._make_one(None) - entry.dataset = DatasetReference(project, dataset_id) + entry.dataset = dataset_ref resource = entry.to_api_repr() exp_resource = { - "dataset": { - "dataset": {"projectId": project, "datasetId": dataset_id}, - "targetTypes": None, - }, + "dataset": {"dataset": dataset_ref, "targetTypes": None,}, "role": None, } self.assertEqual(resource, exp_resource) @@ -268,16 +267,16 @@ def test_dataset_getter_setter_dataset_ref(self): def test_dataset_getter_setter_dataset(self): project = "my-project" dataset_id = "my_dataset" + dataset_ref = DatasetReference(project, dataset_id) + dataset = Dataset(dataset_ref) entry = self._make_one(None) - entry.dataset = Dataset(DatasetReference(project, dataset_id)) + entry.dataset = dataset resource = entry.to_api_repr() exp_resource = { - "dataset": { - "dataset": {"projectId": project, "datasetId": dataset_id}, - "targetTypes": None, - }, "role": None, + "dataset": {"dataset": dataset_ref, "targetTypes": None}, } + print(resource) self.assertEqual(resource, exp_resource) def test_dataset_getter_setter_incorrect_role(self): @@ -429,20 +428,20 @@ def test_target_types_getter_setter(self): entry = self._make_one(None) entry.target_types = target_types self.assertEqual(entry.target_types, target_types) - self.assertEqual(entry.dataset["targetTypes"], target_types) + # self.assertEqual(entry.dataset["targetTypes"], target_types) def test_target_types_getter_setter_none(self): entry = self._make_one(None) self.assertEqual(entry.target_types, None) def test_target_types_getter_setter_w_dataset(self): - dataset = {"dataset": {"projectId": "my-project", "datasetId": "my_dataset"}} + dataset = {"projectId": "my-project", "datasetId": "my_dataset"} target_types = ["VIEWS"] entry = self._make_one(None) entry.dataset = dataset entry.target_types = target_types self.assertEqual(entry.target_types, target_types) - self.assertEqual(entry.dataset["targetTypes"], target_types) + # self.assertEqual(entry.dataset["targetTypes"], target_types) class TestDatasetReference(unittest.TestCase): From ded07b22287980d609ba3f42d7a6ab4ac10e15d1 Mon Sep 17 00:00:00 2001 From: steffnay Date: Wed, 9 Mar 2022 16:41:06 -0800 Subject: [PATCH 26/34] update to use RoutineRef and TableRef --- google/cloud/bigquery/dataset.py | 44 ++++++++++++--------- tests/unit/test_dataset.py | 67 ++++++++++++++++---------------- 2 files changed, 59 insertions(+), 52 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 2237937bb..109efa6cf 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -25,7 +25,7 @@ from google.cloud.bigquery import _helpers from google.cloud.bigquery.model import ModelReference from google.cloud.bigquery.routine import Routine, RoutineReference -from google.cloud.bigquery.table import TableReference, _table_arg_to_table_ref +from google.cloud.bigquery.table import Table, TableReference from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration from typing import Optional, List, Dict, Any, Union @@ -332,8 +332,6 @@ def dataset(self, value): if isinstance(value, (Dataset, DatasetListItem)): value = value.reference - # value = value.to_api_repr() - _helpers._set_sub_prop(self._properties, ["dataset", "dataset"], value) _helpers._set_sub_prop( self._properties, @@ -342,23 +340,23 @@ def dataset(self, value): ) @property - def target_types(self) -> Optional[List[str]]: + def dataset_target_types(self) -> Optional[List[str]]: """Which resources that the dataset in this entry applies to.""" return typing.cast( Optional[List[str]], _helpers._get_sub_prop(self._properties, ["dataset", "targetTypes"]), ) - @target_types.setter - def target_types(self, value): + @dataset_target_types.setter + def dataset_target_types(self, value): if "dataset" not in self._properties: self._properties["dataset"] = {} _helpers._set_sub_prop(self._properties, ["dataset", "targetTypes"], value) @property - def routine(self) -> Optional[Dict[str, Any]]: + def routine(self) -> Optional[RoutineReference]: """API resource representation of a routine reference.""" - return typing.cast(Optional[Dict[str, Any]], self._properties.get("routine")) + return typing.cast(Optional[RoutineReference], self._properties.get("routine")) @routine.setter def routine(self, value): @@ -366,20 +364,22 @@ def routine(self, value): raise ValueError( "Role must be None for a routine. Current " "role: %r" % (self.role) ) - if not isinstance(value, Dict): - if isinstance(value, str): - value = RoutineReference.from_string(value) - if isinstance(value, Routine): - value = value.reference + if isinstance(value, Dict): + value = RoutineReference.from_api_repr(value) + + if isinstance(value, str): + value = RoutineReference.from_string(value) + + if isinstance(value, Routine): + value = value.reference - value = value.to_api_repr() self._properties["routine"] = value @property - def view(self) -> Optional[Dict[str, Any]]: + def view(self) -> Optional[TableReference]: """API resource representation of a view reference.""" - return typing.cast(Optional[Dict[str, Any]], self._properties.get("view")) + return typing.cast(Optional[TableReference], self._properties.get("view")) @view.setter def view(self, value): @@ -387,9 +387,15 @@ def view(self, value): raise ValueError( "Role must be None for a view. Current " "role: %r" % (self.role) ) - if not isinstance(value, dict): - value = _table_arg_to_table_ref(value) - value = value.to_api_repr() + if isinstance(value, dict): + value = TableReference.from_api_repr(value) + + if isinstance(value, str): + value = TableReference.from_string(value) + + if isinstance(value, Table): + value = value.reference + self._properties["view"] = value @property diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index d3ef40313..5f898fe1f 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -119,6 +119,7 @@ def test_to_api_repr_routine(self): "datasetId": "my_dataset", "routineId": "my_routine", } + entry = self._make_one(None, "routine", routine) resource = entry.to_api_repr() exp_resource = {"routine": routine, "role": None} @@ -175,11 +176,12 @@ def test_view_getter_setter(self): "datasetId": "my_dataset", "tableId": "my_table", } + view_ref = TableReference.from_api_repr(view) entry = self._make_one(None) entry.view = view resource = entry.to_api_repr() - exp_resource = {"view": view, "role": None} - self.assertEqual(entry.view, view) + exp_resource = {"view": view_ref, "role": None} + self.assertEqual(entry.view, view_ref) self.assertEqual(resource, exp_resource) def test_view_getter_setter_none(self): @@ -190,22 +192,22 @@ def test_view_getter_setter_string(self): project = "my_project" dataset = "my_dataset" table = "my_table" + view_ref = TableReference.from_string(f"{project}.{dataset}.{table}") entry = self._make_one(None) entry.view = f"{project}.{dataset}.{table}" resource = entry.to_api_repr() - exp_view = {"projectId": project, "datasetId": dataset, "tableId": table} - exp_resource = {"view": exp_view, "role": None} + exp_resource = {"view": view_ref, "role": None} self.assertEqual(resource, exp_resource) def test_view_getter_setter_table_ref(self): project = "my_project" dataset = "my_dataset" table = "my_table" + view_ref = TableReference.from_string(f"{project}.{dataset}.{table}") entry = self._make_one(None) - entry.view = TableReference(DatasetReference(project, dataset), table) + entry.view = view_ref resource = entry.to_api_repr() - exp_view = {"projectId": project, "datasetId": dataset, "tableId": table} - exp_resource = {"view": exp_view, "role": None} + exp_resource = {"view": view_ref, "role": None} self.assertEqual(resource, exp_resource) def test_view_getter_setter_incorrect_role(self): @@ -214,9 +216,10 @@ def test_view_getter_setter_incorrect_role(self): "datasetId": "my_dataset", "tableId": "my_table", } + view_ref = TableReference.from_api_repr(view) entry = self._make_one("READER") with self.assertRaises(ValueError): - entry.view = view + entry.view = view_ref def test_dataset_getter_setter(self): dataset = {"projectId": "my-project", "datasetId": "my_dataset"} @@ -225,11 +228,10 @@ def test_dataset_getter_setter(self): dataset_ref = DatasetReference.from_api_repr(dataset) resource = entry.to_api_repr() exp_resource = { - "dataset": {"dataset": dataset_ref, "targetTypes": None,}, + "dataset": {"dataset": dataset_ref, "targetTypes": None}, "role": None, } prop = entry.dataset - print("prop", prop) self.assertEqual(resource, exp_resource) self.assertEqual(prop, dataset_ref) @@ -246,7 +248,7 @@ def test_dataset_getter_setter_string(self): dataset_ref = DatasetReference.from_string(string_ref) resource = entry.to_api_repr() exp_resource = { - "dataset": {"dataset": dataset_ref, "targetTypes": None,}, + "dataset": {"dataset": dataset_ref, "targetTypes": None}, "role": None, } self.assertEqual(resource, exp_resource) @@ -259,7 +261,7 @@ def test_dataset_getter_setter_dataset_ref(self): entry.dataset = dataset_ref resource = entry.to_api_repr() exp_resource = { - "dataset": {"dataset": dataset_ref, "targetTypes": None,}, + "dataset": {"dataset": dataset_ref, "targetTypes": None}, "role": None, } self.assertEqual(resource, exp_resource) @@ -276,7 +278,6 @@ def test_dataset_getter_setter_dataset(self): "role": None, "dataset": {"dataset": dataset_ref, "targetTypes": None}, } - print(resource) self.assertEqual(resource, exp_resource) def test_dataset_getter_setter_incorrect_role(self): @@ -291,10 +292,11 @@ def test_routine_getter_setter(self): "datasetId": "my_dataset", "routineId": "my_routine", } + routine_ref = RoutineReference.from_api_repr(routine) entry = self._make_one(None) entry.routine = routine resource = entry.to_api_repr() - exp_resource = {"routine": routine, "role": None} + exp_resource = {"routine": routine_ref, "role": None} self.assertEqual(resource, exp_resource) def test_routine_getter_setter_none(self): @@ -307,13 +309,12 @@ def test_routine_getter_setter_string(self): routine_id = "my_routine" entry = self._make_one(None) entry.routine = f"{project}.{dataset_id}.{routine_id}" + routine_ref = RoutineReference.from_string( + f"{project}.{dataset_id}.{routine_id}" + ) resource = entry.to_api_repr() exp_resource = { - "routine": { - "projectId": project, - "datasetId": dataset_id, - "routineId": routine_id, - }, + "routine": routine_ref, "role": None, } self.assertEqual(resource, exp_resource) @@ -324,11 +325,12 @@ def test_routine_getter_setter_routine_ref(self): "datasetId": "my_dataset", "routineId": "my_routine", } + routine_ref = RoutineReference.from_api_repr(routine) entry = self._make_one(None) entry.routine = RoutineReference.from_api_repr(routine) resource = entry.to_api_repr() exp_resource = { - "routine": routine, + "routine": routine_ref, "role": None, } self.assertEqual(resource, exp_resource) @@ -339,14 +341,15 @@ def test_routine_getter_setter_routine(self): "datasetId": "my_dataset", "routineId": "my_routine", } + routine_ref = RoutineReference.from_api_repr(routine) entry = self._make_one(None) - entry.routine = Routine(RoutineReference.from_api_repr(routine)) + entry.routine = Routine(routine_ref) resource = entry.to_api_repr() exp_resource = { - "routine": routine, + "routine": routine_ref, "role": None, } - self.assertEqual(entry.routine, routine) + self.assertEqual(entry.routine, routine_ref) self.assertEqual(resource, exp_resource) def test_routine_getter_setter_incorrect_role(self): @@ -423,25 +426,23 @@ def test_role_getter_setter_none(self): entry = self._make_one(None) self.assertEqual(entry.role, None) - def test_target_types_getter_setter(self): + def test_dataset_target_types_getter_setter(self): target_types = ["VIEWS"] entry = self._make_one(None) - entry.target_types = target_types - self.assertEqual(entry.target_types, target_types) - # self.assertEqual(entry.dataset["targetTypes"], target_types) + entry.dataset_target_types = target_types + self.assertEqual(entry.dataset_target_types, target_types) - def test_target_types_getter_setter_none(self): + def test_dataset_target_types_getter_setter_none(self): entry = self._make_one(None) - self.assertEqual(entry.target_types, None) + self.assertEqual(entry.dataset_target_types, None) - def test_target_types_getter_setter_w_dataset(self): + def test_dataset_target_types_getter_setter_w_dataset(self): dataset = {"projectId": "my-project", "datasetId": "my_dataset"} target_types = ["VIEWS"] entry = self._make_one(None) entry.dataset = dataset - entry.target_types = target_types - self.assertEqual(entry.target_types, target_types) - # self.assertEqual(entry.dataset["targetTypes"], target_types) + entry.dataset_target_types = target_types + self.assertEqual(entry.dataset_target_types, target_types) class TestDatasetReference(unittest.TestCase): From 25365977b54786361a2bd285dc3b76ce9bd46ac7 Mon Sep 17 00:00:00 2001 From: steffnay Date: Fri, 18 Mar 2022 12:48:08 -0700 Subject: [PATCH 27/34] add table test --- tests/unit/test_dataset.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 5f898fe1f..67035f3d2 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -21,6 +21,7 @@ AccessEntry, Dataset, DatasetReference, + Table, TableReference, ) @@ -199,6 +200,17 @@ def test_view_getter_setter_string(self): exp_resource = {"view": view_ref, "role": None} self.assertEqual(resource, exp_resource) + def test_view_getter_setter_table(self): + project = "my_project" + dataset = "my_dataset" + table = "my_table" + view_ref = Table.from_string(f"{project}.{dataset}.{table}") + entry = self._make_one(None) + entry.view = view_ref + resource = entry.to_api_repr() + exp_resource = {"view": view_ref, "role": None} + self.assertEqual(resource, exp_resource) + def test_view_getter_setter_table_ref(self): project = "my_project" dataset = "my_dataset" From 01f3734e5d0efa875d84a85b032dc395948b8546 Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 28 Mar 2022 18:44:42 -0700 Subject: [PATCH 28/34] update to use api_repr --- google/cloud/bigquery/dataset.py | 33 ++++++++------------ tests/unit/test_dataset.py | 52 +++++++++++++++++++++----------- 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 109efa6cf..6a840b2ce 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -312,10 +312,8 @@ def role(self, value): @property def dataset(self) -> Optional[DatasetReference]: """API resource representation of a dataset reference.""" - return typing.cast( - Optional[DatasetReference], - _helpers._get_sub_prop(self._properties, ["dataset", "dataset"]), - ) + value = _helpers._get_sub_prop(self._properties, ["dataset", "dataset"]) + return DatasetReference.from_api_repr(value) if value else None @dataset.setter def dataset(self, value): @@ -323,14 +321,12 @@ def dataset(self, value): raise ValueError( "Role must be None for a dataset. Current " "role: %r" % (self.role) ) - if isinstance(value, dict): - value = DatasetReference.from_api_repr(value) if isinstance(value, str): - value = DatasetReference.from_string(value) + value = DatasetReference.from_string(value).to_api_repr() if isinstance(value, (Dataset, DatasetListItem)): - value = value.reference + value = value.reference.to_api_repr() _helpers._set_sub_prop(self._properties, ["dataset", "dataset"], value) _helpers._set_sub_prop( @@ -356,7 +352,8 @@ def dataset_target_types(self, value): @property def routine(self) -> Optional[RoutineReference]: """API resource representation of a routine reference.""" - return typing.cast(Optional[RoutineReference], self._properties.get("routine")) + value = typing.cast(Optional[Dict], self._properties.get("routine")) + return RoutineReference.from_api_repr(value) if value else None @routine.setter def routine(self, value): @@ -365,21 +362,19 @@ def routine(self, value): "Role must be None for a routine. Current " "role: %r" % (self.role) ) - if isinstance(value, Dict): - value = RoutineReference.from_api_repr(value) - if isinstance(value, str): - value = RoutineReference.from_string(value) + value = RoutineReference.from_string(value).to_api_repr() if isinstance(value, Routine): - value = value.reference + value = value.reference.to_api_repr() self._properties["routine"] = value @property def view(self) -> Optional[TableReference]: """API resource representation of a view reference.""" - return typing.cast(Optional[TableReference], self._properties.get("view")) + value = typing.cast(Optional[Dict],self._properties.get("view")) + return TableReference.from_api_repr(value) if value else None @view.setter def view(self, value): @@ -387,14 +382,12 @@ def view(self, value): raise ValueError( "Role must be None for a view. Current " "role: %r" % (self.role) ) - if isinstance(value, dict): - value = TableReference.from_api_repr(value) if isinstance(value, str): - value = TableReference.from_string(value) + value = TableReference.from_string(value).to_api_repr() if isinstance(value, Table): - value = value.reference + value = value.reference.to_api_repr() self._properties["view"] = value @@ -457,7 +450,7 @@ def __repr__(self): return f"" def _key(self): - """ A tuple key that uniquely describes this field. + """A tuple key that uniquely describes this field. Used to compute this instance's hashcode and evaluate equality. Returns: Tuple: The contents of this :class:`~google.cloud.bigquery.dataset.AccessEntry`. diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 67035f3d2..e758879e6 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -181,7 +181,7 @@ def test_view_getter_setter(self): entry = self._make_one(None) entry.view = view resource = entry.to_api_repr() - exp_resource = {"view": view_ref, "role": None} + exp_resource = {"view": view, "role": None} self.assertEqual(entry.view, view_ref) self.assertEqual(resource, exp_resource) @@ -193,22 +193,31 @@ def test_view_getter_setter_string(self): project = "my_project" dataset = "my_dataset" table = "my_table" - view_ref = TableReference.from_string(f"{project}.{dataset}.{table}") + view = { + "projectId": project, + "datasetId": dataset, + "tableId": table, + } entry = self._make_one(None) entry.view = f"{project}.{dataset}.{table}" resource = entry.to_api_repr() - exp_resource = {"view": view_ref, "role": None} + exp_resource = {"view": view, "role": None} self.assertEqual(resource, exp_resource) def test_view_getter_setter_table(self): project = "my_project" dataset = "my_dataset" table = "my_table" + view = { + "projectId": project, + "datasetId": dataset, + "tableId": table, + } view_ref = Table.from_string(f"{project}.{dataset}.{table}") entry = self._make_one(None) entry.view = view_ref resource = entry.to_api_repr() - exp_resource = {"view": view_ref, "role": None} + exp_resource = {"view": view, "role": None} self.assertEqual(resource, exp_resource) def test_view_getter_setter_table_ref(self): @@ -237,12 +246,12 @@ def test_dataset_getter_setter(self): dataset = {"projectId": "my-project", "datasetId": "my_dataset"} entry = self._make_one(None) entry.dataset = dataset - dataset_ref = DatasetReference.from_api_repr(dataset) resource = entry.to_api_repr() exp_resource = { - "dataset": {"dataset": dataset_ref, "targetTypes": None}, + "dataset": {"dataset": dataset, "targetTypes": None}, "role": None, } + dataset_ref = DatasetReference.from_api_repr(dataset) prop = entry.dataset self.assertEqual(resource, exp_resource) self.assertEqual(prop, dataset_ref) @@ -254,13 +263,16 @@ def test_dataset_getter_setter_none(self): def test_dataset_getter_setter_string(self): project = "my-project" dataset_id = "my_dataset" + dataset = { + "projectId": project, + "datasetId": dataset_id, + } entry = self._make_one(None) string_ref = f"{project}.{dataset_id}" entry.dataset = string_ref - dataset_ref = DatasetReference.from_string(string_ref) resource = entry.to_api_repr() exp_resource = { - "dataset": {"dataset": dataset_ref, "targetTypes": None}, + "dataset": {"dataset": dataset, "targetTypes": None}, "role": None, } self.assertEqual(resource, exp_resource) @@ -281,14 +293,17 @@ def test_dataset_getter_setter_dataset_ref(self): def test_dataset_getter_setter_dataset(self): project = "my-project" dataset_id = "my_dataset" - dataset_ref = DatasetReference(project, dataset_id) - dataset = Dataset(dataset_ref) + dataset_repr = { + "projectId": project, + "datasetId": dataset_id, + } + dataset = Dataset(f"{project}.{dataset_id}") entry = self._make_one(None) entry.dataset = dataset resource = entry.to_api_repr() exp_resource = { "role": None, - "dataset": {"dataset": dataset_ref, "targetTypes": None}, + "dataset": {"dataset": dataset_repr, "targetTypes": None}, } self.assertEqual(resource, exp_resource) @@ -304,11 +319,10 @@ def test_routine_getter_setter(self): "datasetId": "my_dataset", "routineId": "my_routine", } - routine_ref = RoutineReference.from_api_repr(routine) entry = self._make_one(None) entry.routine = routine resource = entry.to_api_repr() - exp_resource = {"routine": routine_ref, "role": None} + exp_resource = {"routine": routine, "role": None} self.assertEqual(resource, exp_resource) def test_routine_getter_setter_none(self): @@ -319,14 +333,16 @@ def test_routine_getter_setter_string(self): project = "my-project" dataset_id = "my_dataset" routine_id = "my_routine" + routine = { + "projectId": project, + "datasetId": dataset_id, + "routineId": routine_id, + } entry = self._make_one(None) entry.routine = f"{project}.{dataset_id}.{routine_id}" - routine_ref = RoutineReference.from_string( - f"{project}.{dataset_id}.{routine_id}" - ) resource = entry.to_api_repr() exp_resource = { - "routine": routine_ref, + "routine": routine, "role": None, } self.assertEqual(resource, exp_resource) @@ -358,7 +374,7 @@ def test_routine_getter_setter_routine(self): entry.routine = Routine(routine_ref) resource = entry.to_api_repr() exp_resource = { - "routine": routine_ref, + "routine": routine, "role": None, } self.assertEqual(entry.routine, routine_ref) From 9c367c9e86493c035a3abb6a358494c35472d5f7 Mon Sep 17 00:00:00 2001 From: steffnay Date: Tue, 29 Mar 2022 09:55:44 -0700 Subject: [PATCH 29/34] lint --- google/cloud/bigquery/dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index c70fdf8ac..a13554fa7 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -77,7 +77,7 @@ def _get_routine_reference(self, routine_id): } ) - + class DatasetReference(object): """DatasetReferences are pointers to datasets. @@ -373,7 +373,7 @@ def routine(self, value): @property def view(self) -> Optional[TableReference]: """API resource representation of a view reference.""" - value = typing.cast(Optional[Dict],self._properties.get("view")) + value = typing.cast(Optional[Dict], self._properties.get("view")) return TableReference.from_api_repr(value) if value else None @view.setter From d9b52036e2f046c1c37d52caef234e2830c9b071 Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Wed, 6 Apr 2022 21:52:09 -0700 Subject: [PATCH 30/34] Update google/cloud/bigquery/dataset.py Co-authored-by: Tim Swast --- google/cloud/bigquery/dataset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 46d735c3c..37d50bd96 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -387,6 +387,9 @@ def view(self, value): if isinstance(value, str): value = TableReference.from_string(value).to_api_repr() + if isinstance(value, TableReference): + value = value.to_api_repr() + if isinstance(value, Table): value = value.reference.to_api_repr() From 0e3b3915a0d9fef66c48ce18763fe88250e5c286 Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Wed, 6 Apr 2022 21:52:15 -0700 Subject: [PATCH 31/34] Update google/cloud/bigquery/dataset.py Co-authored-by: Tim Swast --- google/cloud/bigquery/dataset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 37d50bd96..71e5a4f62 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -366,6 +366,9 @@ def routine(self, value): if isinstance(value, str): value = RoutineReference.from_string(value).to_api_repr() + if isinstance(value, RoutineReference): + value = value.to_api_repr() + if isinstance(value, Routine): value = value.reference.to_api_repr() From 2fa34aec9438faf9144eabaf8b6080cc290eddca Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Wed, 6 Apr 2022 21:52:23 -0700 Subject: [PATCH 32/34] Update google/cloud/bigquery/dataset.py Co-authored-by: Tim Swast --- google/cloud/bigquery/dataset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 71e5a4f62..f8ccdd8b5 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -346,8 +346,7 @@ def dataset_target_types(self) -> Optional[List[str]]: @dataset_target_types.setter def dataset_target_types(self, value): - if "dataset" not in self._properties: - self._properties["dataset"] = {} + self._properties.setdefault("dataset", {}) _helpers._set_sub_prop(self._properties, ["dataset", "targetTypes"], value) @property From 84a4f79a57cde435afc6cc735938a6158b726bcb Mon Sep 17 00:00:00 2001 From: steffnay Date: Fri, 8 Apr 2022 11:35:43 -0700 Subject: [PATCH 33/34] update tests --- tests/unit/test_dataset.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index e758879e6..856674daf 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -224,11 +224,16 @@ def test_view_getter_setter_table_ref(self): project = "my_project" dataset = "my_dataset" table = "my_table" + view = { + "projectId": project, + "datasetId": dataset, + "tableId": table, + } view_ref = TableReference.from_string(f"{project}.{dataset}.{table}") entry = self._make_one(None) entry.view = view_ref resource = entry.to_api_repr() - exp_resource = {"view": view_ref, "role": None} + exp_resource = {"view": view, "role": None} self.assertEqual(resource, exp_resource) def test_view_getter_setter_incorrect_role(self): @@ -353,12 +358,11 @@ def test_routine_getter_setter_routine_ref(self): "datasetId": "my_dataset", "routineId": "my_routine", } - routine_ref = RoutineReference.from_api_repr(routine) entry = self._make_one(None) entry.routine = RoutineReference.from_api_repr(routine) resource = entry.to_api_repr() exp_resource = { - "routine": routine_ref, + "routine": routine, "role": None, } self.assertEqual(resource, exp_resource) From 9885354048c34ae0bb0768a66f985e9075be84eb Mon Sep 17 00:00:00 2001 From: steffnay Date: Fri, 8 Apr 2022 12:06:51 -0700 Subject: [PATCH 34/34] remove repeat type import --- google/cloud/bigquery/dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index f8ccdd8b5..c30204067 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -17,7 +17,6 @@ from __future__ import absolute_import import copy -from typing import Dict, Any import typing