From 0e5261c42104c1e34539752e11d7e37e4f42f776 Mon Sep 17 00:00:00 2001 From: steffnay Date: Tue, 30 Nov 2021 09:17:12 -0800 Subject: [PATCH 1/7] feat: support authorized dataset entity --- google/cloud/bigquery/dataset.py | 66 +++++++++++++++----------------- google/cloud/bigquery/enums.py | 13 +++++++ tests/unit/test_dataset.py | 23 +++++++++-- 3 files changed, 63 insertions(+), 39 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index ff015d605..19209a26c 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -77,10 +77,10 @@ def _get_routine_reference(self, routine_id): class AccessEntry(object): """Represents grant of an access role to an entity. - An entry must have exactly one of the allowed :attr:`ENTITY_TYPES`. If - anything but ``view`` or ``routine`` are set, a ``role`` is also required. - ``role`` is omitted for ``view`` and ``routine``, because they are always - read-only. + An entry must have exactly one of the allowed + :class:`google.cloud.bigquery.enums.EntityTypes`. If anything but ``view``, ``routine``, + or ``dataset`` are set, a ``role`` is also required. ``role`` is omitted for ``view``, + ``routine``, ``dataset``, because they are always read-only. See https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets. @@ -88,17 +88,18 @@ class AccessEntry(object): role (str): Role granted to the entity. The following string values are supported: `'READER'`, `'WRITER'`, `'OWNER'`. It may also be - :data:`None` if the ``entity_type`` is ``view`` or ``routine``. + :data:`None` if the ``entity_type`` is ``view``, ``routine``, or ``dataset``. entity_type (str): - Type of entity being granted the role. One of :attr:`ENTITY_TYPES`. + Type of entity being granted the role. See + :class:`google.cloud.bigquery.enums.EntityTypes` for supported types. entity_id (Union[str, Dict[str, str]]): - If the ``entity_type`` is not 'view' or 'routine', the ``entity_id`` - is the ``str`` ID of the entity being granted the role. If the - ``entity_type`` is 'view' or 'routine', the ``entity_id`` is a ``dict`` - representing the view or routine from a different dataset to grant - access to in the following format for views:: + If the ``entity_type`` is not 'view', 'routine', or 'dataset', the + ``entity_id`` is the ``str`` ID of the entity being granted the role. If + the ``entity_type`` is 'view' or 'routine', the ``entity_id`` is a ``dict`` + representing the view or routine from a different dataset to grant access + to in the following format for views:: { 'projectId': string, @@ -114,11 +115,23 @@ class AccessEntry(object): 'routineId': string } - Raises: - ValueError: - If the ``entity_type`` is not among :attr:`ENTITY_TYPES`, or if a - ``view`` or a ``routine`` has ``role`` set, or a non ``view`` and - non ``routine`` **does not** have a ``role`` set. + If the ``entity_type`` is 'dataset', the ``entity_id`` is a ``dict`` that includes + a 'dataset' field with a ``dict`` representing the dataset and a 'target_types' + field with a ``str`` value of the dataset's resource type:: + + { + 'dataset': { + 'projectId': string, + 'datasetId': string, + }, + 'target_types: 'VIEWS' + } + + - Raises: +- ValueError: +- If the ``entity_type`` is not among :attr:`ENTITY_TYPES`, or if a +- ``view`` or a ``routine`` has ``role`` set, or a non ``view`` and +- non ``routine`` **does not** have a ``role`` set. Examples: >>> entry = AccessEntry('OWNER', 'userByEmail', 'user@example.com') @@ -131,27 +144,8 @@ class AccessEntry(object): >>> entry = AccessEntry(None, 'view', view) """ - ENTITY_TYPES = frozenset( - [ - "userByEmail", - "groupByEmail", - "domain", - "specialGroup", - "view", - "iamMember", - "routine", - ] - ) - """Allowed entity types.""" - def __init__(self, role, entity_type, entity_id): - if entity_type not in self.ENTITY_TYPES: - message = "Entity type %r not among: %s" % ( - entity_type, - ", ".join(self.ENTITY_TYPES), - ) - raise ValueError(message) - if entity_type in ("view", "routine"): + if entity_type in ("view", "routine", "dataset"): if role is not None: raise ValueError( "Role must be None for a %r. Received " diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 0eaaffd2e..0a746e4e4 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -232,6 +232,19 @@ def _make_sql_scalars_enum(): StandardSqlDataTypes = _make_sql_scalars_enum() +class EntityTypes: + """Enum of allowed entity type names in AccessEntry""" + + userByEmail = "userByEmail" + GROUP_BY_EMIAL = "groupByEmail" + DOMAIN = "domain" + DATASET = "dataset" + SPECIAL_GROUP = "specialGroup" + VIEW = "view" + IAM_MEMBER = "iamMember" + ROUTINE = "routine" + + # See also: https://cloud.google.com/bigquery/data-types#legacy_sql_data_types # and https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types class SqlTypeNames(str, enum.Enum): diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index b3a53a08d..4ce587854 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -141,6 +141,16 @@ def test_to_api_repr_routine(self): exp_resource = {"routine": routine} self.assertEqual(resource, exp_resource) + def test_to_api_repr_dataset(self): + dataset = { + "dataset": {"projectId": "my-project", "datasetId": "my_dataset"}, + "target_types": "VIEWS", + } + entry = self._make_one(None, "dataset", dataset) + resource = entry.to_api_repr() + exp_resource = {"dataset": dataset} + self.assertEqual(resource, exp_resource) + def test_from_api_repr(self): resource = {"role": "OWNER", "userByEmail": "salmon@example.com"} entry = self._get_target_class().from_api_repr(resource) @@ -148,10 +158,17 @@ def test_from_api_repr(self): self.assertEqual(entry.entity_type, "userByEmail") self.assertEqual(entry.entity_id, "salmon@example.com") - def test_from_api_repr_w_unknown_entity_type(self): - resource = {"role": "READER", "unknown": "UNKNOWN"} + def test_to_api_value_error(self): + dataset = { + "dataset": { + "projectId": "my-project", + "datasetId": "my_dataset", + "tableId": "my_table", + }, + "target_type": "VIEW", + } with self.assertRaises(ValueError): - self._get_target_class().from_api_repr(resource) + self._make_one("READER", "dataset", dataset) def test_from_api_repr_entries_w_extra_keys(self): resource = { From 261bb2ccd49ef2537f36a9690d4799f2a50dd975 Mon Sep 17 00:00:00 2001 From: steffnay Date: Tue, 30 Nov 2021 10:00:30 -0800 Subject: [PATCH 2/7] cleanup --- google/cloud/bigquery/dataset.py | 9 ++++----- google/cloud/bigquery/enums.py | 4 ++-- tests/unit/test_dataset.py | 16 ++++++++-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 19209a26c..2d6744374 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -127,11 +127,10 @@ class AccessEntry(object): 'target_types: 'VIEWS' } - - Raises: -- ValueError: -- If the ``entity_type`` is not among :attr:`ENTITY_TYPES`, or if a -- ``view`` or a ``routine`` has ``role`` set, or a non ``view`` and -- non ``routine`` **does not** have a ``role`` set. + Raises: + ValueError: + If a ``view``, ``routine``, or ``dataset`` has ``role`` set, or a non ``view``, + non ``routine``, and non ``dataset`` **does not** have a ``role`` set. Examples: >>> entry = AccessEntry('OWNER', 'userByEmail', 'user@example.com') diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 0a746e4e4..dde4253c5 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -235,8 +235,8 @@ def _make_sql_scalars_enum(): class EntityTypes: """Enum of allowed entity type names in AccessEntry""" - userByEmail = "userByEmail" - GROUP_BY_EMIAL = "groupByEmail" + USER_BY_EMAIL = "userByEmail" + GROUP_BY_EMAIL = "groupByEmail" DOMAIN = "domain" DATASET = "dataset" SPECIAL_GROUP = "specialGroup" diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 4ce587854..edcd58bb5 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -151,14 +151,7 @@ def test_to_api_repr_dataset(self): exp_resource = {"dataset": dataset} self.assertEqual(resource, exp_resource) - def test_from_api_repr(self): - resource = {"role": "OWNER", "userByEmail": "salmon@example.com"} - entry = self._get_target_class().from_api_repr(resource) - self.assertEqual(entry.role, "OWNER") - self.assertEqual(entry.entity_type, "userByEmail") - self.assertEqual(entry.entity_id, "salmon@example.com") - - def test_to_api_value_error(self): + def test_to_api_w_incorrect_role(self): dataset = { "dataset": { "projectId": "my-project", @@ -170,6 +163,13 @@ def test_to_api_value_error(self): with self.assertRaises(ValueError): self._make_one("READER", "dataset", dataset) + def test_from_api_repr(self): + resource = {"role": "OWNER", "userByEmail": "salmon@example.com"} + entry = self._get_target_class().from_api_repr(resource) + self.assertEqual(entry.role, "OWNER") + self.assertEqual(entry.entity_type, "userByEmail") + self.assertEqual(entry.entity_id, "salmon@example.com") + def test_from_api_repr_entries_w_extra_keys(self): resource = { "role": "READER", From 4eefa263160ea39818b477cdb1eed24ea63a5630 Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 6 Dec 2021 14:47:55 -0800 Subject: [PATCH 3/7] add test and cache the resource from from_api_repr in a _properties value --- google/cloud/bigquery/dataset.py | 7 +++++-- google/cloud/bigquery/enums.py | 2 +- tests/unit/test_dataset.py | 7 +++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 2d6744374..d974e96a9 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -143,7 +143,8 @@ class AccessEntry(object): >>> entry = AccessEntry(None, 'view', view) """ - def __init__(self, role, entity_type, entity_id): + def __init__(self, role, entity_type=None, entity_id=None): + self._properties = {} if entity_type in ("view", "routine", "dataset"): if role is not None: raise ValueError( @@ -234,7 +235,9 @@ def from_api_repr(cls, resource: dict) -> "AccessEntry": entity_type, entity_id = entry.popitem() if len(entry) != 0: raise ValueError("Entry has unexpected keys remaining.", entry) - return cls(role, entity_type, entity_id) + config = cls(role, entity_type, entity_id) + config._properties = copy.deepcopy(resource) + return config class DatasetReference(object): diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index dde4253c5..7fc0a5fd6 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -232,7 +232,7 @@ def _make_sql_scalars_enum(): StandardSqlDataTypes = _make_sql_scalars_enum() -class EntityTypes: +class EntityTypes(str, enum.Enum): """Enum of allowed entity type names in AccessEntry""" USER_BY_EMAIL = "userByEmail" diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index edcd58bb5..295ff738a 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -179,6 +179,13 @@ def test_from_api_repr_entries_w_extra_keys(self): with self.assertRaises(ValueError): self._get_target_class().from_api_repr(resource) + def test_from_api_repr_w_unknown_entity_type(self): + resource = {"role": "READER", "unknown": "UNKNOWN"} + entry = self._get_target_class().from_api_repr(resource) + self.assertEqual(entry.role, "READER") + self.assertEqual(entry.entity_type, "unknown") + self.assertEqual(entry.entity_id, "UNKNOWN") + class TestDatasetReference(unittest.TestCase): @staticmethod From 49d06d3949c683fbe5ccf3a6dc479b611e4d3aab Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 6 Dec 2021 14:49:38 -0800 Subject: [PATCH 4/7] lint --- tests/unit/test_dataset.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 295ff738a..d5f2e175b 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -170,6 +170,13 @@ def test_from_api_repr(self): self.assertEqual(entry.entity_type, "userByEmail") self.assertEqual(entry.entity_id, "salmon@example.com") + def test_from_api_repr_w_unknown_entity_type(self): + resource = {"role": "READER", "unknown": "UNKNOWN"} + entry = self._get_target_class().from_api_repr(resource) + self.assertEqual(entry.role, "READER") + self.assertEqual(entry.entity_type, "unknown") + self.assertEqual(entry.entity_id, "UNKNOWN") + def test_from_api_repr_entries_w_extra_keys(self): resource = { "role": "READER", @@ -179,13 +186,6 @@ def test_from_api_repr_entries_w_extra_keys(self): with self.assertRaises(ValueError): self._get_target_class().from_api_repr(resource) - def test_from_api_repr_w_unknown_entity_type(self): - resource = {"role": "READER", "unknown": "UNKNOWN"} - entry = self._get_target_class().from_api_repr(resource) - self.assertEqual(entry.role, "READER") - self.assertEqual(entry.entity_type, "unknown") - self.assertEqual(entry.entity_id, "UNKNOWN") - class TestDatasetReference(unittest.TestCase): @staticmethod From 314636d92b8fe1a07d31055ef0686196d9693d29 Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 6 Dec 2021 15:46:14 -0800 Subject: [PATCH 5/7] update samples to use enums --- google/cloud/bigquery/dataset.py | 1 + samples/snippets/authorized_view_tutorial.py | 5 +++-- samples/snippets/update_dataset_access.py | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index d974e96a9..0a5ded285 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -235,6 +235,7 @@ def from_api_repr(cls, resource: dict) -> "AccessEntry": entity_type, entity_id = entry.popitem() if len(entry) != 0: raise ValueError("Entry has unexpected keys remaining.", entry) + config = cls(role, entity_type, entity_id) config._properties = copy.deepcopy(resource) return config diff --git a/samples/snippets/authorized_view_tutorial.py b/samples/snippets/authorized_view_tutorial.py index b6a20c6ec..66810c036 100644 --- a/samples/snippets/authorized_view_tutorial.py +++ b/samples/snippets/authorized_view_tutorial.py @@ -24,6 +24,7 @@ def run_authorized_view_tutorial(override_values={}): # Create a source dataset # [START bigquery_avt_create_source_dataset] from google.cloud import bigquery + from google.cloud.bigquery.enums import EntityTypes client = bigquery.Client() source_dataset_id = "github_source_data" @@ -106,7 +107,7 @@ def run_authorized_view_tutorial(override_values={}): # analyst_group_email = 'data_analysts@example.com' access_entries = shared_dataset.access_entries access_entries.append( - bigquery.AccessEntry("READER", "groupByEmail", analyst_group_email) + bigquery.AccessEntry("READER", EntityTypes.GROUP_BY_EMAIL, analyst_group_email) ) shared_dataset.access_entries = access_entries shared_dataset = client.update_dataset( @@ -118,7 +119,7 @@ def run_authorized_view_tutorial(override_values={}): # [START bigquery_avt_source_dataset_access] access_entries = source_dataset.access_entries access_entries.append( - bigquery.AccessEntry(None, "view", view.reference.to_api_repr()) + bigquery.AccessEntry(None, EntityTypes.VIEW, view.reference.to_api_repr()) ) source_dataset.access_entries = access_entries source_dataset = client.update_dataset( diff --git a/samples/snippets/update_dataset_access.py b/samples/snippets/update_dataset_access.py index fb3bfa14f..1448213a6 100644 --- a/samples/snippets/update_dataset_access.py +++ b/samples/snippets/update_dataset_access.py @@ -27,6 +27,8 @@ def update_dataset_access(dataset_id: str, entity_id: str): # of the entity, such as a view's table reference. entity_id = "user-or-group-to-add@example.com" + from google.cloud.bigquery.enums import EntityTypes + # TODO(developer): Set entity_type to the type of entity you are granting access to. # Common types include: # @@ -37,7 +39,7 @@ def update_dataset_access(dataset_id: str, entity_id: str): # # For a complete reference, see the REST API reference documentation: # https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#Dataset.FIELDS.access - entity_type = "groupByEmail" + entity_type = EntityTypes.GROUP_BY_EMAIL # TODO(developer): Set role to a one of the "Basic roles for datasets" # described here: From dc08e74e9feb45d638b5547ad758bc675916ed3e Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 13 Dec 2021 14:11:56 -0800 Subject: [PATCH 6/7] update to_api_repr and add tests --- google/cloud/bigquery/dataset.py | 8 +++----- tests/unit/test_dataset.py | 8 ++++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 0a5ded285..e8a93b2fd 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -143,7 +143,7 @@ class AccessEntry(object): >>> entry = AccessEntry(None, 'view', view) """ - def __init__(self, role, entity_type=None, entity_id=None): + def __init__(self, role=None, entity_type=None, entity_id=None): self._properties = {} if entity_type in ("view", "routine", "dataset"): if role is not None: @@ -156,7 +156,6 @@ def __init__(self, role, entity_type=None, entity_id=None): raise ValueError( "Role must be set for entity " "type %r" % (entity_type,) ) - self._role = role self._entity_type = entity_type self._entity_id = entity_id @@ -208,7 +207,8 @@ def to_api_repr(self): Returns: Dict[str, object]: Access entry represented as an API resource """ - resource = {self._entity_type: self._entity_id} + resource = copy.deepcopy(self._properties) + resource[self._entity_type] = self._entity_id if self._role is not None: resource["role"] = self._role return resource @@ -233,8 +233,6 @@ def from_api_repr(cls, resource: dict) -> "AccessEntry": entry = resource.copy() role = entry.pop("role", None) entity_type, entity_id = entry.popitem() - if len(entry) != 0: - raise ValueError("Entry has unexpected keys remaining.", entry) config = cls(role, entity_type, entity_id) config._properties = copy.deepcopy(resource) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index d5f2e175b..3ae83e70d 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -176,6 +176,8 @@ def test_from_api_repr_w_unknown_entity_type(self): self.assertEqual(entry.role, "READER") self.assertEqual(entry.entity_type, "unknown") self.assertEqual(entry.entity_id, "UNKNOWN") + exp_resource = entry.to_api_repr() + self.assertEqual(resource, exp_resource) def test_from_api_repr_entries_w_extra_keys(self): resource = { @@ -183,8 +185,10 @@ def test_from_api_repr_entries_w_extra_keys(self): "specialGroup": "projectReaders", "userByEmail": "salmon@example.com", } - with self.assertRaises(ValueError): - self._get_target_class().from_api_repr(resource) + entry = self._get_target_class().from_api_repr(resource) + self.assertEqual(entry.role, "READER") + self.assertEqual(entry.entity_type, "userByEmail") + self.assertEqual(entry._properties["specialGroup"], "projectReaders") class TestDatasetReference(unittest.TestCase): From 17b991d38dc05594ac41d9a807937dc9bfe673ba Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 13 Dec 2021 14:34:52 -0800 Subject: [PATCH 7/7] refactor --- google/cloud/bigquery/dataset.py | 2 ++ tests/unit/test_dataset.py | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index e8a93b2fd..499072de2 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -233,6 +233,8 @@ def from_api_repr(cls, resource: dict) -> "AccessEntry": entry = resource.copy() role = entry.pop("role", None) entity_type, entity_id = entry.popitem() + if len(entry) != 0: + raise ValueError("Entry has unexpected keys remaining.", entry) config = cls(role, entity_type, entity_id) config._properties = copy.deepcopy(resource) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 3ae83e70d..c554782bf 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -179,16 +179,24 @@ def test_from_api_repr_w_unknown_entity_type(self): exp_resource = entry.to_api_repr() self.assertEqual(resource, exp_resource) + def test_to_api_repr_w_extra_properties(self): + resource = { + "role": "READER", + "userByEmail": "salmon@example.com", + } + entry = self._get_target_class().from_api_repr(resource) + entry._properties["specialGroup"] = resource["specialGroup"] = "projectReaders" + exp_resource = entry.to_api_repr() + self.assertEqual(resource, exp_resource) + def test_from_api_repr_entries_w_extra_keys(self): resource = { "role": "READER", "specialGroup": "projectReaders", "userByEmail": "salmon@example.com", } - entry = self._get_target_class().from_api_repr(resource) - self.assertEqual(entry.role, "READER") - self.assertEqual(entry.entity_type, "userByEmail") - self.assertEqual(entry._properties["specialGroup"], "projectReaders") + with self.assertRaises(ValueError): + self._get_target_class().from_api_repr(resource) class TestDatasetReference(unittest.TestCase):