From b62b999e36ff6b7cfc512966d8b6430e6fd4b680 Mon Sep 17 00:00:00 2001 From: Prajin Date: Thu, 30 Nov 2023 08:33:08 +0530 Subject: [PATCH 1/5] Capturing Any file and anonymous function grants --- .../labs/ucx/hive_metastore/grants.py | 9 +++-- .../integration/workspace_access/test_tacl.py | 40 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/databricks/labs/ucx/hive_metastore/grants.py b/src/databricks/labs/ucx/hive_metastore/grants.py index 32a045fd8e..e2edc29b7f 100644 --- a/src/databricks/labs/ucx/hive_metastore/grants.py +++ b/src/databricks/labs/ucx/hive_metastore/grants.py @@ -140,7 +140,7 @@ def _try_load(self): def _crawl(self) -> list[Grant]: """ - Crawls and lists grants for all databases, tables, and views within hive_metastore. + Crawls and lists grants for all databases, tables, views, any file and anonymous function within hive_metastore. Returns: list[Grant]: A list of Grant objects representing the listed grants. @@ -155,13 +155,15 @@ def _crawl(self) -> list[Grant]: Note: - The method assumes that the `_grants` method fetches grants based on the provided parameters (catalog, - database, table, view). + database, table, view, any file, anonymous function). Returns: list[Grant]: A list of Grant objects representing the grants found in hive_metastore. """ catalog = "hive_metastore" tasks = [partial(self._grants, catalog=catalog)] + tasks.append(partial(self._grants, catalog=catalog, any_file=True)) + tasks.append(partial(self._grants, catalog=catalog, anonymous_function=True)) # scan all databases, even empty ones for row in self._fetch(f"SHOW DATABASES FROM {catalog}"): tasks.append(partial(self._grants, catalog=catalog, database=row.databaseName)) @@ -239,7 +241,8 @@ def _grants( ) try: grants = [] - object_type_normalization = {"SCHEMA": "DATABASE", "CATALOG$": "CATALOG"} + object_type_normalization = {"SCHEMA": "DATABASE", "CATALOG$": "CATALOG", "ANY_FILE": "ANY FILE", + "ANONYMOUS_FUNCTION": "ANONYMOUS FUNCTION"} for row in self._fetch(f"SHOW GRANTS ON {on_type} {key}"): (principal, action_type, object_type, _) = row if object_type in object_type_normalization: diff --git a/tests/integration/workspace_access/test_tacl.py b/tests/integration/workspace_access/test_tacl.py index 520ca19c14..a0c57cf98b 100644 --- a/tests/integration/workspace_access/test_tacl.py +++ b/tests/integration/workspace_access/test_tacl.py @@ -9,6 +9,46 @@ logger = logging.getLogger(__name__) +def test_permission_for_files_anonymous_func(sql_backend, inventory_schema, make_group): + group_a = make_group() + group_b = make_group() + group_c = make_group() + group_d = make_group() + + sql_backend.execute(f"GRANT READ_METADATA ON ANY FILE TO `{group_a.display_name}`") + sql_backend.execute(f"GRANT SELECT ON ANONYMOUS FUNCTION TO `{group_c.display_name}`") + + tables = StaticTablesCrawler(sql_backend, inventory_schema, []) + grants = GrantsCrawler(tables) + + tacl_support = TableAclSupport(grants, sql_backend) + + migration_state = MigrationState( + [ + MigratedGroup.partial_info(group_a, group_b), + MigratedGroup.partial_info(group_c, group_d), + ] + ) + for crawler_task in tacl_support.get_crawler_tasks(): + permission = crawler_task() + apply_task = tacl_support.get_apply_task(permission, migration_state) + if not apply_task: + continue + apply_task() + + any_file_actual = {} + for any_file_grant in grants._grants(any_file=True): + any_file_actual[any_file_grant.principal] = any_file_grant.action_type + + assert group_b.display_name in any_file_actual + + anonymous_function_actual = {} + for ano_func_grant in grants._grants(anonymous_function=True): + anonymous_function_actual[ano_func_grant.principal] = ano_func_grant.action_type + + assert group_d.display_name in anonymous_function_actual + + def test_owner_permissions_for_tables_and_schemas(sql_backend, inventory_schema, make_schema, make_table, make_group): group_a = make_group() group_b = make_group() From c07afa2ba7187928c5c623dcedd106f45669f4f9 Mon Sep 17 00:00:00 2001 From: Prajin Date: Thu, 30 Nov 2023 10:01:02 +0530 Subject: [PATCH 2/5] Capturing Any file and anonymous function grants --- src/databricks/labs/ucx/hive_metastore/grants.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/databricks/labs/ucx/hive_metastore/grants.py b/src/databricks/labs/ucx/hive_metastore/grants.py index e2edc29b7f..5e93a6a2f3 100644 --- a/src/databricks/labs/ucx/hive_metastore/grants.py +++ b/src/databricks/labs/ucx/hive_metastore/grants.py @@ -162,6 +162,7 @@ def _crawl(self) -> list[Grant]: """ catalog = "hive_metastore" tasks = [partial(self._grants, catalog=catalog)] + # Scanning ANY FILE and ANONYMOUS FUNCTION grants tasks.append(partial(self._grants, catalog=catalog, any_file=True)) tasks.append(partial(self._grants, catalog=catalog, anonymous_function=True)) # scan all databases, even empty ones @@ -241,6 +242,8 @@ def _grants( ) try: grants = [] + # Added ANY FILE and ANONYMOUS FUNCTION in object_type_normalization + # to capture the same in grants. issue:#623 object_type_normalization = {"SCHEMA": "DATABASE", "CATALOG$": "CATALOG", "ANY_FILE": "ANY FILE", "ANONYMOUS_FUNCTION": "ANONYMOUS FUNCTION"} for row in self._fetch(f"SHOW GRANTS ON {on_type} {key}"): From 7e6c3c84d862c335c64aaa848400de543463fd32 Mon Sep 17 00:00:00 2001 From: Prajin Date: Thu, 30 Nov 2023 11:10:29 +0530 Subject: [PATCH 3/5] Fixed Linting Errors --- src/databricks/labs/ucx/hive_metastore/grants.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/databricks/labs/ucx/hive_metastore/grants.py b/src/databricks/labs/ucx/hive_metastore/grants.py index 5e93a6a2f3..fe3a54f888 100644 --- a/src/databricks/labs/ucx/hive_metastore/grants.py +++ b/src/databricks/labs/ucx/hive_metastore/grants.py @@ -140,7 +140,8 @@ def _try_load(self): def _crawl(self) -> list[Grant]: """ - Crawls and lists grants for all databases, tables, views, any file and anonymous function within hive_metastore. + Crawls and lists grants for all databases, tables, views, any file + and anonymous function within hive_metastore. Returns: list[Grant]: A list of Grant objects representing the listed grants. @@ -244,8 +245,12 @@ def _grants( grants = [] # Added ANY FILE and ANONYMOUS FUNCTION in object_type_normalization # to capture the same in grants. issue:#623 - object_type_normalization = {"SCHEMA": "DATABASE", "CATALOG$": "CATALOG", "ANY_FILE": "ANY FILE", - "ANONYMOUS_FUNCTION": "ANONYMOUS FUNCTION"} + object_type_normalization = { + "SCHEMA": "DATABASE", + "CATALOG$": "CATALOG", + "ANY_FILE": "ANY FILE", + "ANONYMOUS_FUNCTION": "ANONYMOUS FUNCTION", + } for row in self._fetch(f"SHOW GRANTS ON {on_type} {key}"): (principal, action_type, object_type, _) = row if object_type in object_type_normalization: From 3c309f1d792989697df6e0490fc521ce52f1bf7c Mon Sep 17 00:00:00 2001 From: Prajin Date: Thu, 30 Nov 2023 15:03:31 +0530 Subject: [PATCH 4/5] Added few more assertions --- tests/integration/workspace_access/test_tacl.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/workspace_access/test_tacl.py b/tests/integration/workspace_access/test_tacl.py index a0c57cf98b..e02a0704d9 100644 --- a/tests/integration/workspace_access/test_tacl.py +++ b/tests/integration/workspace_access/test_tacl.py @@ -40,13 +40,17 @@ def test_permission_for_files_anonymous_func(sql_backend, inventory_schema, make for any_file_grant in grants._grants(any_file=True): any_file_actual[any_file_grant.principal] = any_file_grant.action_type + assert any_file_actual[group_a.display_name] == "READ_METADATA" assert group_b.display_name in any_file_actual + assert any_file_actual[group_a.display_name] == any_file_actual[group_b.display_name] anonymous_function_actual = {} for ano_func_grant in grants._grants(anonymous_function=True): anonymous_function_actual[ano_func_grant.principal] = ano_func_grant.action_type + assert anonymous_function_actual[group_c.display_name] == "SELECT" assert group_d.display_name in anonymous_function_actual + assert anonymous_function_actual[group_c.display_name] == anonymous_function_actual[group_d.display_name] def test_owner_permissions_for_tables_and_schemas(sql_backend, inventory_schema, make_schema, make_table, make_group): From 7b63b87850da944f89dea6ba4d53835cf3dbc1ee Mon Sep 17 00:00:00 2001 From: Prajin Date: Thu, 30 Nov 2023 15:21:29 +0530 Subject: [PATCH 5/5] Added few more assertions --- tests/integration/workspace_access/test_tacl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/workspace_access/test_tacl.py b/tests/integration/workspace_access/test_tacl.py index e02a0704d9..99eb25805c 100644 --- a/tests/integration/workspace_access/test_tacl.py +++ b/tests/integration/workspace_access/test_tacl.py @@ -40,16 +40,16 @@ def test_permission_for_files_anonymous_func(sql_backend, inventory_schema, make for any_file_grant in grants._grants(any_file=True): any_file_actual[any_file_grant.principal] = any_file_grant.action_type - assert any_file_actual[group_a.display_name] == "READ_METADATA" assert group_b.display_name in any_file_actual + assert any_file_actual[group_b.display_name] == "READ_METADATA" assert any_file_actual[group_a.display_name] == any_file_actual[group_b.display_name] anonymous_function_actual = {} for ano_func_grant in grants._grants(anonymous_function=True): anonymous_function_actual[ano_func_grant.principal] = ano_func_grant.action_type - assert anonymous_function_actual[group_c.display_name] == "SELECT" assert group_d.display_name in anonymous_function_actual + assert anonymous_function_actual[group_d.display_name] == "SELECT" assert anonymous_function_actual[group_c.display_name] == anonymous_function_actual[group_d.display_name]