From 84cd1c2b0479e1787c9d53621e36bebb382db297 Mon Sep 17 00:00:00 2001 From: FastLee Date: Wed, 27 Sep 2023 11:02:55 -0400 Subject: [PATCH 01/16] Added migration_status,migration_target to the "tables" table --- src/databricks/labs/ucx/hive_metastore/tables.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/ucx/hive_metastore/tables.scala b/src/databricks/labs/ucx/hive_metastore/tables.scala index 94098d51dd..bc023d66e1 100644 --- a/src/databricks/labs/ucx/hive_metastore/tables.scala +++ b/src/databricks/labs/ucx/hive_metastore/tables.scala @@ -7,7 +7,7 @@ import org.apache.spark.sql.DataFrame // must follow the same structure as databricks.labs.ucx.hive_metastore.tables.Table case class TableDetails(catalog: String, database: String, name: String, object_type: String, - table_format: String, location: String, view_text: String) + table_format: String, location: String, view_text: String, upgrade_status: Int, upgrade_target: String) // recording error log in the database case class TableError(catalog: String, database: String, name: String, error: String) @@ -36,8 +36,10 @@ def metadataForAllTables(databases: Seq[String], queue: ConcurrentLinkedQueue[Ta failures.add(TableError("hive_metastore", databaseName, tableName, s"result is null")) None } else { + val upgrade_to=table.properties.get("upgraded_to") Some(TableDetails("hive_metastore", databaseName, tableName, table.tableType.name, table.provider.orNull, - table.storage.locationUri.map(_.toString).orNull, table.viewText.orNull)) + table.storage.locationUri.map(_.toString).orNull, table.viewText.orNull, + upgrade_to match {case Some(target) => 1 case None => 0}, upgrade_to match {case Some(target) => target case None => ""})) } } catch { case err: Throwable => From 885d442a630df0dd16c2bf20e39a1ca2c88299e6 Mon Sep 17 00:00:00 2001 From: FastLee Date: Wed, 27 Sep 2023 11:06:05 -0400 Subject: [PATCH 02/16] Updated Dashboard --- src/databricks/labs/ucx/assessment/queries/all_tables.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/ucx/assessment/queries/all_tables.sql b/src/databricks/labs/ucx/assessment/queries/all_tables.sql index 0a8fd95604..b4a89299e5 100644 --- a/src/databricks/labs/ucx/assessment/queries/all_tables.sql +++ b/src/databricks/labs/ucx/assessment/queries/all_tables.sql @@ -1,4 +1,4 @@ --- viz type=table, name=Table Types, columns=database,name,type,format,table_view,storage,is_delta,location +-- viz type=table, name=Table Types, columns=database,name,type,format,table_view,storage,is_delta,location,upgrade_status,upgrade_target -- widget title=Table Types, col=0, row=3, size_x=6, size_y=6 SELECT `database`, name, @@ -12,5 +12,7 @@ SELECT `database`, ELSE "EXTERNAL" END AS storage, IF(format = "delta", "Yes", "No") AS is_delta, - location + location, + upgrade_status, + upgrade_target FROM $inventory.tables From 3601331043546e8e01d167a4e40c8a2623b46ec0 Mon Sep 17 00:00:00 2001 From: FastLee Date: Wed, 27 Sep 2023 11:12:47 -0400 Subject: [PATCH 03/16] Updated Database Dashboard --- .../labs/ucx/assessment/queries/database_summary.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql index 18b16cd409..cb105b1042 100644 --- a/src/databricks/labs/ucx/assessment/queries/database_summary.sql +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -1,11 +1,13 @@ --- viz type=table, name=Database Summary, columns=database,tables,views,dbfs_root,delta_tables,upgrade +-- viz type=table, name=Database Summary, columns=database,tables,views,dbfs_root,delta_tables,upgraded_status,upgrade -- widget title=Database Summary, col=0, row=9, size_x=6, size_y=8 SELECT `database`, SUM(is_table) AS tables, SUM(is_view) AS views, SUM(is_dbfs_root) AS dbfs_root, SUM(is_delta) AS delta_tables, + SUM(upgrade_status) AS upgraded_tables CASE + WHEN (SUM(upgrade_status) = SUM(is_table)) THEN "Fully Upgraded" WHEN (SUM(is_dbfs_root)/SUM(is_table) > .3) THEN "Asset Replication Required" WHEN (SUM(is_delta)/SUM(is_table) < .7) THEN "Some Non Delta Assets" ELSE "In Place Sync" @@ -19,7 +21,8 @@ FROM IF(object_type IN ("MANAGED", "EXTERNAL"), 1, 0) AS is_table, IF(object_type = "VIEW", 1, 0) AS is_view, IF(STARTSWITH(location, "/dbfs/") AND NOT STARTSWITH(location, "/dbfs/mnt"), 1, 0) AS is_dbfs_root, - IF(UPPER(format) = "DELTA", 1, 0) AS is_delta + IF(UPPER(format) = "DELTA", 1, 0) AS is_delta, + upgrade_status FROM $inventory.tables) GROUP BY `database` ORDER BY `database` \ No newline at end of file From bfa17d1f644ba07398c77149a6c329888e48f375 Mon Sep 17 00:00:00 2001 From: FastLee Date: Wed, 27 Sep 2023 11:23:46 -0400 Subject: [PATCH 04/16] Updated Database Dashboard --- src/databricks/labs/ucx/assessment/queries/database_summary.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql index cb105b1042..2a1d9578cf 100644 --- a/src/databricks/labs/ucx/assessment/queries/database_summary.sql +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -5,7 +5,7 @@ SELECT `database`, SUM(is_view) AS views, SUM(is_dbfs_root) AS dbfs_root, SUM(is_delta) AS delta_tables, - SUM(upgrade_status) AS upgraded_tables + SUM(upgrade_status) AS upgraded_tables, CASE WHEN (SUM(upgrade_status) = SUM(is_table)) THEN "Fully Upgraded" WHEN (SUM(is_dbfs_root)/SUM(is_table) > .3) THEN "Asset Replication Required" From 7348b624a93cedbd2b23623212b552ac8a3d60db Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 29 Sep 2023 11:16:39 -0400 Subject: [PATCH 05/16] Reverted Dashboard --- .../ucx/assessment/queries/all_tables.sql | 6 ++-- .../labs/ucx/assessment/queries/clusters.sql | 9 ----- .../assessment/queries/count_total_tables.sql | 4 --- .../assessment/queries/database_summary.sql | 7 ++-- .../labs/ucx/assessment/queries/jobs.sql | 9 ----- .../ucx/assessment/queries/total_tables.sql | 11 ++++++ .../queries/workspace_access_total_groups.sql | 8 +++++ .../labs/ucx/framework/dashboards.py | 36 ++----------------- .../integration/framework/test_dashboards.py | 2 +- 9 files changed, 26 insertions(+), 66 deletions(-) delete mode 100644 src/databricks/labs/ucx/assessment/queries/clusters.sql delete mode 100644 src/databricks/labs/ucx/assessment/queries/count_total_tables.sql delete mode 100644 src/databricks/labs/ucx/assessment/queries/jobs.sql create mode 100644 src/databricks/labs/ucx/assessment/queries/total_tables.sql create mode 100644 src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql diff --git a/src/databricks/labs/ucx/assessment/queries/all_tables.sql b/src/databricks/labs/ucx/assessment/queries/all_tables.sql index b4a89299e5..0a8fd95604 100644 --- a/src/databricks/labs/ucx/assessment/queries/all_tables.sql +++ b/src/databricks/labs/ucx/assessment/queries/all_tables.sql @@ -1,4 +1,4 @@ --- viz type=table, name=Table Types, columns=database,name,type,format,table_view,storage,is_delta,location,upgrade_status,upgrade_target +-- viz type=table, name=Table Types, columns=database,name,type,format,table_view,storage,is_delta,location -- widget title=Table Types, col=0, row=3, size_x=6, size_y=6 SELECT `database`, name, @@ -12,7 +12,5 @@ SELECT `database`, ELSE "EXTERNAL" END AS storage, IF(format = "delta", "Yes", "No") AS is_delta, - location, - upgrade_status, - upgrade_target + location FROM $inventory.tables diff --git a/src/databricks/labs/ucx/assessment/queries/clusters.sql b/src/databricks/labs/ucx/assessment/queries/clusters.sql deleted file mode 100644 index 98aed74b7c..0000000000 --- a/src/databricks/labs/ucx/assessment/queries/clusters.sql +++ /dev/null @@ -1,9 +0,0 @@ --- viz type=table, name=Clusters, columns=cluster_id,cluster_name,creator,compatible,failures --- widget title=Clusters, col=0, row=25, size_x=6, size_y=8 -SELECT cluster_id, - cluster_name, - creator, - IF(success = 1, "Compatible", "Incompatible") AS compatible, - failures -FROM $inventory.clusters -WHERE NOT STARTSWITH(cluster_name, "job-") \ No newline at end of file diff --git a/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql b/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql deleted file mode 100644 index a346849396..0000000000 --- a/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql +++ /dev/null @@ -1,4 +0,0 @@ --- viz type=counter, name=Total Table Count, counter_label=Total Tables, value_column=count_total_tables --- widget col=1, row=0, size_x=1, size_y=3 -SELECT count(*) AS count_total_tables -FROM $inventory.tables diff --git a/src/databricks/labs/ucx/assessment/queries/database_summary.sql b/src/databricks/labs/ucx/assessment/queries/database_summary.sql index 2a1d9578cf..18b16cd409 100644 --- a/src/databricks/labs/ucx/assessment/queries/database_summary.sql +++ b/src/databricks/labs/ucx/assessment/queries/database_summary.sql @@ -1,13 +1,11 @@ --- viz type=table, name=Database Summary, columns=database,tables,views,dbfs_root,delta_tables,upgraded_status,upgrade +-- viz type=table, name=Database Summary, columns=database,tables,views,dbfs_root,delta_tables,upgrade -- widget title=Database Summary, col=0, row=9, size_x=6, size_y=8 SELECT `database`, SUM(is_table) AS tables, SUM(is_view) AS views, SUM(is_dbfs_root) AS dbfs_root, SUM(is_delta) AS delta_tables, - SUM(upgrade_status) AS upgraded_tables, CASE - WHEN (SUM(upgrade_status) = SUM(is_table)) THEN "Fully Upgraded" WHEN (SUM(is_dbfs_root)/SUM(is_table) > .3) THEN "Asset Replication Required" WHEN (SUM(is_delta)/SUM(is_table) < .7) THEN "Some Non Delta Assets" ELSE "In Place Sync" @@ -21,8 +19,7 @@ FROM IF(object_type IN ("MANAGED", "EXTERNAL"), 1, 0) AS is_table, IF(object_type = "VIEW", 1, 0) AS is_view, IF(STARTSWITH(location, "/dbfs/") AND NOT STARTSWITH(location, "/dbfs/mnt"), 1, 0) AS is_dbfs_root, - IF(UPPER(format) = "DELTA", 1, 0) AS is_delta, - upgrade_status + IF(UPPER(format) = "DELTA", 1, 0) AS is_delta FROM $inventory.tables) GROUP BY `database` ORDER BY `database` \ No newline at end of file diff --git a/src/databricks/labs/ucx/assessment/queries/jobs.sql b/src/databricks/labs/ucx/assessment/queries/jobs.sql deleted file mode 100644 index 5842052855..0000000000 --- a/src/databricks/labs/ucx/assessment/queries/jobs.sql +++ /dev/null @@ -1,9 +0,0 @@ --- viz type=table, name=Jobs, columns=job_id,job_name,creator,compatible,failures --- widget title=Jobs, col=0, row=33, size_x=6, size_y=8 -SELECT job_id, - job_name, - creator, - IF(success=1, "Compatible", "Incompatible") AS compatible, - failures -FROM $inventory.jobs -WHERE job_name not like '[UCX]%' diff --git a/src/databricks/labs/ucx/assessment/queries/total_tables.sql b/src/databricks/labs/ucx/assessment/queries/total_tables.sql new file mode 100644 index 0000000000..94074153fa --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/total_tables.sql @@ -0,0 +1,11 @@ +-- viz type=table, name=Table Types, columns=count,managed_tables,managed_pct,external_tables,external_pct,views,view_pct +-- widget title=Table Types, col=0, row=1, size_x=3, size_y=3 +SELECT + COUNT(*) AS count, + SUM(IF(object_type == "MANAGED", 1, 0)) AS managed_tables, + ROUND(100 * SUM(IF(object_type == "MANAGED", 1, 0)) / COUNT(*), 0) AS managed_pct, + SUM(IF(object_type == "EXTERNAL", 1, 0)) AS external_tables, + ROUND(100 * SUM(IF(object_type == "EXTERNAL", 1, 0)) / COUNT(*), 0) AS external_pct, + SUM(IF(object_type == "VIEW", 1, 0)) AS views, + ROUND(100 * SUM(IF(object_type == "VIEW", 1, 0)) / COUNT(*), 0) AS view_pct +FROM $inventory.tables \ No newline at end of file diff --git a/src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql b/src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql new file mode 100644 index 0000000000..04518ae471 --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql @@ -0,0 +1,8 @@ +-- viz type=table, name=Workspace Groups per Object Type, columns=object_type,objects +-- widget title=Object Types, col=0, row=3, size_x=1, size_y=12 +SELECT + object_type, + COUNT(DISTINCT object_id) objects +FROM $inventory.permissions +GROUP BY 1 +ORDER BY objects DESC \ No newline at end of file diff --git a/src/databricks/labs/ucx/framework/dashboards.py b/src/databricks/labs/ucx/framework/dashboards.py index 1418be28c8..44c38eadbc 100644 --- a/src/databricks/labs/ucx/framework/dashboards.py +++ b/src/databricks/labs/ucx/framework/dashboards.py @@ -130,7 +130,7 @@ def _install_widget(self, query: SimpleQuery): def _get_widget_options(self, query: SimpleQuery): self._pos += 1 widget_options = WidgetOptions( - title=query.widget.get("title", ""), + title=query.widget["title"], description=query.widget.get("description", None), position=WidgetPosition( col=int(query.widget.get("col", 0)), @@ -227,7 +227,7 @@ def _install_viz(self, query: SimpleQuery): self._state[query.viz_key] = viz.id def _get_viz_options(self, query: SimpleQuery): - viz_types = {"table": self._table_viz_args, "counter": self._counter_viz_args} + viz_types = {"table": self._table_viz_args} if query.viz_type not in viz_types: msg = f"{query.query}: unknown viz type: {query.viz_type}" raise SyntaxError(msg) @@ -270,38 +270,6 @@ def _table_viz_args( }, } - @staticmethod - def _counter_viz_args( - name: str, - value_column: str, - *, - description: str | None = None, - counter_label: str | None = None, - value_row_number: int = 1, - target_row_number: int = 1, - string_decimal: int = 0, - string_decimal_char: str = ".", - string_thousand_separator: str = ",", - tooltip_format: str = "0,0.000", - count_row: bool = False, - ) -> dict: - return { - "type": "COUNTER", - "name": name, - "description": description, - "options": { - "counterLabel": counter_label, - "counterColName": value_column, - "rowNumber": value_row_number, - "targetRowNumber": target_row_number, - "stringDecimal": string_decimal, - "stringDecChar": string_decimal_char, - "stringThouSep": string_thousand_separator, - "tooltipFormat": tooltip_format, - "countRow": count_row, - }, - } - @staticmethod def _parse_magic_comment(f, magic_comment, text): viz_comment = next(_ for _ in text.splitlines() if _.startswith(magic_comment)) diff --git a/tests/integration/framework/test_dashboards.py b/tests/integration/framework/test_dashboards.py index 6251a15820..9e12767faf 100644 --- a/tests/integration/framework/test_dashboards.py +++ b/tests/integration/framework/test_dashboards.py @@ -15,7 +15,7 @@ # logging.getLogger("databricks").setLevel("DEBUG") -def test_creating_widgets(ws: WorkspaceClient, make_warehouse, make_schema): +def test_creating_widgets(ws: WorkspaceClient): pytest.skip() dashboard_widgets_api = DashboardWidgetsAPI(ws.api_client) query_visualizations_api = QueryVisualizationsExt(ws.api_client) From c14cf24557b406bdd82cfe433715aa012381c6b0 Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 29 Sep 2023 13:58:59 -0400 Subject: [PATCH 06/16] Reverted Dashboard --- .../labs/ucx/framework/dashboards.py | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/ucx/framework/dashboards.py b/src/databricks/labs/ucx/framework/dashboards.py index 44c38eadbc..1418be28c8 100644 --- a/src/databricks/labs/ucx/framework/dashboards.py +++ b/src/databricks/labs/ucx/framework/dashboards.py @@ -130,7 +130,7 @@ def _install_widget(self, query: SimpleQuery): def _get_widget_options(self, query: SimpleQuery): self._pos += 1 widget_options = WidgetOptions( - title=query.widget["title"], + title=query.widget.get("title", ""), description=query.widget.get("description", None), position=WidgetPosition( col=int(query.widget.get("col", 0)), @@ -227,7 +227,7 @@ def _install_viz(self, query: SimpleQuery): self._state[query.viz_key] = viz.id def _get_viz_options(self, query: SimpleQuery): - viz_types = {"table": self._table_viz_args} + viz_types = {"table": self._table_viz_args, "counter": self._counter_viz_args} if query.viz_type not in viz_types: msg = f"{query.query}: unknown viz type: {query.viz_type}" raise SyntaxError(msg) @@ -270,6 +270,38 @@ def _table_viz_args( }, } + @staticmethod + def _counter_viz_args( + name: str, + value_column: str, + *, + description: str | None = None, + counter_label: str | None = None, + value_row_number: int = 1, + target_row_number: int = 1, + string_decimal: int = 0, + string_decimal_char: str = ".", + string_thousand_separator: str = ",", + tooltip_format: str = "0,0.000", + count_row: bool = False, + ) -> dict: + return { + "type": "COUNTER", + "name": name, + "description": description, + "options": { + "counterLabel": counter_label, + "counterColName": value_column, + "rowNumber": value_row_number, + "targetRowNumber": target_row_number, + "stringDecimal": string_decimal, + "stringDecChar": string_decimal_char, + "stringThouSep": string_thousand_separator, + "tooltipFormat": tooltip_format, + "countRow": count_row, + }, + } + @staticmethod def _parse_magic_comment(f, magic_comment, text): viz_comment = next(_ for _ in text.splitlines() if _.startswith(magic_comment)) From cd47002a7446678120ad4dfb80cc5bbb75c9479f Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 29 Sep 2023 14:00:38 -0400 Subject: [PATCH 07/16] Reverted Dashboard --- tests/integration/framework/test_dashboards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/framework/test_dashboards.py b/tests/integration/framework/test_dashboards.py index 9e12767faf..6251a15820 100644 --- a/tests/integration/framework/test_dashboards.py +++ b/tests/integration/framework/test_dashboards.py @@ -15,7 +15,7 @@ # logging.getLogger("databricks").setLevel("DEBUG") -def test_creating_widgets(ws: WorkspaceClient): +def test_creating_widgets(ws: WorkspaceClient, make_warehouse, make_schema): pytest.skip() dashboard_widgets_api = DashboardWidgetsAPI(ws.api_client) query_visualizations_api = QueryVisualizationsExt(ws.api_client) From c58ed241ba0670f7ffb025bb69f71fb0753c8ae7 Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 29 Sep 2023 14:03:15 -0400 Subject: [PATCH 08/16] Reverted Dashboard --- .../labs/ucx/assessment/queries/total_tables.sql | 11 ----------- .../queries/workspace_access_total_groups.sql | 8 -------- 2 files changed, 19 deletions(-) delete mode 100644 src/databricks/labs/ucx/assessment/queries/total_tables.sql delete mode 100644 src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql diff --git a/src/databricks/labs/ucx/assessment/queries/total_tables.sql b/src/databricks/labs/ucx/assessment/queries/total_tables.sql deleted file mode 100644 index 94074153fa..0000000000 --- a/src/databricks/labs/ucx/assessment/queries/total_tables.sql +++ /dev/null @@ -1,11 +0,0 @@ --- viz type=table, name=Table Types, columns=count,managed_tables,managed_pct,external_tables,external_pct,views,view_pct --- widget title=Table Types, col=0, row=1, size_x=3, size_y=3 -SELECT - COUNT(*) AS count, - SUM(IF(object_type == "MANAGED", 1, 0)) AS managed_tables, - ROUND(100 * SUM(IF(object_type == "MANAGED", 1, 0)) / COUNT(*), 0) AS managed_pct, - SUM(IF(object_type == "EXTERNAL", 1, 0)) AS external_tables, - ROUND(100 * SUM(IF(object_type == "EXTERNAL", 1, 0)) / COUNT(*), 0) AS external_pct, - SUM(IF(object_type == "VIEW", 1, 0)) AS views, - ROUND(100 * SUM(IF(object_type == "VIEW", 1, 0)) / COUNT(*), 0) AS view_pct -FROM $inventory.tables \ No newline at end of file diff --git a/src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql b/src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql deleted file mode 100644 index 04518ae471..0000000000 --- a/src/databricks/labs/ucx/assessment/queries/workspace_access_total_groups.sql +++ /dev/null @@ -1,8 +0,0 @@ --- viz type=table, name=Workspace Groups per Object Type, columns=object_type,objects --- widget title=Object Types, col=0, row=3, size_x=1, size_y=12 -SELECT - object_type, - COUNT(DISTINCT object_id) objects -FROM $inventory.permissions -GROUP BY 1 -ORDER BY objects DESC \ No newline at end of file From 9219486ecd23058aeecc35b71617d1d4d04b79f9 Mon Sep 17 00:00:00 2001 From: Liran Bareket Date: Fri, 29 Sep 2023 14:37:14 -0400 Subject: [PATCH 09/16] Ported internal document about UCX persistence schema (#345) Generated the table persistence document in docs --- docs/table_persistence.md | 120 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 docs/table_persistence.md diff --git a/docs/table_persistence.md b/docs/table_persistence.md new file mode 100644 index 0000000000..461f0b2ac0 --- /dev/null +++ b/docs/table_persistence.md @@ -0,0 +1,120 @@ + # UCX Persistence + +Enumeration of all UCX persistence elements + +## Overview +Table Utilization: + +| Table | Generate Assessment | Migrate Local Groups | Migrate External Tables | Migrate SQL Warehouses | Upgrade Jobs | Migrate managed tables | +|--------------------|---------------------|----------------------|-------------------------|------------------------|--------------|------------------------| +| tables | RW | | RO | | | RO | +| grants | | | RW | | | RW | +| mounts | RW | | RO | | RO | RO | +| permissions | | RW | | | | | +| jobs | RW | | | | RO | | +| clusters | RW | +| external_locations | RW | | RO | +| workspace | RW | RO | | | RO | + +**RW** - Read/Write the job that generates the table +**RO** - Read Only + +### Inventory Database +#### _$inventory_.tables + +Holds Inventory of all tables in all databases and their relevant metadata. + +| Column | Datatype | Description | Comments | +|-----------|----------|-------------|----------| +| catalog | string | Original catalog of the table. _hive_metastore_ by default | +| database | string | Original schema of the table | +| name |string|Name of the table| +|object_type|string|MANAGED, EXTERNAL, or VIEW| +|table_format|string|Table provider. Like delta or json or parquet.| +|location|string|Location of the data for table| +|view_text|nullable string|If the table is the view, then this column holds the definition of the view| + +
+ +#### _$inventory_.table_failures +Holds failures that occurred during crawling HMS tables + +| Column | Datatype | Description | Comments | +|-----------|----------|-------------|----------| +|catalog|string|Original catalog of the table. hive_metastore by default| +|database|string|Original schema of the table| +|name|string|Name of the table| +|failures|string|Exception message context| + +
+ +#### _$inventory_.grants +Inventory of all Table ACLs for tables indexed in tables table. + +| Column | Datatype | Description | Comments | +|-----------|----------|-------------|----------| +|principal|string|User name, group name, or service principal name| +|action_type|string|Name of GRANT action| +|catalog|string|Original catalog of the table. hive_metastore by default| +|database|Nullable string|Original schema of the table| +|table|Nullable string|Name of the table| +|view|Nullable string|Name of the view| +|any_file|bool|Any file| +|anonymous_function|string|Grant for the anonymous function| + +
+ +#### _$inventory_.mounts +List of DBFS mount points. + +| Column | Datatype | Description | Comments | +|-----------|----------|-------------|----------| +|name|string|Name of the mount point| +|source|string|Location of the backing dataset| +|instance_profile|Nullable string|This mount point is accessible only with this AWS IAM instance profile| + +
+ +#### _$inventory_.permissions +Workspace object level permissions + +| Column | Datatype | Description | Comments | +|-----------|----------|-------------|----------| +|object_id|string|Either:
Group ID
Workspace Object ID
Redash Object ID
Scope name +|supports|string|One of:
AUTHORIZATION

CLUSTERS
CLUSTER_POLICIES
DIRECTORIES
EXPERIMENTS
FILES
INSTANCE_POOLS
JOBS
NOTEBOOKS
PIPELINES
REGISTERED_MODELS
REPOS
SERVING_ENDPOINTS
SQL_WAREHOUSES +|raw_object_permissions|JSON|JSON-serialized response of:
Generic Permissions
Secret ACL
Group roles and entitlements
Redash permissions| + +
+ +#### _$inventory_.jobs +Holds a list of all jobs with a notation of potential issues. + +| Column | Datatype | Description | Comments | +|-----------|----------|-------------|----------| +|job_id|string|Job ID| +|job_name|string|Job Name| +|job_creator|string|UserID of the Job Creator| +|compatible|int|1 or 0, used for percentage reporting| +|failures|string|List of issues identified by the assessment in JSON format| + + +#### _$inventory_.clusters +Holds a list of all clusters with a notation of potential issues. + +| Column | Datatype | Description | Comments | +|-----------|----------|-------------|----------| +|cluster_id|string|Cluster Id| +|cluster_name|string|Cluster Name| +|cluster_creator|string|UserID of the Cluster Creator| +|compatible|int|1 or 0, used for percentage reporting| +|failures|string|List of issues identified by the assessment in JSON format| + + +#### _$inventory_.external_locations +Holds a list of all external locations that will be required for the migration. + +| Column | Datatype | Description | Comments | +|-----------|----------|-------------|----------| +|external_location|string|External Location URL| + + From 6c84282b41fa713a835eb7cd22c85ecd78d5ee5b Mon Sep 17 00:00:00 2001 From: FastLee Date: Fri, 29 Sep 2023 16:31:52 -0400 Subject: [PATCH 10/16] Updated dataclass and persistence doc --- docs/table_persistence.md | 20 ++++++++++--------- .../labs/ucx/hive_metastore/tables.py | 2 ++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/table_persistence.md b/docs/table_persistence.md index 461f0b2ac0..c5db0044e7 100644 --- a/docs/table_persistence.md +++ b/docs/table_persistence.md @@ -24,15 +24,17 @@ Table Utilization: Holds Inventory of all tables in all databases and their relevant metadata. -| Column | Datatype | Description | Comments | -|-----------|----------|-------------|----------| -| catalog | string | Original catalog of the table. _hive_metastore_ by default | -| database | string | Original schema of the table | -| name |string|Name of the table| -|object_type|string|MANAGED, EXTERNAL, or VIEW| -|table_format|string|Table provider. Like delta or json or parquet.| -|location|string|Location of the data for table| -|view_text|nullable string|If the table is the view, then this column holds the definition of the view| +| Column | Datatype | Description | Comments | +|----------------|----------|-------------|----------| +| catalog | string | Original catalog of the table. _hive_metastore_ by default | +| database | string | Original schema of the table | +| name |string|Name of the table| +| object_type |string|MANAGED, EXTERNAL, or VIEW| +| table_format |string|Table provider. Like delta or json or parquet.| +| location |string|Location of the data for table| +| view_text |nullable string|If the table is the view, then this column holds the definition of the view| +| upgrade_status |int|0-Not Upgraded (default) 1-Upgraded| +| upgrade_target |string|Upgrade Target (3 level namespace)|
diff --git a/src/databricks/labs/ucx/hive_metastore/tables.py b/src/databricks/labs/ucx/hive_metastore/tables.py index 7d15c2c434..d64a62481a 100644 --- a/src/databricks/labs/ucx/hive_metastore/tables.py +++ b/src/databricks/labs/ucx/hive_metastore/tables.py @@ -22,6 +22,8 @@ class Table: location: str = None view_text: str = None + upgrade_status: int = 0 + upgrade_target: str = None @property def is_delta(self) -> bool: From 62ba8a273aae8a94ff23c56283690b4f832d2e98 Mon Sep 17 00:00:00 2001 From: FastLee Date: Mon, 2 Oct 2023 11:30:53 -0400 Subject: [PATCH 11/16] Reintroduce Mistakenly deleted queries --- src/databricks/labs/ucx/assessment/queries/clusters.sql | 9 +++++++++ .../labs/ucx/assessment/queries/count_total_tables.sql | 4 ++++ src/databricks/labs/ucx/assessment/queries/jobs.sql | 9 +++++++++ 3 files changed, 22 insertions(+) create mode 100644 src/databricks/labs/ucx/assessment/queries/clusters.sql create mode 100644 src/databricks/labs/ucx/assessment/queries/count_total_tables.sql create mode 100644 src/databricks/labs/ucx/assessment/queries/jobs.sql diff --git a/src/databricks/labs/ucx/assessment/queries/clusters.sql b/src/databricks/labs/ucx/assessment/queries/clusters.sql new file mode 100644 index 0000000000..98aed74b7c --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/clusters.sql @@ -0,0 +1,9 @@ +-- viz type=table, name=Clusters, columns=cluster_id,cluster_name,creator,compatible,failures +-- widget title=Clusters, col=0, row=25, size_x=6, size_y=8 +SELECT cluster_id, + cluster_name, + creator, + IF(success = 1, "Compatible", "Incompatible") AS compatible, + failures +FROM $inventory.clusters +WHERE NOT STARTSWITH(cluster_name, "job-") \ No newline at end of file diff --git a/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql b/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql new file mode 100644 index 0000000000..a346849396 --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/count_total_tables.sql @@ -0,0 +1,4 @@ +-- viz type=counter, name=Total Table Count, counter_label=Total Tables, value_column=count_total_tables +-- widget col=1, row=0, size_x=1, size_y=3 +SELECT count(*) AS count_total_tables +FROM $inventory.tables diff --git a/src/databricks/labs/ucx/assessment/queries/jobs.sql b/src/databricks/labs/ucx/assessment/queries/jobs.sql new file mode 100644 index 0000000000..5842052855 --- /dev/null +++ b/src/databricks/labs/ucx/assessment/queries/jobs.sql @@ -0,0 +1,9 @@ +-- viz type=table, name=Jobs, columns=job_id,job_name,creator,compatible,failures +-- widget title=Jobs, col=0, row=33, size_x=6, size_y=8 +SELECT job_id, + job_name, + creator, + IF(success=1, "Compatible", "Incompatible") AS compatible, + failures +FROM $inventory.jobs +WHERE job_name not like '[UCX]%' From de6134d9fde3ce0a2eb058aabc95f79fa066ff31 Mon Sep 17 00:00:00 2001 From: FastLee Date: Mon, 2 Oct 2023 14:17:13 -0400 Subject: [PATCH 12/16] Reintroduce Mistakenly deleted queries --- docs/table_persistence.md | 1 - src/databricks/labs/ucx/hive_metastore/tables.py | 13 +++++++++++-- src/databricks/labs/ucx/hive_metastore/tables.scala | 6 +++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/table_persistence.md b/docs/table_persistence.md index c5db0044e7..26c673e397 100644 --- a/docs/table_persistence.md +++ b/docs/table_persistence.md @@ -33,7 +33,6 @@ Holds Inventory of all tables in all databases and their relevant metadata. | table_format |string|Table provider. Like delta or json or parquet.| | location |string|Location of the data for table| | view_text |nullable string|If the table is the view, then this column holds the definition of the view| -| upgrade_status |int|0-Not Upgraded (default) 1-Upgraded| | upgrade_target |string|Upgrade Target (3 level namespace)|
diff --git a/src/databricks/labs/ucx/hive_metastore/tables.py b/src/databricks/labs/ucx/hive_metastore/tables.py index d64a62481a..607bb94c25 100644 --- a/src/databricks/labs/ucx/hive_metastore/tables.py +++ b/src/databricks/labs/ucx/hive_metastore/tables.py @@ -1,4 +1,6 @@ import logging +import re +import string from collections.abc import Iterator from dataclasses import dataclass from functools import partial @@ -22,8 +24,7 @@ class Table: location: str = None view_text: str = None - upgrade_status: int = 0 - upgrade_target: str = None + upgraded_to: str = None @property def is_delta(self) -> bool: @@ -72,6 +73,13 @@ def sql_alter_from(self, catalog): ) +def parse_table_props(tbl_props: string) -> {}: + pattern = r"([^,\[\]]+)=([^,\[\]]+)" + key_value_pairs = re.findall(pattern, tbl_props) + # Convert key-value pairs to dictionary + return dict(key_value_pairs) + + class TablesCrawler(CrawlerBase): def __init__(self, backend: SqlBackend, schema): """ @@ -142,6 +150,7 @@ def _describe(self, catalog: str, database: str, table: str) -> Table | None: table_format=describe.get("Provider", "").upper(), location=describe.get("Location", None), view_text=describe.get("View Text", None), + upgraded_to=parse_table_props(describe.get("Table Properties", "")).get("upgraded_to", ""), ) except Exception as e: logger.error(f"Couldn't fetch information for table {full_name} : {e}") diff --git a/src/databricks/labs/ucx/hive_metastore/tables.scala b/src/databricks/labs/ucx/hive_metastore/tables.scala index bc023d66e1..fa0db7c019 100644 --- a/src/databricks/labs/ucx/hive_metastore/tables.scala +++ b/src/databricks/labs/ucx/hive_metastore/tables.scala @@ -7,7 +7,7 @@ import org.apache.spark.sql.DataFrame // must follow the same structure as databricks.labs.ucx.hive_metastore.tables.Table case class TableDetails(catalog: String, database: String, name: String, object_type: String, - table_format: String, location: String, view_text: String, upgrade_status: Int, upgrade_target: String) + table_format: String, location: String, view_text: String, upgraded_to: String) // recording error log in the database case class TableError(catalog: String, database: String, name: String, error: String) @@ -36,10 +36,10 @@ def metadataForAllTables(databases: Seq[String], queue: ConcurrentLinkedQueue[Ta failures.add(TableError("hive_metastore", databaseName, tableName, s"result is null")) None } else { - val upgrade_to=table.properties.get("upgraded_to") + val upgraded_to=table.properties.get("upgraded_to") Some(TableDetails("hive_metastore", databaseName, tableName, table.tableType.name, table.provider.orNull, table.storage.locationUri.map(_.toString).orNull, table.viewText.orNull, - upgrade_to match {case Some(target) => 1 case None => 0}, upgrade_to match {case Some(target) => target case None => ""})) + upgraded_to match {case Some(target) => target case None => ""})) } } catch { case err: Throwable => From 96a650f9821b13acb2103d5da9d69d39eadfdd28 Mon Sep 17 00:00:00 2001 From: FastLee Date: Tue, 3 Oct 2023 11:16:20 -0400 Subject: [PATCH 13/16] Added null value, refactored function and added tests. --- src/databricks/labs/ucx/hive_metastore/tables.py | 16 ++++++++-------- .../labs/ucx/hive_metastore/tables.scala | 2 +- tests/unit/hive_metastore/test_tables.py | 12 ++++++++++++ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/databricks/labs/ucx/hive_metastore/tables.py b/src/databricks/labs/ucx/hive_metastore/tables.py index 607bb94c25..92d3711e08 100644 --- a/src/databricks/labs/ucx/hive_metastore/tables.py +++ b/src/databricks/labs/ucx/hive_metastore/tables.py @@ -73,13 +73,6 @@ def sql_alter_from(self, catalog): ) -def parse_table_props(tbl_props: string) -> {}: - pattern = r"([^,\[\]]+)=([^,\[\]]+)" - key_value_pairs = re.findall(pattern, tbl_props) - # Convert key-value pairs to dictionary - return dict(key_value_pairs) - - class TablesCrawler(CrawlerBase): def __init__(self, backend: SqlBackend, schema): """ @@ -103,6 +96,13 @@ def snapshot(self) -> list[Table]: """ return self._snapshot(partial(self._try_load), partial(self._crawl)) + @staticmethod + def _parse_table_props(tbl_props: string) -> {}: + pattern = r"([^,\[\]]+)=([^,\[\]]+)" + key_value_pairs = re.findall(pattern, tbl_props) + # Convert key-value pairs to dictionary + return dict(key_value_pairs) + def _try_load(self): """Tries to load table information from the database or throws TABLE_OR_VIEW_NOT_FOUND error""" for row in self._fetch(f"SELECT * FROM {self._full_name}"): @@ -150,7 +150,7 @@ def _describe(self, catalog: str, database: str, table: str) -> Table | None: table_format=describe.get("Provider", "").upper(), location=describe.get("Location", None), view_text=describe.get("View Text", None), - upgraded_to=parse_table_props(describe.get("Table Properties", "")).get("upgraded_to", ""), + upgraded_to=self._parse_table_props(describe.get("Table Properties", "")).get("upgraded_to", None), ) except Exception as e: logger.error(f"Couldn't fetch information for table {full_name} : {e}") diff --git a/src/databricks/labs/ucx/hive_metastore/tables.scala b/src/databricks/labs/ucx/hive_metastore/tables.scala index fa0db7c019..3a42ac647d 100644 --- a/src/databricks/labs/ucx/hive_metastore/tables.scala +++ b/src/databricks/labs/ucx/hive_metastore/tables.scala @@ -39,7 +39,7 @@ def metadataForAllTables(databases: Seq[String], queue: ConcurrentLinkedQueue[Ta val upgraded_to=table.properties.get("upgraded_to") Some(TableDetails("hive_metastore", databaseName, tableName, table.tableType.name, table.provider.orNull, table.storage.locationUri.map(_.toString).orNull, table.viewText.orNull, - upgraded_to match {case Some(target) => target case None => ""})) + upgraded_to match {case Some(target) => target case None => null})) } } catch { case err: Throwable => diff --git a/tests/unit/hive_metastore/test_tables.py b/tests/unit/hive_metastore/test_tables.py index 762fbb7a85..50c08f1e3d 100644 --- a/tests/unit/hive_metastore/test_tables.py +++ b/tests/unit/hive_metastore/test_tables.py @@ -84,6 +84,18 @@ def test_tables_crawler_inventory_table(): assert tc._table == "tables" +def test_tables_crawler_parse_tp(): + tc = TablesCrawler(MockBackend(), "default") + tp1 = tc._parse_table_props( + "[delta.minReaderVersion=1,delta.minWriterVersion=2,upgraded_to=fake_cat.fake_ext.fake_delta]" + ) + tp2 = tc._parse_table_props("[delta.minReaderVersion=1,delta.minWriterVersion=2]") + assert len(tp1) == 3 + assert tp1.get("upgraded_to") == "fake_cat.fake_ext.fake_delta" + assert len(tp2) == 2 + assert tp2.get("upgraded_to") is None + + def test_tables_returning_error_when_describing(): errors = {"DESCRIBE TABLE EXTENDED hive_metastore.database.table1": "error"} rows = { From 82c0327cc5c1fd73f5490f72a8baa06253266a1f Mon Sep 17 00:00:00 2001 From: FastLee Date: Tue, 3 Oct 2023 15:32:14 -0400 Subject: [PATCH 14/16] Fixed table persistence doc --- docs/table_persistence.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/table_persistence.md b/docs/table_persistence.md index 26c673e397..b96925870d 100644 --- a/docs/table_persistence.md +++ b/docs/table_persistence.md @@ -24,16 +24,16 @@ Table Utilization: Holds Inventory of all tables in all databases and their relevant metadata. -| Column | Datatype | Description | Comments | -|----------------|----------|-------------|----------| -| catalog | string | Original catalog of the table. _hive_metastore_ by default | -| database | string | Original schema of the table | -| name |string|Name of the table| -| object_type |string|MANAGED, EXTERNAL, or VIEW| -| table_format |string|Table provider. Like delta or json or parquet.| -| location |string|Location of the data for table| -| view_text |nullable string|If the table is the view, then this column holds the definition of the view| -| upgrade_target |string|Upgrade Target (3 level namespace)| +| Column | Datatype | Description | Comments | +|--------------|----------|-------------|----------| +| catalog | string | Original catalog of the table. _hive_metastore_ by default | +| database | string | Original schema of the table | +| name |string|Name of the table| +| object_type |string|MANAGED, EXTERNAL, or VIEW| +| table_format |string|Table provider. Like delta or json or parquet.| +| location |string|Location of the data for table| +| view_text |nullable string|If the table is the view, then this column holds the definition of the view| +| upgraded_to |string|Upgrade Target (3 level namespace)|
@@ -79,11 +79,11 @@ List of DBFS mount points. #### _$inventory_.permissions Workspace object level permissions -| Column | Datatype | Description | Comments | -|-----------|----------|-------------|----------| -|object_id|string|Either:
Group ID
Workspace Object ID
Redash Object ID
Scope name -|supports|string|One of:
AUTHORIZATION

CLUSTERS
CLUSTER_POLICIES
DIRECTORIES
EXPERIMENTS
FILES
INSTANCE_POOLS
JOBS
NOTEBOOKS
PIPELINES
REGISTERED_MODELS
REPOS
SERVING_ENDPOINTS
SQL_WAREHOUSES -|raw_object_permissions|JSON|JSON-serialized response of:
Generic Permissions
Secret ACL
Group roles and entitlements
Redash permissions| +| Column | Datatype | Description | Comments | +|------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| +| object_id | string | Either:
Group ID
Workspace Object ID
Redash Object ID
Scope name | | +| supports | string | One of:
AUTHORIZATION

CLUSTERS
CLUSTER_POLICIES
DIRECTORIES
EXPERIMENTS
FILES
INSTANCE_POOLS
JOBS
NOTEBOOKS
PIPELINES
REGISTERED_MODELS
REPOS
SERVING_ENDPOINTS
SQL_WAREHOUSES | | +| raw_object_permissions | JSON | JSON-serialized response of:
Generic Permissions
Secret ACL
Group roles and entitlements
Redash permissions | |
From e3066dbee2870b85393766286bede35429a8df75 Mon Sep 17 00:00:00 2001 From: FastLee Date: Tue, 3 Oct 2023 15:42:30 -0400 Subject: [PATCH 15/16] Fixed merge conflict --- docs/table_persistence.md | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/docs/table_persistence.md b/docs/table_persistence.md index b96925870d..461f0b2ac0 100644 --- a/docs/table_persistence.md +++ b/docs/table_persistence.md @@ -24,16 +24,15 @@ Table Utilization: Holds Inventory of all tables in all databases and their relevant metadata. -| Column | Datatype | Description | Comments | -|--------------|----------|-------------|----------| -| catalog | string | Original catalog of the table. _hive_metastore_ by default | -| database | string | Original schema of the table | -| name |string|Name of the table| -| object_type |string|MANAGED, EXTERNAL, or VIEW| -| table_format |string|Table provider. Like delta or json or parquet.| -| location |string|Location of the data for table| -| view_text |nullable string|If the table is the view, then this column holds the definition of the view| -| upgraded_to |string|Upgrade Target (3 level namespace)| +| Column | Datatype | Description | Comments | +|-----------|----------|-------------|----------| +| catalog | string | Original catalog of the table. _hive_metastore_ by default | +| database | string | Original schema of the table | +| name |string|Name of the table| +|object_type|string|MANAGED, EXTERNAL, or VIEW| +|table_format|string|Table provider. Like delta or json or parquet.| +|location|string|Location of the data for table| +|view_text|nullable string|If the table is the view, then this column holds the definition of the view|
@@ -79,11 +78,11 @@ List of DBFS mount points. #### _$inventory_.permissions Workspace object level permissions -| Column | Datatype | Description | Comments | -|------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| -| object_id | string | Either:
Group ID
Workspace Object ID
Redash Object ID
Scope name | | -| supports | string | One of:
AUTHORIZATION

CLUSTERS
CLUSTER_POLICIES
DIRECTORIES
EXPERIMENTS
FILES
INSTANCE_POOLS
JOBS
NOTEBOOKS
PIPELINES
REGISTERED_MODELS
REPOS
SERVING_ENDPOINTS
SQL_WAREHOUSES | | -| raw_object_permissions | JSON | JSON-serialized response of:
Generic Permissions
Secret ACL
Group roles and entitlements
Redash permissions | | +| Column | Datatype | Description | Comments | +|-----------|----------|-------------|----------| +|object_id|string|Either:
Group ID
Workspace Object ID
Redash Object ID
Scope name +|supports|string|One of:
AUTHORIZATION

CLUSTERS
CLUSTER_POLICIES
DIRECTORIES
EXPERIMENTS
FILES
INSTANCE_POOLS
JOBS
NOTEBOOKS
PIPELINES
REGISTERED_MODELS
REPOS
SERVING_ENDPOINTS
SQL_WAREHOUSES +|raw_object_permissions|JSON|JSON-serialized response of:
Generic Permissions
Secret ACL
Group roles and entitlements
Redash permissions|
From f23b7a576b2fc141565c3325c83890a8e2a1a117 Mon Sep 17 00:00:00 2001 From: FastLee Date: Tue, 3 Oct 2023 15:42:45 -0400 Subject: [PATCH 16/16] Fixed merge conflict --- docs/table_persistence.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/table_persistence.md b/docs/table_persistence.md index 461f0b2ac0..b96925870d 100644 --- a/docs/table_persistence.md +++ b/docs/table_persistence.md @@ -24,15 +24,16 @@ Table Utilization: Holds Inventory of all tables in all databases and their relevant metadata. -| Column | Datatype | Description | Comments | -|-----------|----------|-------------|----------| -| catalog | string | Original catalog of the table. _hive_metastore_ by default | -| database | string | Original schema of the table | -| name |string|Name of the table| -|object_type|string|MANAGED, EXTERNAL, or VIEW| -|table_format|string|Table provider. Like delta or json or parquet.| -|location|string|Location of the data for table| -|view_text|nullable string|If the table is the view, then this column holds the definition of the view| +| Column | Datatype | Description | Comments | +|--------------|----------|-------------|----------| +| catalog | string | Original catalog of the table. _hive_metastore_ by default | +| database | string | Original schema of the table | +| name |string|Name of the table| +| object_type |string|MANAGED, EXTERNAL, or VIEW| +| table_format |string|Table provider. Like delta or json or parquet.| +| location |string|Location of the data for table| +| view_text |nullable string|If the table is the view, then this column holds the definition of the view| +| upgraded_to |string|Upgrade Target (3 level namespace)|
@@ -78,11 +79,11 @@ List of DBFS mount points. #### _$inventory_.permissions Workspace object level permissions -| Column | Datatype | Description | Comments | -|-----------|----------|-------------|----------| -|object_id|string|Either:
Group ID
Workspace Object ID
Redash Object ID
Scope name -|supports|string|One of:
AUTHORIZATION

CLUSTERS
CLUSTER_POLICIES
DIRECTORIES
EXPERIMENTS
FILES
INSTANCE_POOLS
JOBS
NOTEBOOKS
PIPELINES
REGISTERED_MODELS
REPOS
SERVING_ENDPOINTS
SQL_WAREHOUSES -|raw_object_permissions|JSON|JSON-serialized response of:
Generic Permissions
Secret ACL
Group roles and entitlements
Redash permissions| +| Column | Datatype | Description | Comments | +|------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| +| object_id | string | Either:
Group ID
Workspace Object ID
Redash Object ID
Scope name | | +| supports | string | One of:
AUTHORIZATION

CLUSTERS
CLUSTER_POLICIES
DIRECTORIES
EXPERIMENTS
FILES
INSTANCE_POOLS
JOBS
NOTEBOOKS
PIPELINES
REGISTERED_MODELS
REPOS
SERVING_ENDPOINTS
SQL_WAREHOUSES | | +| raw_object_permissions | JSON | JSON-serialized response of:
Generic Permissions
Secret ACL
Group roles and entitlements
Redash permissions | |