From b6015a678d368d99efe0400eb2d33304cbb01b37 Mon Sep 17 00:00:00 2001 From: Serge Smertin <259697+nfx@users.noreply.github.com> Date: Fri, 29 Sep 2023 14:22:41 +0200 Subject: [PATCH] Release v0.2.0 (#328) # Version changelog ## 0.2.0 * Added retrieving for all account-level groups with matching names to workspace-level groups in case no explicit configuration ([#277](https://github.com/databricks/ucx/pull/277)). * Added crawler for Azure Service principals used for direct storage access ([#305](https://github.com/databricks/ucx/pull/305)). * Added more SQL queries to the assessment step dashboard ([#269](https://github.com/databricks/ucx/pull/269)). * Added filtering out for job clusters in the clusters crawler ([#298](https://github.com/databricks/ucx/pull/298)). * Added recording errors from `crawl_tables` step in `$inventory.table_failures` table and display counter on the dashboard ([#300](https://github.com/databricks/ucx/pull/300)). * Added comprehensive introduction user manual ([#273](https://github.com/databricks/ucx/pull/273)). * Added interactive tutorial for local group migration readme ([#291](https://github.com/databricks/ucx/pull/291)). * Added tutorial links to the landing page of documentation ([#290](https://github.com/databricks/ucx/pull/290)). * Added (internal) support for account-level configuration and multi-cloud workspace list ([#264](https://github.com/databricks/ucx/pull/264)). * Improved order of tasks in the README notebook ([#286](https://github.com/databricks/ucx/pull/286)). * Improved installation script to run in a Windows Git Bash terminal ([#282](https://github.com/databricks/ucx/pull/282)). * Improved installation script by setting log level to uppercase by default ([#271](https://github.com/databricks/ucx/pull/271)). * Improved installation finish messages within installer script ([#267](https://github.com/databricks/ucx/pull/267)). * Improved automation for `MANAGED` table migration and continued building tables migration component ([#295](https://github.com/databricks/ucx/pull/295)). * Fixed debug notebook code with refactored package structure ([#250](https://github.com/databricks/ucx/pull/250)) ([#265](https://github.com/databricks/ucx/pull/265)). * Fixed replacement of custom configured database to replicate in the report for external locations ([#296](https://github.com/databricks/ucx/pull/296)). * Removed redundant `notebooks` top-level folder ([#263](https://github.com/databricks/ucx/pull/263)). * Split checking for test failures and linting errors into independent GitHub Actions checks ([#287](https://github.com/databricks/ucx/pull/287)). * Verify query metadata for assessment dashboards during unit tests ([#294](https://github.com/databricks/ucx/pull/294)). --- CHANGELOG.md | 22 +++++++++++++++++++ src/databricks/labs/ucx/__about__.py | 2 +- src/databricks/labs/ucx/install.py | 2 +- tests/integration/conftest.py | 7 +++--- .../hive_metastore/test_migrate.py | 2 +- tests/integration/test_installation.py | 12 +++++----- 6 files changed, 34 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 094878ec83..6564e666ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Version changelog +## 0.2.0 + +* Added retrieving for all account-level groups with matching names to workspace-level groups in case no explicit configuration ([#277](https://github.com/databricks/ucx/pull/277)). +* Added crawler for Azure Service principals used for direct storage access ([#305](https://github.com/databricks/ucx/pull/305)). +* Added more SQL queries to the assessment step dashboard ([#269](https://github.com/databricks/ucx/pull/269)). +* Added filtering out for job clusters in the clusters crawler ([#298](https://github.com/databricks/ucx/pull/298)). +* Added recording errors from `crawl_tables` step in `$inventory.table_failures` table and display counter on the dashboard ([#300](https://github.com/databricks/ucx/pull/300)). +* Added comprehensive introduction user manual ([#273](https://github.com/databricks/ucx/pull/273)). +* Added interactive tutorial for local group migration readme ([#291](https://github.com/databricks/ucx/pull/291)). +* Added tutorial links to the landing page of documentation ([#290](https://github.com/databricks/ucx/pull/290)). +* Added (internal) support for account-level configuration and multi-cloud workspace list ([#264](https://github.com/databricks/ucx/pull/264)). +* Improved order of tasks in the README notebook ([#286](https://github.com/databricks/ucx/pull/286)). +* Improved installation script to run in a Windows Git Bash terminal ([#282](https://github.com/databricks/ucx/pull/282)). +* Improved installation script by setting log level to uppercase by default ([#271](https://github.com/databricks/ucx/pull/271)). +* Improved installation finish messages within installer script ([#267](https://github.com/databricks/ucx/pull/267)). +* Improved automation for `MANAGED` table migration and continued building tables migration component ([#295](https://github.com/databricks/ucx/pull/295)). +* Fixed debug notebook code with refactored package structure ([#250](https://github.com/databricks/ucx/pull/250)) ([#265](https://github.com/databricks/ucx/pull/265)). +* Fixed replacement of custom configured database to replicate in the report for external locations ([#296](https://github.com/databricks/ucx/pull/296)). +* Removed redundant `notebooks` top-level folder ([#263](https://github.com/databricks/ucx/pull/263)). +* Split checking for test failures and linting errors into independent GitHub Actions checks ([#287](https://github.com/databricks/ucx/pull/287)). +* Verify query metadata for assessment dashboards during unit tests ([#294](https://github.com/databricks/ucx/pull/294)). + ## 0.1.1 * Added batched iteration for `INSERT INTO` queries in `StatementExecutionBackend` with default `max_records_per_batch=1000` ([#237](https://github.com/databricks/ucx/pull/237)). diff --git a/src/databricks/labs/ucx/__about__.py b/src/databricks/labs/ucx/__about__.py index 485f44ac21..d3ec452c31 100644 --- a/src/databricks/labs/ucx/__about__.py +++ b/src/databricks/labs/ucx/__about__.py @@ -1 +1 @@ -__version__ = "0.1.1" +__version__ = "0.2.0" diff --git a/src/databricks/labs/ucx/install.py b/src/databricks/labs/ucx/install.py index a984bc71c8..5f50bcfc69 100644 --- a/src/databricks/labs/ucx/install.py +++ b/src/databricks/labs/ucx/install.py @@ -393,7 +393,7 @@ def _job_settings(self, step_name: str, dbfs_path: str): tasks = sorted([t for t in _TASKS.values() if t.workflow == step_name], key=lambda _: _.name) return { "name": self._name(step_name), - "tags": {TAG_APP: self._app, TAG_STEP: step_name}, + "tags": {TAG_APP: self._app, TAG_STEP: step_name, "version": f"v{__version__}"}, "job_clusters": self._job_clusters({t.job_cluster for t in tasks}), "email_notifications": email_notifications, "tasks": [self._job_task(task, dbfs_path) for task in tasks], diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 78011ee97e..6f3997602e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -82,11 +82,10 @@ def test_catalog_fixture(make_catalog): @pytest.fixture def make_schema(sql_exec, make_random): - def create(*, catalog="hive_metastore", schema): + def create(*, catalog: str = "hive_metastore", schema: str | None = None): if schema is None: - schema = f"{catalog}.ucx_S{make_random(4)}".lower() - else: - schema = f"{catalog}.{schema}" + schema = f"ucx_S{make_random(4)}" + schema = f"{catalog}.{schema}".lower() sql_exec(f"CREATE SCHEMA {schema}") return schema diff --git a/tests/integration/hive_metastore/test_migrate.py b/tests/integration/hive_metastore/test_migrate.py index 9d1cf58a10..2fa0676178 100644 --- a/tests/integration/hive_metastore/test_migrate.py +++ b/tests/integration/hive_metastore/test_migrate.py @@ -15,7 +15,7 @@ def test_migrate_managed_tables(ws, make_catalog, make_schema, make_table): schema_a = make_schema(catalog="hive_metastore") _, target_schema = schema_a.split(".") - make_schema(catalog=target_catalog, schema_name=target_schema) + make_schema(catalog=target_catalog, schema=target_schema) managed_table = make_table(schema=schema_a) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 2f3c151f15..3c96d2dbfd 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -47,16 +47,16 @@ def test_jobs_with_no_inventory_database( sql_exec(f"GRANT MODIFY ON SCHEMA {schema_b} TO `{ws_group_b.display_name}`") cluster_policy = make_cluster_policy() + make_cluster_policy_permissions( + object_id=cluster_policy.policy_id, + permission_level=PermissionLevel.CAN_USE, + group_name=ws_group_a.display_name, + ) cpp_src = ws.permissions.get("cluster-policies", cluster_policy.policy_id) cluster_policy_src_permissions = sorted( [_ for _ in cpp_src.access_control_list if _.group_name == ws_group_a.display_name], key=lambda p: p.group_name, ) - make_cluster_policy_permissions( - object_id=cluster_policy.policy_id, - permission_level=random.choice([PermissionLevel.CAN_USE]), - group_name=ws_group_a.display_name, - ) job = make_job() make_job_permissions( @@ -86,7 +86,7 @@ def test_jobs_with_no_inventory_database( ) try: - for step in ["assessment", "migrate-groups", "migrate-groups-cleanup"]: + for step in ["assessment", "migrate-groups"]: logger.debug(f"starting {step} job: {ws.config.host}#job/{install._deployed_steps[step]}") ws.jobs.run_now(install._deployed_steps[step]).result()