Skip to content

Commit

Permalink
Test if create-catalogs-schemas works with tables defined as mount …
Browse files Browse the repository at this point in the history
…paths (#1578)

## Changes
Add unit test to show that the `create-catalogs-schemas` logic works
with tables defined as mount paths.

### Linked issues
Resolves #1039

### Functionality 

- [ ] added relevant user documentation
- [ ] added new CLI command
- [ ] modified existing command: `databricks labs ucx ...`
- [ ] added a new workflow
- [ ] modified existing workflow: `...`
- [ ] added a new table
- [ ] modified existing table: `...`

### Tests
<!-- How is this tested? Please see the checklist below and also
describe any other relevant tests -->

- [ ] manually tested
- [x] added unit tests
- [ ] added integration tests
- [ ] verified on staging environment (screenshot attached)
  • Loading branch information
JCZuurmond authored Apr 30, 2024
1 parent 00d2e6e commit 11798e3
Showing 1 changed file with 52 additions and 20 deletions.
72 changes: 52 additions & 20 deletions tests/unit/hive_metastore/test_catalog_schema.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from unittest.mock import create_autospec
from unittest.mock import call, create_autospec

import pytest
from databricks.labs.blueprint.installation import MockInstallation
Expand Down Expand Up @@ -46,6 +46,22 @@ def prepare_test(ws, backend: MockBackend | None = None) -> CatalogSchema:
'src_table': 'table2',
'workspace_name': 'workspace',
},
{
'catalog_name': 'catalog1',
'dst_schema': 'schema2',
'dst_table': 'table3',
'src_schema': 'schema1',
'src_table': 'abfss://container@msft/path/dest1',
'workspace_name': 'workspace',
},
{
'catalog_name': 'catalog3',
'dst_schema': 'schema3',
'dst_table': 'table1',
'src_schema': 'schema1',
'src_table': 'abfss://container@msft/path/dest2',
'workspace_name': 'workspace',
},
]
}
)
Expand All @@ -54,36 +70,43 @@ def prepare_test(ws, backend: MockBackend | None = None) -> CatalogSchema:
grants = [
Grant('user1', 'SELECT', 'catalog1', 'schema3', 'table'),
Grant('user1', 'MODIFY', 'catalog2', 'schema2', 'table'),
Grant('user1', 'SELECY', 'catalog2', 'schema2', 'table2'),
Grant('user1', 'SELECT', 'catalog2', 'schema2', 'table2'),
Grant('user1', 'USAGE', 'hive_metastore', 'schema3'),
Grant('user1', 'USAGE', 'hive_metastore', 'schema2'),
]
principal_acl.get_interactive_cluster_grants.return_value = grants
return CatalogSchema(ws, table_mapping, principal_acl, backend)


def test_create():
@pytest.mark.parametrize("location", ["s3://foo/bar", "s3://foo/bar/test"])
def test_create_all_catalogs_schemas_creates_catalogs(location: str):
"""Catalog 2 and 3 should be created; catalog 1 already exists."""
ws = create_autospec(WorkspaceClient)
mock_prompts = MockPrompts({"Please provide storage location url for catalog: *": "s3://foo/bar"})
mock_prompts = MockPrompts({"Please provide storage location url for catalog: *": location})

catalog_schema = prepare_test(ws)
catalog_schema.create_all_catalogs_schemas(
mock_prompts,
)
ws.catalogs.create.assert_called_once_with("catalog2", storage_root="s3://foo/bar", comment="Created by UCX")
ws.schemas.create.assert_any_call("schema2", "catalog2", comment="Created by UCX")
ws.schemas.create.assert_any_call("schema3", "catalog1", comment="Created by UCX")
catalog_schema.create_all_catalogs_schemas(mock_prompts)

calls = [
call("catalog2", storage_root=location, comment="Created by UCX"),
call("catalog3", storage_root=location, comment="Created by UCX"),
]
ws.catalogs.create.assert_has_calls(calls, any_order=True)


def test_create_sub_location():
@pytest.mark.parametrize(
"catalog,schema",
[("catalog1", "schema2"), ("catalog1", "schema3"), ("catalog2", "schema2"), ("catalog3", "schema3")],
)
def test_create_all_catalogs_schemas_creates_schemas(catalog: str, schema: str):
"""Non-existing schemas should be created."""
ws = create_autospec(WorkspaceClient)
mock_prompts = MockPrompts({"Please provide storage location url for catalog: *": "s3://foo/bar/test"})
mock_prompts = MockPrompts({"Please provide storage location url for catalog: *": "metastore"})

catalog_schema = prepare_test(ws)
catalog_schema.create_all_catalogs_schemas(mock_prompts)
ws.catalogs.create.assert_called_once_with("catalog2", storage_root="s3://foo/bar/test", comment="Created by UCX")
ws.schemas.create.assert_any_call("schema2", "catalog2", comment="Created by UCX")
ws.schemas.create.assert_any_call("schema3", "catalog1", comment="Created by UCX")

ws.schemas.create.assert_any_call(schema, catalog, comment="Created by UCX")


def test_create_bad_location():
Expand All @@ -103,19 +126,28 @@ def test_no_catalog_storage():

catalog_schema = prepare_test(ws)
catalog_schema.create_all_catalogs_schemas(mock_prompts)
ws.catalogs.create.assert_called_once_with("catalog2", comment="Created by UCX")

calls = [
call("catalog2", comment="Created by UCX"),
call("catalog3", comment="Created by UCX"),
]
ws.catalogs.create.assert_has_calls(calls, any_order=True)


def test_catalog_schema_acl():
ws = create_autospec(WorkspaceClient)
backend = MockBackend()
mock_prompts = MockPrompts({"Please provide storage location url for catalog: *": ""})

catalog_schema = prepare_test(ws, backend)
catalog_schema.create_all_catalogs_schemas(
mock_prompts,
)
catalog_schema.create_all_catalogs_schemas(mock_prompts)

calls = [
call("catalog2", comment="Created by UCX"),
call("catalog3", comment="Created by UCX"),
]
ws.catalogs.create.assert_has_calls(calls, any_order=True)
ws.schemas.create.assert_any_call("schema2", "catalog2", comment="Created by UCX")
ws.catalogs.create.assert_called_once_with("catalog2", comment="Created by UCX")
queries = [
'GRANT USE SCHEMA ON DATABASE catalog1.schema3 TO `user1`',
'GRANT USE SCHEMA ON DATABASE catalog2.schema2 TO `user1`',
Expand Down

0 comments on commit 11798e3

Please sign in to comment.