Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add tests to see if pre_model_hook works for tests around snowflake_warehouse #1070

Merged
merged 13 commits into from
Jun 14, 2024
Merged
8 changes: 8 additions & 0 deletions dbt/adapters/snowflake/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from dbt.adapters.base.meta import available
from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability
from dbt.adapters.sql import SQLAdapter
from dbt.adapters.events.logging import AdapterLogger
from dbt.adapters.sql.impl import (
LIST_SCHEMAS_MACRO_NAME,
LIST_RELATIONS_MACRO_NAME,
Expand All @@ -19,6 +20,8 @@
from dbt_common.exceptions import CompilationError, DbtDatabaseError, DbtRuntimeError
from dbt_common.utils import filter_null_values

logger = AdapterLogger("Snowflake")


@dataclass
class SnowflakeConfig(AdapterConfig):
Expand Down Expand Up @@ -99,15 +102,20 @@ def _use_warehouse(self, warehouse: str):
def pre_model_hook(self, config: Mapping[str, Any]) -> Optional[str]:
default_warehouse = self.config.credentials.warehouse
warehouse = config.get("snowflake_warehouse", default_warehouse)
logger.info("Running pre_model_hook:")
logger.info(f"Default warehouse: {default_warehouse}, Selected warehouse: {warehouse}")
if warehouse == default_warehouse or warehouse is None:
return None
previous = self._get_warehouse()
self._use_warehouse(warehouse)
logger.info(f"Changed wareho use from {previous} to {warehouse}")
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want to put these logs behind condition of warehouses being different?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@McKnight-42 I don't think we should add any new logging here. The actual queries being run (select current_warehouse() as warehouse + use warehouse different_warehouse) are already appearing in the logs.

return previous

def post_model_hook(self, config: Mapping[str, Any], context: Optional[str]) -> None:
logger.info(f"Running post_model_hook with config, and context: {context}")
if context is not None:
self._use_warehouse(context)
logger.info(f"Restored warehouse to {context}")

def list_schemas(self, database: str) -> List[str]:
try:
Expand Down
4 changes: 2 additions & 2 deletions dev-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# install latest changes in dbt-core
git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
# install latest changes in dbt-core"
git+https://github.com/dbt-labs/dbt-core.git@mcknight/10198#egg=dbt-core&subdirectory=core
git+https://github.com/dbt-labs/dbt-adapters.git
McKnight-42 marked this conversation as resolved.
Show resolved Hide resolved
git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter
git+https://github.com/dbt-labs/dbt-common.git
Expand Down
31 changes: 26 additions & 5 deletions tests/functional/warehouse_test/test_warehouses.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,35 +3,34 @@

import os


models__override_warehouse_sql = """
{{ config(snowflake_warehouse=env_var('SNOWFLAKE_TEST_ALT_WAREHOUSE', 'DBT_TEST_ALT'), materialized='table') }}
select current_warehouse() as warehouse

"""

models__expected_warehouse_sql = """
{{ config(materialized='table') }}
select '{{ env_var("SNOWFLAKE_TEST_ALT_WAREHOUSE", "DBT_TEST_ALT") }}' as warehouse

"""

models__invalid_warehouse_sql = """
{{ config(snowflake_warehouse='DBT_TEST_DOES_NOT_EXIST') }}
select current_warehouse() as warehouse

"""

project_config_models__override_warehouse_sql = """
{{ config(materialized='table') }}
select current_warehouse() as warehouse

"""

project_config_models__expected_warehouse_sql = """
{{ config(materialized='table') }}
select '{{ env_var("SNOWFLAKE_TEST_ALT_WAREHOUSE", "DBT_TEST_ALT") }}' as warehouse
"""

project_config_models__invalid_warehouse_sql = """
{{ config(materialized='table') }}
select current_warehouse() as warehouse
"""


Expand Down Expand Up @@ -90,3 +89,25 @@ def test_snowflake_override_ok(self, project):
]
)
check_relations_equal(project.adapter, ["OVERRIDE_WAREHOUSE", "EXPECTED_WAREHOUSE"])


class TestInvalidConfigWarehouse:
@pytest.fixture(scope="class")
def models(self):
return {
"invalid_warehouse.sql": project_config_models__invalid_warehouse_sql,
}

@pytest.fixture(scope="class")
def project_config_update(self):
return {
"config-version": 2,
"models": {
"test": {
"snowflake_warehouse": "DBT_TEST_DOES_NOT_EXIST",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, nicely done - we should expect the dbt test to try using a Snowflake warehouse that doesn't exist (use warehouse DBT_TEST_DOES_NOT_EXIST), where previously it wouldn't, and so it should fail with Object does not exist, or operation cannot be performed.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

went ahead and also added a assert to catch if that error message stops showing up.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For the sake of completeness, can we add a positive test case that uses this config option, just to make sure it's setting it? The error message we're getting in the negative test case is pretty general.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added a new test

},
},
}

def test_snowflake_override_invalid(self, project):
run_dbt(["run", "--models", "invalid_warehouse"], expect_pass=False)
Loading