From ed5d07f970dc91282ad629a615eb6a15256e817e Mon Sep 17 00:00:00 2001 From: Colin Date: Mon, 23 Sep 2024 16:40:44 -0700 Subject: [PATCH 1/6] add builder config to node config --- core/dbt/parser/schema_generic_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/parser/schema_generic_tests.py b/core/dbt/parser/schema_generic_tests.py index 14e2dbc862a..01211183afc 100644 --- a/core/dbt/parser/schema_generic_tests.py +++ b/core/dbt/parser/schema_generic_tests.py @@ -200,7 +200,7 @@ def parse_generic_test( # this is the ContextConfig that is used in render_update config: ContextConfig = self.initial_config(fqn) - + config.add_config_call(builder.config) # builder.args contains keyword args for the test macro, # not configs which have been separated out in the builder. # The keyword args are not completely rendered until compilation. From 33d8aef328d613064490d0518b8d72d66e5282af Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 25 Sep 2024 15:46:43 -0700 Subject: [PATCH 2/6] add changie --- .changes/unreleased/Fixes-20240925-154514.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240925-154514.yaml diff --git a/.changes/unreleased/Fixes-20240925-154514.yaml b/.changes/unreleased/Fixes-20240925-154514.yaml new file mode 100644 index 00000000000..8db05cc6a0e --- /dev/null +++ b/.changes/unreleased/Fixes-20240925-154514.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Pass test user config to adapter pre_hook by explicitly adding test builder config to node +time: 2024-09-25T15:45:14.459598-07:00 +custom: + Author: colin-rogers-dbt + Issue: "10484" From 90552ca89e4295beb9e9b8f6bb21e14bc8bbada4 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 25 Sep 2024 19:11:30 -0700 Subject: [PATCH 3/6] raise expected exceptions --- core/dbt/context/context_config.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/dbt/context/context_config.py b/core/dbt/context/context_config.py index caf83d425fe..c77f8d19436 100644 --- a/core/dbt/context/context_config.py +++ b/core/dbt/context/context_config.py @@ -6,9 +6,11 @@ from dbt.adapters.factory import get_config_class_by_name from dbt.config import IsFQNResource, Project, RuntimeConfig from dbt.contracts.graph.model_config import get_config_for +from dbt.exceptions import SchemaConfigError from dbt.node_types import NodeType from dbt.utils import fqn_search from dbt_common.contracts.config.base import BaseConfig, merge_config_dicts +from dbt_common.dataclass_schema import ValidationError from dbt_common.exceptions import DbtInternalError @@ -236,8 +238,12 @@ def calculate_node_config_dict( base=base, patch_config_dict=patch_config_dict, ) - finalized = config.finalize_and_validate() - return finalized.to_dict(omit_none=True) + try: + finalized = config.finalize_and_validate() + return finalized.to_dict(omit_none=True) + except ValidationError as exc: + # we got a ValidationError - probably bad types in config() + raise SchemaConfigError(exc, node=config) from exc class UnrenderedConfigGenerator(BaseContextConfigGenerator[Dict[str, Any]]): From 519f2d9f315ac0fb4697f7b303a7a518c9da41dd Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 8 Oct 2024 12:50:13 -0700 Subject: [PATCH 4/6] add code comment and additional tests --- core/dbt/parser/schema_generic_tests.py | 2 + .../schema_tests/data_test_config.py | 54 ++++++++++++++----- tests/functional/schema_tests/fixtures.py | 19 ++++++- 3 files changed, 60 insertions(+), 15 deletions(-) diff --git a/core/dbt/parser/schema_generic_tests.py b/core/dbt/parser/schema_generic_tests.py index 01211183afc..9f2538f06f0 100644 --- a/core/dbt/parser/schema_generic_tests.py +++ b/core/dbt/parser/schema_generic_tests.py @@ -200,6 +200,8 @@ def parse_generic_test( # this is the ContextConfig that is used in render_update config: ContextConfig = self.initial_config(fqn) + # Adding the builder's config to the ContextConfig + # is needed to ensure the config makes it to the pre_model hook which dbt-snowflake needs config.add_config_call(builder.config) # builder.args contains keyword args for the test macro, # not configs which have been separated out in the builder. diff --git a/tests/functional/schema_tests/data_test_config.py b/tests/functional/schema_tests/data_test_config.py index 99a2dacdca6..a6a16818000 100644 --- a/tests/functional/schema_tests/data_test_config.py +++ b/tests/functional/schema_tests/data_test_config.py @@ -2,6 +2,8 @@ import pytest +from dbt.contracts.graph.manifest import Manifest +from dbt.contracts.graph.nodes import TestNode from dbt.exceptions import CompilationError from dbt.tests.util import get_manifest, run_dbt from tests.functional.schema_tests.fixtures import ( @@ -10,14 +12,44 @@ same_key_error_yml, seed_csv, table_sql, + test_custom_color_from_config, ) +def _select_test_node(manifest: Manifest, pattern: re.Pattern[str]): + # Find the test_id dynamically + test_id = None + for node_id in manifest.nodes: + if pattern.match(node_id): + test_id = node_id + break + + # Ensure the test_id was found + assert test_id is not None, "Test ID matching the pattern was not found in the manifest nodes" + return manifest.nodes[test_id] + + +def get_table_persistence(project, table_name): + sql = f""" + SELECT + relpersistence + FROM pg_class + WHERE relname like '%{table_name}%' + """ + result = project.run_sql(sql, fetch="one") + assert len(result) == 1 + return result[0] + + class BaseDataTestsConfig: @pytest.fixture(scope="class") def seeds(self): return {"seed.csv": seed_csv} + @pytest.fixture(scope="class") + def macros(self): + return {"custom_color_from_config.sql": test_custom_color_from_config} + @pytest.fixture(scope="class") def project_config_update(self): return { @@ -36,29 +68,23 @@ def models(self): def test_custom_config(self, project): run_dbt(["run"]) - run_dbt(["test"], expect_pass=False) + run_dbt(["test", "--log-level", "debug"], expect_pass=False) manifest = get_manifest(project.project_root) # Pattern to match the test_id without the specific suffix pattern = re.compile(r"test\.test\.accepted_values_table_color__blue__red\.\d+") - # Find the test_id dynamically - test_id = None - for node_id in manifest.nodes: - if pattern.match(node_id): - test_id = node_id - break - - # Ensure the test_id was found - assert ( - test_id is not None - ), "Test ID matching the pattern was not found in the manifest nodes" - + test_node: TestNode = _select_test_node(manifest, pattern) # Proceed with the assertions - test_node = manifest.nodes[test_id] assert "custom_config_key" in test_node.config assert test_node.config["custom_config_key"] == "some_value" + # pattern = re.compile(r"test\.test\.custom_color_from_config.*") + # test_node = _select_test_node(manifest, pattern) + persistence = get_table_persistence(project, "custom_color_from_config_table_color") + + assert persistence == "u" + class TestMixedDataTestConfig(BaseDataTestsConfig): @pytest.fixture(scope="class") diff --git a/tests/functional/schema_tests/fixtures.py b/tests/functional/schema_tests/fixtures.py index f07ba1b7ca6..ac266545712 100644 --- a/tests/functional/schema_tests/fixtures.py +++ b/tests/functional/schema_tests/fixtures.py @@ -1285,6 +1285,12 @@ values: ['blue', 'red'] config: custom_config_key: some_value + - custom_color_from_config: + severity: error + config: + test_color: orange + store_failures: true + unlogged: True """ mixed_config_yml = """ @@ -1296,9 +1302,13 @@ data_tests: - accepted_values: values: ['blue', 'red'] - severity: warn config: custom_config_key: some_value + severity: warn + - custom_color_from_config: + severity: error + config: + test_color: blue """ same_key_error_yml = """ @@ -1333,3 +1343,10 @@ -- content of the table.sql select * from {{ ref('seed') }} """ + +test_custom_color_from_config = """ +{% test custom_color_from_config(model, column_name) %} + select * from {{ model }} + where color = '{{ config.get('test_color') }}' +{% endtest %} +""" From cb13b55320372a317c276421afe713efaed53a34 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 10 Oct 2024 13:12:29 -0700 Subject: [PATCH 5/6] update tests --- tests/functional/configs/test_configs.py | 9 +++++---- tests/functional/configs/test_disabled_model.py | 7 +++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/functional/configs/test_configs.py b/tests/functional/configs/test_configs.py index 2bbfac85c5c..3614b851756 100644 --- a/tests/functional/configs/test_configs.py +++ b/tests/functional/configs/test_configs.py @@ -2,6 +2,7 @@ import pytest +from dbt.exceptions import SchemaConfigError from dbt.tests.util import ( check_relations_equal, run_dbt, @@ -81,7 +82,7 @@ def test_tests_materialization_proj_config(self, project): tests_dir = os.path.join(project.project_root, "tests") write_file("select * from foo", tests_dir, "test.sql") - with pytest.raises(ValidationError): + with pytest.raises(SchemaConfigError): run_dbt() @@ -93,7 +94,7 @@ def test_seeds_materialization_proj_config(self, project): seeds_dir = os.path.join(project.project_root, "seeds") write_file("id1, id2\n1, 2", seeds_dir, "seed.csv") - with pytest.raises(ValidationError): + with pytest.raises(SchemaConfigError): run_dbt() @@ -107,7 +108,7 @@ def test_seeds_materialization_schema_config(self, project): ) write_file("id1, id2\n1, 2", seeds_dir, "myseed.csv") - with pytest.raises(ValidationError): + with pytest.raises(SchemaConfigError): run_dbt() @@ -133,5 +134,5 @@ def test_snapshots_materialization_schema_config(self, project): ) write_file(simple_snapshot, snapshots_dir, "mysnapshot.sql") - with pytest.raises(ValidationError): + with pytest.raises(SchemaConfigError): run_dbt() diff --git a/tests/functional/configs/test_disabled_model.py b/tests/functional/configs/test_disabled_model.py index 23cf8fde1e0..a918067ac15 100644 --- a/tests/functional/configs/test_disabled_model.py +++ b/tests/functional/configs/test_disabled_model.py @@ -1,8 +1,7 @@ import pytest -from dbt.exceptions import CompilationError, ParsingError +from dbt.exceptions import CompilationError, ParsingError, SchemaConfigError from dbt.tests.util import get_manifest, run_dbt -from dbt_common.dataclass_schema import ValidationError from tests.functional.configs.fixtures import ( my_model, my_model_2, @@ -394,8 +393,8 @@ def models(self): "my_model.sql": my_model, } - def test_invalis_config(self, project): - with pytest.raises(ValidationError) as exc: + def test_invalid_config(self, project): + with pytest.raises(SchemaConfigError) as exc: run_dbt(["parse"]) exc_str = " ".join(str(exc.value).split()) # flatten all whitespace expected_msg = "'True and False' is not of type 'boolean'" From efecd001c20ceb2fc331c618458c12b7e4ee81d0 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 10 Oct 2024 13:42:42 -0700 Subject: [PATCH 6/6] update tests --- tests/functional/configs/test_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/configs/test_configs.py b/tests/functional/configs/test_configs.py index 3614b851756..952b951e507 100644 --- a/tests/functional/configs/test_configs.py +++ b/tests/functional/configs/test_configs.py @@ -134,5 +134,5 @@ def test_snapshots_materialization_schema_config(self, project): ) write_file(simple_snapshot, snapshots_dir, "mysnapshot.sql") - with pytest.raises(SchemaConfigError): + with pytest.raises(ValidationError): run_dbt()