Skip to content

Commit

Permalink
Support dbt-cloud config dict in dbt_project.yml (#8527)
Browse files Browse the repository at this point in the history
* first pass at adding dbt-cloud config

* changelog

* fix test, add direct validation
  • Loading branch information
emmyoop committed Sep 5, 2023
1 parent 3be943e commit cabb943
Show file tree
Hide file tree
Showing 5 changed files with 69 additions and 1 deletion.
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20230830-212828.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Accept a `dbt-cloud` config in dbt_project.yml
time: 2023-08-30T21:28:28.976746-05:00
custom:
Author: emmyoop
Issue: "8438"
6 changes: 6 additions & 0 deletions core/dbt/config/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,7 @@ def create_project(self, rendered: RenderComponents) -> "Project":
metrics: Dict[str, Any]
exposures: Dict[str, Any]
vars_value: VarProvider
dbt_cloud: Dict[str, Any]

dispatch = cfg.dispatch
models = cfg.models
Expand Down Expand Up @@ -433,6 +434,8 @@ def create_project(self, rendered: RenderComponents) -> "Project":
manifest_selectors = SelectorDict.parse_from_selectors_list(
rendered.selectors_dict["selectors"]
)
dbt_cloud = cfg.dbt_cloud

project = Project(
project_name=name,
version=version,
Expand Down Expand Up @@ -470,6 +473,7 @@ def create_project(self, rendered: RenderComponents) -> "Project":
config_version=cfg.config_version,
unrendered=unrendered,
project_env_vars=project_env_vars,
dbt_cloud=dbt_cloud,
)
# sanity check - this means an internal issue
project.validate()
Expand Down Expand Up @@ -572,6 +576,7 @@ class Project:
config_version: int
unrendered: RenderComponents
project_env_vars: Dict[str, Any]
dbt_cloud: Dict[str, Any]

@property
def all_source_paths(self) -> List[str]:
Expand Down Expand Up @@ -640,6 +645,7 @@ def to_project_config(self, with_packages=False):
"vars": self.vars.to_dict(),
"require-dbt-version": [v.to_version_string() for v in self.dbt_version],
"config-version": self.config_version,
"dbt-cloud": self.dbt_cloud,
}
)
if self.query_comment:
Expand Down
1 change: 1 addition & 0 deletions core/dbt/config/runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ def from_parts(
args=args,
cli_vars=cli_vars,
dependencies=dependencies,
dbt_cloud=project.dbt_cloud,
)

# Called by 'load_projects' in this class
Expand Down
5 changes: 5 additions & 0 deletions core/dbt/contracts/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@ class Project(HyphenatedDbtClassMixin, Replaceable):
)
packages: List[PackageSpec] = field(default_factory=list)
query_comment: Optional[Union[QueryComment, NoValue, str]] = field(default_factory=NoValue)
dbt_cloud: Optional[Dict[str, Any]] = None

@classmethod
def validate(cls, data):
Expand All @@ -239,6 +240,10 @@ def validate(cls, data):
or not isinstance(entry["search_order"], list)
):
raise ValidationError(f"Invalid project dispatch config: {entry}")
if "dbt_cloud" in data and not isinstance(data["dbt_cloud"], dict):
raise ValidationError(
f"Invalid dbt_cloud config. Expected a 'dict' but got '{type(data['dbt_cloud'])}'"
)


@dataclass
Expand Down
52 changes: 51 additions & 1 deletion tests/functional/basic/test_project.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import os
import pytest
from dbt.tests.util import run_dbt, update_config_file
import yaml
from pathlib import Path
from dbt.tests.util import run_dbt, update_config_file, write_config_file
from dbt.exceptions import ProjectContractError


Expand Down Expand Up @@ -62,3 +65,50 @@ def test_invalid_version(self, project):
assert "at path ['version']: 'invalid' is not valid under any of the given schemas" in str(
excinfo.value
)


class TestProjectDbtCloudConfig:
@pytest.fixture(scope="class")
def models(self):
return {"simple_model.sql": simple_model_sql, "simple_model.yml": simple_model_yml}

def test_dbt_cloud(self, project):
run_dbt(["parse"], expect_pass=True)
conf = yaml.safe_load(
Path(os.path.join(project.project_root, "dbt_project.yml")).read_text()
)
assert conf == {"name": "test", "profile": "test"}

config = {
"name": "test",
"profile": "test",
"dbt-cloud": {
"account_id": "123",
"application": "test",
"environment": "test",
"api_key": "test",
},
}
write_config_file(config, project.project_root, "dbt_project.yml")
run_dbt(["parse"], expect_pass=True)
conf = yaml.safe_load(
Path(os.path.join(project.project_root, "dbt_project.yml")).read_text()
)
assert conf == config


class TestProjectDbtCloudConfigString:
@pytest.fixture(scope="class")
def models(self):
return {"simple_model.sql": simple_model_sql, "simple_model.yml": simple_model_yml}

def test_dbt_cloud_invalid(self, project):
run_dbt()
config = {"name": "test", "profile": "test", "dbt-cloud": "Some string"}
update_config_file(config, "dbt_project.yml")
expected_err = (
"at path ['dbt-cloud']: 'Some string' is not valid under any of the given schemas"
)
with pytest.raises(ProjectContractError) as excinfo:
run_dbt()
assert expected_err in str(excinfo.value)

0 comments on commit cabb943

Please sign in to comment.