Skip to content

Commit

Permalink
instead of a special "config" key, use a "+" prefix and only require …
Browse files Browse the repository at this point in the history 
…it on dict values

Make the package-duplicates test use a local dependency
  • Loading branch information
Jacob Beck committed Apr 20, 2020
1 parent c6b2346 commit 51baece
Show file tree
Hide file tree
Showing 38 changed files with 181 additions and 309 deletions.
11 changes: 5 additions & 6 deletions core/dbt/config/project.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -651,7 +651,7 @@ def as_v1(self):
# remove this from the v1 form
mutated.pop('vars')
# ok, now we want to look through all the existing cfgkeys and mirror
# it, except anything under 'config' gets included directly.
# it, except expand the '+' prefix.
for cfgkey in common_config_keys:
if cfgkey not in dct:
continue
Expand All @@ -666,11 +666,10 @@ def as_v1(self):
def _flatten_config(dct: Dict[str, Any]):
result = {}
for key, value in dct.items():
if isinstance(value, dict):
if key == 'config':
result.update(value)
else:
result[key] = _flatten_config(value)
if isinstance(value, dict) and not key.startswith('+'):
result[key] = _flatten_config(value)
else:
if key.startswith('+'):
key = key[1:]
result[key] = value
return result
10 changes: 4 additions & 6 deletions core/dbt/config/runtime.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,12 +228,10 @@ def _get_v2_config_paths(
paths: MutableSet[FQNPath],
) -> PathSet:
for key, value in config.items():
if isinstance(value, dict):
if key == 'config':
paths.add(path)
else:
self._get_v2_config_paths(value, path + (key,), paths)

if isinstance(value, dict) and not key.startswith('+'):
self._get_v2_config_paths(value, path + (key,), paths)
else:
paths.add(path)
return frozenset(paths)

def _get_v1_config_paths(
Expand Down
14 changes: 8 additions & 6 deletions core/dbt/context/context_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,12 +114,14 @@ def project_configs(
else:
model_configs = project.models
for level_config in fqn_search(model_configs, fqn):
if 'config' not in level_config:
continue

level_value = level_config['config']
if isinstance(level_value, dict):
yield deepcopy(level_value)
result = {}
for key, value in level_config.items():
if key.startswith('+'):
result[key[1:]] = deepcopy(value)
elif not isinstance(value, dict):
result[key] = deepcopy(value)

yield result

def active_project_configs(
self, fqn: List[str], resource_type: NodeType
Expand Down
6 changes: 3 additions & 3 deletions core/dbt/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -540,15 +540,15 @@ def executor(config: HasThreadingConfig) -> concurrent.futures.Executor:

def fqn_search(
root: Dict[str, Any], fqn: List[str]
) -> Iterator[Any]:
) -> Iterator[Dict[str, Any]]:
"""Iterate into a nested dictionary, looking for keys in the fqn as levels.
Yield level name, level config pairs.
Yield the level config.
"""
yield root

for level in fqn:
level_config = root.get(level, None)
if level_config is None:
if not isinstance(level_config, dict):
break
yield copy.deepcopy(level_config)
root = level_config
Expand Down
12 changes: 3 additions & 9 deletions test/integration/001_simple_copy_test/test_simple_copy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,7 @@ def seed_quote_cfg_with(self, extra):
cfg = {
'config-version': 2,
'seeds': {
'config': {
'quote_columns': False,
},
'quote_columns': False,
}
}
cfg.update(extra)
Expand Down Expand Up @@ -112,9 +110,7 @@ def test__snowflake__simple_copy(self):
self.use_default_project({
"data-paths": [self.dir("seed-initial")],
"seeds": {
'config': {
'quote_columns': False,
},
'quote_columns': False,
}
})
results = self.run_dbt(["seed"])
Expand Down Expand Up @@ -328,9 +324,7 @@ def test__snowflake__incremental_overwrite(self):
# Setting the incremental_strategy should make this succeed
self.use_default_project({
"models": {
'config': {
"incremental_strategy": "delete+insert"
},
"incremental_strategy": "delete+insert"
},
"data-paths": [self.dir("snowflake-seed-update")],
})
Expand Down
24 changes: 9 additions & 15 deletions test/integration/004_simple_snapshot_test/test_simple_snapshot.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,9 +114,7 @@ def project_config(self):
'macro-paths': ['custom-snapshot-macros', 'macros'],
'snapshot-paths': ['test-snapshots-checkall'],
'seeds': {
'config': {
'quote_columns': False,
},
'quote_columns': False,
}
}

Expand Down Expand Up @@ -316,12 +314,10 @@ def project_config(self):
"snapshot-paths": ['test-snapshots-select-noconfig'],
"snapshots": {
"test": {
"config": {
"target_schema": self.unique_schema(),
"unique_key": "id || '-' || first_name",
'strategy': 'timestamp',
'updated_at': 'updated_at',
},
"target_schema": self.unique_schema(),
"unique_key": "id || '-' || first_name",
'strategy': 'timestamp',
'updated_at': 'updated_at',
},
},
'macro-paths': ['macros'],
Expand Down Expand Up @@ -561,12 +557,10 @@ def project_config(self):
"snapshot-paths": ['test-check-col-snapshots-noconfig'],
"snapshots": {
"test": {
"config": {
"target_schema": self.unique_schema(),
"unique_key": "id || '-' || first_name",
"strategy": "check",
"check_cols": ["email"],
},
"target_schema": self.unique_schema(),
"unique_key": "id || '-' || first_name",
"strategy": "check",
"check_cols": ["email"],
},
},
'macro-paths': ['macros'],
Expand Down
18 changes: 6 additions & 12 deletions test/integration/005_simple_seed_test/test_seed_type_override.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,21 +15,15 @@ def project_config(self):
'macro-paths': ['macros'],
'seeds': {
'test': {
'config': {
'enabled': False,
'quote_columns': True,
},
'enabled': False,
'quote_columns': True,
'seed_enabled': {
'config': {
'enabled': True,
'column_types': self.seed_enabled_types()
},
'enabled': True,
'+column_types': self.seed_enabled_types()
},
'seed_tricky': {
'config': {
'enabled': True,
'column_types': self.seed_tricky_types(),
},
'enabled': True,
'+column_types': self.seed_tricky_types(),
},
},
},
Expand Down
34 changes: 9 additions & 25 deletions test/integration/005_simple_seed_test/test_simple_seed.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@ def project_config(self):
'config-version': 2,
"data-paths": ['data'],
'seeds': {
'config': {
'quote_columns': False,
},
'quote_columns': False,
}
}

Expand Down Expand Up @@ -73,10 +71,8 @@ def project_config(self):
'config-version': 2,
"data-paths": ['data'],
'seeds': {
'config': {
"schema": "custom_schema",
'quote_columns': False,
},
"schema": "custom_schema",
'quote_columns': False,
},
}

Expand Down Expand Up @@ -126,19 +122,13 @@ def project_config(self):
'seeds': {
"test": {
"seed_enabled": {
'config': {
"enabled": True
},
"enabled": True
},
"seed_disabled": {
'config': {
"enabled": False
},
"enabled": False
}
},
'config': {
'quote_columns': False,
},
'quote_columns': False,
},
}

Expand Down Expand Up @@ -185,9 +175,7 @@ def project_config(self):
'config-version': 2,
"data-paths": ['data-bad'],
'seeds': {
'config': {
'quote_columns': False,
},
'quote_columns': False,
},
}

Expand Down Expand Up @@ -220,9 +208,7 @@ def project_config(self):
'config-version': 2,
"data-paths": ['data-bom'],
'seeds': {
'config': {
'quote_columns': False,
},
'quote_columns': False,
},
}

Expand Down Expand Up @@ -254,9 +240,7 @@ def project_config(self):
'config-version': 2,
"data-paths": ['data-unicode'],
'seeds': {
'config': {
'quote_columns': False,
},
'quote_columns': False,
}
}

Expand Down
8 changes: 2 additions & 6 deletions test/integration/006_simple_dependency_test/test_local_dependency.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,14 +129,10 @@ def project_config(self):
'config-version': 2,
'macro-paths': ['schema_override_macros'],
'models': {
'config': {
'schema': 'dbt_test',
},
'schema': 'dbt_test',
},
'seeds': {
'config': {
'schema': 'dbt_test',
},
'schema': 'dbt_test',
}
}

Expand Down
8 changes: 2 additions & 6 deletions test/integration/007_graph_selection_tests/test_tag_selection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,10 @@ def project_config(self):
"models": {
"test": {
"users": {
'config': {
"tags": "specified_as_string"
},
"tags": "specified_as_string"
},
"users_rollup": {
'config': {
"tags": ["specified_in_project"],
},
"tags": ["specified_in_project"],
}
}
}
Expand Down
Loading

0 comments on commit 51baece

Please sign in to comment.