From 8442fb66a591c4f5353f8baaaaf62f846c5f5171 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 8 Nov 2021 19:09:54 +0100 Subject: [PATCH] Reorganize global project (macros) (#4154) * Add integration tests * Reorganize + dispatch more global macros * Reorg materializations subdir * Move around + document generic tests * Fix failing tests * Fix merge conflict * Grab fix from #4148 * PR feedback * Fixup * Add load_relation back, it was nice * Last few test fixes * Rm incremental_upsert, now unused * Add changelog entry --- CHANGELOG.md | 1 + .../include/global_project/dbt_project.yml | 3 +- .../include/global_project/docs/overview.md | 4 +- .../macros/adapters/columns.sql | 89 +++++ .../global_project/macros/adapters/common.sql | 344 ------------------ .../macros/adapters/freshness.sql | 26 ++ .../macros/adapters/indexes.sql | 23 ++ .../macros/adapters/metadata.sql | 65 ++++ .../macros/adapters/persist_docs.sql | 33 ++ .../macros/adapters/relation.sql | 84 +++++ .../global_project/macros/adapters/schema.sql | 20 + .../global_project/macros/etc/datetime.sql | 4 +- .../global_project/macros/etc/query.sql | 8 - .../macros/{core.sql => etc/statement.sql} | 11 + .../accepted_values.sql | 6 - .../macros/generic_test_sql/not_null.sql | 8 + .../relationships.sql | 7 - .../unique.sql | 5 - .../macros/generic_tests/not_null.sql | 13 - .../get_custom_alias.sql | 1 + .../get_custom_database.sql | 0 .../get_custom_schema.sql | 2 +- .../macros/materializations/configs.sql | 21 ++ .../macros/materializations/helpers.sql | 83 ----- .../macros/materializations/hooks.sql | 35 ++ .../materializations/incremental/helpers.sql | 21 -- .../models/incremental/column_helpers.sql | 52 +++ .../{ => models}/incremental/incremental.sql | 0 .../models/incremental}/is_incremental.sql | 0 .../{common => models/incremental}/merge.sql | 34 +- .../incremental/on_schema_change.sql | 33 -- .../models/table/create_table_as.sql | 25 ++ .../{ => models}/table/table.sql | 2 +- .../view/create_or_replace_view.sql | 17 +- .../models/view/create_view_as.sql | 22 ++ .../materializations/models/view/helpers.sql | 8 + .../{ => models}/view/view.sql | 0 .../{seed/seed.sql => seeds/helpers.sql} | 63 +--- .../macros/materializations/seeds/seed.sql | 55 +++ .../snapshot.sql => snapshots/helpers.sql} | 126 +------ .../materializations/snapshots/snapshot.sql | 98 +++++ .../snapshot_merge.sql | 0 .../{snapshot => snapshots}/strategies.sql | 2 - .../macros/materializations/tests/helpers.sql | 14 + .../materializations/{ => tests}/test.sql | 16 - .../tests}/where_subquery.sql | 0 .../global_project/tests/generic/builtin.sql | 30 ++ .../test_docs_generate.py | 10 +- 48 files changed, 760 insertions(+), 764 deletions(-) create mode 100644 core/dbt/include/global_project/macros/adapters/columns.sql delete mode 100644 core/dbt/include/global_project/macros/adapters/common.sql create mode 100644 core/dbt/include/global_project/macros/adapters/freshness.sql create mode 100644 core/dbt/include/global_project/macros/adapters/indexes.sql create mode 100644 core/dbt/include/global_project/macros/adapters/metadata.sql create mode 100644 core/dbt/include/global_project/macros/adapters/persist_docs.sql create mode 100644 core/dbt/include/global_project/macros/adapters/relation.sql create mode 100644 core/dbt/include/global_project/macros/adapters/schema.sql delete mode 100644 core/dbt/include/global_project/macros/etc/query.sql rename core/dbt/include/global_project/macros/{core.sql => etc/statement.sql} (78%) rename core/dbt/include/global_project/macros/{generic_tests => generic_test_sql}/accepted_values.sql (71%) create mode 100644 core/dbt/include/global_project/macros/generic_test_sql/not_null.sql rename core/dbt/include/global_project/macros/{generic_tests => generic_test_sql}/relationships.sql (68%) rename core/dbt/include/global_project/macros/{generic_tests => generic_test_sql}/unique.sql (61%) delete mode 100644 core/dbt/include/global_project/macros/generic_tests/not_null.sql rename core/dbt/include/global_project/macros/{etc => get_custom_name}/get_custom_alias.sql (99%) rename core/dbt/include/global_project/macros/{etc => get_custom_name}/get_custom_database.sql (100%) rename core/dbt/include/global_project/macros/{etc => get_custom_name}/get_custom_schema.sql (95%) create mode 100644 core/dbt/include/global_project/macros/materializations/configs.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/helpers.sql create mode 100644 core/dbt/include/global_project/macros/materializations/hooks.sql delete mode 100644 core/dbt/include/global_project/macros/materializations/incremental/helpers.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/incremental/column_helpers.sql rename core/dbt/include/global_project/macros/materializations/{ => models}/incremental/incremental.sql (100%) rename core/dbt/include/global_project/macros/{etc => materializations/models/incremental}/is_incremental.sql (100%) rename core/dbt/include/global_project/macros/materializations/{common => models/incremental}/merge.sql (87%) rename core/dbt/include/global_project/macros/materializations/{ => models}/incremental/on_schema_change.sql (82%) create mode 100644 core/dbt/include/global_project/macros/materializations/models/table/create_table_as.sql rename core/dbt/include/global_project/macros/materializations/{ => models}/table/table.sql (97%) rename core/dbt/include/global_project/macros/materializations/{ => models}/view/create_or_replace_view.sql (77%) create mode 100644 core/dbt/include/global_project/macros/materializations/models/view/create_view_as.sql create mode 100644 core/dbt/include/global_project/macros/materializations/models/view/helpers.sql rename core/dbt/include/global_project/macros/materializations/{ => models}/view/view.sql (100%) rename core/dbt/include/global_project/macros/materializations/{seed/seed.sql => seeds/helpers.sql} (64%) create mode 100644 core/dbt/include/global_project/macros/materializations/seeds/seed.sql rename core/dbt/include/global_project/macros/materializations/{snapshot/snapshot.sql => snapshots/helpers.sql} (51%) create mode 100644 core/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql rename core/dbt/include/global_project/macros/materializations/{snapshot => snapshots}/snapshot_merge.sql (100%) rename core/dbt/include/global_project/macros/materializations/{snapshot => snapshots}/strategies.sql (99%) create mode 100644 core/dbt/include/global_project/macros/materializations/tests/helpers.sql rename core/dbt/include/global_project/macros/materializations/{ => tests}/test.sql (69%) rename core/dbt/include/global_project/macros/{etc => materializations/tests}/where_subquery.sql (100%) create mode 100644 core/dbt/include/global_project/tests/generic/builtin.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index be52475eda4..7dd47a7682f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Breaking changes - Replace `greedy` flag/property for test selection with `indirect_selection: eager/cautious` flag/property. Set to `eager` by default. **Note:** This reverts test selection to its pre-v0.20 behavior by default. `dbt test -s my_model` _will_ select multi-parent tests, such as `relationships`, that depend on unselected resources. To achieve the behavior change in v0.20 + v0.21, set `--indirect-selection=cautious` on the CLI or `indirect_selection: cautious` in yaml selectors. ([#4082](https://github.com/dbt-labs/dbt-core/issues/4082), [#4104](https://github.com/dbt-labs/dbt-core/pull/4104)) - In v1.0.0, **`pip install dbt` will raise an explicit error.** Instead, please use `pip install dbt-` (to use dbt with that database adapter), or `pip install dbt-core` (for core functionality). For parity with the previous behavior of `pip install dbt`, you can use: `pip install dbt-core dbt-postgres dbt-redshift dbt-snowflake dbt-bigquery` ([#4100](https://github.com/dbt-labs/dbt-core/issues/4100), [#4133](https://github.com/dbt-labs/dbt-core/pull/4133)) +- Reorganize the `global_project` (macros) into smaller files with clearer names. Remove unused global macros: `column_list`, `column_list_for_create_table`, `incremental_upsert` ([#4154](https://github.com/dbt-labs/dbt-core/pull/4154)) ### Features - Allow nullable `error_after` in source freshness ([#3874](https://github.com/dbt-labs/dbt-core/issues/3874), [#3955](https://github.com/dbt-labs/dbt-core/pull/3955)) diff --git a/core/dbt/include/global_project/dbt_project.yml b/core/dbt/include/global_project/dbt_project.yml index dec6d7d452f..fe15d183c58 100644 --- a/core/dbt/include/global_project/dbt_project.yml +++ b/core/dbt/include/global_project/dbt_project.yml @@ -2,5 +2,6 @@ config-version: 2 name: dbt version: 1.0 -docs-paths: ['docs'] +docs-paths: ["docs"] macro-paths: ["macros"] +test-paths: ["tests"] diff --git a/core/dbt/include/global_project/docs/overview.md b/core/dbt/include/global_project/docs/overview.md index 1c295ff2efd..0eebb63354a 100644 --- a/core/dbt/include/global_project/docs/overview.md +++ b/core/dbt/include/global_project/docs/overview.md @@ -26,7 +26,7 @@ On model pages, you'll see the immediate parents and children of the model you'r button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build, or are built from, the model you're exploring. -Once expanded, you'll be able to use the `--models` and `--exclude` model selection syntax to filter the +Once expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax). Note that you can also right-click on models to interactively filter and explore the graph. @@ -38,6 +38,6 @@ Note that you can also right-click on models to interactively filter and explore - [What is dbt](https://docs.getdbt.com/docs/overview)? - Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint) - [Installation](https://docs.getdbt.com/docs/installation) -- Join the [chat](https://community.getdbt.com/) on Slack for live questions and support. +- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion {% enddocs %} diff --git a/core/dbt/include/global_project/macros/adapters/columns.sql b/core/dbt/include/global_project/macros/adapters/columns.sql new file mode 100644 index 00000000000..9f992028617 --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/columns.sql @@ -0,0 +1,89 @@ +{% macro get_columns_in_relation(relation) -%} + {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }} +{% endmacro %} + +{% macro default__get_columns_in_relation(relation) -%} + {{ exceptions.raise_not_implemented( + 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + +{# helper for adapter-specific implementations of get_columns_in_relation #} +{% macro sql_convert_columns_in_relation(table) -%} + {% set columns = [] %} + {% for row in table %} + {% do columns.append(api.Column(*row)) %} + {% endfor %} + {{ return(columns) }} +{% endmacro %} + + +{% macro get_columns_in_query(select_sql) -%} + {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }} +{% endmacro %} + +{% macro default__get_columns_in_query(select_sql) %} + {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%} + select * from ( + {{ select_sql }} + ) as __dbt_sbq + where false + limit 0 + {% endcall %} + + {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }} +{% endmacro %} + + +{% macro alter_column_type(relation, column_name, new_column_type) -%} + {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }} +{% endmacro %} + +{% macro default__alter_column_type(relation, column_name, new_column_type) -%} + {# + 1. Create a new column (w/ temp name and correct type) + 2. Copy data over to it + 3. Drop the existing column (cascade!) + 4. Rename the new column to existing column + #} + {%- set tmp_column = column_name + "__dbt_alter" -%} + + {% call statement('alter_column_type') %} + alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }}; + update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }}; + alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade; + alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }} + {% endcall %} + +{% endmacro %} + + +{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%} + {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }} +{% endmacro %} + +{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %} + + {% if add_columns is none %} + {% set add_columns = [] %} + {% endif %} + {% if remove_columns is none %} + {% set remove_columns = [] %} + {% endif %} + + {% set sql -%} + + alter {{ relation.type }} {{ relation }} + + {% for column in add_columns %} + add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} + {% endfor %}{{ ',' if add_columns and remove_columns }} + + {% for column in remove_columns %} + drop column {{ column.name }}{{ ',' if not loop.last }} + {% endfor %} + + {%- endset -%} + + {% do run_query(sql) %} + +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/common.sql b/core/dbt/include/global_project/macros/adapters/common.sql deleted file mode 100644 index 5a26ef06ca6..00000000000 --- a/core/dbt/include/global_project/macros/adapters/common.sql +++ /dev/null @@ -1,344 +0,0 @@ -{% macro get_columns_in_query(select_sql) -%} - {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }} -{% endmacro %} - -{% macro default__get_columns_in_query(select_sql) %} - {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%} - select * from ( - {{ select_sql }} - ) as __dbt_sbq - where false - limit 0 - {% endcall %} - - {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }} -{% endmacro %} - -{% macro create_schema(relation) -%} - {{ adapter.dispatch('create_schema', 'dbt')(relation) }} -{% endmacro %} - -{% macro default__create_schema(relation) -%} - {%- call statement('create_schema') -%} - create schema if not exists {{ relation.without_identifier() }} - {% endcall %} -{% endmacro %} - -{% macro drop_schema(relation) -%} - {{ adapter.dispatch('drop_schema', 'dbt')(relation) }} -{% endmacro %} - -{% macro default__drop_schema(relation) -%} - {%- call statement('drop_schema') -%} - drop schema if exists {{ relation.without_identifier() }} cascade - {% endcall %} -{% endmacro %} - -{% macro create_table_as(temporary, relation, sql) -%} - {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, sql) }} -{%- endmacro %} - -{% macro default__create_table_as(temporary, relation, sql) -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - - create {% if temporary: -%}temporary{%- endif %} table - {{ relation.include(database=(not temporary), schema=(not temporary)) }} - as ( - {{ sql }} - ); - -{% endmacro %} - -{% macro get_create_index_sql(relation, index_dict) -%} - {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }} -{% endmacro %} - -{% macro default__get_create_index_sql(relation, index_dict) -%} - {% do return(None) %} -{% endmacro %} - -{% macro create_indexes(relation) -%} - {{ adapter.dispatch('create_indexes', 'dbt')(relation) }} -{%- endmacro %} - -{% macro default__create_indexes(relation) -%} - {%- set _indexes = config.get('indexes', default=[]) -%} - - {% for _index_dict in _indexes %} - {% set create_index_sql = get_create_index_sql(relation, _index_dict) %} - {% if create_index_sql %} - {% do run_query(create_index_sql) %} - {% endif %} - {% endfor %} -{% endmacro %} - -{% macro create_view_as(relation, sql) -%} - {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }} -{%- endmacro %} - -{% macro default__create_view_as(relation, sql) -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - create view {{ relation }} as ( - {{ sql }} - ); -{% endmacro %} - - -{% macro get_catalog(information_schema, schemas) -%} - {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }} -{%- endmacro %} - -{% macro default__get_catalog(information_schema, schemas) -%} - - {% set typename = adapter.type() %} - {% set msg -%} - get_catalog not implemented for {{ typename }} - {%- endset %} - - {{ exceptions.raise_compiler_error(msg) }} -{% endmacro %} - - -{% macro get_columns_in_relation(relation) -%} - {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }} -{% endmacro %} - -{% macro sql_convert_columns_in_relation(table) -%} - {% set columns = [] %} - {% for row in table %} - {% do columns.append(api.Column(*row)) %} - {% endfor %} - {{ return(columns) }} -{% endmacro %} - -{% macro default__get_columns_in_relation(relation) -%} - {{ exceptions.raise_not_implemented( - 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }} -{% endmacro %} - -{% macro alter_column_type(relation, column_name, new_column_type) -%} - {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }} -{% endmacro %} - - - -{% macro alter_column_comment(relation, column_dict) -%} - {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }} -{% endmacro %} - -{% macro default__alter_column_comment(relation, column_dict) -%} - {{ exceptions.raise_not_implemented( - 'alter_column_comment macro not implemented for adapter '+adapter.type()) }} -{% endmacro %} - -{% macro alter_relation_comment(relation, relation_comment) -%} - {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }} -{% endmacro %} - -{% macro default__alter_relation_comment(relation, relation_comment) -%} - {{ exceptions.raise_not_implemented( - 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }} -{% endmacro %} - -{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%} - {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }} -{% endmacro %} - -{% macro default__persist_docs(relation, model, for_relation, for_columns) -%} - {% if for_relation and config.persist_relation_docs() and model.description %} - {% do run_query(alter_relation_comment(relation, model.description)) %} - {% endif %} - - {% if for_columns and config.persist_column_docs() and model.columns %} - {% do run_query(alter_column_comment(relation, model.columns)) %} - {% endif %} -{% endmacro %} - - - -{% macro default__alter_column_type(relation, column_name, new_column_type) -%} - {# - 1. Create a new column (w/ temp name and correct type) - 2. Copy data over to it - 3. Drop the existing column (cascade!) - 4. Rename the new column to existing column - #} - {%- set tmp_column = column_name + "__dbt_alter" -%} - - {% call statement('alter_column_type') %} - alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }}; - update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }}; - alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade; - alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }} - {% endcall %} - -{% endmacro %} - - -{% macro drop_relation(relation) -%} - {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }} -{% endmacro %} - - -{% macro default__drop_relation(relation) -%} - {% call statement('drop_relation', auto_begin=False) -%} - drop {{ relation.type }} if exists {{ relation }} cascade - {%- endcall %} -{% endmacro %} - -{% macro truncate_relation(relation) -%} - {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }} -{% endmacro %} - - -{% macro default__truncate_relation(relation) -%} - {% call statement('truncate_relation') -%} - truncate table {{ relation }} - {%- endcall %} -{% endmacro %} - -{% macro rename_relation(from_relation, to_relation) -%} - {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }} -{% endmacro %} - -{% macro default__rename_relation(from_relation, to_relation) -%} - {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %} - {% call statement('rename_relation') -%} - alter table {{ from_relation }} rename to {{ target_name }} - {%- endcall %} -{% endmacro %} - - -{% macro information_schema_name(database) %} - {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }} -{% endmacro %} - -{% macro default__information_schema_name(database) -%} - {%- if database -%} - {{ database }}.INFORMATION_SCHEMA - {%- else -%} - INFORMATION_SCHEMA - {%- endif -%} -{%- endmacro %} - - -{% macro list_schemas(database) -%} - {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }} -{% endmacro %} - -{% macro default__list_schemas(database) -%} - {% set sql %} - select distinct schema_name - from {{ information_schema_name(database) }}.SCHEMATA - where catalog_name ilike '{{ database }}' - {% endset %} - {{ return(run_query(sql)) }} -{% endmacro %} - - -{% macro check_schema_exists(information_schema, schema) -%} - {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }} -{% endmacro %} - -{% macro default__check_schema_exists(information_schema, schema) -%} - {% set sql -%} - select count(*) - from {{ information_schema.replace(information_schema_view='SCHEMATA') }} - where catalog_name='{{ information_schema.database }}' - and schema_name='{{ schema }}' - {%- endset %} - {{ return(run_query(sql)) }} -{% endmacro %} - - -{% macro list_relations_without_caching(schema_relation) %} - {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }} -{% endmacro %} - - -{% macro default__list_relations_without_caching(schema_relation) %} - {{ exceptions.raise_not_implemented( - 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }} -{% endmacro %} - - -{% macro current_timestamp() -%} - {{ adapter.dispatch('current_timestamp', 'dbt')() }} -{%- endmacro %} - - -{% macro default__current_timestamp() -%} - {{ exceptions.raise_not_implemented( - 'current_timestamp macro not implemented for adapter '+adapter.type()) }} -{%- endmacro %} - - -{% macro collect_freshness(source, loaded_at_field, filter) %} - {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}} -{% endmacro %} - - -{% macro default__collect_freshness(source, loaded_at_field, filter) %} - {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%} - select - max({{ loaded_at_field }}) as max_loaded_at, - {{ current_timestamp() }} as snapshotted_at - from {{ source }} - {% if filter %} - where {{ filter }} - {% endif %} - {% endcall %} - {{ return(load_result('collect_freshness').table) }} -{% endmacro %} - -{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %} - {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix))}} -{% endmacro %} - -{% macro default__make_temp_relation(base_relation, suffix) %} - {% set tmp_identifier = base_relation.identifier ~ suffix %} - {% set tmp_relation = base_relation.incorporate( - path={"identifier": tmp_identifier}) -%} - - {% do return(tmp_relation) %} -{% endmacro %} - -{% macro set_sql_header(config) -%} - {{ config.set('sql_header', caller()) }} -{%- endmacro %} - - -{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%} - {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }} -{% endmacro %} - -{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %} - - {% if add_columns is none %} - {% set add_columns = [] %} - {% endif %} - {% if remove_columns is none %} - {% set remove_columns = [] %} - {% endif %} - - {% set sql -%} - - alter {{ relation.type }} {{ relation }} - - {% for column in add_columns %} - add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} - {% endfor %}{{ ',' if add_columns and remove_columns }} - - {% for column in remove_columns %} - drop column {{ column.name }}{{ ',' if not loop.last }} - {% endfor %} - - {%- endset -%} - - {% do run_query(sql) %} - -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/freshness.sql b/core/dbt/include/global_project/macros/adapters/freshness.sql new file mode 100644 index 00000000000..fe8e7d7a1fe --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/freshness.sql @@ -0,0 +1,26 @@ +{% macro current_timestamp() -%} + {{ adapter.dispatch('current_timestamp', 'dbt')() }} +{%- endmacro %} + +{% macro default__current_timestamp() -%} + {{ exceptions.raise_not_implemented( + 'current_timestamp macro not implemented for adapter '+adapter.type()) }} +{%- endmacro %} + + +{% macro collect_freshness(source, loaded_at_field, filter) %} + {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}} +{% endmacro %} + +{% macro default__collect_freshness(source, loaded_at_field, filter) %} + {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%} + select + max({{ loaded_at_field }}) as max_loaded_at, + {{ current_timestamp() }} as snapshotted_at + from {{ source }} + {% if filter %} + where {{ filter }} + {% endif %} + {% endcall %} + {{ return(load_result('collect_freshness').table) }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/indexes.sql b/core/dbt/include/global_project/macros/adapters/indexes.sql new file mode 100644 index 00000000000..a4fdd9cee1b --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/indexes.sql @@ -0,0 +1,23 @@ +{% macro get_create_index_sql(relation, index_dict) -%} + {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }} +{% endmacro %} + +{% macro default__get_create_index_sql(relation, index_dict) -%} + {% do return(None) %} +{% endmacro %} + + +{% macro create_indexes(relation) -%} + {{ adapter.dispatch('create_indexes', 'dbt')(relation) }} +{%- endmacro %} + +{% macro default__create_indexes(relation) -%} + {%- set _indexes = config.get('indexes', default=[]) -%} + + {% for _index_dict in _indexes %} + {% set create_index_sql = get_create_index_sql(relation, _index_dict) %} + {% if create_index_sql %} + {% do run_query(create_index_sql) %} + {% endif %} + {% endfor %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/metadata.sql b/core/dbt/include/global_project/macros/adapters/metadata.sql new file mode 100644 index 00000000000..3c334dae6b8 --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/metadata.sql @@ -0,0 +1,65 @@ +{% macro get_catalog(information_schema, schemas) -%} + {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }} +{%- endmacro %} + +{% macro default__get_catalog(information_schema, schemas) -%} + + {% set typename = adapter.type() %} + {% set msg -%} + get_catalog not implemented for {{ typename }} + {%- endset %} + + {{ exceptions.raise_compiler_error(msg) }} +{% endmacro %} + + +{% macro information_schema_name(database) %} + {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }} +{% endmacro %} + +{% macro default__information_schema_name(database) -%} + {%- if database -%} + {{ database }}.INFORMATION_SCHEMA + {%- else -%} + INFORMATION_SCHEMA + {%- endif -%} +{%- endmacro %} + + +{% macro list_schemas(database) -%} + {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }} +{% endmacro %} + +{% macro default__list_schemas(database) -%} + {% set sql %} + select distinct schema_name + from {{ information_schema_name(database) }}.SCHEMATA + where catalog_name ilike '{{ database }}' + {% endset %} + {{ return(run_query(sql)) }} +{% endmacro %} + + +{% macro check_schema_exists(information_schema, schema) -%} + {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }} +{% endmacro %} + +{% macro default__check_schema_exists(information_schema, schema) -%} + {% set sql -%} + select count(*) + from {{ information_schema.replace(information_schema_view='SCHEMATA') }} + where catalog_name='{{ information_schema.database }}' + and schema_name='{{ schema }}' + {%- endset %} + {{ return(run_query(sql)) }} +{% endmacro %} + + +{% macro list_relations_without_caching(schema_relation) %} + {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }} +{% endmacro %} + +{% macro default__list_relations_without_caching(schema_relation) %} + {{ exceptions.raise_not_implemented( + 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/persist_docs.sql b/core/dbt/include/global_project/macros/adapters/persist_docs.sql new file mode 100644 index 00000000000..8749e59f606 --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/persist_docs.sql @@ -0,0 +1,33 @@ +{% macro alter_column_comment(relation, column_dict) -%} + {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }} +{% endmacro %} + +{% macro default__alter_column_comment(relation, column_dict) -%} + {{ exceptions.raise_not_implemented( + 'alter_column_comment macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + + +{% macro alter_relation_comment(relation, relation_comment) -%} + {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }} +{% endmacro %} + +{% macro default__alter_relation_comment(relation, relation_comment) -%} + {{ exceptions.raise_not_implemented( + 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + + +{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%} + {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }} +{% endmacro %} + +{% macro default__persist_docs(relation, model, for_relation, for_columns) -%} + {% if for_relation and config.persist_relation_docs() and model.description %} + {% do run_query(alter_relation_comment(relation, model.description)) %} + {% endif %} + + {% if for_columns and config.persist_column_docs() and model.columns %} + {% do run_query(alter_column_comment(relation, model.columns)) %} + {% endif %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/relation.sql b/core/dbt/include/global_project/macros/adapters/relation.sql new file mode 100644 index 00000000000..f5976b25b31 --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/relation.sql @@ -0,0 +1,84 @@ +{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %} + {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix))}} +{% endmacro %} + +{% macro default__make_temp_relation(base_relation, suffix) %} + {% set tmp_identifier = base_relation.identifier ~ suffix %} + {% set tmp_relation = base_relation.incorporate( + path={"identifier": tmp_identifier}) -%} + + {% do return(tmp_relation) %} +{% endmacro %} + + +{% macro drop_relation(relation) -%} + {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }} +{% endmacro %} + +{% macro default__drop_relation(relation) -%} + {% call statement('drop_relation', auto_begin=False) -%} + drop {{ relation.type }} if exists {{ relation }} cascade + {%- endcall %} +{% endmacro %} + + +{% macro truncate_relation(relation) -%} + {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }} +{% endmacro %} + +{% macro default__truncate_relation(relation) -%} + {% call statement('truncate_relation') -%} + truncate table {{ relation }} + {%- endcall %} +{% endmacro %} + + +{% macro rename_relation(from_relation, to_relation) -%} + {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }} +{% endmacro %} + +{% macro default__rename_relation(from_relation, to_relation) -%} + {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %} + {% call statement('rename_relation') -%} + alter table {{ from_relation }} rename to {{ target_name }} + {%- endcall %} +{% endmacro %} + + +{% macro get_or_create_relation(database, schema, identifier, type) -%} + {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }} +{% endmacro %} + +{% macro default__get_or_create_relation(database, schema, identifier, type) %} + {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} + + {% if target_relation %} + {% do return([true, target_relation]) %} + {% endif %} + + {%- set new_relation = api.Relation.create( + database=database, + schema=schema, + identifier=identifier, + type=type + ) -%} + {% do return([false, new_relation]) %} +{% endmacro %} + + +{# a user-friendly interface into adapter.get_relation #} +{% macro load_relation(relation) %} + {% do return(adapter.get_relation( + database=relation.database, + schema=relation.schema, + identifier=relation.identifier + )) -%} +{% endmacro %} + + +{# not used much, here for backwards compatibility #} +{% macro drop_relation_if_exists(relation) %} + {% if relation is not none %} + {{ adapter.drop_relation(relation) }} + {% endif %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/adapters/schema.sql b/core/dbt/include/global_project/macros/adapters/schema.sql new file mode 100644 index 00000000000..9e0c7559286 --- /dev/null +++ b/core/dbt/include/global_project/macros/adapters/schema.sql @@ -0,0 +1,20 @@ +{% macro create_schema(relation) -%} + {{ adapter.dispatch('create_schema', 'dbt')(relation) }} +{% endmacro %} + +{% macro default__create_schema(relation) -%} + {%- call statement('create_schema') -%} + create schema if not exists {{ relation.without_identifier() }} + {% endcall %} +{% endmacro %} + + +{% macro drop_schema(relation) -%} + {{ adapter.dispatch('drop_schema', 'dbt')(relation) }} +{% endmacro %} + +{% macro default__drop_schema(relation) -%} + {%- call statement('drop_schema') -%} + drop schema if exists {{ relation.without_identifier() }} cascade + {% endcall %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/etc/datetime.sql b/core/dbt/include/global_project/macros/etc/datetime.sql index 43fd731d1da..c6299c57ca8 100644 --- a/core/dbt/include/global_project/macros/etc/datetime.sql +++ b/core/dbt/include/global_project/macros/etc/datetime.sql @@ -1,4 +1,3 @@ - {% macro convert_datetime(date_str, date_fmt) %} {% set error_msg -%} @@ -10,6 +9,7 @@ {% endmacro %} + {% macro dates_in_range(start_date_str, end_date_str=none, in_fmt="%Y%m%d", out_fmt="%Y%m%d") %} {% set end_date_str = start_date_str if end_date_str is none else end_date_str %} @@ -38,6 +38,7 @@ {{ return(date_list) }} {% endmacro %} + {% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %} {% set partition_range = (raw_partition_date | string).split(",") %} @@ -54,6 +55,7 @@ {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }} {% endmacro %} + {% macro py_current_timestring() %} {% set dt = modules.datetime.datetime.now() %} {% do return(dt.strftime("%Y%m%d%H%M%S%f")) %} diff --git a/core/dbt/include/global_project/macros/etc/query.sql b/core/dbt/include/global_project/macros/etc/query.sql deleted file mode 100644 index aa0bb6990f0..00000000000 --- a/core/dbt/include/global_project/macros/etc/query.sql +++ /dev/null @@ -1,8 +0,0 @@ - -{% macro run_query(sql) %} - {% call statement("run_query_statement", fetch_result=true, auto_begin=false) %} - {{ sql }} - {% endcall %} - - {% do return(load_result("run_query_statement").table) %} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/core.sql b/core/dbt/include/global_project/macros/etc/statement.sql similarity index 78% rename from core/dbt/include/global_project/macros/core.sql rename to core/dbt/include/global_project/macros/etc/statement.sql index 8b46d11d9b0..f13b5a9079c 100644 --- a/core/dbt/include/global_project/macros/core.sql +++ b/core/dbt/include/global_project/macros/etc/statement.sql @@ -15,6 +15,7 @@ {%- endif -%} {%- endmacro %} + {% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%} {%- set sql = caller() -%} @@ -28,3 +29,13 @@ {%- endif -%} {%- endmacro %} + + +{# a user-friendly interface into statements #} +{% macro run_query(sql) %} + {% call statement("run_query_statement", fetch_result=true, auto_begin=false) %} + {{ sql }} + {% endcall %} + + {% do return(load_result("run_query_statement").table) %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/generic_tests/accepted_values.sql b/core/dbt/include/global_project/macros/generic_test_sql/accepted_values.sql similarity index 71% rename from core/dbt/include/global_project/macros/generic_tests/accepted_values.sql rename to core/dbt/include/global_project/macros/generic_test_sql/accepted_values.sql index 3997eacb9a0..9b5a0b0e4dd 100644 --- a/core/dbt/include/global_project/macros/generic_tests/accepted_values.sql +++ b/core/dbt/include/global_project/macros/generic_test_sql/accepted_values.sql @@ -25,9 +25,3 @@ where value_field not in ( ) {% endmacro %} - - -{% test accepted_values(model, column_name, values, quote=True) %} - {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %} - {{ macro(model, column_name, values, quote) }} -{% endtest %} diff --git a/core/dbt/include/global_project/macros/generic_test_sql/not_null.sql b/core/dbt/include/global_project/macros/generic_test_sql/not_null.sql new file mode 100644 index 00000000000..021e3a167c6 --- /dev/null +++ b/core/dbt/include/global_project/macros/generic_test_sql/not_null.sql @@ -0,0 +1,8 @@ +{% macro default__test_not_null(model, column_name) %} + +select * +from {{ model }} +where {{ column_name }} is null + +{% endmacro %} + diff --git a/core/dbt/include/global_project/macros/generic_tests/relationships.sql b/core/dbt/include/global_project/macros/generic_test_sql/relationships.sql similarity index 68% rename from core/dbt/include/global_project/macros/generic_tests/relationships.sql rename to core/dbt/include/global_project/macros/generic_test_sql/relationships.sql index b2f43b2c0a1..db779a43e7d 100644 --- a/core/dbt/include/global_project/macros/generic_tests/relationships.sql +++ b/core/dbt/include/global_project/macros/generic_test_sql/relationships.sql @@ -1,4 +1,3 @@ - {% macro default__test_relationships(model, column_name, to, field) %} with child as ( @@ -22,9 +21,3 @@ left join parent where parent.to_field is null {% endmacro %} - - -{% test relationships(model, column_name, to, field) %} - {% set macro = adapter.dispatch('test_relationships', 'dbt') %} - {{ macro(model, column_name, to, field) }} -{% endtest %} diff --git a/core/dbt/include/global_project/macros/generic_tests/unique.sql b/core/dbt/include/global_project/macros/generic_test_sql/unique.sql similarity index 61% rename from core/dbt/include/global_project/macros/generic_tests/unique.sql rename to core/dbt/include/global_project/macros/generic_test_sql/unique.sql index 72c8b6d3b15..111918ebda7 100644 --- a/core/dbt/include/global_project/macros/generic_tests/unique.sql +++ b/core/dbt/include/global_project/macros/generic_test_sql/unique.sql @@ -11,8 +11,3 @@ having count(*) > 1 {% endmacro %} - -{% test unique(model, column_name) %} - {% set macro = adapter.dispatch('test_unique', 'dbt') %} - {{ macro(model, column_name) }} -{% endtest %} diff --git a/core/dbt/include/global_project/macros/generic_tests/not_null.sql b/core/dbt/include/global_project/macros/generic_tests/not_null.sql deleted file mode 100644 index 1ded013de6a..00000000000 --- a/core/dbt/include/global_project/macros/generic_tests/not_null.sql +++ /dev/null @@ -1,13 +0,0 @@ -{% macro default__test_not_null(model, column_name) %} - -select * -from {{ model }} -where {{ column_name }} is null - -{% endmacro %} - - -{% test not_null(model, column_name) %} - {% set macro = adapter.dispatch('test_not_null', 'dbt') %} - {{ macro(model, column_name) }} -{% endtest %} diff --git a/core/dbt/include/global_project/macros/etc/get_custom_alias.sql b/core/dbt/include/global_project/macros/get_custom_name/get_custom_alias.sql similarity index 99% rename from core/dbt/include/global_project/macros/etc/get_custom_alias.sql rename to core/dbt/include/global_project/macros/get_custom_name/get_custom_alias.sql index bc3f7a17b77..d2954facccc 100644 --- a/core/dbt/include/global_project/macros/etc/get_custom_alias.sql +++ b/core/dbt/include/global_project/macros/get_custom_name/get_custom_alias.sql @@ -12,6 +12,7 @@ node: The available node that an alias is being generated for, or none #} + {% macro generate_alias_name(custom_alias_name=none, node=none) -%} {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %} {%- endmacro %} diff --git a/core/dbt/include/global_project/macros/etc/get_custom_database.sql b/core/dbt/include/global_project/macros/get_custom_name/get_custom_database.sql similarity index 100% rename from core/dbt/include/global_project/macros/etc/get_custom_database.sql rename to core/dbt/include/global_project/macros/get_custom_name/get_custom_database.sql diff --git a/core/dbt/include/global_project/macros/etc/get_custom_schema.sql b/core/dbt/include/global_project/macros/get_custom_name/get_custom_schema.sql similarity index 95% rename from core/dbt/include/global_project/macros/etc/get_custom_schema.sql rename to core/dbt/include/global_project/macros/get_custom_name/get_custom_schema.sql index d711fd3d45e..20348ec3e7b 100644 --- a/core/dbt/include/global_project/macros/etc/get_custom_schema.sql +++ b/core/dbt/include/global_project/macros/get_custom_name/get_custom_schema.sql @@ -14,7 +14,7 @@ node: The node the schema is being generated for #} -{% macro generate_schema_name(custom_schema_name, node) -%} +{% macro generate_schema_name(custom_schema_name=none, node=none) -%} {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/configs.sql b/core/dbt/include/global_project/macros/materializations/configs.sql new file mode 100644 index 00000000000..d15ccb8e603 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/configs.sql @@ -0,0 +1,21 @@ +{% macro set_sql_header(config) -%} + {{ config.set('sql_header', caller()) }} +{%- endmacro %} + + +{% macro should_full_refresh() %} + {% set config_full_refresh = config.get('full_refresh') %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {% do return(config_full_refresh) %} +{% endmacro %} + + +{% macro should_store_failures() %} + {% set config_store_failures = config.get('store_failures') %} + {% if config_store_failures is none %} + {% set config_store_failures = flags.STORE_FAILURES %} + {% endif %} + {% do return(config_store_failures) %} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/helpers.sql b/core/dbt/include/global_project/macros/materializations/helpers.sql deleted file mode 100644 index 7010f384112..00000000000 --- a/core/dbt/include/global_project/macros/materializations/helpers.sql +++ /dev/null @@ -1,83 +0,0 @@ -{% macro run_hooks(hooks, inside_transaction=True) %} - {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} - {% if not inside_transaction and loop.first %} - {% call statement(auto_begin=inside_transaction) %} - commit; - {% endcall %} - {% endif %} - {% set rendered = render(hook.get('sql')) | trim %} - {% if (rendered | length) > 0 %} - {% call statement(auto_begin=inside_transaction) %} - {{ rendered }} - {% endcall %} - {% endif %} - {% endfor %} -{% endmacro %} - - -{% macro column_list(columns) %} - {%- for col in columns %} - {{ col.name }} {% if not loop.last %},{% endif %} - {% endfor -%} -{% endmacro %} - - -{% macro column_list_for_create_table(columns) %} - {%- for col in columns %} - {{ col.name }} {{ col.data_type }} {%- if not loop.last %},{% endif %} - {% endfor -%} -{% endmacro %} - - -{% macro make_hook_config(sql, inside_transaction) %} - {{ tojson({"sql": sql, "transaction": inside_transaction}) }} -{% endmacro %} - - -{% macro before_begin(sql) %} - {{ make_hook_config(sql, inside_transaction=False) }} -{% endmacro %} - - -{% macro in_transaction(sql) %} - {{ make_hook_config(sql, inside_transaction=True) }} -{% endmacro %} - - -{% macro after_commit(sql) %} - {{ make_hook_config(sql, inside_transaction=False) }} -{% endmacro %} - - -{% macro drop_relation_if_exists(relation) %} - {% if relation is not none %} - {{ adapter.drop_relation(relation) }} - {% endif %} -{% endmacro %} - - -{% macro load_relation(relation) %} - {% do return(adapter.get_relation( - database=relation.database, - schema=relation.schema, - identifier=relation.identifier - )) -%} -{% endmacro %} - - -{% macro should_full_refresh() %} - {% set config_full_refresh = config.get('full_refresh') %} - {% if config_full_refresh is none %} - {% set config_full_refresh = flags.FULL_REFRESH %} - {% endif %} - {% do return(config_full_refresh) %} -{% endmacro %} - - -{% macro should_store_failures() %} - {% set config_store_failures = config.get('store_failures') %} - {% if config_store_failures is none %} - {% set config_store_failures = flags.STORE_FAILURES %} - {% endif %} - {% do return(config_store_failures) %} -{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/hooks.sql b/core/dbt/include/global_project/macros/materializations/hooks.sql new file mode 100644 index 00000000000..2e198196c4f --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/hooks.sql @@ -0,0 +1,35 @@ +{% macro run_hooks(hooks, inside_transaction=True) %} + {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} + {% if not inside_transaction and loop.first %} + {% call statement(auto_begin=inside_transaction) %} + commit; + {% endcall %} + {% endif %} + {% set rendered = render(hook.get('sql')) | trim %} + {% if (rendered | length) > 0 %} + {% call statement(auto_begin=inside_transaction) %} + {{ rendered }} + {% endcall %} + {% endif %} + {% endfor %} +{% endmacro %} + + +{% macro make_hook_config(sql, inside_transaction) %} + {{ tojson({"sql": sql, "transaction": inside_transaction}) }} +{% endmacro %} + + +{% macro before_begin(sql) %} + {{ make_hook_config(sql, inside_transaction=False) }} +{% endmacro %} + + +{% macro in_transaction(sql) %} + {{ make_hook_config(sql, inside_transaction=True) }} +{% endmacro %} + + +{% macro after_commit(sql) %} + {{ make_hook_config(sql, inside_transaction=False) }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/incremental/helpers.sql b/core/dbt/include/global_project/macros/materializations/incremental/helpers.sql deleted file mode 100644 index 3528167f361..00000000000 --- a/core/dbt/include/global_project/macros/materializations/incremental/helpers.sql +++ /dev/null @@ -1,21 +0,0 @@ - -{% macro incremental_upsert(tmp_relation, target_relation, unique_key=none, statement_name="main") %} - - {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%} - {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%} - - {%- if unique_key is not none -%} - delete - from {{ target_relation }} - where ({{ unique_key }}) in ( - select ({{ unique_key }}) - from {{ tmp_relation }} - ); - {%- endif %} - - insert into {{ target_relation }} ({{ dest_cols_csv }}) - ( - select {{ dest_cols_csv }} - from {{ tmp_relation }} - ); -{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/incremental/column_helpers.sql b/core/dbt/include/global_project/macros/materializations/models/incremental/column_helpers.sql new file mode 100644 index 00000000000..44835814af9 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/incremental/column_helpers.sql @@ -0,0 +1,52 @@ +/* {# + Helper macros for internal use with incremental materializations. + Use with care if calling elsewhere. +#} */ + + +{% macro get_quoted_csv(column_names) %} + + {% set quoted = [] %} + {% for col in column_names -%} + {%- do quoted.append(adapter.quote(col)) -%} + {%- endfor %} + + {%- set dest_cols_csv = quoted | join(', ') -%} + {{ return(dest_cols_csv) }} + +{% endmacro %} + + +{% macro diff_columns(source_columns, target_columns) %} + + {% set result = [] %} + {% set source_names = source_columns | map(attribute = 'column') | list %} + {% set target_names = target_columns | map(attribute = 'column') | list %} + + {# --check whether the name attribute exists in the target - this does not perform a data type check #} + {% for sc in source_columns %} + {% if sc.name not in target_names %} + {{ result.append(sc) }} + {% endif %} + {% endfor %} + + {{ return(result) }} + +{% endmacro %} + + +{% macro diff_column_data_types(source_columns, target_columns) %} + + {% set result = [] %} + {% for sc in source_columns %} + {% set tc = target_columns | selectattr("name", "equalto", sc.name) | list | first %} + {% if tc %} + {% if sc.data_type != tc.data_type %} + {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }} + {% endif %} + {% endif %} + {% endfor %} + + {{ return(result) }} + +{% endmacro %} \ No newline at end of file diff --git a/core/dbt/include/global_project/macros/materializations/incremental/incremental.sql b/core/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql similarity index 100% rename from core/dbt/include/global_project/macros/materializations/incremental/incremental.sql rename to core/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql diff --git a/core/dbt/include/global_project/macros/etc/is_incremental.sql b/core/dbt/include/global_project/macros/materializations/models/incremental/is_incremental.sql similarity index 100% rename from core/dbt/include/global_project/macros/etc/is_incremental.sql rename to core/dbt/include/global_project/macros/materializations/models/incremental/is_incremental.sql diff --git a/core/dbt/include/global_project/macros/materializations/common/merge.sql b/core/dbt/include/global_project/macros/materializations/models/incremental/merge.sql similarity index 87% rename from core/dbt/include/global_project/macros/materializations/common/merge.sql rename to core/dbt/include/global_project/macros/materializations/models/incremental/merge.sql index 979bc7f27f6..d5cda4f969c 100644 --- a/core/dbt/include/global_project/macros/materializations/common/merge.sql +++ b/core/dbt/include/global_project/macros/materializations/models/incremental/merge.sql @@ -1,20 +1,7 @@ - - {% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%} {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, predicates) }} {%- endmacro %} - -{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%} - {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns) }} -{%- endmacro %} - - -{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%} - {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }} -{%- endmacro %} - - {% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%} {%- set predicates = [] if predicates is none else [] + predicates -%} {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} @@ -52,18 +39,11 @@ {% endmacro %} -{% macro get_quoted_csv(column_names) %} - {% set quoted = [] %} - {% for col in column_names -%} - {%- do quoted.append(adapter.quote(col)) -%} - {%- endfor %} - - {%- set dest_cols_csv = quoted | join(', ') -%} - {{ return(dest_cols_csv) }} -{% endmacro %} - +{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%} + {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns) }} +{%- endmacro %} -{% macro common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%} +{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%} {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} @@ -83,10 +63,10 @@ {%- endmacro %} -{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%} - {{ common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) }} -{% endmacro %} +{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%} + {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }} +{%- endmacro %} {% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%} {%- set predicates = [] if predicates is none else [] + predicates -%} diff --git a/core/dbt/include/global_project/macros/materializations/incremental/on_schema_change.sql b/core/dbt/include/global_project/macros/materializations/models/incremental/on_schema_change.sql similarity index 82% rename from core/dbt/include/global_project/macros/materializations/incremental/on_schema_change.sql rename to core/dbt/include/global_project/macros/materializations/models/incremental/on_schema_change.sql index 4367ad80eb1..a6c9a8275e4 100644 --- a/core/dbt/include/global_project/macros/materializations/incremental/on_schema_change.sql +++ b/core/dbt/include/global_project/macros/materializations/models/incremental/on_schema_change.sql @@ -15,39 +15,6 @@ {% endmacro %} -{% macro diff_columns(source_columns, target_columns) %} - - {% set result = [] %} - {% set source_names = source_columns | map(attribute = 'column') | list %} - {% set target_names = target_columns | map(attribute = 'column') | list %} - - {# --check whether the name attribute exists in the target - this does not perform a data type check #} - {% for sc in source_columns %} - {% if sc.name not in target_names %} - {{ result.append(sc) }} - {% endif %} - {% endfor %} - - {{ return(result) }} - -{% endmacro %} - -{% macro diff_column_data_types(source_columns, target_columns) %} - - {% set result = [] %} - {% for sc in source_columns %} - {% set tc = target_columns | selectattr("name", "equalto", sc.name) | list | first %} - {% if tc %} - {% if sc.data_type != tc.data_type %} - {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }} - {% endif %} - {% endif %} - {% endfor %} - - {{ return(result) }} - -{% endmacro %} - {% macro check_for_schema_changes(source_relation, target_relation) %} diff --git a/core/dbt/include/global_project/macros/materializations/models/table/create_table_as.sql b/core/dbt/include/global_project/macros/materializations/models/table/create_table_as.sql new file mode 100644 index 00000000000..8e976ea2066 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/table/create_table_as.sql @@ -0,0 +1,25 @@ +{% macro get_create_table_as_sql(temporary, relation, sql) -%} + {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }} +{%- endmacro %} + +{% macro default__get_create_table_as_sql(temporary, relation, sql) -%} + {{ return(create_table_as(temporary, relation, sql)) }} +{% endmacro %} + + +/* {# keep logic under old macro name for backwards compatibility #} */ +{% macro create_table_as(temporary, relation, sql) -%} + {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, sql) }} +{%- endmacro %} + +{% macro default__create_table_as(temporary, relation, sql) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + + create {% if temporary: -%}temporary{%- endif %} table + {{ relation.include(database=(not temporary), schema=(not temporary)) }} + as ( + {{ sql }} + ); +{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/table/table.sql b/core/dbt/include/global_project/macros/materializations/models/table/table.sql similarity index 97% rename from core/dbt/include/global_project/macros/materializations/table/table.sql rename to core/dbt/include/global_project/macros/materializations/models/table/table.sql index cb1fb1a6c6f..88fc2bf713b 100644 --- a/core/dbt/include/global_project/macros/materializations/table/table.sql +++ b/core/dbt/include/global_project/macros/materializations/models/table/table.sql @@ -43,7 +43,7 @@ -- build model {% call statement('main') -%} - {{ create_table_as(False, intermediate_relation, sql) }} + {{ get_create_table_as_sql(False, intermediate_relation, sql) }} {%- endcall %} -- cleanup diff --git a/core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql b/core/dbt/include/global_project/macros/materializations/models/view/create_or_replace_view.sql similarity index 77% rename from core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql rename to core/dbt/include/global_project/macros/materializations/models/view/create_or_replace_view.sql index d74e14981b4..27ec438b9b9 100644 --- a/core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql +++ b/core/dbt/include/global_project/macros/materializations/models/view/create_or_replace_view.sql @@ -1,14 +1,4 @@ - -{% macro handle_existing_table(full_refresh, old_relation) %} - {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }} -{% endmacro %} - -{% macro default__handle_existing_table(full_refresh, old_relation) %} - {{ log("Dropping relation " ~ old_relation ~ " because it is of type " ~ old_relation.type) }} - {{ adapter.drop_relation(old_relation) }} -{% endmacro %} - -{# /* +/* {# Core materialization implementation. BigQuery and Snowflake are similar because both can use `create or replace view` where the resulting view schema is not necessarily the same as the existing view. On Redshift, this would @@ -17,8 +7,7 @@ This implementation is superior to the create_temp, swap_with_existing, drop_old paradigm because transactions don't run DDL queries atomically on Snowflake. By using `create or replace view`, the materialization becomes atomic in nature. - */ -#} +#} */ {% macro create_or_replace_view() %} {%- set identifier = model['alias'] -%} @@ -42,7 +31,7 @@ -- build model {% call statement('main') -%} - {{ create_view_as(target_relation, sql) }} + {{ get_create_view_as_sql(target_relation, sql) }} {%- endcall %} {{ run_hooks(post_hooks) }} diff --git a/core/dbt/include/global_project/macros/materializations/models/view/create_view_as.sql b/core/dbt/include/global_project/macros/materializations/models/view/create_view_as.sql new file mode 100644 index 00000000000..241e40cd12c --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/view/create_view_as.sql @@ -0,0 +1,22 @@ +{% macro get_create_view_as_sql(relation, sql) -%} + {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }} +{%- endmacro %} + +{% macro default__get_create_view_as_sql(relation, sql) -%} + {{ return(create_view_as(relation, sql)) }} +{% endmacro %} + + +/* {# keep logic under old name for backwards compatibility #} */ +{% macro create_view_as(relation, sql) -%} + {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }} +{%- endmacro %} + +{% macro default__create_view_as(relation, sql) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + create view {{ relation }} as ( + {{ sql }} + ); +{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/models/view/helpers.sql b/core/dbt/include/global_project/macros/materializations/models/view/helpers.sql new file mode 100644 index 00000000000..98f57018730 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/models/view/helpers.sql @@ -0,0 +1,8 @@ +{% macro handle_existing_table(full_refresh, old_relation) %} + {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }} +{% endmacro %} + +{% macro default__handle_existing_table(full_refresh, old_relation) %} + {{ log("Dropping relation " ~ old_relation ~ " because it is of type " ~ old_relation.type) }} + {{ adapter.drop_relation(old_relation) }} +{% endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/view/view.sql b/core/dbt/include/global_project/macros/materializations/models/view/view.sql similarity index 100% rename from core/dbt/include/global_project/macros/materializations/view/view.sql rename to core/dbt/include/global_project/macros/materializations/models/view/view.sql diff --git a/core/dbt/include/global_project/macros/materializations/seed/seed.sql b/core/dbt/include/global_project/macros/materializations/seeds/helpers.sql similarity index 64% rename from core/dbt/include/global_project/macros/materializations/seed/seed.sql rename to core/dbt/include/global_project/macros/materializations/seeds/helpers.sql index 3924ec9e2a9..bb6090eedab 100644 --- a/core/dbt/include/global_project/macros/materializations/seed/seed.sql +++ b/core/dbt/include/global_project/macros/materializations/seeds/helpers.sql @@ -25,6 +25,7 @@ {{ return(sql) }} {% endmacro %} + {% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%} {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }} {%- endmacro %} @@ -42,6 +43,7 @@ {{ return(sql) }} {% endmacro %} + {% macro get_binding_char() -%} {{ adapter.dispatch('get_binding_char', 'dbt')() }} {%- endmacro %} @@ -50,14 +52,16 @@ {{ return('%s') }} {% endmacro %} + {% macro get_batch_size() -%} - {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }} + {{ adapter.dispatch('get_batch_size', 'dbt')() }} {%- endmacro %} {% macro default__get_batch_size() %} {{ return(10000) }} {% endmacro %} + {% macro get_seed_column_quoted_csv(model, column_names) %} {%- set quote_seed_column = model['config'].get('quote_columns', None) -%} {% set quoted = [] %} @@ -69,6 +73,7 @@ {{ return(dest_cols_csv) }} {% endmacro %} + {% macro load_csv_rows(model, agate_table) -%} {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }} {%- endmacro %} @@ -110,59 +115,3 @@ {# Return SQL so we can render it out into the compiled files #} {{ return(statements[0]) }} {% endmacro %} - -{% materialization seed, default %} - - {%- set identifier = model['alias'] -%} - {%- set full_refresh_mode = (should_full_refresh()) -%} - - {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} - - {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} - {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} - - {%- set agate_table = load_agate_table() -%} - {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%} - - {{ run_hooks(pre_hooks, inside_transaction=False) }} - - -- `BEGIN` happens here: - {{ run_hooks(pre_hooks, inside_transaction=True) }} - - -- build model - {% set create_table_sql = "" %} - {% if exists_as_view %} - {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation)) }} - {% elif exists_as_table %} - {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %} - {% else %} - {% set create_table_sql = create_csv_table(model, agate_table) %} - {% endif %} - - {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %} - {% set rows_affected = (agate_table.rows | length) %} - {% set sql = load_csv_rows(model, agate_table) %} - - {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %} - {{ create_table_sql }}; - -- dbt seed -- - {{ sql }} - {% endcall %} - - {% set target_relation = this.incorporate(type='table') %} - {% do persist_docs(target_relation, model) %} - - {% if full_refresh_mode or not exists_as_table %} - {% do create_indexes(target_relation) %} - {% endif %} - - {{ run_hooks(post_hooks, inside_transaction=True) }} - - -- `COMMIT` happens here - {{ adapter.commit() }} - - {{ run_hooks(post_hooks, inside_transaction=False) }} - - {{ return({'relations': [target_relation]}) }} - -{% endmaterialization %} diff --git a/core/dbt/include/global_project/macros/materializations/seeds/seed.sql b/core/dbt/include/global_project/macros/materializations/seeds/seed.sql new file mode 100644 index 00000000000..a6eedcfbff8 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/seeds/seed.sql @@ -0,0 +1,55 @@ +{% materialization seed, default %} + + {%- set identifier = model['alias'] -%} + {%- set full_refresh_mode = (should_full_refresh()) -%} + + {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} + + {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} + {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%} + + {%- set agate_table = load_agate_table() -%} + {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% set create_table_sql = "" %} + {% if exists_as_view %} + {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation)) }} + {% elif exists_as_table %} + {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %} + {% else %} + {% set create_table_sql = create_csv_table(model, agate_table) %} + {% endif %} + + {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %} + {% set rows_affected = (agate_table.rows | length) %} + {% set sql = load_csv_rows(model, agate_table) %} + + {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %} + {{ create_table_sql }}; + -- dbt seed -- + {{ sql }} + {% endcall %} + + {% set target_relation = this.incorporate(type='table') %} + {% do persist_docs(target_relation, model) %} + + {% if full_refresh_mode or not exists_as_table %} + {% do create_indexes(target_relation) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + -- `COMMIT` happens here + {{ adapter.commit() }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{% endmaterialization %} diff --git a/core/dbt/include/global_project/macros/materializations/snapshot/snapshot.sql b/core/dbt/include/global_project/macros/materializations/snapshots/helpers.sql similarity index 51% rename from core/dbt/include/global_project/macros/materializations/snapshot/snapshot.sql rename to core/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 03b477e98c2..7c17b9e7621 100644 --- a/core/dbt/include/global_project/macros/materializations/snapshot/snapshot.sql +++ b/core/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -24,6 +24,10 @@ {% macro snapshot_staging_table(strategy, source_sql, target_relation) -%} + {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }} +{% endmacro %} + +{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} with snapshot_query as ( @@ -139,7 +143,11 @@ {%- endmacro %} -{% macro build_snapshot_table(strategy, sql) %} +{% macro build_snapshot_table(strategy, sql) -%} + {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }} +{% endmacro %} + +{% macro default__build_snapshot_table(strategy, sql) %} select *, {{ strategy.scd_id }} as dbt_scd_id, @@ -153,22 +161,6 @@ {% endmacro %} -{% macro get_or_create_relation(database, schema, identifier, type) %} - {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} - - {% if target_relation %} - {% do return([true, target_relation]) %} - {% endif %} - - {%- set new_relation = api.Relation.create( - database=database, - schema=schema, - identifier=identifier, - type=type - ) -%} - {% do return([false, new_relation]) %} -{% endmacro %} - {% macro build_snapshot_staging_table(strategy, sql, target_relation) %} {% set tmp_relation = make_temp_relation(target_relation) %} @@ -180,103 +172,3 @@ {% do return(tmp_relation) %} {% endmacro %} - - -{% materialization snapshot, default %} - {%- set config = model['config'] -%} - - {%- set target_table = model.get('alias', model.get('name')) -%} - - {%- set strategy_name = config.get('strategy') -%} - {%- set unique_key = config.get('unique_key') %} - - {% if not adapter.check_schema_exists(model.database, model.schema) %} - {% do create_schema(model.database, model.schema) %} - {% endif %} - - {% set target_relation_exists, target_relation = get_or_create_relation( - database=model.database, - schema=model.schema, - identifier=target_table, - type='table') -%} - - {%- if not target_relation.is_table -%} - {% do exceptions.relation_wrong_type(target_relation, 'table') %} - {%- endif -%} - - - {{ run_hooks(pre_hooks, inside_transaction=False) }} - - {{ run_hooks(pre_hooks, inside_transaction=True) }} - - {% set strategy_macro = strategy_dispatch(strategy_name) %} - {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", config, target_relation_exists) %} - - {% if not target_relation_exists %} - - {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %} - {% set final_sql = create_table_as(False, target_relation, build_sql) %} - - {% else %} - - {{ adapter.valid_snapshot_target(target_relation) }} - - {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %} - - -- this may no-op if the database does not require column expansion - {% do adapter.expand_target_column_types(from_relation=staging_table, - to_relation=target_relation) %} - - {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation) - | rejectattr('name', 'equalto', 'dbt_change_type') - | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') - | rejectattr('name', 'equalto', 'dbt_unique_key') - | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') - | list %} - - {% do create_columns(target_relation, missing_columns) %} - - {% set source_columns = adapter.get_columns_in_relation(staging_table) - | rejectattr('name', 'equalto', 'dbt_change_type') - | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') - | rejectattr('name', 'equalto', 'dbt_unique_key') - | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') - | list %} - - {% set quoted_source_columns = [] %} - {% for column in source_columns %} - {% do quoted_source_columns.append(adapter.quote(column.name)) %} - {% endfor %} - - {% set final_sql = snapshot_merge_sql( - target = target_relation, - source = staging_table, - insert_cols = quoted_source_columns - ) - %} - - {% endif %} - - {% call statement('main') %} - {{ final_sql }} - {% endcall %} - - {% do persist_docs(target_relation, model) %} - - {% if not target_relation_exists %} - {% do create_indexes(target_relation) %} - {% endif %} - - {{ run_hooks(post_hooks, inside_transaction=True) }} - - {{ adapter.commit() }} - - {% if staging_table is defined %} - {% do post_snapshot(staging_table) %} - {% endif %} - - {{ run_hooks(post_hooks, inside_transaction=False) }} - - {{ return({'relations': [target_relation]}) }} - -{% endmaterialization %} diff --git a/core/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql b/core/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql new file mode 100644 index 00000000000..76a9b5898f7 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql @@ -0,0 +1,98 @@ +{% materialization snapshot, default %} + {%- set config = model['config'] -%} + + {%- set target_table = model.get('alias', model.get('name')) -%} + + {%- set strategy_name = config.get('strategy') -%} + {%- set unique_key = config.get('unique_key') %} + + {% if not adapter.check_schema_exists(model.database, model.schema) %} + {% do create_schema(model.database, model.schema) %} + {% endif %} + + {% set target_relation_exists, target_relation = get_or_create_relation( + database=model.database, + schema=model.schema, + identifier=target_table, + type='table') -%} + + {%- if not target_relation.is_table -%} + {% do exceptions.relation_wrong_type(target_relation, 'table') %} + {%- endif -%} + + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {% set strategy_macro = strategy_dispatch(strategy_name) %} + {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", config, target_relation_exists) %} + + {% if not target_relation_exists %} + + {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %} + {% set final_sql = create_table_as(False, target_relation, build_sql) %} + + {% else %} + + {{ adapter.valid_snapshot_target(target_relation) }} + + {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %} + + -- this may no-op if the database does not require column expansion + {% do adapter.expand_target_column_types(from_relation=staging_table, + to_relation=target_relation) %} + + {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation) + | rejectattr('name', 'equalto', 'dbt_change_type') + | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') + | rejectattr('name', 'equalto', 'dbt_unique_key') + | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') + | list %} + + {% do create_columns(target_relation, missing_columns) %} + + {% set source_columns = adapter.get_columns_in_relation(staging_table) + | rejectattr('name', 'equalto', 'dbt_change_type') + | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') + | rejectattr('name', 'equalto', 'dbt_unique_key') + | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') + | list %} + + {% set quoted_source_columns = [] %} + {% for column in source_columns %} + {% do quoted_source_columns.append(adapter.quote(column.name)) %} + {% endfor %} + + {% set final_sql = snapshot_merge_sql( + target = target_relation, + source = staging_table, + insert_cols = quoted_source_columns + ) + %} + + {% endif %} + + {% call statement('main') %} + {{ final_sql }} + {% endcall %} + + {% do persist_docs(target_relation, model) %} + + {% if not target_relation_exists %} + {% do create_indexes(target_relation) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {% if staging_table is defined %} + {% do post_snapshot(staging_table) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{% endmaterialization %} diff --git a/core/dbt/include/global_project/macros/materializations/snapshot/snapshot_merge.sql b/core/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql similarity index 100% rename from core/dbt/include/global_project/macros/materializations/snapshot/snapshot_merge.sql rename to core/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql diff --git a/core/dbt/include/global_project/macros/materializations/snapshot/strategies.sql b/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql similarity index 99% rename from core/dbt/include/global_project/macros/materializations/snapshot/strategies.sql rename to core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql index 5c4615ac259..aeb2e09d965 100644 --- a/core/dbt/include/global_project/macros/materializations/snapshot/strategies.sql +++ b/core/dbt/include/global_project/macros/materializations/snapshots/strategies.sql @@ -39,7 +39,6 @@ {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }} {%- endmacro %} - {% macro default__snapshot_hash_arguments(args) -%} md5({%- for arg in args -%} coalesce(cast({{ arg }} as varchar ), '') @@ -97,7 +96,6 @@ {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }} {%- endmacro %} - {% macro default__snapshot_string_as_time(timestamp) %} {% do exceptions.raise_not_implemented( 'snapshot_string_as_time macro not implemented for adapter '+adapter.type() diff --git a/core/dbt/include/global_project/macros/materializations/tests/helpers.sql b/core/dbt/include/global_project/macros/materializations/tests/helpers.sql new file mode 100644 index 00000000000..efc55288076 --- /dev/null +++ b/core/dbt/include/global_project/macros/materializations/tests/helpers.sql @@ -0,0 +1,14 @@ +{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} + {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }} +{%- endmacro %} + +{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} + select + {{ fail_calc }} as failures, + {{ fail_calc }} {{ warn_if }} as should_warn, + {{ fail_calc }} {{ error_if }} as should_error + from ( + {{ main_sql }} + {{ "limit " ~ limit if limit != none }} + ) dbt_internal_test +{%- endmacro %} diff --git a/core/dbt/include/global_project/macros/materializations/test.sql b/core/dbt/include/global_project/macros/materializations/tests/test.sql similarity index 69% rename from core/dbt/include/global_project/macros/materializations/test.sql rename to core/dbt/include/global_project/macros/materializations/tests/test.sql index 5b73fdc87e7..8f6654f8e36 100644 --- a/core/dbt/include/global_project/macros/materializations/test.sql +++ b/core/dbt/include/global_project/macros/materializations/tests/test.sql @@ -1,19 +1,3 @@ -{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} - {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }} -{%- endmacro %} - - -{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%} - select - {{ fail_calc }} as failures, - {{ fail_calc }} {{ warn_if }} as should_warn, - {{ fail_calc }} {{ error_if }} as should_error - from ( - {{ main_sql }} - {{ "limit " ~ limit if limit != none }} - ) dbt_internal_test -{%- endmacro %} - {%- materialization test, default -%} {% set relations = [] %} diff --git a/core/dbt/include/global_project/macros/etc/where_subquery.sql b/core/dbt/include/global_project/macros/materializations/tests/where_subquery.sql similarity index 100% rename from core/dbt/include/global_project/macros/etc/where_subquery.sql rename to core/dbt/include/global_project/macros/materializations/tests/where_subquery.sql diff --git a/core/dbt/include/global_project/tests/generic/builtin.sql b/core/dbt/include/global_project/tests/generic/builtin.sql new file mode 100644 index 00000000000..23a7507fa0a --- /dev/null +++ b/core/dbt/include/global_project/tests/generic/builtin.sql @@ -0,0 +1,30 @@ +/* {# + Generic tests can be defined in `macros/` or in `tests/generic`. + These four tests are built into the dbt-core global project. + To support extensibility to other adapters and SQL dialects, + they call 'dispatched' macros. By default, they will use + the SQL defined in `global_project/macros/generic_test_sql` +#} */ + +{% test unique(model, column_name) %} + {% set macro = adapter.dispatch('test_unique', 'dbt') %} + {{ macro(model, column_name) }} +{% endtest %} + + +{% test not_null(model, column_name) %} + {% set macro = adapter.dispatch('test_not_null', 'dbt') %} + {{ macro(model, column_name) }} +{% endtest %} + + +{% test accepted_values(model, column_name, values, quote=True) %} + {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %} + {{ macro(model, column_name, values, quote) }} +{% endtest %} + + +{% test relationships(model, column_name, to, field) %} + {% set macro = adapter.dispatch('test_relationships', 'dbt') %} + {{ macro(model, column_name, to, field) }} +{% endtest %} diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 0e192af1cde..4d7cb887a99 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -556,8 +556,8 @@ def unrendered_tst_config(self, **updates): def _verify_generic_macro_structure(self, manifest): # just test a known global macro to avoid having to update this every # time they change. - self.assertIn('macro.dbt.column_list', manifest['macros']) - macro = manifest['macros']['macro.dbt.column_list'] + self.assertIn('macro.dbt.get_quoted_csv', manifest['macros']) + macro = manifest['macros']['macro.dbt.get_quoted_csv'] self.assertEqual( set(macro), { @@ -574,7 +574,7 @@ def _verify_generic_macro_structure(self, manifest): if k not in {'macro_sql'} } # Windows means we can't hard-code these. - helpers_path = Normalized('macros/materializations/helpers.sql') + helpers_path = Normalized('macros/materializations/models/incremental/column_helpers.sql') root_path = Normalized(os.path.join( self.dbt_core_install_root, 'include', 'global_project' )) @@ -584,8 +584,8 @@ def _verify_generic_macro_structure(self, manifest): 'original_file_path': helpers_path, 'package_name': 'dbt', 'root_path': root_path, - 'name': 'column_list', - 'unique_id': 'macro.dbt.column_list', + 'name': 'get_quoted_csv', + 'unique_id': 'macro.dbt.get_quoted_csv', 'created_at': ANY, 'tags': [], 'resource_type': 'macro',