diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 23c8a3c3..5f3ed012 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -98,7 +98,7 @@ jobs: uses: actions/checkout@v2 - name: Install Python packages - run: python -m pip install dbt-snowflake~=1.1.0 sqlfluff-templater-dbt + run: python -m pip install dbt-snowflake~=1.2.0 sqlfluff-templater-dbt - name: Test database connection run: dbt debug diff --git a/.gitignore b/.gitignore index c1cd9634..2381230b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ Pipfile.lock env.sh .python_version +.vscode diff --git a/README.md b/README.md index 4884d568..473663b9 100644 --- a/README.md +++ b/README.md @@ -36,30 +36,33 @@ packages: 3. Add an on-run-end hook to your `dbt_project.yml`: `on-run-end: "{{ dbt_artifacts.upload_results(results) }}"` (We recommend adding a conditional here so that the upload only occurs in your production environment, such as `on-run-end: "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}"`) -4. Create the tables dbt_artifacts uploads to with `dbt run-operation create_dbt_artifacts_tables` +4. If you are using [selectors](https://docs.getdbt.com/reference/node-selection/syntax), be sure to include the `dbt_artifacts` models in your dbt invocation step. 5. Run your project! +> :construction_worker: Always run the dbt_artifacts models in every dbt invocation which uses the `upload_results` macro. This ensures that the source models always have the correct fields in case of an update. + ## Configuration -The following configuration can be used to specify where the raw data is uploaded, and where the dbt models are created: +The following configuration can be used to specify where the raw (sources) data is uploaded, and where the dbt models are created: ```yml -vars: - dbt_artifacts_database: your_db # optional, default is your target database - dbt_artifacts_schema: your_schema # optional, default is your target schema - dbt_artifacts_create_schema: true|false # optional, set to false if you don't have privileges to create schema, default is true - models: ... dbt_artifacts: - +schema: your_destination_schema # optional, default is your target database + +database: your_destination_database # optional, default is your target database + +schema: your_destination_schema # optional, default is your target schema staging: + +database: your_destination_database # optional, default is your target database +schema: your_destination_schema # optional, default is your target schema - ... + sources: + +database: your_sources_database # optional, default is your target database + +schema: your sources_database # optional, default is your target schema ``` -Note that the model materializations are defined in this package's `dbt_project.yml`, so do not set them in your project. +Note that model materializations and `on_schema_change` configs are defined in this package's `dbt_project.yml`, so do not set them globally in your `dbt_project.yml` ([see docs on configuring packages](https://docs.getdbt.com/docs/building-a-dbt-project/package-management#configuring-packages)): + +> Configurations made in your dbt_project.yml file will override any configurations in a package (either in the dbt_project.yml file of the package, or in config blocks). ### Environment Variables @@ -92,6 +95,28 @@ vars: ] ``` +## Upgrading from 1.x to >=2.0.0 +If you were using the following variables: + +```yml +vars: + dbt_artifacts_database: your_db + dbt_artifacts_schema: your_schema +``` + +You must now move these to the following model configs: + +```yml +models: + ... + dbt_artifacts: + sources: + +database: your_db + +schema: your_schema +``` + +That's because the raw tables are now managed as dbt models. Be aware of any impact that [generate_database_name](https://docs.getdbt.com/docs/building-a-dbt-project/building-models/using-custom-databases#generate_database_name) and [generate_schema_name](https://docs.getdbt.com/docs/building-a-dbt-project/building-models/using-custom-schemas#how-does-dbt-generate-a-models-schema-name) macros may have on the final database/schema. + ## Migrating From <1.0.0 to >=1.0.0 To migrate your existing data from the `dbt-artifacts` versions <=0.8.0, a helper macro and guide is provided. This migration uses the old `fct_*` and `dim_*` models' data to populate the new sources. The steps to use the macro are as follows: diff --git a/dbt_project.yml b/dbt_project.yml index 2f27fbe0..5b4e25a9 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ name: 'dbt_artifacts' version: '1.2.0' config-version: 2 -require-dbt-version: ">=1.0.0" +require-dbt-version: ">=1.2.0" profile: "dbt_artifacts" clean-targets: # folders to be removed by `dbt clean` @@ -13,3 +13,6 @@ models: dbt_artifacts: +materialized: view +file_format: delta + sources: + +materialized: incremental + +on_schema_change: append_new_columns diff --git a/macros/create_dbt_artifacts_tables.sql b/macros/create_dbt_artifacts_tables.sql deleted file mode 100644 index bd515bf9..00000000 --- a/macros/create_dbt_artifacts_tables.sql +++ /dev/null @@ -1,45 +0,0 @@ -{% macro create_dbt_artifacts_tables() %} - - {% if target.type in ('spark', 'databricks') and target.database is not defined %} - {% set database_name = None %} - {% else %} - {% set database_name = var('dbt_artifacts_database', target.database) %} - {% endif %} - - {%- if var('dbt_artifacts_create_schema', true) -%} - {%- do adapter.create_schema(api.Relation.create(database=database_name, schema=var('dbt_artifacts_schema', target.schema))) -%} - {%- endif -%} - - {% set src_dbt_exposures = source('dbt_artifacts', 'exposures') %} - {{ dbt_artifacts.create_exposures_table_if_not_exists(src_dbt_exposures.database, src_dbt_exposures.schema, src_dbt_exposures.identifier) }} - - {% set src_dbt_model_executions = source('dbt_artifacts', 'model_executions') %} - {{ dbt_artifacts.create_model_executions_table_if_not_exists(src_dbt_model_executions.database, src_dbt_model_executions.schema, src_dbt_model_executions.identifier) }} - - {% set src_dbt_models = source('dbt_artifacts', 'models') %} - {{ dbt_artifacts.create_models_table_if_not_exists(src_dbt_models.database, src_dbt_models.schema, src_dbt_models.identifier) }} - - {% set src_dbt_seed_executions = source('dbt_artifacts', 'seed_executions') %} - {{ dbt_artifacts.create_seed_executions_table_if_not_exists(src_dbt_seed_executions.database, src_dbt_seed_executions.schema, src_dbt_seed_executions.identifier) }} - - {% set src_dbt_seeds = source('dbt_artifacts', 'seeds') %} - {{ dbt_artifacts.create_seeds_table_if_not_exists(src_dbt_seeds.database, src_dbt_seeds.schema, src_dbt_seeds.identifier) }} - - {% set src_dbt_snapshot_executions = source('dbt_artifacts', 'snapshot_executions') %} - {{ dbt_artifacts.create_snapshot_executions_table_if_not_exists(src_dbt_snapshot_executions.database, src_dbt_snapshot_executions.schema, src_dbt_snapshot_executions.identifier) }} - - {% set src_dbt_snapshots = source('dbt_artifacts', 'snapshots') %} - {{ dbt_artifacts.create_snapshots_table_if_not_exists(src_dbt_snapshots.database, src_dbt_snapshots.schema, src_dbt_snapshots.identifier) }} - - {% set src_dbt_sources = source('dbt_artifacts', 'sources') %} - {{ dbt_artifacts.create_sources_table_if_not_exists(src_dbt_sources.database, src_dbt_sources.schema, src_dbt_sources.identifier) }} - - {% set src_dbt_test_executions = source('dbt_artifacts', 'test_executions') %} - {{ dbt_artifacts.create_test_executions_table_if_not_exists(src_dbt_test_executions.database, src_dbt_test_executions.schema, src_dbt_test_executions.identifier) }} - - {% set src_dbt_tests = source('dbt_artifacts', 'tests') %} - {{ dbt_artifacts.create_tests_table_if_not_exists(src_dbt_tests.database, src_dbt_tests.schema, src_dbt_tests.identifier) }} - - {% set src_dbt_invocations = source('dbt_artifacts', 'invocations') %} - {{ dbt_artifacts.create_invocations_table_if_not_exists(src_dbt_invocations.database, src_dbt_invocations.schema, src_dbt_invocations.identifier) }} -{% endmacro %} diff --git a/macros/create_exposures_table_if_not_exists.sql b/macros/create_exposures_table_if_not_exists.sql deleted file mode 100644 index 12a429ac..00000000 --- a/macros/create_exposures_table_if_not_exists.sql +++ /dev/null @@ -1,86 +0,0 @@ -{% macro create_exposures_table_if_not_exists(database_name, schema_name, table_name) -%} - - {%- if adapter.get_relation(database=database_name, schema=schema_name, identifier=table_name) is none -%} - {% if database_name %} - {{ log("Creating table " ~ adapter.quote(database_name ~ "." ~ schema_name ~ "." ~ table_name), info=true) }} - {% else %} - {{ log("Creating table " ~ adapter.quote(schema_name ~ "." ~ table_name), info=true) }} - {% endif %} - {%- set query -%} - {{ adapter.dispatch('get_create_exposures_table_if_not_exists_statement', 'dbt_artifacts')(database_name, schema_name, table_name) }} - {% endset %} - {%- call statement(auto_begin=True) -%} - {{ query }} - {%- endcall -%} - {%- endif -%} - -{%- endmacro %} - -{% macro spark__get_create_exposures_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - name STRING, - type STRING, - owner STRING, - maturity STRING, - path STRING, - description STRING, - url STRING, - package_name STRING, - depends_on_nodes STRING - ) - using delta -{%- endmacro %} - -{% macro snowflake__get_create_exposures_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP_TZ, - name STRING, - type STRING, - owner VARIANT, - maturity STRING, - path STRING, - description STRING, - url STRING, - package_name STRING, - depends_on_nodes ARRAY - ) -{%- endmacro %} - -{% macro bigquery__get_create_exposures_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - name STRING, - type STRING, - owner JSON, - maturity STRING, - path STRING, - description STRING, - url STRING, - package_name STRING, - depends_on_nodes ARRAY - ) -{%- endmacro %} - -{% macro default__get_create_exposures_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - name STRING, - type STRING, - owner STRING, - maturity STRING, - path STRING, - description STRING, - url STRING, - package_name STRING, - depends_on_nodes STRING - ) -{%- endmacro %} diff --git a/macros/create_invocations_table_if_not_exists.sql b/macros/create_invocations_table_if_not_exists.sql deleted file mode 100644 index aafd29aa..00000000 --- a/macros/create_invocations_table_if_not_exists.sql +++ /dev/null @@ -1,106 +0,0 @@ -{% macro create_invocations_table_if_not_exists(database_name, schema_name, table_name) -%} - - {%- if adapter.get_relation(database=database_name, schema=schema_name, identifier=table_name) is none -%} - {% if database_name %} - {{ log("Creating table " ~ adapter.quote(database_name ~ "." ~ schema_name ~ "." ~ table_name), info=true) }} - {% else %} - {{ log("Creating table " ~ adapter.quote(schema_name ~ "." ~ table_name), info=true) }} - {% endif %} - {%- set query -%} - {{ adapter.dispatch('get_create_invocations_table_if_not_exists_statement', 'dbt_artifacts')(database_name, schema_name, table_name) }} - {% endset %} - {%- call statement(auto_begin=True) -%} - {{ query }} - {%- endcall -%} - {%- endif -%} - -{%- endmacro %} - -{% macro spark__get_create_invocations_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - dbt_version STRING, - project_name STRING, - run_started_at TIMESTAMP, - dbt_command STRING, - full_refresh_flag BOOLEAN, - target_profile_name STRING, - target_name STRING, - target_schema STRING, - target_threads INTEGER, - dbt_cloud_project_id STRING, - dbt_cloud_job_id STRING, - dbt_cloud_run_id STRING, - dbt_cloud_run_reason_category STRING, - dbt_cloud_run_reason STRING, - env_vars STRING, - dbt_vars STRING - ) - using delta -{%- endmacro %} - -{% macro snowflake__get_create_invocations_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - dbt_version STRING, - project_name STRING, - run_started_at TIMESTAMP_TZ, - dbt_command STRING, - full_refresh_flag BOOLEAN, - target_profile_name STRING, - target_name STRING, - target_schema STRING, - target_threads INTEGER, - dbt_cloud_project_id STRING, - dbt_cloud_job_id STRING, - dbt_cloud_run_id STRING, - dbt_cloud_run_reason_category STRING, - dbt_cloud_run_reason STRING, - env_vars OBJECT, - dbt_vars OBJECT - ) -{%- endmacro %} - -{% macro bigquery__get_create_invocations_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - dbt_version STRING, - project_name STRING, - run_started_at TIMESTAMP, - dbt_command STRING, - full_refresh_flag BOOLEAN, - target_profile_name STRING, - target_name STRING, - target_schema STRING, - target_threads INTEGER, - dbt_cloud_project_id STRING, - dbt_cloud_job_id STRING, - dbt_cloud_run_id STRING, - dbt_cloud_run_reason_category STRING, - dbt_cloud_run_reason STRING, - env_vars JSON, - dbt_vars JSON - ) -{%- endmacro %} - -{% macro default__get_create_invocations_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - dbt_version STRING, - project_name STRING, - run_started_at TIMESTAMP, - dbt_command STRING, - full_refresh_flag BOOLEAN, - target_profile_name STRING, - target_name STRING, - target_schema STRING, - target_threads INTEGER, - dbt_cloud_project_id STRING, - dbt_cloud_job_id STRING, - dbt_cloud_run_id STRING, - dbt_cloud_run_reason_category STRING, - dbt_cloud_run_reason STRING, - env_vars STRING, - dbt_vars STRING - ) -{%- endmacro %} diff --git a/macros/create_model_executions_table_if_not_exists.sql b/macros/create_model_executions_table_if_not_exists.sql deleted file mode 100644 index cf0e7d43..00000000 --- a/macros/create_model_executions_table_if_not_exists.sql +++ /dev/null @@ -1,91 +0,0 @@ -{% macro create_model_executions_table_if_not_exists(database_name, schema_name, table_name) -%} - - {%- if adapter.get_relation(database=database_name, schema=schema_name, identifier=table_name) is none -%} - {% if database_name %} - {{ log("Creating table " ~ adapter.quote(database_name ~ "." ~ schema_name ~ "." ~ table_name), info=true) }} - {% else %} - {{ log("Creating table " ~ adapter.quote(schema_name ~ "." ~ table_name), info=true) }} - {% endif %} - {%- set query -%} - {{ adapter.dispatch('get_create_model_executions_table_if_not_exists_statement', 'dbt_artifacts')(database_name, schema_name, table_name) }} - {% endset %} - {%- call statement(auto_begin=True) -%} - {{ query }} - {%- endcall -%} - {%- endif -%} - -{%- endmacro %} - -{% macro spark__get_create_model_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime DOUBLE, - rows_affected INTEGER, - materialization STRING, - schema STRING, - name STRING - ) - using delta -{%- endmacro %} - -{% macro snowflake__get_create_model_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP_TZ, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP_TZ, - query_completed_at TIMESTAMP_TZ, - total_node_runtime DOUBLE, - rows_affected INTEGER, - materialization STRING, - schema STRING, - name STRING - ) -{%- endmacro %} - -{% macro bigquery__get_create_model_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime FLOAT64, - rows_affected INTEGER, - bytes_processed INTEGER, - materialization STRING, - schema STRING, - name STRING - ) -{%- endmacro %} - -{% macro default__get_create_model_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime INTEGER, - rows_affected INTEGER, - materialization STRING, - schema STRING, - name STRING - ) -{%- endmacro %} diff --git a/macros/create_models_table_if_not_exists.sql b/macros/create_models_table_if_not_exists.sql deleted file mode 100644 index f4053ef2..00000000 --- a/macros/create_models_table_if_not_exists.sql +++ /dev/null @@ -1,82 +0,0 @@ -{% macro create_models_table_if_not_exists(database_name, schema_name, table_name) -%} - - {%- if adapter.get_relation(database=database_name, schema=schema_name, identifier=table_name) is none -%} - {% if database_name %} - {{ log("Creating table " ~ adapter.quote(database_name ~ "." ~ schema_name ~ "." ~ table_name), info=true) }} - {% else %} - {{ log("Creating table " ~ adapter.quote(schema_name ~ "." ~ table_name), info=true) }} - {% endif %} - {%- set query -%} - {{ adapter.dispatch('get_create_models_table_if_not_exists_statement', 'dbt_artifacts')(database_name, schema_name, table_name) }} - {% endset %} - {%- call statement(auto_begin=True) -%} - {{ query }} - {%- endcall -%} - {%- endif -%} - -{%- endmacro %} - -{% macro spark__get_create_models_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - name STRING, - depends_on_nodes STRING, - package_name STRING, - path STRING, - checksum STRING, - materialization STRING - ) - using delta -{%- endmacro %} - -{% macro snowflake__get_create_models_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP_TZ, - database STRING, - schema STRING, - name STRING, - depends_on_nodes ARRAY, - package_name STRING, - path STRING, - checksum STRING, - materialization STRING - ) -{%- endmacro %} - -{% macro bigquery__get_create_models_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - name STRING, - depends_on_nodes ARRAY, - package_name STRING, - path STRING, - checksum STRING, - materialization STRING - ) -{%- endmacro %} - -{% macro default__get_create_models_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - name STRING, - depends_on_nodes STRING, - package_name STRING, - path STRING, - checksum STRING, - materialization STRING - ) -{%- endmacro %} diff --git a/macros/create_seed_executions_table_if_not_exists.sql b/macros/create_seed_executions_table_if_not_exists.sql deleted file mode 100644 index 98f1eb71..00000000 --- a/macros/create_seed_executions_table_if_not_exists.sql +++ /dev/null @@ -1,90 +0,0 @@ -{% macro create_seed_executions_table_if_not_exists(database_name, schema_name, table_name) -%} - - {%- if adapter.get_relation(database=database_name, schema=schema_name, identifier=table_name) is none -%} - {% if database_name %} - {{ log("Creating table " ~ adapter.quote(database_name ~ "." ~ schema_name ~ "." ~ table_name), info=true) }} - {% else %} - {{ log("Creating table " ~ adapter.quote(schema_name ~ "." ~ table_name), info=true) }} - {% endif %} - {%- set query -%} - {{ adapter.dispatch('get_create_seed_executions_table_if_not_exists_statement', 'dbt_artifacts')(database_name, schema_name, table_name) }} - {% endset %} - {%- call statement(auto_begin=True) -%} - {{ query }} - {%- endcall -%} - {%- endif -%} - -{%- endmacro %} - -{% macro spark__get_create_seed_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime DOUBLE, - rows_affected INTEGER, - materialization STRING, - schema STRING, - name STRING - ) - using delta -{%- endmacro %} - -{% macro snowflake__get_create_seed_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP_TZ, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP_TZ, - query_completed_at TIMESTAMP_TZ, - total_node_runtime INTEGER, - rows_affected INTEGER, - materialization STRING, - schema STRING, - name STRING - ) -{%- endmacro %} - -{% macro bigquery__get_create_seed_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime FLOAT64, - rows_affected INTEGER, - materialization STRING, - schema STRING, - name STRING - ) -{%- endmacro %} - -{% macro default__get_create_seed_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime INTEGER, - rows_affected INTEGER, - materialization STRING, - schema STRING, - name STRING - ) -{%- endmacro %} diff --git a/macros/create_seeds_table_if_not_exists.sql b/macros/create_seeds_table_if_not_exists.sql deleted file mode 100644 index f23dc72e..00000000 --- a/macros/create_seeds_table_if_not_exists.sql +++ /dev/null @@ -1,74 +0,0 @@ -{% macro create_seeds_table_if_not_exists(database_name, schema_name, table_name) -%} - - {%- if adapter.get_relation(database=database_name, schema=schema_name, identifier=table_name) is none -%} - {% if database_name %} - {{ log("Creating table " ~ adapter.quote(database_name ~ "." ~ schema_name ~ "." ~ table_name), info=true) }} - {% else %} - {{ log("Creating table " ~ adapter.quote(schema_name ~ "." ~ table_name), info=true) }} - {% endif %} - {%- set query -%} - {{ adapter.dispatch('get_create_seeds_table_if_not_exists_statement', 'dbt_artifacts')(database_name, schema_name, table_name) }} - {% endset %} - {%- call statement(auto_begin=True) -%} - {{ query }} - {%- endcall -%} - {%- endif -%} - -{%- endmacro %} - -{% macro spark__get_create_seeds_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - name STRING, - package_name STRING, - path STRING, - checksum STRING - ) - using delta -{%- endmacro %} - -{% macro snowflake__get_create_seeds_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP_TZ, - database STRING, - schema STRING, - name STRING, - package_name STRING, - path STRING, - checksum STRING - ) -{%- endmacro %} - -{% macro bigquery__get_create_seeds_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - name STRING, - package_name STRING, - path STRING, - checksum STRING - ) -{%- endmacro %} - -{% macro default__get_create_seeds_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - name STRING, - package_name STRING, - path STRING, - checksum STRING - ) -{%- endmacro %} diff --git a/macros/create_snapshot_executions_table_if_not_exists.sql b/macros/create_snapshot_executions_table_if_not_exists.sql deleted file mode 100644 index f742f94d..00000000 --- a/macros/create_snapshot_executions_table_if_not_exists.sql +++ /dev/null @@ -1,90 +0,0 @@ -{% macro create_snapshot_executions_table_if_not_exists(database_name, schema_name, table_name) -%} - - {%- if adapter.get_relation(database=database_name, schema=schema_name, identifier=table_name) is none -%} - {% if database_name %} - {{ log("Creating table " ~ adapter.quote(database_name ~ "." ~ schema_name ~ "." ~ table_name), info=true) }} - {% else %} - {{ log("Creating table " ~ adapter.quote(schema_name ~ "." ~ table_name), info=true) }} - {% endif %} - {%- set query -%} - {{ adapter.dispatch('get_create_snapshot_executions_table_if_not_exists_statement', 'dbt_artifacts')(database_name, schema_name, table_name) }} - {% endset %} - {%- call statement(auto_begin=True) -%} - {{ query }} - {%- endcall -%} - {%- endif -%} - -{%- endmacro %} - -{% macro spark__get_create_snapshot_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create or replace table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime DOUBLE, - rows_affected INTEGER, - materialization STRING, - schema STRING, - name STRING - ) - using delta -{%- endmacro %} - -{% macro snowflake__get_create_snapshot_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create or replace table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP_TZ, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP_TZ, - query_completed_at TIMESTAMP_TZ, - total_node_runtime DOUBLE, - rows_affected INTEGER, - materialization STRING, - schema STRING, - name STRING - ) -{%- endmacro %} - -{% macro bigquery__get_create_snapshot_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create or replace table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime FLOAT64, - rows_affected INTEGER, - materialization STRING, - schema STRING, - name STRING - ) -{%- endmacro %} - -{% macro default__get_create_snapshot_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime INTEGER, - rows_affected INTEGER, - materialization STRING, - schema STRING, - name STRING - ) -{%- endmacro %} diff --git a/macros/create_snapshots_table_if_not_exists.sql b/macros/create_snapshots_table_if_not_exists.sql deleted file mode 100644 index 2d45a2f6..00000000 --- a/macros/create_snapshots_table_if_not_exists.sql +++ /dev/null @@ -1,82 +0,0 @@ -{% macro create_snapshots_table_if_not_exists(database_name, schema_name, table_name) -%} - - {%- if adapter.get_relation(database=database_name, schema=schema_name, identifier=table_name) is none -%} - {% if database_name %} - {{ log("Creating table " ~ adapter.quote(database_name ~ "." ~ schema_name ~ "." ~ table_name), info=true) }} - {% else %} - {{ log("Creating table " ~ adapter.quote(schema_name ~ "." ~ table_name), info=true) }} - {% endif %} - {%- set query -%} - {{ adapter.dispatch('get_create_snapshots_table_if_not_exists_statement', 'dbt_artifacts')(database_name, schema_name, table_name) }} - {% endset %} - {%- call statement(auto_begin=True) -%} - {{ query }} - {%- endcall -%} - {%- endif -%} - -{%- endmacro %} - -{% macro spark__get_create_snapshots_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - name STRING, - depends_on_nodes STRING, - package_name STRING, - path STRING, - checksum STRING, - strategy STRING - ) - using delta -{%- endmacro %} - -{% macro snowflake__get_create_snapshots_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP_TZ, - database STRING, - schema STRING, - name STRING, - depends_on_nodes ARRAY, - package_name STRING, - path STRING, - checksum STRING, - strategy STRING - ) -{%- endmacro %} - -{% macro bigquery__get_create_snapshots_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - name STRING, - depends_on_nodes ARRAY, - package_name STRING, - path STRING, - checksum STRING, - strategy STRING - ) -{%- endmacro %} - -{% macro default__get_create_snapshots_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - name STRING, - depends_on_nodes STRING, - package_name STRING, - path STRING, - checksum STRING, - strategy STRING - ) -{%- endmacro %} diff --git a/macros/create_sources_table_if_not_exists.sql b/macros/create_sources_table_if_not_exists.sql deleted file mode 100644 index d44b3e52..00000000 --- a/macros/create_sources_table_if_not_exists.sql +++ /dev/null @@ -1,82 +0,0 @@ -{% macro create_sources_table_if_not_exists(database_name, schema_name, table_name) -%} - - {%- if adapter.get_relation(database=database_name, schema=schema_name, identifier=table_name) is none -%} - {% if database_name %} - {{ log("Creating table " ~ adapter.quote(database_name ~ "." ~ schema_name ~ "." ~ table_name), info=true) }} - {% else %} - {{ log("Creating table " ~ adapter.quote(schema_name ~ "." ~ table_name), info=true) }} - {% endif %} - {%- set query -%} - {{ adapter.dispatch('get_create_sources_table_if_not_exists_statement', 'dbt_artifacts')(database_name, schema_name, table_name) }} - {% endset %} - {%- call statement(auto_begin=True) -%} - {{ query }} - {%- endcall -%} - {%- endif -%} - -{%- endmacro %} - -{% macro spark__get_create_sources_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - source_name STRING, - loader STRING, - name STRING, - identifier STRING, - loaded_at_field STRING, - freshness STRING - ) - using delta -{%- endmacro %} - -{% macro snowflake__get_create_sources_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP_TZ, - database STRING, - schema STRING, - source_name STRING, - loader STRING, - name STRING, - identifier STRING, - loaded_at_field STRING, - freshness ARRAY - ) -{%- endmacro %} - -{% macro bigquery__get_create_sources_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - source_name STRING, - loader STRING, - name STRING, - identifier STRING, - loaded_at_field STRING, - freshness JSON - ) -{%- endmacro %} - -{% macro default__get_create_sources_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - database STRING, - schema STRING, - source_name STRING, - loader STRING, - name STRING, - identifier STRING, - loaded_at_field STRING, - freshness STRING - ) -{%- endmacro %} diff --git a/macros/create_test_executions_table_if_not_exists.sql b/macros/create_test_executions_table_if_not_exists.sql deleted file mode 100644 index b7bea249..00000000 --- a/macros/create_test_executions_table_if_not_exists.sql +++ /dev/null @@ -1,82 +0,0 @@ -{% macro create_test_executions_table_if_not_exists(database_name, schema_name, table_name) -%} - - {%- if adapter.get_relation(database=database_name, schema=schema_name, identifier=table_name) is none -%} - {% if database_name %} - {{ log("Creating table " ~ adapter.quote(database_name ~ "." ~ schema_name ~ "." ~ table_name), info=true) }} - {% else %} - {{ log("Creating table " ~ adapter.quote(schema_name ~ "." ~ table_name), info=true) }} - {% endif %} - {%- set query -%} - {{ adapter.dispatch('get_create_test_executions_table_if_not_exists_statement', 'dbt_artifacts')(database_name, schema_name, table_name) }} - {% endset %} - {%- call statement(auto_begin=True) -%} - {{ query }} - {%- endcall -%} - {%- endif -%} - -{%- endmacro %} - -{% macro spark__get_create_test_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime DOUBLE, - rows_affected INTEGER, - failures INTEGER - ) - using delta -{%- endmacro %} - -{% macro snowflake__get_create_test_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP_TZ, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP_TZ, - query_completed_at TIMESTAMP_TZ, - total_node_runtime DOUBLE, - rows_affected INTEGER, - failures INTEGER - ) -{%- endmacro %} - -{% macro bigquery__get_create_test_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime FLOAT64, - rows_affected INTEGER, - failures INTEGER - ) -{%- endmacro %} - -{% macro default__get_create_test_executions_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - was_full_refresh BOOLEAN, - thread_id STRING, - status STRING, - compile_started_at TIMESTAMP, - query_completed_at TIMESTAMP, - total_node_runtime INTEGER, - rows_affected INTEGER, - failures INTEGER - ) -{%- endmacro %} diff --git a/macros/create_tests_table_if_not_exists.sql b/macros/create_tests_table_if_not_exists.sql deleted file mode 100644 index e876d6bd..00000000 --- a/macros/create_tests_table_if_not_exists.sql +++ /dev/null @@ -1,70 +0,0 @@ -{% macro create_tests_table_if_not_exists(database_name, schema_name, table_name) -%} - - {%- if adapter.get_relation(database=database_name, schema=schema_name, identifier=table_name) is none -%} - {% if database_name %} - {{ log("Creating table " ~ adapter.quote(database_name ~ "." ~ schema_name ~ "." ~ table_name), info=true) }} - {% else %} - {{ log("Creating table " ~ adapter.quote(schema_name ~ "." ~ table_name), info=true) }} - {% endif %} - {%- set query -%} - {{ adapter.dispatch('get_create_tests_table_if_not_exists_statement', 'dbt_artifacts')(database_name, schema_name, table_name) }} - {% endset %} - {%- call statement(auto_begin=True) -%} - {{ query }} - {%- endcall -%} - {%- endif -%} - -{%- endmacro %} - -{% macro spark__get_create_tests_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - name STRING, - depends_on_nodes STRING, - package_name STRING, - test_path STRING, - tags STRING - ) - using delta -{%- endmacro %} - -{% macro snowflake__get_create_tests_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP_TZ, - name STRING, - depends_on_nodes ARRAY, - package_name STRING, - test_path STRING, - tags ARRAY - ) -{%- endmacro %} - -{% macro bigquery__get_create_tests_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - name STRING, - depends_on_nodes ARRAY, - package_name STRING, - test_path STRING, - tags ARRAY - ) -{%- endmacro %} - -{% macro default__get_create_tests_table_if_not_exists_statement(database_name, schema_name, table_name) -%} - create table {{database_name}}.{{schema_name}}.{{table_name}} ( - command_invocation_id STRING, - node_id STRING, - run_started_at TIMESTAMP, - name STRING, - depends_on_nodes STRING, - package_name STRING, - test_path STRING, - tags STRING - ) -{%- endmacro %} diff --git a/macros/surrogate_key.sql b/macros/surrogate_key.sql new file mode 100644 index 00000000..e951167f --- /dev/null +++ b/macros/surrogate_key.sql @@ -0,0 +1,56 @@ +{# + Since folks commonly install dbt_artifacts alongside a myriad of other packages, + we copy the dbt_utils implementation of the surrogate_key macro so we don't have + any dependencies to make conflicts worse! +#} + +{%- macro surrogate_key(field_list) -%} + {# needed for safe_add to allow for non-keyword arguments see SO post #} + {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} + {% set frustrating_jinja_feature = varargs %} + {{ return(adapter.dispatch('surrogate_key', 'dbt_artifacts')(field_list, *varargs)) }} +{% endmacro %} + +{%- macro default__surrogate_key(field_list) -%} + +{%- if varargs|length >= 1 or field_list is string %} + +{%- set error_message = ' +Warning: the `surrogate_key` macro now takes a single list argument instead of \ +multiple string arguments. The {}.{} model triggered this warning. \ +'.format(model.package_name, model.name) -%} + +{%- do exceptions.warn(error_message) -%} + +{# first argument is not included in varargs, so add first element to field_list_xf #} +{%- set field_list_xf = [field_list] -%} + +{%- for field in varargs %} +{%- set _ = field_list_xf.append(field) -%} +{%- endfor -%} + +{%- else -%} + +{# if using list, just set field_list_xf as field_list #} +{%- set field_list_xf = field_list -%} + +{%- endif -%} + + +{%- set fields = [] -%} + +{%- for field in field_list_xf -%} + + {%- set _ = fields.append( + "coalesce(cast(" ~ field ~ " as " ~ type_string() ~ "), '')" + ) -%} + + {%- if not loop.last %} + {%- set _ = fields.append("'-'") -%} + {%- endif -%} + +{%- endfor -%} + +{{ hash(concat(fields)) }} + +{%- endmacro -%} diff --git a/macros/type_helpers.sql b/macros/type_helpers.sql new file mode 100644 index 00000000..19c3a718 --- /dev/null +++ b/macros/type_helpers.sql @@ -0,0 +1,45 @@ +{#- BOOLEAN -#} + +{% macro type_boolean() %} + {{ return(adapter.dispatch('type_boolean', 'dbt_artifacts')()) }} +{% endmacro %} + +{% macro default__type_boolean() %} + {{ return(api.Column.translate_type("boolean")) }} +{% endmacro %} + +{#- JSON -#} + +{% macro type_json() %} + {{ return(adapter.dispatch('type_json', 'dbt_artifacts')()) }} +{% endmacro %} + +{% macro default__type_json() %} + {{ return(api.Column.translate_type("string")) }} +{% endmacro %} + +{% macro snowflake__type_json() %} + OBJECT +{% endmacro %} + +{% macro bigquery__type_json() %} + JSON +{% endmacro %} + +{#- ARRAY -#} + +{% macro type_array() %} + {{ return(adapter.dispatch('type_array', 'dbt_artifacts')()) }} +{% endmacro %} + +{% macro default__type_array() %} + {{ return(api.Column.translate_type("string")) }} +{% endmacro %} + +{% macro snowflake__type_array() %} + ARRAY +{% endmacro %} + +{% macro bigquery__type_array() %} + ARRAY +{% endmacro %} diff --git a/macros/upload_results.sql b/macros/upload_results.sql index 12d32020..b8fd9646 100644 --- a/macros/upload_results.sql +++ b/macros/upload_results.sql @@ -1,46 +1,64 @@ +{# dbt doesn't like us ref'ing in an operation so we fetch the info from the graph #} +{% macro get_relation(get_relation_name) %} + {% if execute %} + {% set model_get_relation_node = graph.nodes.values() | selectattr('name', 'equalto', get_relation_name) | first %} + {% set relation = api.Relation.create( + database = model_get_relation_node.database, + schema = model_get_relation_node.schema, + identifier = model_get_relation_node.alias + ) + %} + {% do return(relation) %} + {% else %} + {% do return(api.Relation.create()) %} + {% endif %} +{% endmacro %} + {% macro upload_results(results) -%} + {% if execute %} + {% if results != [] %} {% do log("Uploading model executions", true) %} - {% set src_dbt_model_executions = source('dbt_artifacts', 'model_executions') %} + {% set model_executions = dbt_artifacts.get_relation('model_executions') %} {% set content_model_executions = dbt_artifacts.upload_model_executions(results) %} {{ dbt_artifacts.insert_into_metadata_table( - database_name=src_dbt_model_executions.database, - schema_name=src_dbt_model_executions.schema, - table_name=src_dbt_model_executions.identifier, + database_name=model_executions.database, + schema_name=model_executions.schema, + table_name=model_executions.identifier, content=content_model_executions ) }} {% do log("Uploading seed executions", true) %} - {% set src_dbt_seed_executions = source('dbt_artifacts', 'seed_executions') %} + {% set seed_executions = dbt_artifacts.get_relation('seed_executions') %} {% set content_seed_executions = dbt_artifacts.upload_seed_executions(results) %} {{ dbt_artifacts.insert_into_metadata_table( - database_name=src_dbt_seed_executions.database, - schema_name=src_dbt_seed_executions.schema, - table_name=src_dbt_seed_executions.identifier, + database_name=seed_executions.database, + schema_name=seed_executions.schema, + table_name=seed_executions.identifier, content=content_seed_executions ) }} {% do log("Uploading snapshot executions", true) %} - {% set src_dbt_snapshot_executions = source('dbt_artifacts', 'snapshot_executions') %} + {% set snapshot_executions = dbt_artifacts.get_relation('snapshot_executions') %} {% set content_snapshot_executions = dbt_artifacts.upload_snapshot_executions(results) %} {{ dbt_artifacts.insert_into_metadata_table( - database_name=src_dbt_snapshot_executions.database, - schema_name=src_dbt_snapshot_executions.schema, - table_name=src_dbt_snapshot_executions.identifier, + database_name=snapshot_executions.database, + schema_name=snapshot_executions.schema, + table_name=snapshot_executions.identifier, content=content_snapshot_executions ) }} {% do log("Uploading test executions", true) %} - {% set src_dbt_test_executions = source('dbt_artifacts', 'test_executions') %} + {% set test_executions = dbt_artifacts.get_relation('test_executions') %} {% set content_test_executions = dbt_artifacts.upload_test_executions(results) %} {{ dbt_artifacts.insert_into_metadata_table( - database_name=src_dbt_test_executions.database, - schema_name=src_dbt_test_executions.schema, - table_name=src_dbt_test_executions.identifier, + database_name=test_executions.database, + schema_name=test_executions.schema, + table_name=test_executions.identifier, content=content_test_executions ) }} @@ -48,78 +66,78 @@ {% endif %} {% do log("Uploading exposures", true) %} - {% set src_dbt_exposures = source('dbt_artifacts', 'exposures') %} + {% set exposures = dbt_artifacts.get_relation('exposures') %} {% set content_exposures = dbt_artifacts.upload_exposures(graph) %} {{ dbt_artifacts.insert_into_metadata_table( - database_name=src_dbt_exposures.database, - schema_name=src_dbt_exposures.schema, - table_name=src_dbt_exposures.identifier, + database_name=exposures.database, + schema_name=exposures.schema, + table_name=exposures.identifier, content=content_exposures ) }} {% do log("Uploading tests", true) %} - {% set src_dbt_tests = source('dbt_artifacts', 'tests') %} + {% set tests = dbt_artifacts.get_relation('tests') %} {% set content_tests = dbt_artifacts.upload_tests(graph) %} {{ dbt_artifacts.insert_into_metadata_table( - database_name=src_dbt_tests.database, - schema_name=src_dbt_tests.schema, - table_name=src_dbt_tests.identifier, + database_name=tests.database, + schema_name=tests.schema, + table_name=tests.identifier, content=content_tests ) }} {% do log("Uploading seeds", true) %} - {% set src_dbt_seeds = source('dbt_artifacts', 'seeds') %} + {% set seeds = dbt_artifacts.get_relation('seeds') %} {% set content_seeds = dbt_artifacts.upload_seeds(graph) %} {{ dbt_artifacts.insert_into_metadata_table( - database_name=src_dbt_seeds.database, - schema_name=src_dbt_seeds.schema, - table_name=src_dbt_seeds.identifier, + database_name=seeds.database, + schema_name=seeds.schema, + table_name=seeds.identifier, content=content_seeds ) }} {% do log("Uploading models", true) %} - {% set src_dbt_models = source('dbt_artifacts', 'models') %} + {% set models = dbt_artifacts.get_relation('models') %} {% set content_models = dbt_artifacts.upload_models(graph) %} {{ dbt_artifacts.insert_into_metadata_table( - database_name=src_dbt_models.database, - schema_name=src_dbt_models.schema, - table_name=src_dbt_models.identifier, + database_name=models.database, + schema_name=models.schema, + table_name=models.identifier, content=content_models ) }} {% do log("Uploading sources", true) %} - {% set src_dbt_sources = source('dbt_artifacts', 'sources') %} + {% set sources = dbt_artifacts.get_relation('sources') %} {% set content_sources = dbt_artifacts.upload_sources(graph) %} {{ dbt_artifacts.insert_into_metadata_table( - database_name=src_dbt_sources.database, - schema_name=src_dbt_sources.schema, - table_name=src_dbt_sources.identifier, + database_name=sources.database, + schema_name=sources.schema, + table_name=sources.identifier, content=content_sources ) }} {% do log("Uploading snapshots", true) %} - {% set src_dbt_snapshots = source('dbt_artifacts', 'snapshots') %} + {% set snapshots = dbt_artifacts.get_relation('snapshots') %} {% set content_snapshots = dbt_artifacts.upload_snapshots(graph) %} {{ dbt_artifacts.insert_into_metadata_table( - database_name=src_dbt_snapshots.database, - schema_name=src_dbt_snapshots.schema, - table_name=src_dbt_snapshots.identifier, + database_name=snapshots.database, + schema_name=snapshots.schema, + table_name=snapshots.identifier, content=content_snapshots ) }} {% do log("Uploading invocations", true) %} - {% set src_dbt_invocations = source('dbt_artifacts', 'invocations') %} + {% set invocations = dbt_artifacts.get_relation('invocations') %} {% set content_invocations = dbt_artifacts.upload_invocations() %} {{ dbt_artifacts.insert_into_metadata_table( - database_name=src_dbt_invocations.database, - schema_name=src_dbt_invocations.schema, - table_name=src_dbt_invocations.identifier, + database_name=invocations.database, + schema_name=invocations.schema, + table_name=invocations.identifier, content=content_invocations ) }} diff --git a/macros/upload_snapshot_executions.sql b/macros/upload_snapshot_executions.sql index b349d9dd..e689a571 100644 --- a/macros/upload_snapshot_executions.sql +++ b/macros/upload_snapshot_executions.sql @@ -1,5 +1,4 @@ {% macro upload_snapshot_executions(results) -%} - {% set src_dbt_snapshot_executions = source('dbt_artifacts', 'snapshot_executions') %} {% set snapshots = [] %} {% for result in results %} {% if result.node.resource_type == "snapshot" %} diff --git a/models/sources/exposures.sql b/models/sources/exposures.sql new file mode 100644 index 00000000..6e9f1e22 --- /dev/null +++ b/models/sources/exposures.sql @@ -0,0 +1,20 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as type, + cast(null as {{ type_json() }}) as owner, + cast(null as {{ type_string() }}) as maturity, + cast(null as {{ type_string() }}) as path, + cast(null as {{ type_string() }}) as description, + cast(null as {{ type_string() }}) as url, + cast(null as {{ type_string() }}) as package_name, + cast(null as {{ type_array() }}) as depends_on_nodes +from dummy_cte +where 1 = 0 diff --git a/models/sources/invocations.sql b/models/sources/invocations.sql new file mode 100644 index 00000000..b176d2b3 --- /dev/null +++ b/models/sources/invocations.sql @@ -0,0 +1,25 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as dbt_version, + cast(null as {{ type_string() }}) as project_name, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as dbt_command, + cast(null as {{ type_boolean() }}) as full_refresh_flag, + cast(null as {{ type_string() }}) as target_profile_name, + cast(null as {{ type_string() }}) as target_name, + cast(null as {{ type_string() }}) as target_schema, + cast(null as {{ type_int() }}) as target_threads, + cast(null as {{ type_string() }}) as dbt_cloud_project_id, + cast(null as {{ type_string() }}) as dbt_cloud_job_id, + cast(null as {{ type_string() }}) as dbt_cloud_run_id, + cast(null as {{ type_string() }}) as dbt_cloud_run_reason_category, + cast(null as {{ type_string() }}) as dbt_cloud_run_reason, + cast(null as {{ type_json() }}) as env_vars, + cast(null as {{ type_json() }}) as dbt_vars +from dummy_cte +where 1 = 0 diff --git a/models/sources/model_executions.sql b/models/sources/model_executions.sql new file mode 100644 index 00000000..a5675f93 --- /dev/null +++ b/models/sources/model_executions.sql @@ -0,0 +1,24 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_boolean() }}) as was_full_refresh, + cast(null as {{ type_string() }}) as thread_id, + cast(null as {{ type_string() }}) as status, + cast(null as {{ type_timestamp() }}) as compile_started_at, + cast(null as {{ type_timestamp() }}) as query_completed_at, + cast(null as {{ type_float() }}) as total_node_runtime, + cast(null as {{ type_int() }}) as rows_affected, + {% if target.type == 'bigquery' %} + cast(null as {{ type_int() }}) as bytes_processed, + {% endif %} + cast(null as {{ type_string() }}) as materialization, + cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as name +from dummy_cte +where 1 = 0 diff --git a/models/sources/models.sql b/models/sources/models.sql new file mode 100644 index 00000000..51c43c4d --- /dev/null +++ b/models/sources/models.sql @@ -0,0 +1,19 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as database, + cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_array() }}) as depends_on_nodes, + cast(null as {{ type_string() }}) as package_name, + cast(null as {{ type_string() }}) as path, + cast(null as {{ type_string() }}) as checksum, + cast(null as {{ type_string() }}) as materialization +from dummy_cte +where 1 = 0 diff --git a/models/sources/seed_executions.sql b/models/sources/seed_executions.sql new file mode 100644 index 00000000..08e44c41 --- /dev/null +++ b/models/sources/seed_executions.sql @@ -0,0 +1,21 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_boolean() }}) as was_full_refresh, + cast(null as {{ type_string() }}) as thread_id, + cast(null as {{ type_string() }}) as status, + cast(null as {{ type_timestamp() }}) as compile_started_at, + cast(null as {{ type_timestamp() }}) as query_completed_at, + cast(null as {{ type_float() }}) as total_node_runtime, + cast(null as {{ type_int() }}) as rows_affected, + cast(null as {{ type_string() }}) as materialization, + cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as name +from dummy_cte +where 1 = 0 diff --git a/models/sources/seeds.sql b/models/sources/seeds.sql new file mode 100644 index 00000000..095c0ad1 --- /dev/null +++ b/models/sources/seeds.sql @@ -0,0 +1,17 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as database, + cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as package_name, + cast(null as {{ type_string() }}) as path, + cast(null as {{ type_string() }}) as checksum +from dummy_cte +where 1 = 0 diff --git a/models/sources/snapshot_executions.sql b/models/sources/snapshot_executions.sql new file mode 100644 index 00000000..08e44c41 --- /dev/null +++ b/models/sources/snapshot_executions.sql @@ -0,0 +1,21 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_boolean() }}) as was_full_refresh, + cast(null as {{ type_string() }}) as thread_id, + cast(null as {{ type_string() }}) as status, + cast(null as {{ type_timestamp() }}) as compile_started_at, + cast(null as {{ type_timestamp() }}) as query_completed_at, + cast(null as {{ type_float() }}) as total_node_runtime, + cast(null as {{ type_int() }}) as rows_affected, + cast(null as {{ type_string() }}) as materialization, + cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as name +from dummy_cte +where 1 = 0 diff --git a/models/sources/snapshots.sql b/models/sources/snapshots.sql new file mode 100644 index 00000000..c8f37e21 --- /dev/null +++ b/models/sources/snapshots.sql @@ -0,0 +1,19 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as database, + cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_array() }}) as depends_on_nodes, + cast(null as {{ type_string() }}) as package_name, + cast(null as {{ type_string() }}) as path, + cast(null as {{ type_string() }}) as checksum, + cast(null as {{ type_string() }}) as strategy +from dummy_cte +where 1 = 0 diff --git a/models/sources/sources.sql b/models/sources/sources.sql new file mode 100644 index 00000000..13f69ddc --- /dev/null +++ b/models/sources/sources.sql @@ -0,0 +1,23 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as database, + cast(null as {{ type_string() }}) as schema, + cast(null as {{ type_string() }}) as source_name, + cast(null as {{ type_string() }}) as loader, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_string() }}) as identifier, + cast(null as {{ type_string() }}) as loaded_at_field, + {% if target.type == 'snowflake'%} + cast(null as {{ type_array() }}) as freshness + {% else %} + cast(null as {{ type_json() }}) as freshness + {% endif %} +from dummy_cte +where 1 = 0 diff --git a/models/sources/test_executions.sql b/models/sources/test_executions.sql new file mode 100644 index 00000000..f7262099 --- /dev/null +++ b/models/sources/test_executions.sql @@ -0,0 +1,19 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_boolean() }}) as was_full_refresh, + cast(null as {{ type_string() }}) as thread_id, + cast(null as {{ type_string() }}) as status, + cast(null as {{ type_timestamp() }}) as compile_started_at, + cast(null as {{ type_timestamp() }}) as query_completed_at, + cast(null as {{ type_float() }}) as total_node_runtime, + cast(null as {{ type_int() }}) as rows_affected, + cast(null as {{ type_int() }}) as failures +from dummy_cte +where 1 = 0 diff --git a/models/sources/tests.sql b/models/sources/tests.sql new file mode 100644 index 00000000..c23a666f --- /dev/null +++ b/models/sources/tests.sql @@ -0,0 +1,16 @@ +/* Bigquery won't let us `where` without `from` so we use this workaround */ +with dummy_cte as ( + select 1 as foo +) + +select + cast(null as {{ type_string() }}) as command_invocation_id, + cast(null as {{ type_string() }}) as node_id, + cast(null as {{ type_timestamp() }}) as run_started_at, + cast(null as {{ type_string() }}) as name, + cast(null as {{ type_array() }}) as depends_on_nodes, + cast(null as {{ type_string() }}) as package_name, + cast(null as {{ type_string() }}) as test_path, + cast(null as {{ type_array() }}) as tags +from dummy_cte +where 1 = 0 diff --git a/models/staging/sources.yml b/models/staging/sources.yml deleted file mode 100644 index 06a766eb..00000000 --- a/models/staging/sources.yml +++ /dev/null @@ -1,300 +0,0 @@ -version: 2 - -sources: - - name: dbt_artifacts - database: "{% if target.type not in ('spark', 'databricks') %}{{ var('dbt_artifacts_database', target.database) }}{% endif %}" - schema: "{{ var('dbt_artifacts_schema', target.schema) }}" - tables: - - name: model_executions - identifier: "{{ var('dbt_model_executions_table', 'model_executions') }}" - columns: - - name: command_invocation_id - description: '{{ doc("command_invocation_id") }}' - - name: compile_started_at - description: '{{ doc("compile_started_at") }}' - - name: materialization - description: '{{ doc("materialization") }}' - - name: name - description: '{{ doc("name") }}' - - name: node_id - description: '{{ doc("node_id") }}' - - name: query_completed_at - description: '{{ doc("query_completed_at") }}' - - name: rows_affected - description: '{{ doc("rows_affected") }}' - - name: schema - description: '{{ doc("schema") }}' - - name: status - description: '{{ doc("status") }}' - - name: thread_id - description: '{{ doc("thread_id") }}' - - name: total_node_runtime - description: '{{ doc("total_node_runtime") }}' - - name: was_full_refresh - description: '{{ doc("was_full_refresh") }}' - - - name: tests - identifier: "{{ var('dbt_tests_table', 'tests') }}" - columns: - - name: command_invocation_id - description: '{{ doc("command_invocation_id") }}' - - name: depends_on_nodes - description: '{{ doc("depends_on_nodes") }}' - - name: name - description: '{{ doc("name") }}' - - name: node_id - description: '{{ doc("node_id") }}' - - name: package_name - description: '{{ doc("package_name") }}' - - name: run_started_at - description: '{{ doc("run_started_at") }}' - - name: tags - description: '{{ doc("tags") }}' - - name: test_path - description: '{{ doc("test_path") }}' - - - name: test_executions - identifier: "{{ var('dbt_test_executions_table', 'test_executions') }}" - columns: - - name: command_invocation_id - description: '{{ doc("command_invocation_id") }}' - - name: compile_started_at - description: '{{ doc("compile_started_at") }}' - - name: failures - description: '{{ doc("failures") }}' - - name: node_id - description: '{{ doc("node_id") }}' - - name: query_completed_at - description: '{{ doc("query_completed_at") }}' - - name: rows_affected - description: '{{ doc("rows_affected") }}' - - name: status - description: '{{ doc("status") }}' - - name: thread_id - description: '{{ doc("thread_id") }}' - - name: total_node_runtime - description: '{{ doc("total_node_runtime") }}' - - name: was_full_refresh - description: '{{ doc("was_full_refresh") }}' - - - name: models - identifier: "{{ var('dbt_models_table', 'models') }}" - columns: - - name: checksum - description: '{{ doc("checksum") }}' - - name: command_invocation_id - description: '{{ doc("command_invocation_id") }}' - - name: database - description: '{{ doc("database") }}' - - name: depends_on_nodes - description: '{{ doc("depends_on_nodes") }}' - - name: materialization - description: '{{ doc("materialization") }}' - - name: name - description: '{{ doc("name") }}' - - name: node_id - description: '{{ doc("node_id") }}' - - name: package_name - description: '{{ doc("package_name") }}' - - name: path - description: '{{ doc("path") }}' - - name: run_started_at - description: '{{ doc("run_started_at") }}' - - name: schema - description: '{{ doc("schema") }}' - - - name: seeds - identifier: "{{ var('dbt_seeds_table', 'seeds') }}" - columns: - - name: checksum - description: '{{ doc("checksum") }}' - - name: command_invocation_id - description: '{{ doc("command_invocation_id") }}' - - name: database - description: '{{ doc("database") }}' - - name: name - description: '{{ doc("name") }}' - - name: node_id - description: '{{ doc("node_id") }}' - - name: package_name - description: '{{ doc("package_name") }}' - - name: path - description: '{{ doc("path") }}' - - name: schema - description: '{{ doc("schema") }}' - - - name: seed_executions - identifier: "{{ var('dbt_seed_executions_table', 'seed_executions') }}" - columns: - - name: command_invocation_id - description: '{{ doc("command_invocation_id") }}' - - name: compile_started_at - description: '{{ doc("compile_started_at") }}' - - name: materialization - description: '{{ doc("materialization") }}' - - name: name - description: '{{ doc("name") }}' - - name: node_id - description: '{{ doc("node_id") }}' - - name: query_completed_at - description: '{{ doc("query_completed_at") }}' - - name: rows_affected - description: '{{ doc("rows_affected") }}' - - name: run_started_at - description: '{{ doc("run_started_at") }}' - - name: schema - description: '{{ doc("schema") }}' - - name: status - description: '{{ doc("status") }}' - - name: thread_id - description: '{{ doc("thread_id") }}' - - name: total_node_runtime - description: '{{ doc("total_node_runtime") }}' - - name: was_full_refresh - description: '{{ doc("was_full_refresh") }}' - - - name: exposures - identifier: "{{ var('dbt_exposures_table', 'exposures') }}" - columns: - - name: command_invocation_id - description: '{{ doc("command_invocation_id") }}' - - name: depends_on_nodes - description: '{{ doc("depends_on_nodes") }}' - - name: description - description: '{{ doc("description") }}' - - name: maturity - description: '{{ doc("maturity") }}' - - name: name - description: '{{ doc("name") }}' - - name: node_id - description: '{{ doc("node_id") }}' - - name: owner - description: '{{ doc("owner") }}' - - name: package_name - description: '{{ doc("package_name") }}' - - name: path - description: '{{ doc("path") }}' - - name: run_started_at - description: '{{ doc("run_started_at") }}' - - name: type - description: '{{ doc("type") }}' - - name: url - description: '{{ doc("url") }}' - - - name: snapshots - identifier: "{{ var('dbt_snapshots_table', 'snapshots') }}" - columns: - - name: checksum - description: '{{ doc("checksum") }}' - - name: command_invocation_id - description: '{{ doc("command_invocation_id") }}' - - name: database - description: '{{ doc("database") }}' - - name: depends_on_nodes - description: '{{ doc("depends_on_nodes") }}' - - name: name - description: '{{ doc("name") }}' - - name: node_id - description: '{{ doc("node_id") }}' - - name: package_name - description: '{{ doc("package_name") }}' - - name: path - description: '{{ doc("path") }}' - - name: run_started_at - description: '{{ doc("run_started_at") }}' - - name: schema - description: '{{ doc("schema") }}' - - name: strategy - description: '{{ doc("strategy") }}' - - - name: snapshot_executions - identifier: "{{ var('dbt_snapshot_executions_table', 'snapshot_executions') }}" - columns: - - name: command_invocation_id - description: '{{ doc("command_invocation_id") }}' - - name: compile_started_at - description: '{{ doc("compile_started_at") }}' - - name: materialization - description: '{{ doc("materialization") }}' - - name: name - description: '{{ doc("name") }}' - - name: node_id - description: '{{ doc("node_id") }}' - - name: query_completed_at - description: '{{ doc("query_completed_at") }}' - - name: rows_affected - description: '{{ doc("rows_affected") }}' - - name: schema - description: '{{ doc("schema") }}' - - name: status - description: '{{ doc("status") }}' - - name: thread_id - description: '{{ doc("thread_id") }}' - - name: total_node_runtime - description: '{{ doc("total_node_runtime") }}' - - name: was_full_refresh - description: '{{ doc("was_full_refresh") }}' - - - name: sources - identifier: "{{ var('dbt_sources_table', 'sources') }}" - columns: - - name: command_invocation_id - description: '{{ doc("command_invocation_id") }}' - - name: database - description: '{{ doc("database") }}' - - name: freshness - description: '{{ doc("freshness") }}' - - name: identifier - description: '{{ doc("identifier") }}' - - name: loaded_at_field - description: '{{ doc("loaded_at_field") }}' - - name: loader - description: '{{ doc("loader") }}' - - name: name - description: '{{ doc("name") }}' - - name: node_id - description: '{{ doc("node_id") }}' - - name: run_started_at - description: '{{ doc("run_started_at") }}' - - name: schema - description: '{{ doc("schema") }}' - - name: source_name - description: '{{ doc("source_name") }}' - - - name: invocations - identifier: "{{ var('dbt_invocations_table', 'invocations') }}" - columns: - - name: command_invocation_id - description: '{{ doc("command_invocation_id") }}' - - name: dbt_version - description: '{{ doc("dbt_version") }}' - - name: project_name - description: '{{ doc("project_name") }}' - - name: run_started_at - description: '{{ doc("run_started_at") }}' - - name: dbt_command - description: '{{ doc("dbt_command") }}' - - name: full_refresh_flag - description: '{{ doc("full_refresh_flag") }}' - - name: target_profile_name - description: '{{ doc("target_profile_name") }}' - - name: target_name - description: '{{ doc("target_name") }}' - - name: target_schema - description: '{{ doc("target_schema") }}' - - name: target_threads - description: '{{ doc("target_threads") }}' - - name: dbt_cloud_project_id - description: '{{ doc("dbt_cloud_project_id") }}' - - name: dbt_cloud_job_id - description: '{{ doc("dbt_cloud_job_id") }}' - - name: dbt_cloud_run_id - description: '{{ doc("dbt_cloud_run_id") }}' - - name: dbt_cloud_run_reason_category - description: '{{ doc("dbt_cloud_run_reason_category") }}' - - name: dbt_cloud_run_reason - description: '{{ doc("dbt_cloud_run_reason") }}' - - name: env_vars - description: '{{ doc("env_vars") }}' - - name: dbt_vars - description: '{{ doc("dbt_vars") }}' diff --git a/models/staging/stg_dbt__exposures.sql b/models/staging/stg_dbt__exposures.sql index e9a0b482..6aeecd4d 100644 --- a/models/staging/stg_dbt__exposures.sql +++ b/models/staging/stg_dbt__exposures.sql @@ -1,14 +1,14 @@ with base as ( select * - from {{ source('dbt_artifacts', 'exposures') }} + from {{ ref('exposures') }} ), enhanced as ( select - {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as exposure_execution_id, + {{ dbt_artifacts.surrogate_key(['command_invocation_id', 'node_id']) }} as exposure_execution_id, command_invocation_id, node_id, run_started_at, diff --git a/models/staging/stg_dbt__invocations.sql b/models/staging/stg_dbt__invocations.sql index 5dae587c..d642a3fa 100644 --- a/models/staging/stg_dbt__invocations.sql +++ b/models/staging/stg_dbt__invocations.sql @@ -1,7 +1,7 @@ with base as ( select * - from {{ source('dbt_artifacts', 'invocations') }} + from {{ ref('invocations') }} ), diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index 06b4e753..865cb955 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -1,19 +1,19 @@ with base as ( select * - from {{ source('dbt_artifacts', 'model_executions') }} + from {{ ref('model_executions') }} ), enhanced as ( select - {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id, + {{ dbt_artifacts.surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id, command_invocation_id, node_id, run_started_at, was_full_refresh, - {{ dbt_utils.split_part('thread_id', "'-'", 2) }} as thread_id, -- noqa + {{ split_part('thread_id', "'-'", 2) }} as thread_id, status, compile_started_at, query_completed_at, diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql index 6819693a..0314038d 100644 --- a/models/staging/stg_dbt__models.sql +++ b/models/staging/stg_dbt__models.sql @@ -1,14 +1,14 @@ with base as ( select * - from {{ source('dbt_artifacts', 'models') }} + from {{ ref('models') }} ), enhanced as ( select - {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id, + {{ dbt_artifacts.surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id, command_invocation_id, node_id, run_started_at, diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index 35f2214a..2af030db 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -1,19 +1,19 @@ with base as ( select * - from {{ source('dbt_artifacts', 'seed_executions') }} + from {{ ref('seed_executions') }} ), enhanced as ( select - {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id, + {{ dbt_artifacts.surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id, command_invocation_id, node_id, run_started_at, was_full_refresh, - {{ dbt_utils.split_part('thread_id', "'-'", 2) }} as thread_id, -- noqa + {{ split_part('thread_id', "'-'", 2) }} as thread_id, status, compile_started_at, query_completed_at, diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql index 79ae62c0..b6853259 100644 --- a/models/staging/stg_dbt__seeds.sql +++ b/models/staging/stg_dbt__seeds.sql @@ -1,14 +1,14 @@ with base as ( select * - from {{ source('dbt_artifacts', 'seeds') }} + from {{ ref('seeds') }} ), enhanced as ( select - {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id, + {{ dbt_artifacts.surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id, command_invocation_id, node_id, run_started_at, diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index cb897069..0f07369b 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -1,19 +1,19 @@ with base as ( select * - from {{ source('dbt_artifacts', 'snapshot_executions') }} + from {{ ref('snapshot_executions') }} ), enhanced as ( select - {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id, + {{ dbt_artifacts.surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id, command_invocation_id, node_id, run_started_at, was_full_refresh, - {{ dbt_utils.split_part('thread_id', "'-'", 2) }} as thread_id, -- noqa + {{ split_part('thread_id', "'-'", 2) }} as thread_id, status, compile_started_at, query_completed_at, diff --git a/models/staging/stg_dbt__snapshots.sql b/models/staging/stg_dbt__snapshots.sql index 83564261..bdd86a0f 100644 --- a/models/staging/stg_dbt__snapshots.sql +++ b/models/staging/stg_dbt__snapshots.sql @@ -1,14 +1,14 @@ with base as ( select * - from {{ source('dbt_artifacts', 'snapshots') }} + from {{ ref('snapshots') }} ), enhanced as ( select - {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id, + {{ dbt_artifacts.surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id, command_invocation_id, node_id, run_started_at, diff --git a/models/staging/stg_dbt__sources.sql b/models/staging/stg_dbt__sources.sql index 561e7d50..d35a923a 100644 --- a/models/staging/stg_dbt__sources.sql +++ b/models/staging/stg_dbt__sources.sql @@ -1,14 +1,14 @@ with base as ( select * - from {{ source('dbt_artifacts', 'sources') }} + from {{ ref('sources') }} ), enhanced as ( select - {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as source_execution_id, + {{ dbt_artifacts.surrogate_key(['command_invocation_id', 'node_id']) }} as source_execution_id, command_invocation_id, node_id, run_started_at, diff --git a/models/staging/stg_dbt__test_executions.sql b/models/staging/stg_dbt__test_executions.sql index 5479fc53..1c72639c 100644 --- a/models/staging/stg_dbt__test_executions.sql +++ b/models/staging/stg_dbt__test_executions.sql @@ -1,19 +1,19 @@ with base as ( select * - from {{ source('dbt_artifacts', 'test_executions') }} + from {{ ref('test_executions') }} ), enhanced as ( select - {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as test_execution_id, + {{ dbt_artifacts.surrogate_key(['command_invocation_id', 'node_id']) }} as test_execution_id, command_invocation_id, node_id, run_started_at, was_full_refresh, - {{ dbt_utils.split_part('thread_id', "'-'", 2) }} as thread_id, -- noqa + {{ split_part('thread_id', "'-'", 2) }} as thread_id, status, compile_started_at, query_completed_at, diff --git a/models/staging/stg_dbt__tests.sql b/models/staging/stg_dbt__tests.sql index a69b4088..f301d5f7 100644 --- a/models/staging/stg_dbt__tests.sql +++ b/models/staging/stg_dbt__tests.sql @@ -1,14 +1,14 @@ with base as ( select * - from {{ source('dbt_artifacts', 'tests') }} + from {{ ref('tests') }} ), enhanced as ( select - {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as test_execution_id, + {{ dbt_artifacts.surrogate_key(['command_invocation_id', 'node_id']) }} as test_execution_id, command_invocation_id, node_id, run_started_at, diff --git a/packages.yml b/packages.yml deleted file mode 100644 index 2af079a0..00000000 --- a/packages.yml +++ /dev/null @@ -1,3 +0,0 @@ -packages: - - package: dbt-labs/dbt_utils - version: [">=0.6.0", "<0.9.0"] diff --git a/tox.ini b/tox.ini index d3ca66bf..9d8fce62 100644 --- a/tox.ini +++ b/tox.ini @@ -43,7 +43,7 @@ rules = L001,L003,L004,L005,L006,L007,L008,L010,L011,L012,L014,L015,L017,L018,L0 deps = sqlfluff-templater-dbt - dbt-snowflake~=1.1.0 + dbt-snowflake~=1.2.0 [sqlfluff:rules:L004] indent_unit = space @@ -112,42 +112,34 @@ deps = {[sqlfluff]deps} commands = sqlfluff fix models --ignore parsing [testenv:generate_docs] -deps = dbt-snowflake~=1.1.0 +deps = dbt-snowflake~=1.2.0 commands = dbt docs generate --profiles-dir integration_test_project [testenv:integration_snowflake] changedir = integration_test_project -deps = dbt-snowflake~=1.1.0 +deps = dbt-snowflake~=1.2.0 commands = dbt deps - dbt run-operation create_dbt_artifacts_tables --target snowflake - dbt build --exclude dbt_artifacts --target snowflake - dbt build -s dbt_artifacts --target snowflake + dbt build --target snowflake [testenv:integration_databricks] changedir = integration_test_project -deps = dbt-databricks~=1.1.0 +deps = dbt-databricks~=1.2.0 commands = dbt deps - dbt run-operation create_dbt_artifacts_tables --target databricks - dbt build --exclude dbt_artifacts --target databricks - dbt build -s dbt_artifacts --target databricks + dbt build --target databricks [testenv:integration_bigquery] changedir = integration_test_project deps = dbt-bigquery~=1.2.0 commands = dbt deps - dbt run-operation create_dbt_artifacts_tables --target bigquery - dbt build --exclude dbt_artifacts --target bigquery - dbt build -s dbt_artifacts --target bigquery + dbt build --target bigquery [testenv:integration_spark] changedir = integration_test_project -deps = dbt-spark[ODBC]~=1.1.0 +deps = dbt-spark[ODBC]~=1.2.0 commands = dbt deps - dbt run-operation create_dbt_artifacts_tables --target spark - dbt build --exclude dbt_artifacts tag:snapshot --target spark - dbt build -s dbt_artifacts --target spark + dbt build --target spark