Skip to content

Commit

Permalink
Add adapter response (#289)
Browse files Browse the repository at this point in the history 
* Add doc block

* Update macro to include adapter response

* Update source tables to include new field

* Update staging tables to include new field

* Remove from test executions

* Should fix snowflake integration test error

* Add bigquery parse_json

* Add parse_json to bigquery.... not snowflake........

* Add back in TEST_ENV_VAR for old release

* Add docs for new field

---------

Co-authored-by: dluftspring <daniel.luftspring@gmail.com>
  • Loading branch information
@glsdown @dluftspring
glsdown and dluftspring authored May 12, 2023
1 parent 6d7625d commit 61af6c9
Show file tree
Hide file tree
Showing 19 changed files with 72 additions and 26 deletions.
17 changes: 11 additions & 6 deletions macros/upload_model_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }},
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }}

from values
{% for model in models -%}
(
Expand Down Expand Up @@ -70,8 +72,8 @@
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
'{{ model.message }}' {# message #}

'{{ model.message }}', {# message #}
'{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
Expand Down Expand Up @@ -128,7 +130,8 @@
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
'{{ model.message }}' {# message #}
'{{ model.message }}', {# message #}
parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
Expand Down Expand Up @@ -157,7 +160,8 @@
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }},
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }}
from values
{% for model in models -%}
(
Expand Down Expand Up @@ -201,7 +205,8 @@
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
'{{ model.message }}' {# message #}
'{{ model.message }}', {# message #}
'{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
Expand Down
15 changes: 10 additions & 5 deletions macros/upload_seed_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }},
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }}
from values
{% for model in seeds -%}
(
Expand Down Expand Up @@ -70,7 +71,8 @@
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
'{{ model.message }}' {# message #}
'{{ model.message }}', {# message #}
'{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
Expand Down Expand Up @@ -126,7 +128,8 @@
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
'{{ model.message }}' {# message #}
'{{ model.message }}', {# message #}
parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
Expand Down Expand Up @@ -155,7 +158,8 @@
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }},
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }}
from values
{% for model in seeds -%}
(
Expand Down Expand Up @@ -199,7 +203,8 @@
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
'{{ model.message }}' {# message #}
'{{ model.message }}', {# message #}
'{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
Expand Down
15 changes: 10 additions & 5 deletions macros/upload_snapshot_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }},
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }}
from values
{% for model in snapshots -%}
(
Expand Down Expand Up @@ -70,7 +71,8 @@
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
'{{ model.message }}' {# message #}
'{{ model.message }}', {# message #}
'{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
Expand Down Expand Up @@ -126,7 +128,8 @@
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
'{{ model.message }}' {# message #}
'{{ model.message }}', {# message #}
parse_json('{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
Expand Down Expand Up @@ -155,7 +158,8 @@
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(13) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(14) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }}
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(15) }},
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(16)) }}
from values
{% for model in snapshots -%}
(
Expand Down Expand Up @@ -199,7 +203,8 @@
'{{ model.node.schema }}', {# schema #}
'{{ model.node.name }}', {# name #}
'{{ model.node.alias }}', {# alias #}
'{{ model.message }}' {# message #}
'{{ model.message }}', {# message #}
'{{ tojson(model.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
Expand Down
10 changes: 7 additions & 3 deletions macros/upload_test_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }},
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(13)) }}
from values
{% for test in tests -%}
(
Expand Down Expand Up @@ -64,7 +65,8 @@
{{ test.execution_time }}, {# total_node_runtime #}
null, {# rows_affected not available in Databricks #}
{{ 'null' if test.failures is none else test.failures }}, {# failures #}
'{{ test.message }}' {# message #}
'{{ test.message }}', {# message #}
'{{ tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
Expand Down Expand Up @@ -117,7 +119,9 @@
{{ test.execution_time }}, {# total_node_runtime #}
null, {# rows_affected not available in Databricks #}
{{ 'null' if test.failures is none else test.failures }}, {# failures #}
'{{ test.message }}' {# message #}
'{{ test.message }}', {# message #}
parse_json('{{ tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}') {# adapter_response #}

)
{%- if not loop.last %},{%- endif %}

Expand Down
6 changes: 6 additions & 0 deletions models/docs.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,3 +400,9 @@ Alias of the node.
Result report, based on information returned from the database

{% enddocs %}

{% docs adapter_response %}

Response provided by the adapter as JSON.

{% enddocs %}
3 changes: 2 additions & 1 deletion models/sources/model_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ select
cast(null as {{ type_string() }}) as schema,
cast(null as {{ type_string() }}) as name,
cast(null as {{ type_string() }}) as alias,
cast(null as {{ type_string() }}) as message
cast(null as {{ type_string() }}) as message,
cast(null as {{ type_json() }}) as adapter_response
from dummy_cte
where 1 = 0
2 changes: 2 additions & 0 deletions models/sources/model_executions.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,5 @@ models:
description: '{{ doc("alias") }}'
- name: message
description: '{{ doc("message") }}'
- name: adapter_response
description: '{{ doc("adapter_response") }}'
3 changes: 2 additions & 1 deletion models/sources/seed_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ select
cast(null as {{ type_string() }}) as schema,
cast(null as {{ type_string() }}) as name,
cast(null as {{ type_string() }}) as alias,
cast(null as {{ type_string() }}) as message
cast(null as {{ type_string() }}) as message,
cast(null as {{ type_json() }}) as adapter_response
from dummy_cte
where 1 = 0
2 changes: 2 additions & 0 deletions models/sources/seed_executions.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,5 @@ models:
description: '{{ doc("alias") }}'
- name: message
description: '{{ doc("message") }}'
- name: adapter_response
description: '{{ doc("adapter_response") }}'
3 changes: 2 additions & 1 deletion models/sources/snapshot_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ select
cast(null as {{ type_string() }}) as schema,
cast(null as {{ type_string() }}) as name,
cast(null as {{ type_string() }}) as alias,
cast(null as {{ type_string() }}) as message
cast(null as {{ type_string() }}) as message,
cast(null as {{ type_json() }}) as adapter_response
from dummy_cte
where 1 = 0
2 changes: 2 additions & 0 deletions models/sources/snapshot_executions.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,5 @@ models:
description: '{{ doc("alias") }}'
- name: message
description: '{{ doc("message") }}'
- name: adapter_response
description: '{{ doc("adapter_response") }}'
3 changes: 2 additions & 1 deletion models/sources/test_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ select
cast(null as {{ type_float() }}) as total_node_runtime,
cast(null as {{ type_int() }}) as rows_affected,
cast(null as {{ type_int() }}) as failures,
cast(null as {{ type_string() }}) as message
cast(null as {{ type_string() }}) as message,
cast(null as {{ type_json() }}) as adapter_response
from dummy_cte
where 1 = 0
2 changes: 2 additions & 0 deletions models/sources/test_executions.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,5 @@ models:
description: '{{ doc("failures") }}'
- name: message
description: '{{ doc("message") }}'
- name: adapter_response
description: '{{ doc("adapter_response") }}'
3 changes: 2 additions & 1 deletion models/staging/stg_dbt__model_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ enhanced as (
schema, -- noqa
name,
alias,
message
message,
adapter_response
from base

)
Expand Down
2 changes: 2 additions & 0 deletions models/staging/stg_dbt__model_executions.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,5 @@ models:
description: '{{ doc("was_full_refresh") }}'
- name: message
description: '{{ doc("message") }}'
- name: adapter_response
description: '{{ doc("adapter_response") }}'
3 changes: 2 additions & 1 deletion models/staging/stg_dbt__seed_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ enhanced as (
schema, -- noqa
name,
alias,
message
message,
adapter_response
from base

)
Expand Down
2 changes: 2 additions & 0 deletions models/staging/stg_dbt__seed_executions.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,5 @@ models:
description: '{{ doc("was_full_refresh") }}'
- name: message
description: '{{ doc("message") }}'
- name: adapter_response
description: '{{ doc("adapter_response") }}'
3 changes: 2 additions & 1 deletion models/staging/stg_dbt__snapshot_executions.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ enhanced as (
schema, -- noqa
name,
alias,
message
message,
adapter_response
from base

)
Expand Down
2 changes: 2 additions & 0 deletions models/staging/stg_dbt__snapshot_executions.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,5 @@ models:
description: '{{ doc("was_full_refresh") }}'
- name: message
description: '{{ doc("message") }}'
- name: adapter_response
description: '{{ doc("adapter_response") }}'

0 comments on commit 61af6c9

Please sign in to comment.