From e1aa8c209a79321f25bde25f5b2439a096aa5160 Mon Sep 17 00:00:00 2001 From: outinim Date: Thu, 4 Apr 2024 19:04:38 +0200 Subject: [PATCH 1/4] fix: snapshots failing because of wrong mysql colum quoting --- .../materializations/snapshot/create_columns.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 dbt/include/mariadb/macros/materializations/snapshot/create_columns.sql diff --git a/dbt/include/mariadb/macros/materializations/snapshot/create_columns.sql b/dbt/include/mariadb/macros/materializations/snapshot/create_columns.sql new file mode 100644 index 0000000..6b13956 --- /dev/null +++ b/dbt/include/mariadb/macros/materializations/snapshot/create_columns.sql @@ -0,0 +1,14 @@ +{# + Add new columns to the table if applicable +#} +{% macro create_columns(relation, columns) %} + {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }} +{% endmacro %} + +{% macro default__create_columns(relation, columns) %} + {% for column in columns %} + {% call statement() %} + alter table {{ relation }} add column {{ column.quoted() }} {{ column.data_type }}; + {% endcall %} + {% endfor %} +{% endmacro %} \ No newline at end of file From 021e256c9dd56d639e9bb24b54dc7a0a1660d34f Mon Sep 17 00:00:00 2001 From: outinim Date: Sun, 7 Apr 2024 23:36:01 +0200 Subject: [PATCH 2/4] adding create columns for three DBs --- .../materializations/snapshot/create_columns.sql | 14 -------------- .../macros/materializations/snapshot/snapshot.sql | 13 ++++++++++++- .../macros/materializations/snapshot/snapshot.sql | 13 ++++++++++++- .../macros/materializations/snapshot/snapshot.sql | 13 ++++++++++++- 4 files changed, 36 insertions(+), 17 deletions(-) delete mode 100644 dbt/include/mariadb/macros/materializations/snapshot/create_columns.sql diff --git a/dbt/include/mariadb/macros/materializations/snapshot/create_columns.sql b/dbt/include/mariadb/macros/materializations/snapshot/create_columns.sql deleted file mode 100644 index 6b13956..0000000 --- a/dbt/include/mariadb/macros/materializations/snapshot/create_columns.sql +++ /dev/null @@ -1,14 +0,0 @@ -{# - Add new columns to the table if applicable -#} -{% macro create_columns(relation, columns) %} - {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }} -{% endmacro %} - -{% macro default__create_columns(relation, columns) %} - {% for column in columns %} - {% call statement() %} - alter table {{ relation }} add column {{ column.quoted() }} {{ column.data_type }}; - {% endcall %} - {% endfor %} -{% endmacro %} \ No newline at end of file diff --git a/dbt/include/mariadb/macros/materializations/snapshot/snapshot.sql b/dbt/include/mariadb/macros/materializations/snapshot/snapshot.sql index 7bac593..9defeea 100644 --- a/dbt/include/mariadb/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/mariadb/macros/materializations/snapshot/snapshot.sql @@ -1,3 +1,14 @@ +{# + Add new columns to the table if applicable +#} + +{% macro mariadb__create_columns(relation, columns) %} + {% for column in columns %} + {% call statement() %} + alter table {{ relation }} add column {{ column.quoted() }} {{ column.data_type }}; + {% endcall %} + {% endfor %} +{% endmacro %} {% macro mariadb__snapshot_string_as_time(timestamp) -%} {%- set result = "str_to_date('" ~ timestamp ~ "', '%Y-%m-%d %T')" -%} @@ -59,7 +70,7 @@ | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') | list %} - {% do create_columns(target_relation, missing_columns) %} + {% do mariadb__create_columns(target_relation, missing_columns) %} {% set source_columns = adapter.get_columns_in_relation(staging_table) | rejectattr('name', 'equalto', 'dbt_change_type') diff --git a/dbt/include/mysql/macros/materializations/snapshot/snapshot.sql b/dbt/include/mysql/macros/materializations/snapshot/snapshot.sql index 8719fc2..19d5552 100644 --- a/dbt/include/mysql/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/mysql/macros/materializations/snapshot/snapshot.sql @@ -1,3 +1,14 @@ +{# + Add new columns to the table if applicable +#} + +{% macro mysql__create_columns(relation, columns) %} + {% for column in columns %} + {% call statement() %} + alter table {{ relation }} add column {{ column.quoted() }} {{ column.data_type }}; + {% endcall %} + {% endfor %} +{% endmacro %} {% macro mysql__snapshot_string_as_time(timestamp) -%} {%- set result = "str_to_date('" ~ timestamp ~ "', '%Y-%m-%d %T')" -%} @@ -59,7 +70,7 @@ | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') | list %} - {% do create_columns(target_relation, missing_columns) %} + {% do mysql__create_columns(target_relation, missing_columns) %} {% set source_columns = adapter.get_columns_in_relation(staging_table) | rejectattr('name', 'equalto', 'dbt_change_type') diff --git a/dbt/include/mysql5/macros/materializations/snapshot/snapshot.sql b/dbt/include/mysql5/macros/materializations/snapshot/snapshot.sql index b3fa1fe..cd32c2b 100644 --- a/dbt/include/mysql5/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/mysql5/macros/materializations/snapshot/snapshot.sql @@ -1,3 +1,14 @@ +{# + Add new columns to the table if applicable +#} + +{% macro mysql5__create_columns(relation, columns) %} + {% for column in columns %} + {% call statement() %} + alter table {{ relation }} add column {{ column.quoted() }} {{ column.data_type }}; + {% endcall %} + {% endfor %} +{% endmacro %} {% macro mysql5__snapshot_string_as_time(timestamp) -%} {%- set result = "str_to_date('" ~ timestamp ~ "', '%Y-%m-%d %T')" -%} @@ -59,7 +70,7 @@ | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') | list %} - {% do create_columns(target_relation, missing_columns) %} + {% do mysql5__create_columns(target_relation, missing_columns) %} {% set source_columns = adapter.get_columns_in_relation(staging_table) | rejectattr('name', 'equalto', 'dbt_change_type') From 44f46eba473c4458d8c70afe1cc7d235f5e13d4d Mon Sep 17 00:00:00 2001 From: outinim Date: Sun, 7 Apr 2024 23:38:46 +0200 Subject: [PATCH 3/4] update changelogs --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a58f452..e049be3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Support dbt v1.5 ([#145](https://github.com/dbeatty10/dbt-mysql/issues/145)) - Support connecting via UNIX sockets ([#164](https://github.com/dbeatty10/dbt-mysql/issues/164)) - Support Black & MyPy pre-commit hooks ([#138](https://github.com/dbeatty10/dbt-mysql/issues/138)) +- Support dbt snapshots if table DDL is evolving ([#171](https://github.com/dbeatty10/dbt-mysql/pull/171)) ### Fixes - Fix incremental composite keys ([#144](https://github.com/dbeatty10/dbt-mysql/issues/144)) From 29bd8c435b920c733683198886a8a2ed1b0afdc6 Mon Sep 17 00:00:00 2001 From: Matthew Wallace Date: Mon, 27 May 2024 17:29:34 -0600 Subject: [PATCH 4/4] Fix create_columns macro invocation to be more general. --- .../mariadb/macros/materializations/snapshot/snapshot.sql | 4 ++-- .../mysql/macros/materializations/snapshot/snapshot.sql | 4 ++-- .../mysql5/macros/materializations/snapshot/snapshot.sql | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbt/include/mariadb/macros/materializations/snapshot/snapshot.sql b/dbt/include/mariadb/macros/materializations/snapshot/snapshot.sql index 9defeea..984bc8c 100644 --- a/dbt/include/mariadb/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/mariadb/macros/materializations/snapshot/snapshot.sql @@ -5,7 +5,7 @@ {% macro mariadb__create_columns(relation, columns) %} {% for column in columns %} {% call statement() %} - alter table {{ relation }} add column {{ column.quoted() }} {{ column.data_type }}; + alter table {{ relation }} add column {{ column.quoted() }} {{ column.data_type }}; {% endcall %} {% endfor %} {% endmacro %} @@ -70,7 +70,7 @@ | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') | list %} - {% do mariadb__create_columns(target_relation, missing_columns) %} + {% do create_columns(target_relation, missing_columns) %} {% set source_columns = adapter.get_columns_in_relation(staging_table) | rejectattr('name', 'equalto', 'dbt_change_type') diff --git a/dbt/include/mysql/macros/materializations/snapshot/snapshot.sql b/dbt/include/mysql/macros/materializations/snapshot/snapshot.sql index 19d5552..8dafda9 100644 --- a/dbt/include/mysql/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/mysql/macros/materializations/snapshot/snapshot.sql @@ -5,7 +5,7 @@ {% macro mysql__create_columns(relation, columns) %} {% for column in columns %} {% call statement() %} - alter table {{ relation }} add column {{ column.quoted() }} {{ column.data_type }}; + alter table {{ relation }} add column {{ column.quoted() }} {{ column.data_type }}; {% endcall %} {% endfor %} {% endmacro %} @@ -70,7 +70,7 @@ | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') | list %} - {% do mysql__create_columns(target_relation, missing_columns) %} + {% do create_columns(target_relation, missing_columns) %} {% set source_columns = adapter.get_columns_in_relation(staging_table) | rejectattr('name', 'equalto', 'dbt_change_type') diff --git a/dbt/include/mysql5/macros/materializations/snapshot/snapshot.sql b/dbt/include/mysql5/macros/materializations/snapshot/snapshot.sql index cd32c2b..0441cff 100644 --- a/dbt/include/mysql5/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/mysql5/macros/materializations/snapshot/snapshot.sql @@ -5,7 +5,7 @@ {% macro mysql5__create_columns(relation, columns) %} {% for column in columns %} {% call statement() %} - alter table {{ relation }} add column {{ column.quoted() }} {{ column.data_type }}; + alter table {{ relation }} add column {{ column.quoted() }} {{ column.data_type }}; {% endcall %} {% endfor %} {% endmacro %} @@ -70,7 +70,7 @@ | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') | list %} - {% do mysql5__create_columns(target_relation, missing_columns) %} + {% do create_columns(target_relation, missing_columns) %} {% set source_columns = adapter.get_columns_in_relation(staging_table) | rejectattr('name', 'equalto', 'dbt_change_type')