Lift + shift for cross-db macros (#597)

* Initialize lift + shift, dateadd + datediff

* Install branches of all plugins

* Use dispatch for passthrough behavior

* Only support dbt-core >= 1.2

* Switch to alternative dev branches

* Lift and shift cross-database macros, fixtures, and tests into dbt-core and adapters

* Test `current_timestamp()` adapter definition vs. original dbt-utils

* Split out the `current_timestamp` work into its own separate effort (#599)

* Reinstate original `current_timestamp` implementation

* Reinstate original `type_*` implementations

* Reinstate original `current_timestamp_in_utc` implementation

* Kick out the `current_timestamp` + `type_*` tests

* Preserve `dbt_utils` as one more step along the dispatch train

* Remove branch identifiers

Co-authored-by: Jeremy Cohen <jeremy@dbtlabs.com>

dev-requirements.txt

@@ -1,3 +1,8 @@
 pytest
 pytest-dotenv
-dbt-tests-adapter
+git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
+git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter
+git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugins/postgres
+git+https://github.com/dbt-labs/dbt-redshift.git
+git+https://github.com/dbt-labs/dbt-snowflake.git
+git+https://github.com/dbt-labs/dbt-bigquery.git

macros/cross_db_utils/any_value.sql

@@ -1,17 +1,9 @@
+{# This is here for backwards compatibility only #}
+
 {% macro any_value(expression) -%}
     {{ return(adapter.dispatch('any_value', 'dbt_utils') (expression)) }}
 {% endmacro %}
 
-
 {% macro default__any_value(expression) -%}
-
-    any_value({{ expression }})
-
-{%- endmacro %}
-
-
-{% macro postgres__any_value(expression) -%}
-    {#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#}
-    min({{ expression }})
-
-{%- endmacro %}
+    {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}
+{% endmacro %}

macros/cross_db_utils/bool_or.sql

@@ -1,24 +1,9 @@
+{# This is here for backwards compatibility only #}
+
 {% macro bool_or(expression) -%}
     {{ return(adapter.dispatch('bool_or', 'dbt_utils') (expression)) }}
 {% endmacro %}
 
-
 {% macro default__bool_or(expression) -%}
-
-    bool_or({{ expression }})
-
-{%- endmacro %}
-
-
-{% macro snowflake__bool_or(expression) -%}
-
-    boolor_agg({{ expression }})
-
-{%- endmacro %}
-
-
-{% macro bigquery__bool_or(expression) -%}
-
-    logical_or({{ expression }})
-
-{%- endmacro %}
+    {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}
+{% endmacro %}

macros/cross_db_utils/cast_bool_to_text.sql

@@ -1,15 +1,9 @@
+{# This is here for backwards compatibility only #}
+
 {% macro cast_bool_to_text(field) %}
   {{ adapter.dispatch('cast_bool_to_text', 'dbt_utils') (field) }}
 {% endmacro %}
 
-
 {% macro default__cast_bool_to_text(field) %}
-    cast({{ field }} as {{ dbt_utils.type_string() }})
-{% endmacro %}
-
-{% macro redshift__cast_bool_to_text(field) %}
-    case
-        when {{ field }} is true then 'true'
-        when {{ field }} is false then 'false'
-    end::text
+  {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}
 {% endmacro %}

macros/cross_db_utils/concat.sql

@@ -1,9 +1,9 @@
-
+{# This is here for backwards compatibility only #}
 
 {% macro concat(fields) -%}
   {{ return(adapter.dispatch('concat', 'dbt_utils')(fields)) }}
 {%- endmacro %}
 
 {% macro default__concat(fields) -%}
-    {{ fields|join(' || ') }}
+  {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}
 {%- endmacro %}

macros/cross_db_utils/date_trunc.sql

@@ -1,15 +1,9 @@
+{# This is here for backwards compatibility only #}
+
 {% macro date_trunc(datepart, date) -%}
   {{ return(adapter.dispatch('date_trunc', 'dbt_utils') (datepart, date)) }}
 {%- endmacro %}
 
 {% macro default__date_trunc(datepart, date) -%}
-    date_trunc('{{datepart}}', {{date}})
-{%- endmacro %}
-
-{% macro bigquery__date_trunc(datepart, date) -%}
-    timestamp_trunc(
-        cast({{date}} as timestamp),
-        {{datepart}}
-    )
-
+  {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}
 {%- endmacro %}

macros/cross_db_utils/dateadd.sql

@@ -1,37 +1,9 @@
+{# This is here for backwards compatibility only #}
+
 {% macro dateadd(datepart, interval, from_date_or_timestamp) %}
   {{ return(adapter.dispatch('dateadd', 'dbt_utils')(datepart, interval, from_date_or_timestamp)) }}
 {% endmacro %}
 
-
 {% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}
-
-    dateadd(
-        {{ datepart }},
-        {{ interval }},
-        {{ from_date_or_timestamp }}
-        )
-
-{% endmacro %}
-
-
-{% macro bigquery__dateadd(datepart, interval, from_date_or_timestamp) %}
-
-        datetime_add(
-            cast( {{ from_date_or_timestamp }} as datetime),
-        interval {{ interval }} {{ datepart }}
-        )
-
-{% endmacro %}
-
-{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}
-
-    {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))
-
-{% endmacro %}
-
-{# redshift should use default instead of postgres #}
-{% macro redshift__dateadd(datepart, interval, from_date_or_timestamp) %}
-
-    {{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }}
-
+  {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}
 {% endmacro %}

macros/cross_db_utils/datediff.sql

@@ -1,66 +1,9 @@
+{# This is here for backwards compatibility only #}
+
 {% macro datediff(first_date, second_date, datepart) %}
   {{ return(adapter.dispatch('datediff', 'dbt_utils')(first_date, second_date, datepart)) }}
 {% endmacro %}
 
-
-{% macro default__datediff(first_date, second_date, datepart) -%}
-
-    datediff(
-        {{ datepart }},
-        {{ first_date }},
-        {{ second_date }}
-        )
-
-{%- endmacro %}
-
-
-{% macro bigquery__datediff(first_date, second_date, datepart) -%}
-
-    datetime_diff(
-        cast({{second_date}} as datetime),
-        cast({{first_date}} as datetime),
-        {{datepart}}
-    )
-
-{%- endmacro %}
-
-{% macro postgres__datediff(first_date, second_date, datepart) -%}
-
-    {% if datepart == 'year' %}
-        (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))
-    {% elif datepart == 'quarter' %}
-        ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))
-    {% elif datepart == 'month' %}
-        ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))
-    {% elif datepart == 'day' %}
-        (({{second_date}})::date - ({{first_date}})::date)
-    {% elif datepart == 'week' %}
-        ({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case
-            when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then
-                case when {{first_date}} <= {{second_date}} then 0 else -1 end
-            else
-                case when {{first_date}} <= {{second_date}} then 1 else 0 end
-        end)
-    {% elif datepart == 'hour' %}
-        ({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))
-    {% elif datepart == 'minute' %}
-        ({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))
-    {% elif datepart == 'second' %}
-        ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))
-    {% elif datepart == 'millisecond' %}
-        ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))
-    {% elif datepart == 'microsecond' %}
-        ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))
-    {% else %}
-        {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }}
-    {% endif %}
-
-{%- endmacro %}
-
-
-{# redshift should use default instead of postgres #}
-{% macro redshift__datediff(first_date, second_date, datepart) -%}
-
-    {{ return(dbt_utils.default__datediff(first_date, second_date, datepart)) }}
-
-{%- endmacro %}
+{% macro default__datediff(first_date, second_date, datepart) %}
+  {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}
+{% endmacro %}

macros/cross_db_utils/escape_single_quotes.sql

@@ -1,18 +1,9 @@
+{# This is here for backwards compatibility only #}
+
 {% macro escape_single_quotes(expression) %}
       {{ return(adapter.dispatch('escape_single_quotes', 'dbt_utils') (expression)) }}
 {% endmacro %}
 
-{# /*Default to replacing a single apostrophe with two apostrophes: they're -> they''re*/ #}
-{% macro default__escape_single_quotes(expression) -%}
-{{ expression | replace("'","''") }}
-{%- endmacro %}
-
-{# /*Snowflake uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #}
-{% macro snowflake__escape_single_quotes(expression) -%}
-{{ expression | replace("'", "\\'") }}
-{%- endmacro %}
-
-{# /*BigQuery uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #}
-{% macro bigquery__escape_single_quotes(expression) -%}
-{{ expression | replace("'", "\\'") }}
-{%- endmacro %}
+{% macro default__escape_single_quotes(expression) %}
+      {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}
+{% endmacro %}

macros/cross_db_utils/except.sql

@@ -1,16 +1,9 @@
+{# This is here for backwards compatibility only #}
+
 {% macro except() %}
   {{ return(adapter.dispatch('except', 'dbt_utils')()) }}
 {% endmacro %}
 
-
 {% macro default__except() %}
-
-    except
-
+  {{ return(adapter.dispatch('except', 'dbt')()) }}
 {% endmacro %}
-
-{% macro bigquery__except() %}
-
-    except distinct
-
-{% endmacro %}

macros/cross_db_utils/hash.sql

@@ -1,13 +1,9 @@
+{# This is here for backwards compatibility only #}
+
 {% macro hash(field) -%}
   {{ return(adapter.dispatch('hash', 'dbt_utils') (field)) }}
 {%- endmacro %}
 
-
 {% macro default__hash(field) -%}
-    md5(cast({{field}} as {{dbt_utils.type_string()}}))
-{%- endmacro %}
-
-
-{% macro bigquery__hash(field) -%}
-    to_hex({{dbt_utils.default__hash(field)}})
+  {{ return(adapter.dispatch('hash', 'dbt') (field)) }}
 {%- endmacro %}

macros/cross_db_utils/intersect.sql

@@ -1,16 +1,9 @@
+{# This is here for backwards compatibility only #}
+
 {% macro intersect() %}
   {{ return(adapter.dispatch('intersect', 'dbt_utils')()) }}
 {% endmacro %}
 
-
 {% macro default__intersect() %}
-
-    intersect
-
-{% endmacro %}
-
-{% macro bigquery__intersect() %}
-
-    intersect distinct
-
+  {{ return(adapter.dispatch('intersect', 'dbt')()) }}
 {% endmacro %}

macros/cross_db_utils/last_day.sql

@@ -1,3 +1,5 @@
+{# This is here for backwards compatibility only #}
+
 /*
 This function has been tested with dateparts of month and quarters. Further
 testing is required to validate that it will work on other dateparts.
@@ -7,39 +9,6 @@ testing is required to validate that it will work on other dateparts.
   {{ return(adapter.dispatch('last_day', 'dbt_utils') (date, datepart)) }}
 {% endmacro %}
 
-
-{%- macro default_last_day(date, datepart) -%}
-    cast(
-        {{dbt_utils.dateadd('day', '-1',
-        dbt_utils.dateadd(datepart, '1', dbt_utils.date_trunc(datepart, date))
-        )}}
-        as date)
-{%- endmacro -%}
-
-
-{% macro default__last_day(date, datepart) -%}
-    {{dbt_utils.default_last_day(date, datepart)}}
-{%- endmacro %}
-
-
-{% macro postgres__last_day(date, datepart) -%}
-
-    {%- if datepart == 'quarter' -%}
-    -- postgres dateadd does not support quarter interval.
-    cast(
-        {{dbt_utils.dateadd('day', '-1',
-        dbt_utils.dateadd('month', '3', dbt_utils.date_trunc(datepart, date))
-        )}}
-        as date)
-    {%- else -%}
-    {{dbt_utils.default_last_day(date, datepart)}}
-    {%- endif -%}
-
-{%- endmacro %}
-
-{# redshift should use default instead of postgres #}
-{% macro redshift__last_day(date, datepart) %}
-
-    {{ return(dbt_utils.default__last_day(date, datepart)) }}
-
+{% macro default__last_day(date, datepart) %}
+  {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}
 {% endmacro %}

macros/cross_db_utils/length.sql

@@ -1,21 +1,9 @@
+{# This is here for backwards compatibility only #}
+
 {% macro length(expression) -%}
     {{ return(adapter.dispatch('length', 'dbt_utils') (expression)) }}
 {% endmacro %}
 
-
-{% macro default__length(expression) %}
-
-    length(
-        {{ expression }}
-    )
-
-{%- endmacro -%}
-
-
-{% macro redshift__length(expression) %}
-
-    len(
-        {{ expression }}
-    )
-
-{%- endmacro -%}
+{% macro default__length(expression) -%}
+    {{ return(adapter.dispatch('length', 'dbt') (expression)) }}
+{% endmacro %}