Add new macros for diff calculation, and unit tests (#99)

.circleci/config.yml

@@ -33,7 +33,7 @@ jobs:
             . dbt_venv/bin/activate
 
             python -m pip install --upgrade pip setuptools
-            python -m pip install --pre dbt-core dbt-postgres dbt-redshift dbt-snowflake dbt-bigquery
+            python -m pip install --pre dbt-core dbt-postgres dbt-redshift dbt-snowflake dbt-bigquery dbt-databricks
 
             mkdir -p ~/.dbt
             cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
@@ -51,9 +51,8 @@ jobs:
             cd integration_tests
             dbt deps --target postgres
             dbt seed --target postgres --full-refresh
-            dbt compile --target postgres
-            dbt run --target postgres
-            dbt test --target postgres
+            dbt run --target postgres --exclude tag:skip+ tag:temporary_skip+
+            dbt test --target postgres --exclude tag:skip+ tag:temporary_skip+
 
       - run:
           name: "Run Tests - Redshift"
@@ -63,9 +62,8 @@ jobs:
             cd integration_tests
             dbt deps --target redshift
             dbt seed --target redshift --full-refresh
-            dbt compile --target redshift
-            dbt run --target redshift
-            dbt test --target redshift
+            dbt run --target redshift --exclude tag:skip+ tag:temporary_skip+
+            dbt test --target redshift --exclude tag:skip+ tag:temporary_skip+
 
       - run:
           name: "Run Tests - Snowflake"
@@ -75,9 +73,8 @@ jobs:
             cd integration_tests
             dbt deps --target snowflake
             dbt seed --target snowflake --full-refresh
-            dbt compile --target snowflake
-            dbt run --target snowflake
-            dbt test --target snowflake
+            dbt run --target snowflake --exclude tag:skip+ tag:temporary_skip+
+            dbt test --target snowflake --exclude tag:skip+ tag:temporary_skip+
 
       - run:
           name: "Run Tests - BigQuery"
@@ -90,10 +87,19 @@ jobs:
             cd integration_tests
             dbt deps --target bigquery
             dbt seed --target bigquery --full-refresh
-            dbt compile --target bigquery
-            dbt run --target bigquery --full-refresh
-            dbt test --target bigquery
+            dbt run --target bigquery --full-refresh --exclude tag:skip+ tag:temporary_skip+
+            dbt test --target bigquery --exclude tag:skip+ tag:temporary_skip+
 
+      - run:
+          name: "Run Tests - Databricks"
+          command: |
+            . dbt_venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps --target databricks
+            dbt seed --target databricks --full-refresh
+            dbt run --target databricks --exclude tag:skip+ tag:temporary_skip+
+            dbt test --target databricks --exclude tag:skip+ tag:temporary_skip+
 
       - save_cache:
           key: deps1-{{ .Branch }}
@@ -115,3 +121,4 @@ workflows:
             - profile-redshift
             - profile-snowflake
             - profile-bigquery
+            - profile-databricks

.gitignore

@@ -1,4 +1,7 @@
 target/
 dbt_packages/
 logs/
-logfile
+logfile
+.DS_Store
+package-lock.yml
+integration_tests/package-lock.yml

.vscode/settings.json

@@ -0,0 +1,21 @@
+{    
+    "yaml.schemas": {
+        "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/latest/dbt_yml_files-latest.json": [
+            "/**/*.yml",
+            "!profiles.yml",
+            "!dbt_project.yml",
+            "!packages.yml",
+            "!selectors.yml",
+            "!profile_template.yml"
+        ],
+        "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/latest/dbt_project-latest.json": [
+            "dbt_project.yml"
+        ],
+        "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/latest/selectors-latest.json": [
+            "selectors.yml"
+        ],
+        "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/latest/packages-latest.json": [
+            "packages.yml"
+        ]
+    },
+}

integration_tests/ci/sample.profiles.yml

@@ -2,10 +2,6 @@
 # HEY! This file is used in the dbt-audit-helper integrations tests with CircleCI.
 # You should __NEVER__ check credentials into version control. Thanks for reading :)
 
-config:
-    send_anonymous_usage_stats: False
-    use_colors: True
-
 integration_tests:
   target: postgres
   outputs:
@@ -27,15 +23,15 @@ integration_tests:
       dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}"
       port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}"
       schema: audit_helper_integration_tests_redshift
-      threads: 1
+      threads: 8
 
     bigquery:
       type: bigquery
       method: service-account
       keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}"
       project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
       schema: audit_helper_integration_tests_bigquery
-      threads: 1
+      threads: 8
 
     snowflake:
       type: snowflake
@@ -46,4 +42,12 @@ integration_tests:
       database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}"
       warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
       schema: audit_helper_integration_tests_snowflake
-      threads: 1
+      threads: 8
+
+    databricks:
+      type: databricks
+      schema: dbt_project_evaluator_integration_tests_databricks
+      host: "{{ env_var('DATABRICKS_TEST_HOST') }}"
+      http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}"
+      token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}"
+      threads: 10

integration_tests/dbt_project.yml

@@ -17,3 +17,14 @@ clean-targets:         # directories to be removed by `dbt clean`
 
 seeds:
   +quote_columns: false
+
+vars:
+  compare_queries_summarize: true
+  primary_key_columns_var: ['col1']
+  columns_var: ['col1']
+  event_time_var:
+  quick_are_queries_identical_cols: ['col1']
+
+flags:
+  send_anonymous_usage_stats: False
+  use_colors: True

integration_tests/macros/unit_tests/struct_generation_macros.sql

@@ -0,0 +1,26 @@
+{%- macro _basic_json_function() -%}
+    {%- if target.type == 'snowflake' -%}
+        object_construct
+    {%- elif target.type == 'bigquery' -%}
+        json_object
+    {%- elif target.type == 'databricks' -%}
+        map
+    {%- elif execute -%}
+        {# Only raise exception if it's actually being called, not during parsing #}
+        {%- do exceptions.raise_compiler_error("Unknown adapter '"~ target.type ~ "'") -%}
+    {%- endif -%}
+{%- endmacro -%}
+
+{% macro _complex_json_function(json) %}
+
+    {% if target.type == 'redshift' %}
+        json_parse({{ json }})
+    {% elif target.type == 'databricks' %}
+        from_json({{ json }}, schema_of_json({{ json }}))
+    {% elif target.type in ['snowflake', 'bigquery'] %}
+        parse_json({{ json }})
+    {% elif execute %}
+        {# Only raise exception if it's actually being called, not during parsing #}
+        {%- do exceptions.raise_compiler_error("Unknown adapter '"~ target.type ~ "'") -%}    
+    {% endif %}
+{% endmacro %}

integration_tests/models/data_tests/compare_and_classify_query_results.sql

@@ -0,0 +1,11 @@
+-- this has no tests, it's just making sure that the introspecive queries for event_time actually run
+
+{{
+    audit_helper.compare_and_classify_query_results(
+        a_query="select * from " ~ ref('unit_test_model_a') ~ " where 1=1",
+        b_query="select * from " ~ ref('unit_test_model_b') ~ " where 1=1",
+        primary_key_columns=['id'],
+        columns=['id', 'col1', 'col2'],
+        event_time='created_at'
+    )
+}}

integration_tests/models/compare_which_columns_differ.sql → integration_tests/models/data_tests/compare_which_columns_differ.sql

@@ -9,9 +9,9 @@ select
     has_difference
 from (
 
-    {{ audit_helper.compare_which_columns_differ(
+    {{ audit_helper.compare_which_relation_columns_differ(
         a_relation=a_relation,
         b_relation=b_relation,
-        primary_key="id"
+        primary_key_columns=["id"]
     ) }}
 ) as macro_output

integration_tests/models/data_tests/compare_which_columns_differ_exclude_cols.sql

@@ -0,0 +1,25 @@
+{% set a_relation=ref('data_compare_which_columns_differ_a')%}
+
+{% set b_relation=ref('data_compare_which_columns_differ_b') %}
+
+{% set pk_cols = ['id'] %}
+{% set cols = ['id','value_changes','becomes_not_null','does_not_change'] %}
+
+{% if target.type == 'snowflake' %}
+    {% set pk_cols = pk_cols | map("upper") | list %}
+    {% set cols = cols | map("upper") | list %}
+{% endif %}
+
+select 
+    lower(column_name) as column_name,
+    has_difference
+from (
+
+    {{ audit_helper.compare_which_relation_columns_differ(
+        a_relation=a_relation,
+        b_relation=b_relation,
+        primary_key_columns=pk_cols,
+        columns=cols
+    ) }}
+
+) as macro_output

integration_tests/models/schema.yml → integration_tests/models/data_tests/schema.yml

@@ -2,96 +2,96 @@ version: 2
 
 models:
   - name: compare_queries
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_relations_without_exclude')
 
   - name: compare_queries_concat_pk_without_summary
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_without_summary')
 
   - name: compare_queries_with_summary
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_with_summary')
 
   - name: compare_queries_without_summary
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_without_summary')
 
   - name: compare_relations_with_summary
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_with_summary')
 
   - name: compare_relations_without_summary
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_without_summary')
 
   - name: compare_relations_with_exclude
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_relations_with_exclude')
 
   - name: compare_relations_without_exclude
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_relations_without_exclude')
 
   - name: compare_all_columns_with_summary
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_all_columns_with_summary')
 
   - name: compare_all_columns_without_summary
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_all_columns_without_summary')
 
   - name: compare_all_columns_concat_pk_with_summary
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_all_columns_concat_pk_with_summary')
 
   - name: compare_all_columns_concat_pk_without_summary
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_all_columns_concat_pk_without_summary')
 
   - name: compare_all_columns_with_summary_and_exclude
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_all_columns_with_summary_and_exclude')
 
   - name: compare_all_columns_where_clause
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_all_columns_where_clause')
 
   - name: compare_relation_columns
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_relation_columns')
 
   - name: compare_relations_concat_pk_without_summary
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_without_summary')
 
   - name: compare_which_columns_differ
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_which_columns_differ')
 
   - name: compare_which_columns_differ_exclude_cols
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_which_columns_differ_exclude_cols')
 
   - name: compare_row_counts
-    tests:
+    data_tests:
       - dbt_utils.equality:
           compare_model: ref('expected_results__compare_row_counts')

integration_tests/models/unit_test_placeholder_models/unit_test_model_a.sql

@@ -0,0 +1 @@
+select 12 as id, 22 as id_2, 'xyz' as col1, 'tuv' as col2, 123 as col3, {{ dbt.current_timestamp() }} as created_at

integration_tests/models/unit_test_placeholder_models/unit_test_model_b.sql

@@ -0,0 +1 @@
+select 12 as id, 22 as id_2, 'xyz' as col1, 'tuv' as col2, 123 as col3, {{ dbt.current_timestamp() }} as created_at