From 9241c385b1561e58aa262eff7c4782d2a174faf1 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 31 Jan 2023 10:46:57 -0600 Subject: [PATCH 1/3] change type to string (#611) * change type to string * reduce required length to 1 --- .changie.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changie.yaml b/.changie.yaml index e6cc19642..6a670c6d0 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -73,8 +73,8 @@ custom: minLength: 3 - key: Issue label: GitHub Issue Number (separated by a single space if multiple) - type: int - minLength: 4 + type: string + minLength: 1 footerFormat: | {{- $contributorDict := dict }} From 5d27961798e2ddc8f86433dbfa865ad5d5cb4946 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 1 Feb 2023 13:47:24 -0600 Subject: [PATCH 2/3] fix bug with contributor list (#620) --- .changie.yaml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.changie.yaml b/.changie.yaml index 6a670c6d0..09dfd3cf3 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -88,19 +88,28 @@ footerFormat: | {{- /* we only want to include non-core team contributors */}} {{- if not (has $authorLower $core_team)}} {{- $changeList := splitList " " $change.Custom.Author }} + {{- $IssueList := list }} {{- $changeLink := $change.Kind }} {{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }} - {{- $changeLink = "[#nbr](https://github.com/dbt-labs/dbt-spark/pull/nbr)" | replace "nbr" $change.Custom.PR }} + {{- $changes := splitList " " $change.Custom.PR }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-spark/pull/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} {{- else }} - {{- $changeLink = "[#nbr](https://github.com/dbt-labs/dbt-spark/issues/nbr)" | replace "nbr" $change.Custom.Issue }} + {{- $changes := splitList " " $change.Custom.Issue }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-spark/issues/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} {{- end }} {{- /* check if this contributor has other changes associated with them already */}} {{- if hasKey $contributorDict $author }} {{- $contributionList := get $contributorDict $author }} - {{- $contributionList = append $contributionList $changeLink }} + {{- $contributionList = concat $contributionList $IssueList }} {{- $contributorDict := set $contributorDict $author $contributionList }} {{- else }} - {{- $contributionList := list $changeLink }} + {{- $contributionList := $IssueList }} {{- $contributorDict := set $contributorDict $author $contributionList }} {{- end }} {{- end}} From f877d1e7bc9721c200c9712ca897379fa35c2554 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Wed, 1 Feb 2023 22:23:52 -0800 Subject: [PATCH 3/3] Convert Spark persist docs test (#612) * Begin conversion and get ready for CI testing. * Uncheck models * Change core index. * Pair test down for minimal test * Adjust code with some guessing. * Forgot a version tag * Make test conversion work. Finally * Fix up the code. * Attempt to fix test conversion with profile skips. * Add missing column test and cleanup code. * Remove shas from the requirements now that base conversion is live. * Revert whitespace change. --------- Co-authored-by: Mila Page --- .../adapter/persist_docs/fixtures.py} | 93 +++++++------- .../adapter/persist_docs/test_persist_docs.py | 119 ++++++++++++++++++ .../models/incremental_delta_model.sql | 2 - .../persist_docs/models/my_fun_docs.md | 10 -- .../persist_docs/models/no_docs_model.sql | 1 - .../persist_docs/models/table_delta_model.sql | 2 - .../persist_docs/models/view_model.sql | 2 - tests/integration/persist_docs/seeds/seed.csv | 3 - .../integration/persist_docs/seeds/seeds.yml | 26 ---- .../persist_docs/test_persist_docs.py | 68 ---------- 10 files changed, 163 insertions(+), 163 deletions(-) rename tests/{integration/persist_docs/models/schema.yml => functional/adapter/persist_docs/fixtures.py} (60%) create mode 100644 tests/functional/adapter/persist_docs/test_persist_docs.py delete mode 100644 tests/integration/persist_docs/models/incremental_delta_model.sql delete mode 100644 tests/integration/persist_docs/models/my_fun_docs.md delete mode 100644 tests/integration/persist_docs/models/no_docs_model.sql delete mode 100644 tests/integration/persist_docs/models/table_delta_model.sql delete mode 100644 tests/integration/persist_docs/models/view_model.sql delete mode 100644 tests/integration/persist_docs/seeds/seed.csv delete mode 100644 tests/integration/persist_docs/seeds/seeds.yml delete mode 100644 tests/integration/persist_docs/test_persist_docs.py diff --git a/tests/integration/persist_docs/models/schema.yml b/tests/functional/adapter/persist_docs/fixtures.py similarity index 60% rename from tests/integration/persist_docs/models/schema.yml rename to tests/functional/adapter/persist_docs/fixtures.py index 6680f392e..3c351ab55 100644 --- a/tests/integration/persist_docs/models/schema.yml +++ b/tests/functional/adapter/persist_docs/fixtures.py @@ -1,31 +1,35 @@ +_MODELS__MY_FUN_DOCS = """ +{% docs my_fun_doc %} +name Column description "with double quotes" +and with 'single quotes' as welll as other; +'''abc123''' +reserved -- characters +-- +/* comment */ +Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting + +{% enddocs %} +""" + +_MODELS__INCREMENTAL_DELTA = """ +{{ config(materialized='incremental', file_format='delta') }} +select 1 as id, 'Joe' as name +""" + +_MODELS__TABLE_DELTA_MODEL = """ +{{ config(materialized='table', file_format='delta') }} +select 1 as id, 'Joe' as name +""" + +_MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN = """ +{{ config(materialized='table', file_format='delta') }} +select 1 as id, 'Joe' as different_name +""" + +_PROPERTIES__MODELS = """ version: 2 models: - - - name: table_parquet_model - description: | - Table model description "with double quotes" - and with 'single quotes' as welll as other; - '''abc123''' - reserved -- characters - -- - /* comment */ - Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting - columns: - - name: id - description: | - id Column description "with double quotes" - and with 'single quotes' as welll as other; - '''abc123''' - reserved -- characters - -- - /* comment */ - Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting - - name: name - description: | - Some stuff here and then a call to - {{ doc('my_fun_doc')}} - - name: table_delta_model description: | Table model description "with double quotes" @@ -50,9 +54,9 @@ Some stuff here and then a call to {{ doc('my_fun_doc')}} - - name: table_hudi_model + - name: incremental_delta_model description: | - Table model description "with double quotes" + Incremental model description "with double quotes" and with 'single quotes' as welll as other; '''abc123''' reserved -- characters @@ -73,30 +77,15 @@ description: | Some stuff here and then a call to {{ doc('my_fun_doc')}} - - - name: view_model - description: | - View model description "with double quotes" - and with 'single quotes' as welll as other; - '''abc123''' - reserved -- characters - -- - /* comment */ - Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting - columns: - - name: id - description: | - id Column description "with double quotes" - and with 'single quotes' as welll as other; - '''abc123''' - reserved -- characters - -- - /* comment */ - Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting +""" - - name: incremental_delta_model +_PROPERTIES__SEEDS = """ +version: 2 + +seeds: + - name: seed description: | - Incremental model description "with double quotes" + Seed model description "with double quotes" and with 'single quotes' as welll as other; '''abc123''' reserved -- characters @@ -117,3 +106,9 @@ description: | Some stuff here and then a call to {{ doc('my_fun_doc')}} +""" + +_SEEDS__BASIC = """id,name +1,Alice +2,Bob +""" diff --git a/tests/functional/adapter/persist_docs/test_persist_docs.py b/tests/functional/adapter/persist_docs/test_persist_docs.py new file mode 100644 index 000000000..3ddab6df7 --- /dev/null +++ b/tests/functional/adapter/persist_docs/test_persist_docs.py @@ -0,0 +1,119 @@ +import json +import os +import pytest + +from dbt.tests.util import run_dbt + +from fixtures import ( + _MODELS__MY_FUN_DOCS, + _MODELS__INCREMENTAL_DELTA, + _MODELS__TABLE_DELTA_MODEL, + _MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN, + _PROPERTIES__MODELS, + _PROPERTIES__SEEDS, + _SEEDS__BASIC, +) + + +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestPersistDocsDeltaTable: + @pytest.fixture(scope="class") + def models(self): + return { + "incremental_delta_model.sql": _MODELS__INCREMENTAL_DELTA, + "my_fun_docs.md": _MODELS__MY_FUN_DOCS, + "table_delta_model.sql": _MODELS__TABLE_DELTA_MODEL, + "schema.yml": _PROPERTIES__MODELS + } + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": _SEEDS__BASIC, + "seed.yml": _PROPERTIES__SEEDS + } + + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + 'models': { + 'test': { + '+persist_docs': { + "relation": True, + "columns": True, + }, + } + }, + 'seeds': { + 'test': { + '+persist_docs': { + "relation": True, + "columns": True, + }, + '+file_format': 'delta', + '+quote_columns': True + } + }, + } + + def test_delta_comments(self, project): + run_dbt(['seed']) + run_dbt(['run']) + + for table, whatis in [ + ('table_delta_model', 'Table'), + ('seed', 'Seed'), + ('incremental_delta_model', 'Incremental') + ]: + results = project.run_sql( + 'describe extended {schema}.{table}'.format(schema=project.test_schema, table=table), + fetch='all' + ) + + for result in results: + if result[0] == 'Comment': + assert result[1].startswith(f'{whatis} model description') + if result[0] == 'id': + assert result[2].startswith('id Column description') + if result[0] == 'name': + assert result[2].startswith('Some stuff here and then a call to') + + +@pytest.mark.skip_profile("apache_spark", "spark_session") +class TestPersistDocsMissingColumn: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "+persist_docs": { + "columns": True, + }, + } + } + } + + @pytest.fixture(scope="class") + def seeds(self): + return { + "seed.csv": _SEEDS__BASIC, + "seed.yml": _PROPERTIES__SEEDS + } + + @pytest.fixture(scope="class") + def models(self): + return { + "table_delta_model.sql": _MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN, + "my_fun_docs.md": _MODELS__MY_FUN_DOCS, + } + + @pytest.fixture(scope="class") + def properties(self): + return {"schema.yml": _PROPERTIES__MODELS} + + def test_missing_column(self, project): + '''spark will use our schema to verify all columns exist rather than fail silently''' + run_dbt(["seed"]) + res = run_dbt(["run"], expect_pass=False) + assert "Missing field name in table" in res[0].message diff --git a/tests/integration/persist_docs/models/incremental_delta_model.sql b/tests/integration/persist_docs/models/incremental_delta_model.sql deleted file mode 100644 index c3f325ea3..000000000 --- a/tests/integration/persist_docs/models/incremental_delta_model.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(materialized='incremental', file_format='delta') }} -select 1 as id, 'Joe' as name diff --git a/tests/integration/persist_docs/models/my_fun_docs.md b/tests/integration/persist_docs/models/my_fun_docs.md deleted file mode 100644 index f3c0fbf55..000000000 --- a/tests/integration/persist_docs/models/my_fun_docs.md +++ /dev/null @@ -1,10 +0,0 @@ -{% docs my_fun_doc %} -name Column description "with double quotes" -and with 'single quotes' as welll as other; -'''abc123''' -reserved -- characters --- -/* comment */ -Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting - -{% enddocs %} diff --git a/tests/integration/persist_docs/models/no_docs_model.sql b/tests/integration/persist_docs/models/no_docs_model.sql deleted file mode 100644 index e39a7a156..000000000 --- a/tests/integration/persist_docs/models/no_docs_model.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as id, 'Alice' as name diff --git a/tests/integration/persist_docs/models/table_delta_model.sql b/tests/integration/persist_docs/models/table_delta_model.sql deleted file mode 100644 index 06e40e0c6..000000000 --- a/tests/integration/persist_docs/models/table_delta_model.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(materialized='table', file_format='delta') }} -select 1 as id, 'Joe' as name diff --git a/tests/integration/persist_docs/models/view_model.sql b/tests/integration/persist_docs/models/view_model.sql deleted file mode 100644 index a6f96a16d..000000000 --- a/tests/integration/persist_docs/models/view_model.sql +++ /dev/null @@ -1,2 +0,0 @@ -{{ config(materialized='view') }} -select 2 as id, 'Bob' as name diff --git a/tests/integration/persist_docs/seeds/seed.csv b/tests/integration/persist_docs/seeds/seed.csv deleted file mode 100644 index 4a295177c..000000000 --- a/tests/integration/persist_docs/seeds/seed.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,name -1,Alice -2,Bob \ No newline at end of file diff --git a/tests/integration/persist_docs/seeds/seeds.yml b/tests/integration/persist_docs/seeds/seeds.yml deleted file mode 100644 index 7ab82fa6b..000000000 --- a/tests/integration/persist_docs/seeds/seeds.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 2 - -seeds: - - name: seed - description: | - Seed model description "with double quotes" - and with 'single quotes' as welll as other; - '''abc123''' - reserved -- characters - -- - /* comment */ - Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting - columns: - - name: id - description: | - id Column description "with double quotes" - and with 'single quotes' as welll as other; - '''abc123''' - reserved -- characters - -- - /* comment */ - Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting - - name: name - description: | - Some stuff here and then a call to - {{ doc('my_fun_doc')}} diff --git a/tests/integration/persist_docs/test_persist_docs.py b/tests/integration/persist_docs/test_persist_docs.py deleted file mode 100644 index 527f548a3..000000000 --- a/tests/integration/persist_docs/test_persist_docs.py +++ /dev/null @@ -1,68 +0,0 @@ -from cProfile import run -from tests.integration.base import DBTIntegrationTest, use_profile - - -class TestPersistDocsDelta(DBTIntegrationTest): - @property - def schema(self): - return "persist_docs_columns" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'models': { - 'test': { - '+persist_docs': { - "relation": True, - "columns": True, - }, - } - }, - 'seeds': { - 'test': { - '+persist_docs': { - "relation": True, - "columns": True, - }, - '+file_format': 'delta', - '+quote_columns': True - } - }, - } - - def test_delta_comments(self): - self.run_dbt(['seed']) - self.run_dbt(['run']) - - for table, whatis in [ - ('table_delta_model', 'Table'), - ('seed', 'Seed'), - ('incremental_delta_model', 'Incremental') - ]: - results = self.run_sql( - 'describe extended {schema}.{table}'.format(schema=self.unique_schema(), table=table), - fetch='all' - ) - - for result in results: - if result[0] == 'Comment': - assert result[1].startswith(f'{whatis} model description') - if result[0] == 'id': - assert result[2].startswith('id Column description') - if result[0] == 'name': - assert result[2].startswith('Some stuff here and then a call to') - - # runs on Spark v3.0 - @use_profile("databricks_cluster") - def test_delta_comments_databricks_cluster(self): - self.test_delta_comments() - - # runs on Spark v3.0 - @use_profile("databricks_sql_endpoint") - def test_delta_comments_databricks_sql_endpoint(self): - self.test_delta_comments()