Convert Spark persist docs test (dbt-labs#612)

* Begin conversion and get ready for CI testing.

* Uncheck models

* Change core index.

* Pair test down for minimal test

* Adjust code with some guessing.

* Forgot a version tag

* Make test conversion work. Finally

* Fix up the code.

* Attempt to fix test conversion with profile skips.

* Add missing column test and cleanup code.

* Remove shas from the requirements now that base conversion is live.

* Revert whitespace change.

---------

Co-authored-by: Mila Page <versusfacit@users.noreply.github.com>

tests/integration/persist_docs/models/schema.yml → tests/functional/adapter/persist_docs/fixtures.py

@@ -1,31 +1,35 @@
+_MODELS__MY_FUN_DOCS = """
+{% docs my_fun_doc %}
+name Column description "with double quotes"
+and with 'single  quotes' as welll as other;
+'''abc123'''
+reserved -- characters
+--
+/* comment */
+Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+
+{% enddocs %}
+"""
+
+_MODELS__INCREMENTAL_DELTA = """
+{{ config(materialized='incremental', file_format='delta') }}
+select 1 as id, 'Joe' as name
+"""
+
+_MODELS__TABLE_DELTA_MODEL = """
+{{ config(materialized='table', file_format='delta') }}
+select 1 as id, 'Joe' as name
+"""
+
+_MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN = """
+{{ config(materialized='table', file_format='delta') }}
+select 1 as id, 'Joe' as different_name
+"""
+
+_PROPERTIES__MODELS = """
 version: 2
 
 models:
-
-  - name: table_parquet_model
-    description: |
-      Table model description "with double quotes"
-      and with 'single  quotes' as welll as other;
-      '''abc123'''
-      reserved -- characters
-      --
-      /* comment */
-      Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
-    columns:
-      - name: id
-        description: |
-          id Column description "with double quotes"
-          and with 'single  quotes' as welll as other;
-          '''abc123'''
-          reserved -- characters
-          --
-          /* comment */
-          Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
-      - name: name
-        description: |
-          Some stuff here and then a call to
-          {{ doc('my_fun_doc')}}
-
   - name: table_delta_model
     description: |
       Table model description "with double quotes"
@@ -50,9 +54,9 @@
           Some stuff here and then a call to
           {{ doc('my_fun_doc')}}
 
-  - name: table_hudi_model
+  - name: incremental_delta_model
     description: |
-      Table model description "with double quotes"
+      Incremental model description "with double quotes"
       and with 'single  quotes' as welll as other;
       '''abc123'''
       reserved -- characters
@@ -73,30 +77,15 @@
         description: |
           Some stuff here and then a call to
           {{ doc('my_fun_doc')}}
-  
-  - name: view_model
-    description: |
-      View model description "with double quotes"
-      and with 'single  quotes' as welll as other;
-      '''abc123'''
-      reserved -- characters
-      --
-      /* comment */
-      Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
-    columns:
-      - name: id
-        description: |
-          id Column description "with double quotes"
-          and with 'single  quotes' as welll as other;
-          '''abc123'''
-          reserved -- characters
-          --
-          /* comment */
-          Some $lbl$ labeled $lbl$ and $$ unlabeled $$ dollar-quoting
+"""
 
-  - name: incremental_delta_model
+_PROPERTIES__SEEDS = """
+version: 2
+
+seeds:
+  - name: seed
     description: |
-      Incremental model description "with double quotes"
+      Seed model description "with double quotes"
       and with 'single  quotes' as welll as other;
       '''abc123'''
       reserved -- characters
@@ -117,3 +106,9 @@
         description: |
           Some stuff here and then a call to
           {{ doc('my_fun_doc')}}
+"""
+
+_SEEDS__BASIC = """id,name
+1,Alice
+2,Bob
+"""

tests/functional/adapter/persist_docs/test_persist_docs.py

@@ -0,0 +1,119 @@
+import json
+import os
+import pytest
+
+from dbt.tests.util import run_dbt
+
+from fixtures import (
+    _MODELS__MY_FUN_DOCS,
+    _MODELS__INCREMENTAL_DELTA,
+    _MODELS__TABLE_DELTA_MODEL,
+    _MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN,
+    _PROPERTIES__MODELS,
+    _PROPERTIES__SEEDS,
+    _SEEDS__BASIC,
+)
+
+
+@pytest.mark.skip_profile("apache_spark", "spark_session")
+class TestPersistDocsDeltaTable:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "incremental_delta_model.sql": _MODELS__INCREMENTAL_DELTA,
+            "my_fun_docs.md": _MODELS__MY_FUN_DOCS,
+            "table_delta_model.sql": _MODELS__TABLE_DELTA_MODEL,
+            "schema.yml": _PROPERTIES__MODELS
+        }
+
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {
+            "seed.csv": _SEEDS__BASIC,
+            "seed.yml": _PROPERTIES__SEEDS
+        }
+
+
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            'models': {
+                'test': {
+                    '+persist_docs': {
+                        "relation": True,
+                        "columns": True,
+                    },
+                }
+            },
+            'seeds': {
+                'test': {
+                    '+persist_docs': {
+                        "relation": True,
+                        "columns": True,
+                    },
+                    '+file_format': 'delta',
+                    '+quote_columns': True
+                }
+            },
+        }
+
+    def test_delta_comments(self, project):
+        run_dbt(['seed'])
+        run_dbt(['run'])
+
+        for table, whatis in [
+            ('table_delta_model', 'Table'),
+            ('seed', 'Seed'),
+            ('incremental_delta_model', 'Incremental')
+        ]:
+            results = project.run_sql(
+                'describe extended {schema}.{table}'.format(schema=project.test_schema, table=table),
+                fetch='all'
+            )
+
+            for result in results:
+                if result[0] == 'Comment':
+                    assert result[1].startswith(f'{whatis} model description')
+                if result[0] == 'id':
+                    assert result[2].startswith('id Column description')
+                if result[0] == 'name':
+                    assert result[2].startswith('Some stuff here and then a call to')
+
+
+@pytest.mark.skip_profile("apache_spark", "spark_session")
+class TestPersistDocsMissingColumn:
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "models": {
+                "test": {
+                    "+persist_docs": {
+                        "columns": True,
+                    },
+                }
+            }
+        }
+
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {
+            "seed.csv": _SEEDS__BASIC,
+            "seed.yml": _PROPERTIES__SEEDS
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "table_delta_model.sql": _MODELS__TABLE_DELTA_MODEL_MISSING_COLUMN,
+            "my_fun_docs.md": _MODELS__MY_FUN_DOCS,
+    }
+
+    @pytest.fixture(scope="class")
+    def properties(self):
+        return {"schema.yml": _PROPERTIES__MODELS}
+
+    def test_missing_column(self, project):
+        '''spark will use our schema to verify all columns exist rather than fail silently'''
+        run_dbt(["seed"])
+        res = run_dbt(["run"], expect_pass=False)
+        assert "Missing field name in table" in res[0].message