From f02243506df2a9b6851abe6bcfe10af8808d41cc Mon Sep 17 00:00:00 2001 From: Stu Kilgore Date: Tue, 8 Nov 2022 15:30:29 -0600 Subject: [PATCH] Convert postgres index tests (#6228) --- .../Under the Hood-20221108-115633.yaml | 7 + .../models-invalid/invalid_columns_type.sql | 10 -- .../models-invalid/invalid_type.sql | 10 -- .../models-invalid/invalid_unique_config.sql | 10 -- .../models-invalid/missing_columns.sql | 10 -- .../models/incremental.sql | 18 --- .../065_postgres_index_tests/models/table.sql | 14 -- .../065_postgres_index_tests/seeds/seed.csv | 4 - .../snapshots/colors.sql | 29 ---- .../test_postgres_indexes.py | 134 ---------------- tests/functional/postgres/fixtures.py | 134 ++++++++++++++++ .../postgres/test_postgres_indexes.py | 149 ++++++++++++++++++ 12 files changed, 290 insertions(+), 239 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20221108-115633.yaml delete mode 100644 test/integration/065_postgres_index_tests/models-invalid/invalid_columns_type.sql delete mode 100644 test/integration/065_postgres_index_tests/models-invalid/invalid_type.sql delete mode 100644 test/integration/065_postgres_index_tests/models-invalid/invalid_unique_config.sql delete mode 100644 test/integration/065_postgres_index_tests/models-invalid/missing_columns.sql delete mode 100644 test/integration/065_postgres_index_tests/models/incremental.sql delete mode 100644 test/integration/065_postgres_index_tests/models/table.sql delete mode 100644 test/integration/065_postgres_index_tests/seeds/seed.csv delete mode 100644 test/integration/065_postgres_index_tests/snapshots/colors.sql delete mode 100644 test/integration/065_postgres_index_tests/test_postgres_indexes.py create mode 100644 tests/functional/postgres/fixtures.py create mode 100644 tests/functional/postgres/test_postgres_indexes.py diff --git a/.changes/unreleased/Under the Hood-20221108-115633.yaml b/.changes/unreleased/Under the Hood-20221108-115633.yaml new file mode 100644 index 00000000000..2ba10536728 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20221108-115633.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Convert postgres index tests to pytest +time: 2022-11-08T11:56:33.743042-06:00 +custom: + Author: stu-k + Issue: "5770" + PR: "6228" diff --git a/test/integration/065_postgres_index_tests/models-invalid/invalid_columns_type.sql b/test/integration/065_postgres_index_tests/models-invalid/invalid_columns_type.sql deleted file mode 100644 index 10f41526abd..00000000000 --- a/test/integration/065_postgres_index_tests/models-invalid/invalid_columns_type.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ - config( - materialized = "table", - indexes=[ - {'columns': 'column_a, column_b'}, - ] - ) -}} - -select 1 as column_a, 2 as column_b diff --git a/test/integration/065_postgres_index_tests/models-invalid/invalid_type.sql b/test/integration/065_postgres_index_tests/models-invalid/invalid_type.sql deleted file mode 100644 index 824ca36595f..00000000000 --- a/test/integration/065_postgres_index_tests/models-invalid/invalid_type.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ - config( - materialized = "table", - indexes=[ - {'columns': ['column_a'], 'type': 'non_existent_type'}, - ] - ) -}} - -select 1 as column_a, 2 as column_b diff --git a/test/integration/065_postgres_index_tests/models-invalid/invalid_unique_config.sql b/test/integration/065_postgres_index_tests/models-invalid/invalid_unique_config.sql deleted file mode 100644 index ca0113272ea..00000000000 --- a/test/integration/065_postgres_index_tests/models-invalid/invalid_unique_config.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ - config( - materialized = "table", - indexes=[ - {'columns': ['column_a'], 'unique': 'yes'}, - ] - ) -}} - -select 1 as column_a, 2 as column_b diff --git a/test/integration/065_postgres_index_tests/models-invalid/missing_columns.sql b/test/integration/065_postgres_index_tests/models-invalid/missing_columns.sql deleted file mode 100644 index 9b47943e6cf..00000000000 --- a/test/integration/065_postgres_index_tests/models-invalid/missing_columns.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ - config( - materialized = "table", - indexes=[ - {'unique': True}, - ] - ) -}} - -select 1 as column_a, 2 as column_b diff --git a/test/integration/065_postgres_index_tests/models/incremental.sql b/test/integration/065_postgres_index_tests/models/incremental.sql deleted file mode 100644 index 7cd24bdcf8c..00000000000 --- a/test/integration/065_postgres_index_tests/models/incremental.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ - config( - materialized = "incremental", - indexes=[ - {'columns': ['column_a'], 'type': 'hash'}, - {'columns': ['column_a', 'column_b'], 'unique': True}, - ] - ) -}} - -select * -from ( - select 1 as column_a, 2 as column_b -) t - -{% if is_incremental() %} - where column_a > (select max(column_a) from {{this}}) -{% endif %} diff --git a/test/integration/065_postgres_index_tests/models/table.sql b/test/integration/065_postgres_index_tests/models/table.sql deleted file mode 100644 index 39fccc14b15..00000000000 --- a/test/integration/065_postgres_index_tests/models/table.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ - config( - materialized = "table", - indexes=[ - {'columns': ['column_a']}, - {'columns': ['column_b']}, - {'columns': ['column_a', 'column_b']}, - {'columns': ['column_b', 'column_a'], 'type': 'btree', 'unique': True}, - {'columns': ['column_a'], 'type': 'hash'} - ] - ) -}} - -select 1 as column_a, 2 as column_b diff --git a/test/integration/065_postgres_index_tests/seeds/seed.csv b/test/integration/065_postgres_index_tests/seeds/seed.csv deleted file mode 100644 index e744edef675..00000000000 --- a/test/integration/065_postgres_index_tests/seeds/seed.csv +++ /dev/null @@ -1,4 +0,0 @@ -country_code,country_name -US,United States -CA,Canada -GB,United Kingdom diff --git a/test/integration/065_postgres_index_tests/snapshots/colors.sql b/test/integration/065_postgres_index_tests/snapshots/colors.sql deleted file mode 100644 index f3a901d615f..00000000000 --- a/test/integration/065_postgres_index_tests/snapshots/colors.sql +++ /dev/null @@ -1,29 +0,0 @@ -{% snapshot colors %} - - {{ - config( - target_database=database, - target_schema=schema, - unique_key='id', - strategy='check', - check_cols=['color'], - indexes=[ - {'columns': ['id'], 'type': 'hash'}, - {'columns': ['id', 'color'], 'unique': True}, - ] - ) - }} - - {% if var('version') == 1 %} - - select 1 as id, 'red' as color union all - select 2 as id, 'green' as color - - {% else %} - - select 1 as id, 'blue' as color union all - select 2 as id, 'green' as color - - {% endif %} - -{% endsnapshot %} diff --git a/test/integration/065_postgres_index_tests/test_postgres_indexes.py b/test/integration/065_postgres_index_tests/test_postgres_indexes.py deleted file mode 100644 index 56dc557d5ac..00000000000 --- a/test/integration/065_postgres_index_tests/test_postgres_indexes.py +++ /dev/null @@ -1,134 +0,0 @@ -import re - -from test.integration.base import DBTIntegrationTest, use_profile - - -INDEX_DEFINITION_PATTERN = re.compile(r'using\s+(\w+)\s+\((.+)\)\Z') - -class TestPostgresIndex(DBTIntegrationTest): - @property - def schema(self): - return "postgres_index_065" - - @property - def models(self): - return "models" - - @property - def project_config(self): - return { - 'config-version': 2, - 'seeds': { - 'quote_columns': False, - 'indexes': [ - {'columns': ['country_code'], 'unique': False, 'type': 'hash'}, - {'columns': ['country_code', 'country_name'], 'unique': True}, - ], - }, - 'vars': { - 'version': 1 - }, - } - - @use_profile('postgres') - def test__postgres__table(self): - results = self.run_dbt(['run', '--models', 'table']) - self.assertEqual(len(results), 1) - - indexes = self.get_indexes('table') - self.assertCountEqual( - indexes, - [ - {'columns': 'column_a', 'unique': False, 'type': 'btree'}, - {'columns': 'column_b', 'unique': False, 'type': 'btree'}, - {'columns': 'column_a, column_b', 'unique': False, 'type': 'btree'}, - {'columns': 'column_b, column_a', 'unique': True, 'type': 'btree'}, - {'columns': 'column_a', 'unique': False, 'type': 'hash'} - ] - ) - - @use_profile('postgres') - def test__postgres__incremental(self): - for additional_argument in [[], [], ['--full-refresh']]: - results = self.run_dbt(['run', '--models', 'incremental'] + additional_argument) - self.assertEqual(len(results), 1) - - indexes = self.get_indexes('incremental') - self.assertCountEqual( - indexes, - [ - {'columns': 'column_a', 'unique': False, 'type': 'hash'}, - {'columns': 'column_a, column_b', 'unique': True, 'type': 'btree'}, - ] - ) - - @use_profile('postgres') - def test__postgres__seed(self): - for additional_argument in [[], [], ['--full-refresh']]: - results = self.run_dbt(["seed"] + additional_argument) - self.assertEqual(len(results), 1) - - indexes = self.get_indexes('seed') - self.assertCountEqual( - indexes, - [ - {'columns': 'country_code', 'unique': False, 'type': 'hash'}, - {'columns': 'country_code, country_name', 'unique': True, 'type': 'btree'}, - ] - ) - - @use_profile('postgres') - def test__postgres__snapshot(self): - for version in [1, 2]: - results = self.run_dbt(["snapshot", '--vars', 'version: {}'.format(version)]) - self.assertEqual(len(results), 1) - - indexes = self.get_indexes('colors') - self.assertCountEqual( - indexes, - [ - {'columns': 'id', 'unique': False, 'type': 'hash'}, - {'columns': 'id, color', 'unique': True, 'type': 'btree'}, - ] - ) - - def get_indexes(self, table_name): - sql = """ - SELECT - pg_get_indexdef(idx.indexrelid) as index_definition - FROM pg_index idx - JOIN pg_class tab ON tab.oid = idx.indrelid - WHERE - tab.relname = '{table}' - AND tab.relnamespace = ( - SELECT oid FROM pg_namespace WHERE nspname = '{schema}' - ); - """ - - sql = sql.format(table=table_name, schema=self.unique_schema()) - results = self.run_sql(sql, fetch='all') - return [self.parse_index_definition(row[0]) for row in results] - - def parse_index_definition(self, index_definition): - index_definition = index_definition.lower() - is_unique = 'unique' in index_definition - m = INDEX_DEFINITION_PATTERN.search(index_definition) - return {'columns': m.group(2), 'unique': is_unique, 'type': m.group(1)} - -class TestPostgresInvalidIndex(DBTIntegrationTest): - @property - def schema(self): - return "postgres_index_065" - - @property - def models(self): - return "models-invalid" - - @use_profile('postgres') - def test__postgres__invalid_index_configs(self): - results, output = self.run_dbt_and_capture(expect_pass=False) - self.assertEqual(len(results), 4) - self.assertRegex(output, r'columns.*is not of type \'array\'') - self.assertRegex(output, r'unique.*is not of type \'boolean\'') - self.assertRegex(output, r'\'columns\' is a required property') - self.assertRegex(output, r'Database Error in model invalid_type') diff --git a/tests/functional/postgres/fixtures.py b/tests/functional/postgres/fixtures.py new file mode 100644 index 00000000000..93b26b4f31b --- /dev/null +++ b/tests/functional/postgres/fixtures.py @@ -0,0 +1,134 @@ +models__incremental_sql = """ +{{ + config( + materialized = "incremental", + indexes=[ + {'columns': ['column_a'], 'type': 'hash'}, + {'columns': ['column_a', 'column_b'], 'unique': True}, + ] + ) +}} + +select * +from ( + select 1 as column_a, 2 as column_b +) t + +{% if is_incremental() %} + where column_a > (select max(column_a) from {{this}}) +{% endif %} + +""" + +models__table_sql = """ +{{ + config( + materialized = "table", + indexes=[ + {'columns': ['column_a']}, + {'columns': ['column_b']}, + {'columns': ['column_a', 'column_b']}, + {'columns': ['column_b', 'column_a'], 'type': 'btree', 'unique': True}, + {'columns': ['column_a'], 'type': 'hash'} + ] + ) +}} + +select 1 as column_a, 2 as column_b + +""" + +models_invalid__invalid_columns_type_sql = """ +{{ + config( + materialized = "table", + indexes=[ + {'columns': 'column_a, column_b'}, + ] + ) +}} + +select 1 as column_a, 2 as column_b + +""" + +models_invalid__invalid_type_sql = """ +{{ + config( + materialized = "table", + indexes=[ + {'columns': ['column_a'], 'type': 'non_existent_type'}, + ] + ) +}} + +select 1 as column_a, 2 as column_b + +""" + +models_invalid__invalid_unique_config_sql = """ +{{ + config( + materialized = "table", + indexes=[ + {'columns': ['column_a'], 'unique': 'yes'}, + ] + ) +}} + +select 1 as column_a, 2 as column_b + +""" + +models_invalid__missing_columns_sql = """ +{{ + config( + materialized = "table", + indexes=[ + {'unique': True}, + ] + ) +}} + +select 1 as column_a, 2 as column_b + +""" + +snapshots__colors_sql = """ +{% snapshot colors %} + + {{ + config( + target_database=database, + target_schema=schema, + unique_key='id', + strategy='check', + check_cols=['color'], + indexes=[ + {'columns': ['id'], 'type': 'hash'}, + {'columns': ['id', 'color'], 'unique': True}, + ] + ) + }} + + {% if var('version') == 1 %} + + select 1 as id, 'red' as color union all + select 2 as id, 'green' as color + + {% else %} + + select 1 as id, 'blue' as color union all + select 2 as id, 'green' as color + + {% endif %} + +{% endsnapshot %} + +""" + +seeds__seed_csv = """country_code,country_name +US,United States +CA,Canada +GB,United Kingdom +""" diff --git a/tests/functional/postgres/test_postgres_indexes.py b/tests/functional/postgres/test_postgres_indexes.py new file mode 100644 index 00000000000..64d61d2df87 --- /dev/null +++ b/tests/functional/postgres/test_postgres_indexes.py @@ -0,0 +1,149 @@ +import pytest +import re +from dbt.tests.util import ( + run_dbt, + run_dbt_and_capture, +) +from tests.functional.postgres.fixtures import ( + models__incremental_sql, + models__table_sql, + models_invalid__missing_columns_sql, + models_invalid__invalid_columns_type_sql, + models_invalid__invalid_type_sql, + models_invalid__invalid_unique_config_sql, + seeds__seed_csv, + snapshots__colors_sql, +) + + +INDEX_DEFINITION_PATTERN = re.compile(r"using\s+(\w+)\s+\((.+)\)\Z") + + +class TestPostgresIndex: + @pytest.fixture(scope="class") + def models(self): + return { + "table.sql": models__table_sql, + "incremental.sql": models__incremental_sql, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"seed.csv": seeds__seed_csv} + + @pytest.fixture(scope="class") + def snapshots(self): + return {"colors.sql": snapshots__colors_sql} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "config-version": 2, + "seeds": { + "quote_columns": False, + "indexes": [ + {"columns": ["country_code"], "unique": False, "type": "hash"}, + {"columns": ["country_code", "country_name"], "unique": True}, + ], + }, + "vars": { + "version": 1, + }, + } + + def test_table(self, project, unique_schema): + results = run_dbt(["run", "--models", "table"]) + assert len(results) == 1 + + indexes = self.get_indexes("table", project, unique_schema) + expected = [ + {"columns": "column_a", "unique": False, "type": "btree"}, + {"columns": "column_b", "unique": False, "type": "btree"}, + {"columns": "column_a, column_b", "unique": False, "type": "btree"}, + {"columns": "column_b, column_a", "unique": True, "type": "btree"}, + {"columns": "column_a", "unique": False, "type": "hash"}, + ] + assert len(indexes) == len(expected) + + def test_incremental(self, project, unique_schema): + for additional_argument in [[], [], ["--full-refresh"]]: + results = run_dbt(["run", "--models", "incremental"] + additional_argument) + assert len(results) == 1 + + indexes = self.get_indexes('incremental', project, unique_schema) + expected = [ + {"columns": "column_a", "unique": False, "type": "hash"}, + {"columns": "column_a, column_b", "unique": True, "type": "btree"}, + ] + assert len(indexes) == len(expected) + + def test_seed(self, project, unique_schema): + for additional_argument in [[], [], ['--full-refresh']]: + results = run_dbt(["seed"] + additional_argument) + assert len(results) == 1 + + indexes = self.get_indexes('seed', project, unique_schema) + expected = [ + {"columns": "country_code", "unique": False, "type": "hash"}, + {"columns": "country_code, country_name", "unique": True, "type": "btree"}, + ] + assert len(indexes) == len(expected) + + def test_snapshot(self, project, unique_schema): + for version in [1, 2]: + results = run_dbt(["snapshot", "--vars", f"version: {version}"]) + assert len(results) == 1 + + indexes = self.get_indexes('colors', project, unique_schema) + expected = [ + {"columns": "id", "unique": False, "type": "hash"}, + {"columns": "id, color", "unique": True, "type": "btree"}, + ] + assert len(indexes) == len(expected) + + def get_indexes(self, table_name, project, unique_schema): + sql = f""" + SELECT + pg_get_indexdef(idx.indexrelid) as index_definition + FROM pg_index idx + JOIN pg_class tab ON tab.oid = idx.indrelid + WHERE + tab.relname = '{table_name}' + AND tab.relnamespace = ( + SELECT oid FROM pg_namespace WHERE nspname = '{unique_schema}' + ); + """ + results = project.run_sql(sql, fetch="all") + return [self.parse_index_definition(row[0]) for row in results] + + def parse_index_definition(self, index_definition): + index_definition = index_definition.lower() + is_unique = "unique" in index_definition + m = INDEX_DEFINITION_PATTERN.search(index_definition) + return { + "columns": m.group(2), + "unique": is_unique, + "type": m.group(1), + } + + def assertCountEqual(self, a, b): + assert len(a) == len(b) + + +class TestPostgresInvalidIndex(): + @pytest.fixture(scope="class") + def models(self): + return { + "invalid_unique_config.sql": models_invalid__invalid_unique_config_sql, + "invalid_type.sql": models_invalid__invalid_type_sql, + "invalid_columns_type.sql": models_invalid__invalid_columns_type_sql, + "missing_columns.sql": models_invalid__missing_columns_sql, + } + + def test_invalid_index_configs(self, project): + results, output = run_dbt_and_capture(expect_pass=False) + assert len(results) == 4 + assert re.search(r"columns.*is not of type 'array'", output) + assert re.search(r"unique.*is not of type 'boolean'", output) + assert re.search(r"'columns' is a required property", output) + assert re.search(r"Database Error in model invalid_type", output)