From 64518d079d1e7c7113d5795fad5efd4a4b3c00e3 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Mon, 18 Sep 2023 15:00:10 +0100 Subject: [PATCH] add TestLargeEphemeralCompilation (#8376) --- .../Under the Hood-20230912-230619.yaml | 6 + tests/functional/materializations/fixtures.py | 245 ++++++++++++++++++ .../test_ephemeral_compilation.py | 83 +++--- 3 files changed, 291 insertions(+), 43 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20230912-230619.yaml create mode 100644 tests/functional/materializations/fixtures.py diff --git a/.changes/unreleased/Under the Hood-20230912-230619.yaml b/.changes/unreleased/Under the Hood-20230912-230619.yaml new file mode 100644 index 00000000000..bc4936730fd --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230912-230619.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: add a test for ephemeral cte injection +time: 2023-09-12T23:06:19.938207+01:00 +custom: + Author: michelleark + Issue: "8376" diff --git a/tests/functional/materializations/fixtures.py b/tests/functional/materializations/fixtures.py new file mode 100644 index 00000000000..b799d08e611 --- /dev/null +++ b/tests/functional/materializations/fixtures.py @@ -0,0 +1,245 @@ +fct_eph_first_sql = """ +-- fct_eph_first.sql +{{ config(materialized='ephemeral') }} + +with int_eph_first as( + select * from {{ ref('int_eph_first') }} +) + +select * from int_eph_first +""" + +int_eph_first_sql = """ +-- int_eph_first.sql +{{ config(materialized='ephemeral') }} + +select + 1 as first_column, + 2 as second_column +""" + +schema_yml = """ +version: 2 + +models: + - name: int_eph_first + columns: + - name: first_column + tests: + - not_null + - name: second_column + tests: + - not_null + + - name: fct_eph_first + columns: + - name: first_column + tests: + - not_null + - name: second_column + tests: + - not_null + +""" + +bar_sql = """ +{{ config(materialized = 'table') }} + +WITH foo AS ( + + SELECT * FROM {{ ref('foo') }} + +), foo_1 AS ( + + SELECT * FROM {{ ref('foo_1') }} + +), foo_2 AS ( + + SELECT * FROM {{ ref('foo_2') }} + +) + +SELECT * FROM foo +UNION ALL +SELECT * FROM foo_1 +UNION ALL +SELECT * FROM foo_2 +""" + +bar1_sql = """ +{{ config(materialized = 'table') }} + +WITH foo AS ( + + SELECT * FROM {{ ref('foo') }} + +), foo_1 AS ( + + SELECT * FROM {{ ref('foo_1') }} + +), foo_2 AS ( + + SELECT * FROM {{ ref('foo_2') }} + +) + +SELECT * FROM foo +UNION ALL +SELECT * FROM foo_1 +UNION ALL +SELECT * FROM foo_2 +""" + +bar2_sql = """ +{{ config(materialized = 'table') }} + +WITH foo AS ( + + SELECT * FROM {{ ref('foo') }} + +), foo_1 AS ( + + SELECT * FROM {{ ref('foo_1') }} + +), foo_2 AS ( + + SELECT * FROM {{ ref('foo_2') }} + +) + +SELECT * FROM foo +UNION ALL +SELECT * FROM foo_1 +UNION ALL +SELECT * FROM foo_2 +""" + +bar3_sql = """ +{{ config(materialized = 'table') }} + +WITH foo AS ( + + SELECT * FROM {{ ref('foo') }} + +), foo_1 AS ( + + SELECT * FROM {{ ref('foo_1') }} + +), foo_2 AS ( + + SELECT * FROM {{ ref('foo_2') }} + +) + +SELECT * FROM foo +UNION ALL +SELECT * FROM foo_1 +UNION ALL +SELECT * FROM foo_2 +""" + +bar4_sql = """ +{{ config(materialized = 'table') }} + +WITH foo AS ( + + SELECT * FROM {{ ref('foo') }} + +), foo_1 AS ( + + SELECT * FROM {{ ref('foo_1') }} + +), foo_2 AS ( + + SELECT * FROM {{ ref('foo_2') }} + +) + +SELECT * FROM foo +UNION ALL +SELECT * FROM foo_1 +UNION ALL +SELECT * FROM foo_2 +""" + +bar5_sql = """ +{{ config(materialized = 'table') }} + +WITH foo AS ( + + SELECT * FROM {{ ref('foo') }} + +), foo_1 AS ( + + SELECT * FROM {{ ref('foo_1') }} + +), foo_2 AS ( + + SELECT * FROM {{ ref('foo_2') }} + +) + +SELECT * FROM foo +UNION ALL +SELECT * FROM foo_1 +UNION ALL +SELECT * FROM foo_2 +""" + +baz_sql = """ +{{ config(materialized = 'table') }} +SELECT * FROM {{ ref('bar') }} +""" + +baz1_sql = """ +{{ config(materialized = 'table') }} +SELECT * FROM {{ ref('bar_1') }} +""" + +foo_sql = """ +{{ config(materialized = 'ephemeral') }} + +with source as ( + + select 1 as id + +), renamed as ( + + select id as uid from source + +) + +select * from renamed +""" + +foo1_sql = """ +{{ config(materialized = 'ephemeral') }} + +WITH source AS ( + + SELECT 1 AS id + +), RENAMED as ( + + SELECT id as UID FROM source + +) + +SELECT * FROM renamed +""" + +foo2_sql = """ +{{ config(materialized = 'ephemeral') }} + +WITH source AS ( + + SELECT 1 AS id + +), RENAMED as ( + + SELECT id as UID FROM source + +) + +SELECT * FROM renamed +""" diff --git a/tests/functional/materializations/test_ephemeral_compilation.py b/tests/functional/materializations/test_ephemeral_compilation.py index c9f17d3e00c..f8419e40fd5 100644 --- a/tests/functional/materializations/test_ephemeral_compilation.py +++ b/tests/functional/materializations/test_ephemeral_compilation.py @@ -9,51 +9,23 @@ # fails fairly regularly if that is broken, but does occasionally work (depending # on the order in which things are compiled). It requires multi-threading to fail. - -fct_eph_first_sql = """ --- fct_eph_first.sql -{{ config(materialized='ephemeral') }} - -with int_eph_first as( - select * from {{ ref('int_eph_first') }} +from tests.functional.materializations.fixtures import ( + fct_eph_first_sql, + int_eph_first_sql, + schema_yml, + bar_sql, + bar1_sql, + bar2_sql, + bar3_sql, + bar4_sql, + bar5_sql, + baz_sql, + baz1_sql, + foo_sql, + foo1_sql, + foo2_sql, ) -select * from int_eph_first -""" - -int_eph_first_sql = """ --- int_eph_first.sql -{{ config(materialized='ephemeral') }} - -select - 1 as first_column, - 2 as second_column -""" - -schema_yml = """ -version: 2 - -models: - - name: int_eph_first - columns: - - name: first_column - tests: - - not_null - - name: second_column - tests: - - not_null - - - name: fct_eph_first - columns: - - name: first_column - tests: - - not_null - - name: second_column - tests: - - not_null - -""" - SUPPRESSED_CTE_EXPECTED_OUTPUT = """-- fct_eph_first.sql @@ -89,3 +61,28 @@ def test__suppress_injected_ctes(self, project): node = node_result.node assert isinstance(node, ModelNode) assert node.compiled_code == SUPPRESSED_CTE_EXPECTED_OUTPUT + + +# From: https://github.com/jeremyyeo/ephemeral-invalid-sql-repro/tree/main/models +class TestLargeEphemeralCompilation: + @pytest.fixture(scope="class") + def models(self): + + return { + "bar.sql": bar_sql, + "bar_1.sql": bar1_sql, + "bar_2.sql": bar2_sql, + "bar_3.sql": bar3_sql, + "bar_4.sql": bar4_sql, + "bar_5.sql": bar5_sql, + "baz.sql": baz_sql, + "baz_1.sql": baz1_sql, + "foo.sql": foo_sql, + "foo_1.sql": foo1_sql, + "foo_2.sql": foo2_sql, + } + + def test_ephemeral_compilation(self, project): + # 8/11 table models are built as expected. no compilation errors + results = run_dbt(["build"]) + assert len(results) == 8