Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initialize sqlparse lexer and tweak order of setting compilation fields #8215

Merged
merged 2 commits into from
Jul 26, 2023
Merged

Initialize sqlparse lexer and tweak order of setting compilation fields #8215

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
33d9437
Initialize sqlparse lexer and tweak order of setting compilation fields
gshank Jul 26, 2023
393cd90
Move initialization of sqlparse lexer
gshank Jul 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20230726-104448.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Improve handling of CTE injection with ephemeral models
time: 2023-07-26T10:44:48.888451-04:00
custom:
Author: gshank
Issue: "8213"
18 changes: 12 additions & 6 deletions core/dbt/compilation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import networkx as nx # type: ignore
import os
import pickle
import sqlparse

from collections import defaultdict
from typing import List, Dict, Any, Tuple, Optional
Expand Down Expand Up @@ -36,6 +35,7 @@
from dbt.events.format import pluralize
import dbt.tracking
import dbt.task.list as list_task
import sqlparse

graph_file_name = "graph.gpickle"

Expand Down Expand Up @@ -378,16 +378,16 @@ def _recursively_prepend_ctes(

_add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql))

injected_sql = inject_ctes_into_sql(
model.compiled_code,
prepended_ctes,
)
# Check again before updating for multi-threading
if not model.extra_ctes_injected:
injected_sql = inject_ctes_into_sql(
model.compiled_code,
prepended_ctes,
)
model.extra_ctes_injected = True
model._pre_injected_sql = model.compiled_code
model.compiled_code = injected_sql
model.extra_ctes = prepended_ctes
model.extra_ctes_injected = True

# if model.extra_ctes is not set to prepended ctes, something went wrong
return model, model.extra_ctes
Expand Down Expand Up @@ -523,6 +523,12 @@ def compile_node(
the node's raw_code into compiled_code, and then calls the
recursive method to "prepend" the ctes.
"""
# Make sure Lexer for sqlparse 0.4.4 is initialized
from sqlparse.lexer import Lexer # type: ignore

if hasattr(Lexer, "get_default_instance"):
Lexer.get_default_instance()

node = self._compile_code(node, manifest, extra_context)

node, _ = self._recursively_prepend_ctes(node, manifest, extra_context)
Expand Down