diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index 23f5a590..03ceb3d3 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -24,7 +24,9 @@ env: DBT_CLOUD_PROJECT_ID: 123 DBT_CLOUD_JOB_ID: ABC DBT_CLOUD_RUN_REASON: "String with 'quotes' !" - TEST_ENV_VAR_1: TEST_VALUE + TEST_ENV_VAR_NUMBER: 3 + TEST_ENV_VAR_EMPTY: "" + TEST_ENV_VAR_WITH_QUOTE: "Triggered via Apache Airflow by task 'trigger_dbt_cloud_job_run' in the airtable_ingest DAG." DBT_ENV_CUSTOM_ENV_FAVOURITE_DBT_PACKAGE: dbt_artifacts jobs: diff --git a/integration_test_project/dbt_project.yml b/integration_test_project/dbt_project.yml index cd1bbe79..a6550009 100644 --- a/integration_test_project/dbt_project.yml +++ b/integration_test_project/dbt_project.yml @@ -1,8 +1,8 @@ -name: 'artifacts_integration_tests' -version: '1.0' +name: "artifacts_integration_tests" +version: "1.0" config-version: 2 -profile: 'dbt_artifacts' +profile: "dbt_artifacts" model-paths: ["models"] analysis-paths: ["analysis"] @@ -18,15 +18,9 @@ vars: test_dbt_vars_1: dbt_vars_1 test_dbt_vars_2: dbt_vars_2 test_dbt_vars_3: dbt_vars_3 - env_vars: [ - 'TEST_ENV_VAR_1' - ] - dbt_vars: [ - 'test_dbt_vars_1', - 'test_dbt_vars_2', - 'test_dbt_vars_3' - ] - + env_vars: + ["TEST_ENV_VAR_NUMBER", "TEST_ENV_VAR_EMPTY", "TEST_ENV_VAR_WITH_QUOTE"] + dbt_vars: ["test_dbt_vars_1", "test_dbt_vars_2", "test_dbt_vars_3"] models: +persist_docs: diff --git a/integration_test_project/example-env.sh b/integration_test_project/example-env.sh index e37616ca..51450cf3 100755 --- a/integration_test_project/example-env.sh +++ b/integration_test_project/example-env.sh @@ -23,5 +23,7 @@ export DBT_CLOUD_JOB_ID= export DBT_CLOUD_RUN_ID= export DBT_CLOUD_RUN_REASON_CATEGORY= export DBT_CLOUD_RUN_REASON= -export TEST_ENV_VAR_1= +export TEST_ENV_VAR_NUMBER=3 +export TEST_ENV_VAR_EMPTY= export DBT_ENV_CUSTOM_ENV_FAVOURITE_DBT_PACKAGE=dbt_artifacts +export TEST_ENV_VAR_WITH_QUOTE="Triggered via Apache Airflow by task 'trigger_dbt_cloud_job_run' in the airtable_ingest DAG." diff --git a/macros/upload_invocations.sql b/macros/upload_invocations.sql index 0e4b713b..21418fd2 100644 --- a/macros/upload_invocations.sql +++ b/macros/upload_invocations.sql @@ -46,7 +46,7 @@ {% if var('env_vars', none) %} {% set env_vars_dict = {} %} {% for env_variable in var('env_vars') %} - {% do env_vars_dict.update({env_variable: env_var(env_variable)}) %} + {% do env_vars_dict.update({env_variable: (env_var(env_variable, '') | replace("'", "''"))}) %} {% endfor %} '{{ tojson(env_vars_dict) }}', {# env_vars #} {% else %} @@ -56,7 +56,7 @@ {% if var('dbt_vars', none) %} {% set dbt_vars_dict = {} %} {% for dbt_var in var('dbt_vars') %} - {% do dbt_vars_dict.update({dbt_var: var(dbt_var)}) %} + {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, '') | replace("'", "''"))}) %} {% endfor %} '{{ tojson(dbt_vars_dict) }}', {# dbt_vars #} {% else %} @@ -64,7 +64,12 @@ {% endif %} '{{ tojson(invocation_args_dict) | replace('\\', '\\\\') }}', {# invocation_args #} - '{{ tojson(dbt_metadata_envs) | replace("'", "\\'") }}' {# dbt_custom_envs #} + + {% set metadata_env = {} %} + {% for key, value in dbt_metadata_envs.items() %} + {% do metadata_env.update({key: (value | replace("'", "''"))}) %} + {% endfor %} + '{{ tojson(metadata_env) | replace('\\', '\\\\') }}' {# dbt_custom_envs #} ) {% endset %} @@ -95,7 +100,7 @@ {% if var('env_vars', none) %} {% set env_vars_dict = {} %} {% for env_variable in var('env_vars') %} - {% do env_vars_dict.update({env_variable: env_var(env_variable)}) %} + {% do env_vars_dict.update({env_variable: (env_var(env_variable, '') | replace("'", "''"))}) %} {% endfor %} parse_json('{{ tojson(env_vars_dict) }}'), {# env_vars #} {% else %} @@ -105,7 +110,7 @@ {% if var('dbt_vars', none) %} {% set dbt_vars_dict = {} %} {% for dbt_var in var('dbt_vars') %} - {% do dbt_vars_dict.update({dbt_var: var(dbt_var)}) %} + {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, '') | replace("'", "''"))}) %} {% endfor %} parse_json('{{ tojson(dbt_vars_dict) }}'), {# dbt_vars #} {% else %} @@ -119,7 +124,11 @@ {% endif %} {# invocation_args_dict.vars, in the absence of any vars, results in the value "{}\n" as a string which results in an error. safe.parse_json accomodates for this gracefully. #} safe.parse_json('{{ tojson(invocation_args_dict) }}'), {# invocation_args #} - parse_json('{{ tojson(dbt_metadata_envs) }}') {# dbt_custom_envs #} + {% set metadata_env = {} %} + {% for key, value in dbt_metadata_envs.items() %} + {% do metadata_env.update({key: (value | replace("'", "''"))}) %} + {% endfor %} + parse_json('{{ tojson(metadata_env) | replace('\\', '\\\\') }}') {# dbt_custom_envs #} ) {% endset %} diff --git a/tox.ini b/tox.ini index 3c8437b1..9d5ae1d5 100644 --- a/tox.ini +++ b/tox.ini @@ -90,7 +90,9 @@ passenv = DBT_CLOUD_RUN_ID DBT_CLOUD_RUN_REASON_CATEGORY DBT_CLOUD_RUN_REASON - TEST_ENV_VAR_1 + TEST_ENV_VAR_NUMBER + TEST_ENV_VAR_EMPTY + TEST_ENV_VAR_WITH_QUOTE [testenv:lint] deps = {[sqlfluff]deps}