From 74e3754f558a0993dd23c28ee909d104e218d845 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 14 Apr 2021 09:27:01 -0700 Subject: [PATCH 01/18] low-end serverless tier has no columnstore --- test/integration/dbt_project.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration/dbt_project.yml b/test/integration/dbt_project.yml index f02f89c5..d443367d 100644 --- a/test/integration/dbt_project.yml +++ b/test/integration/dbt_project.yml @@ -3,4 +3,8 @@ name: 'sqlserver_integration_tests' version: '1.0' config-version: 2 -profile: 'integration_tests' \ No newline at end of file +profile: 'integration_tests' + +models: + sqlserver_integration_tests: + +as_columnstore: false \ No newline at end of file From a33f5424dea3a4ea1d6639342dba7e5217bdea5c Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 14 Apr 2021 09:27:28 -0700 Subject: [PATCH 02/18] use cxn test to wake up serverless db --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 458c236b..6bf85b5c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -113,4 +113,7 @@ workflows: - DBT_SYNAPSE_PROFILE - connection-sqlserver: *profile - integration-sqlserver: *profile - - integration-azuresql: *profile + - integration-azuresql: + <<: *profile + requires: + - connection-azuresql From e388c7ec6c3b697a176319214f23c611c816278e Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 14 Apr 2021 17:37:34 -0700 Subject: [PATCH 03/18] no columnstore --- test/integration/azuresql.dbtspec | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/integration/azuresql.dbtspec b/test/integration/azuresql.dbtspec index ed464781..79c3b8a1 100644 --- a/test/integration/azuresql.dbtspec +++ b/test/integration/azuresql.dbtspec @@ -11,6 +11,22 @@ target: encrypt: yes trust_cert: yes threads: 1 +projects: + - overrides: base + dbt_project_yml: &override-project + name: dbt_test_project + config-version: 2 + version: '1.0.0' + models: + +as_columnstore: false + - overrides: ephemeral + dbt_project_yml: *override-project + - overrides: incremental + dbt_project_yml: *override-project + - overrides: snapshot_strategy_timestamp + dbt_project_yml: *override-project + - overrides: schema_test + dbt_project_yml: *override-project sequences: test_dbt_empty: empty test_dbt_base: base From b0630c69010fc05a4d969bdea05b1e658c57b07b Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 14 Apr 2021 17:49:07 -0700 Subject: [PATCH 04/18] include project name? --- test/integration/azuresql.dbtspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/azuresql.dbtspec b/test/integration/azuresql.dbtspec index 79c3b8a1..1f92c66f 100644 --- a/test/integration/azuresql.dbtspec +++ b/test/integration/azuresql.dbtspec @@ -18,7 +18,8 @@ projects: config-version: 2 version: '1.0.0' models: - +as_columnstore: false + dbt_test_project: + +as_columnstore: false - overrides: ephemeral dbt_project_yml: *override-project - overrides: incremental From 1c9b3417d6f268ea7ef8950926fa656463cec3be Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 14 Apr 2021 17:56:50 -0700 Subject: [PATCH 05/18] json --- test/integration/azuresql.dbtspec | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/integration/azuresql.dbtspec b/test/integration/azuresql.dbtspec index 1f92c66f..494e6255 100644 --- a/test/integration/azuresql.dbtspec +++ b/test/integration/azuresql.dbtspec @@ -13,13 +13,17 @@ target: threads: 1 projects: - overrides: base - dbt_project_yml: &override-project - name: dbt_test_project - config-version: 2 - version: '1.0.0' - models: - dbt_test_project: - +as_columnstore: false + dbt_project_yml: | &override-project + { + "name":"dbt_test_project", + "config-version": 2, + "version": "1.0.0", + "models": { + "dbt_test_project": { + "as_columnstore": "false" + } + } + } - overrides: ephemeral dbt_project_yml: *override-project - overrides: incremental From 941ae6826108a6dc3eb3197c41923ed6b6f3eb4f Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 14 Apr 2021 17:58:48 -0700 Subject: [PATCH 06/18] upause server --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6bf85b5c..7a371e74 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,6 +79,11 @@ jobs: - run: *install-dbt-sqlserver - azure-cli/install - run: *prep=connect + - run: + name: wake up server + command: | + cd test/integration + dbt debug --target azuresql_sqlcred - run: name: cnxn -- Azure SQL - SQL CRED user+pass command: | From 8685cb274580b7858030b9355fcf79713a3e9b42 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 14 Apr 2021 17:59:15 -0700 Subject: [PATCH 07/18] not needed --- test/integration/dbt_project.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/integration/dbt_project.yml b/test/integration/dbt_project.yml index d443367d..f02f89c5 100644 --- a/test/integration/dbt_project.yml +++ b/test/integration/dbt_project.yml @@ -3,8 +3,4 @@ name: 'sqlserver_integration_tests' version: '1.0' config-version: 2 -profile: 'integration_tests' - -models: - sqlserver_integration_tests: - +as_columnstore: false \ No newline at end of file +profile: 'integration_tests' \ No newline at end of file From 49620bda26f7b91ff48894205ea93a8e6b59879a Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 14 Apr 2021 18:01:40 -0700 Subject: [PATCH 08/18] document --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4219883e..7f45de67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### v0.19.0.3 + +#### under the hood +- allow CI to work with the lower-cost serverless Azure SQL [#132](https://github.com/dbt-msft/dbt-sqlserver/pull/132) ### v0.19.0.2 #### fixes From cd07e21aa1ebb55754ccb3fcac37043f8e420b78 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 14 Apr 2021 18:05:28 -0700 Subject: [PATCH 09/18] plz --- test/integration/azuresql.dbtspec | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/test/integration/azuresql.dbtspec b/test/integration/azuresql.dbtspec index 494e6255..249b098a 100644 --- a/test/integration/azuresql.dbtspec +++ b/test/integration/azuresql.dbtspec @@ -13,17 +13,7 @@ target: threads: 1 projects: - overrides: base - dbt_project_yml: | &override-project - { - "name":"dbt_test_project", - "config-version": 2, - "version": "1.0.0", - "models": { - "dbt_test_project": { - "as_columnstore": "false" - } - } - } + dbt_project_yml: '{"name":"dbt_test_project","config-version": 2, "version": "1.0.0", "models": {"dbt_test_project": {"as_columnstore": "false"}}}' &override-project - overrides: ephemeral dbt_project_yml: *override-project - overrides: incremental From 8ccc83feafa5d28ec08caab94f6a437318894a0f Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 15 Apr 2021 07:14:55 -0700 Subject: [PATCH 10/18] correct name --- test/integration/azuresql.dbtspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/azuresql.dbtspec b/test/integration/azuresql.dbtspec index 249b098a..caab2359 100644 --- a/test/integration/azuresql.dbtspec +++ b/test/integration/azuresql.dbtspec @@ -13,7 +13,7 @@ target: threads: 1 projects: - overrides: base - dbt_project_yml: '{"name":"dbt_test_project","config-version": 2, "version": "1.0.0", "models": {"dbt_test_project": {"as_columnstore": "false"}}}' &override-project + dbt_project_yml: '{"name":"schema_tests","config-version": 2, "version": "1.0.0", "models": {"dbt_test_project": {"as_columnstore": "false"}}}' &override-project - overrides: ephemeral dbt_project_yml: *override-project - overrides: incremental From 6c6245566fd46c0acb8e3a13dd7f5b30f4145c56 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 15 Apr 2021 07:34:05 -0700 Subject: [PATCH 11/18] try YAML again --- test/integration/azuresql.dbtspec | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/integration/azuresql.dbtspec b/test/integration/azuresql.dbtspec index caab2359..515786e8 100644 --- a/test/integration/azuresql.dbtspec +++ b/test/integration/azuresql.dbtspec @@ -13,8 +13,13 @@ target: threads: 1 projects: - overrides: base - dbt_project_yml: '{"name":"schema_tests","config-version": 2, "version": "1.0.0", "models": {"dbt_test_project": {"as_columnstore": "false"}}}' &override-project - - overrides: ephemeral + dbt_project_yml: &override-project + name: schema_tests + config-version: 2 + version: '1.0.0' + models: + dbt_test_project: + +as_columnstore: false - overrides: ephemeral dbt_project_yml: *override-project - overrides: incremental dbt_project_yml: *override-project From ace60ca6c24f37c4fd4d0eee293082cef4b2932f Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 15 Apr 2021 08:26:33 -0700 Subject: [PATCH 12/18] just one please --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a371e74..7425e4d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,13 +112,13 @@ jobs: workflows: main: jobs: - - unit - - connection-azuresql: &profile - context: - - DBT_SYNAPSE_PROFILE - - connection-sqlserver: *profile - - integration-sqlserver: *profile + # - unit + # - connection-azuresql: &profile + # context: + # - DBT_SYNAPSE_PROFILE + # - connection-sqlserver: *profile + # - integration-sqlserver: *profile - integration-azuresql: <<: *profile - requires: - - connection-azuresql + # requires: + # - connection-azuresql From b94067a1a5c457146b87090372272bdd069f2c1b Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 15 Apr 2021 08:35:57 -0700 Subject: [PATCH 13/18] typo --- test/integration/azuresql.dbtspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/azuresql.dbtspec b/test/integration/azuresql.dbtspec index 515786e8..7e4fa68b 100644 --- a/test/integration/azuresql.dbtspec +++ b/test/integration/azuresql.dbtspec @@ -25,7 +25,7 @@ projects: dbt_project_yml: *override-project - overrides: snapshot_strategy_timestamp dbt_project_yml: *override-project - - overrides: schema_test + - overrides: schema_tests dbt_project_yml: *override-project sequences: test_dbt_empty: empty From 53e208f0d8ff4c2273fc2d589bc601c7494f2b73 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 15 Apr 2021 08:36:29 -0700 Subject: [PATCH 14/18] typo --- test/integration/azuresql.dbtspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/azuresql.dbtspec b/test/integration/azuresql.dbtspec index 7e4fa68b..ad0cdab2 100644 --- a/test/integration/azuresql.dbtspec +++ b/test/integration/azuresql.dbtspec @@ -19,7 +19,8 @@ projects: version: '1.0.0' models: dbt_test_project: - +as_columnstore: false - overrides: ephemeral + +as_columnstore: false + - overrides: ephemeral dbt_project_yml: *override-project - overrides: incremental dbt_project_yml: *override-project From e301dff4e488e269fa91eeef51e97d58fcb88f1f Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 15 Apr 2021 08:37:33 -0700 Subject: [PATCH 15/18] typo --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7425e4d4..74a3c180 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -113,9 +113,9 @@ workflows: main: jobs: # - unit - # - connection-azuresql: &profile - # context: - # - DBT_SYNAPSE_PROFILE + - connection-azuresql: &profile + context: + - DBT_SYNAPSE_PROFILE # - connection-sqlserver: *profile # - integration-sqlserver: *profile - integration-azuresql: From 91938a3edaf5dd0601bec013925b71be551f9618 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 15 Apr 2021 08:41:34 -0700 Subject: [PATCH 16/18] add new tests --- test/integration/azuresql.dbtspec | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/azuresql.dbtspec b/test/integration/azuresql.dbtspec index ad0cdab2..daef69a8 100644 --- a/test/integration/azuresql.dbtspec +++ b/test/integration/azuresql.dbtspec @@ -26,6 +26,8 @@ projects: dbt_project_yml: *override-project - overrides: snapshot_strategy_timestamp dbt_project_yml: *override-project + - overrides: snapshot_strategy_check_cols + dbt_project_yml: *override-project - overrides: schema_tests dbt_project_yml: *override-project sequences: @@ -34,7 +36,7 @@ sequences: test_dbt_ephemeral: ephemeral test_dbt_incremental: incremental test_dbt_snapshot_strategy_timestamp: snapshot_strategy_timestamp - # test_dbt_snapshot_strategy_check_cols: snapshot_strategy_check_cols + test_dbt_snapshot_strategy_check_cols: snapshot_strategy_check_cols test_dbt_data_test: data_test test_dbt_schema_test: schema_test # test_dbt_ephemeral_data_tests: data_test_ephemeral_models From 69834255532244203a791e978409711e1078a715 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 15 Apr 2021 08:42:20 -0700 Subject: [PATCH 17/18] re-enable --- .circleci/config.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 74a3c180..362d736b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,13 +112,11 @@ jobs: workflows: main: jobs: - # - unit + - unit - connection-azuresql: &profile context: - DBT_SYNAPSE_PROFILE - # - connection-sqlserver: *profile - # - integration-sqlserver: *profile + - connection-sqlserver: *profile + - integration-sqlserver: *profile - integration-azuresql: <<: *profile - # requires: - # - connection-azuresql From 4dc51c19baed28347c755c3426f96610fba11dd2 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Thu, 15 Apr 2021 08:55:26 -0700 Subject: [PATCH 18/18] last step --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 362d736b..7a371e74 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,3 +120,5 @@ workflows: - integration-sqlserver: *profile - integration-azuresql: <<: *profile + requires: + - connection-azuresql