diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml index da996d80..d3094a62 100644 --- a/.github/workflows/dbt_alter_gha_task.yml +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -11,14 +11,14 @@ on: description: Name of the workflow to perform the action on, no .yml extension required: true task_action: - type: choice + type: choice description: Action to perform required: true options: - SUSPEND - RESUME default: SUSPEND - + env: DBT_PROFILES_DIR: ./ @@ -36,11 +36,11 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 with: workflow_name: | - ${{ inputs.workflow_name }} + ${{ inputs.workflow_name }} task_action: | - ${{ inputs.task_action }} + ${{ inputs.task_action }} environment: workflow_prod - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index 51f2d88e..b6b463d0 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -23,5 +23,5 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@main - secrets: inherit + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@AN-4374/upgrade-dbt-1.7 + secrets: inherit diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 8178217d..96b5f322 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -7,7 +7,7 @@ on: - "main" inputs: environment: - type: choice + type: choice description: DBT Run Environment required: true options: @@ -15,9 +15,9 @@ on: - prod default: dev warehouse: - type: choice + type: choice description: Snowflake warehouse - required: true + required: true options: - DBT - DBT_CLOUD @@ -25,9 +25,9 @@ on: default: DBT dbt_command: type: string - description: 'DBT Run Command' + description: "DBT Run Command" required: true - + env: USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" @@ -47,15 +47,15 @@ concurrency: jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_${{ inputs.environment }} steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/.github/workflows/dbt_run_daily_clusters.yml b/.github/workflows/dbt_run_daily_clusters.yml index e59e8cd1..06578a1a 100644 --- a/.github/workflows/dbt_run_daily_clusters.yml +++ b/.github/workflows/dbt_run_daily_clusters.yml @@ -5,8 +5,8 @@ on: workflow_dispatch: schedule: # Runs "at 00:00 UTC" (see https://crontab.guru) - - cron: '0 0 * * *' - + - cron: "0 0 * * *" + env: USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" @@ -26,15 +26,15 @@ concurrency: jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: - name: workflow_prod + environment: + name: workflow_prod steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml index e0cc5a5f..dc5a8d52 100644 --- a/.github/workflows/dbt_run_deployment.yml +++ b/.github/workflows/dbt_run_deployment.yml @@ -7,9 +7,9 @@ on: - "main" inputs: warehouse: - type: choice + type: choice description: Snowflake warehouse - required: true + required: true options: - DBT - DBT_CLOUD @@ -17,7 +17,7 @@ on: default: DBT dbt_command: type: string - description: 'DBT Run Command' + description: "DBT Run Command" required: true env: @@ -33,15 +33,14 @@ env: WAREHOUSE: "${{ inputs.WAREHOUSE }}" SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: | - ${{ inputs.dbt_command }} + ${{ inputs.dbt_command }} warehouse: ${{ inputs.WAREHOUSE }} - secrets: inherit + secrets: inherit diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 9362e7a4..886f3b54 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -5,8 +5,8 @@ on: workflow_dispatch: schedule: # Runs "at 9:00 UTC" (see https://crontab.guru) - - cron: '0 9 * * *' - + - cron: "0 9 * * *" + env: USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" @@ -25,10 +25,10 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run-operation run_sp_create_prod_clone environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + secrets: inherit diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml index dca99b24..87b4763d 100644 --- a/.github/workflows/dbt_run_observability.yml +++ b/.github/workflows/dbt_run_observability.yml @@ -25,10 +25,10 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run -s tag:observability environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + secrets: inherit diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index 500263c6..0af8a03b 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -25,10 +25,10 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run -s "bitcoin_models,tag:scheduled_core" environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + secrets: inherit diff --git a/.github/workflows/dbt_run_scheduled_non_core.yml b/.github/workflows/dbt_run_scheduled_non_core.yml index bdb0cc8a..fde8c572 100644 --- a/.github/workflows/dbt_run_scheduled_non_core.yml +++ b/.github/workflows/dbt_run_scheduled_non_core.yml @@ -25,10 +25,10 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run -s "bitcoin_models,tag:scheduled_non_core" environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + secrets: inherit diff --git a/.github/workflows/dbt_run_streamline_blocks_hash_realtime.yml b/.github/workflows/dbt_run_streamline_blocks_hash_realtime.yml index 8b3a0017..915b98f8 100644 --- a/.github/workflows/dbt_run_streamline_blocks_hash_realtime.yml +++ b/.github/workflows/dbt_run_streamline_blocks_hash_realtime.yml @@ -25,10 +25,10 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__blocks_hash_realtime.sql + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__blocks_hash_realtime.sql environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + secrets: inherit diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml index 463101b7..240ed0f6 100644 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -25,10 +25,10 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/ + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/ environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + secrets: inherit diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index 60963988..55fd35e8 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: schedule: # Run at 1400 UTC daily (see https://crontab.guru) - - cron: '0 14 * * *' + - cron: "0 14 * * *" env: USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" @@ -26,15 +26,15 @@ concurrency: jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml index 976440bd..594148bb 100644 --- a/.github/workflows/dbt_test_tasks.yml +++ b/.github/workflows/dbt_test_tasks.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: branches: - "main" - + env: DBT_PROFILES_DIR: ./ @@ -23,5 +23,5 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main - secrets: inherit \ No newline at end of file + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7 + secrets: inherit diff --git a/dbt_project.yml b/dbt_project.yml index 9684558d..18c7ba94 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -5,7 +5,7 @@ name: "bitcoin_models" version: "1.0.0" config-version: 2 -require-dbt-version: ">=1.4.0" +require-dbt-version: ">=1.7.0" # This setting configures which "profile" dbt uses for this project. profile: "bitcoin" @@ -54,11 +54,11 @@ tests: +store_failures: true # all tests on-run-start: - - '{{create_sps()}}' - - '{{create_udfs()}}' + - "{{create_sps()}}" + - "{{create_udfs()}}" on-run-end: - - '{{ apply_meta_as_tags(results) }}' + - "{{ apply_meta_as_tags(results) }}" dispatch: - macro_namespace: dbt @@ -68,5 +68,5 @@ dispatch: - dbt query-comment: - comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}' + comment: "{{ dbt_snowflake_query_tags.get_query_comment(node) }}" append: true # Snowflake removes prefixed comments. diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 00000000..f64551db --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,16 @@ +packages: +- package: calogica/dbt_expectations + version: 0.8.2 +- package: dbt-labs/dbt_external_tables + version: 0.8.2 +- package: dbt-labs/dbt_utils + version: 1.0.0 +- git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: c84d623aa09ce2acb9451e6aedf5fa70c19b0b95 +- package: get-select/dbt_snowflake_query_tags + version: 2.3.2 +- package: calogica/dbt_date + version: 0.7.2 +- git: https://github.com/FlipsideCrypto/livequery-models.git + revision: bca494102fbd2d621d32746e9a7fe780678044f8 +sha1_hash: ce25373a314499826f1bef1d1fca885141e945e9 diff --git a/requirements.txt b/requirements.txt index 54648210..c61d3e1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -dbt-snowflake~=1.4 +dbt-snowflake>=1.7,<1.8