Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

1.7 update #66

Merged
merged 1 commit into from
Jan 30, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .github/workflows/dbt_alter_gha_task.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@ on:
description: Name of the workflow to perform the action on, no .yml extension
required: true
task_action:
type: choice
type: choice
description: Action to perform
required: true
options:
- SUSPEND
- RESUME
default: SUSPEND

env:
DBT_PROFILES_DIR: ./

Expand All @@ -36,11 +36,11 @@ concurrency:

jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7
with:
workflow_name: |
${{ inputs.workflow_name }}
${{ inputs.workflow_name }}
task_action: |
${{ inputs.task_action }}
${{ inputs.task_action }}
environment: workflow_prod
secrets: inherit
secrets: inherit
4 changes: 2 additions & 2 deletions .github/workflows/dbt_docs_update.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,5 @@ concurrency:

jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@main
secrets: inherit
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@AN-4374/upgrade-dbt-1.7
secrets: inherit
16 changes: 8 additions & 8 deletions .github/workflows/dbt_run_adhoc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,27 @@ on:
- "main"
inputs:
environment:
type: choice
type: choice
description: DBT Run Environment
required: true
options:
- dev
- prod
default: dev
warehouse:
type: choice
type: choice
description: Snowflake warehouse
required: true
required: true
options:
- DBT
- DBT_CLOUD
- DBT_EMERGENCY
default: DBT
dbt_command:
type: string
description: 'DBT Run Command'
description: "DBT Run Command"
required: true

env:
USE_VARS: "${{ vars.USE_VARS }}"
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
Expand All @@ -47,15 +47,15 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_${{ inputs.environment }}

steps:
- uses: actions/checkout@v3

- uses: actions/setup-python@v1
- uses: actions/setup-python@v4
with:
python-version: "3.7.x"
python-version: "3.10"

- name: install dependencies
run: |
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/dbt_run_daily_clusters.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ on:
workflow_dispatch:
schedule:
# Runs "at 00:00 UTC" (see https://crontab.guru)
- cron: '0 0 * * *'
- cron: "0 0 * * *"

env:
USE_VARS: "${{ vars.USE_VARS }}"
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
Expand All @@ -26,15 +26,15 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
name: workflow_prod
environment:
name: workflow_prod

steps:
- uses: actions/checkout@v3

- uses: actions/setup-python@v1
- uses: actions/setup-python@v4
with:
python-version: "3.7.x"
python-version: "3.10"

- name: install dependencies
run: |
Expand Down
13 changes: 6 additions & 7 deletions .github/workflows/dbt_run_deployment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,17 @@ on:
- "main"
inputs:
warehouse:
type: choice
type: choice
description: Snowflake warehouse
required: true
required: true
options:
- DBT
- DBT_CLOUD
- DBT_EMERGENCY
default: DBT
dbt_command:
type: string
description: 'DBT Run Command'
description: "DBT Run Command"
required: true

env:
Expand All @@ -33,15 +33,14 @@ env:
WAREHOUSE: "${{ inputs.WAREHOUSE }}"
SCHEMA: "${{ vars.SCHEMA }}"


concurrency:
group: ${{ github.workflow }}

jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@main
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: |
${{ inputs.dbt_command }}
${{ inputs.dbt_command }}
warehouse: ${{ inputs.WAREHOUSE }}
secrets: inherit
secrets: inherit
8 changes: 4 additions & 4 deletions .github/workflows/dbt_run_dev_refresh.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ on:
workflow_dispatch:
schedule:
# Runs "at 9:00 UTC" (see https://crontab.guru)
- cron: '0 9 * * *'
- cron: "0 9 * * *"

env:
USE_VARS: "${{ vars.USE_VARS }}"
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
Expand All @@ -25,10 +25,10 @@ concurrency:

jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt run-operation run_sp_create_prod_clone
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
secrets: inherit
4 changes: 2 additions & 2 deletions .github/workflows/dbt_run_observability.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ concurrency:

jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt run -s tag:observability
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
secrets: inherit
4 changes: 2 additions & 2 deletions .github/workflows/dbt_run_scheduled.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ concurrency:

jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt run -s "bitcoin_models,tag:scheduled_core"
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
secrets: inherit
4 changes: 2 additions & 2 deletions .github/workflows/dbt_run_scheduled_non_core.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ concurrency:

jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt run -s "bitcoin_models,tag:scheduled_non_core"
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
secrets: inherit
6 changes: 3 additions & 3 deletions .github/workflows/dbt_run_streamline_blocks_hash_realtime.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ concurrency:

jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__blocks_hash_realtime.sql
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__blocks_hash_realtime.sql
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
secrets: inherit
6 changes: 3 additions & 3 deletions .github/workflows/dbt_run_streamline_realtime.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ concurrency:

jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
secrets: inherit
8 changes: 4 additions & 4 deletions .github/workflows/dbt_test_daily.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:
workflow_dispatch:
schedule:
# Run at 1400 UTC daily (see https://crontab.guru)
- cron: '0 14 * * *'
- cron: "0 14 * * *"
env:
USE_VARS: "${{ vars.USE_VARS }}"
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
Expand All @@ -26,15 +26,15 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_prod

steps:
- uses: actions/checkout@v3

- uses: actions/setup-python@v1
- uses: actions/setup-python@v4
with:
python-version: "3.7.x"
python-version: "3.10"

- name: install dependencies
run: |
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/dbt_test_tasks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ on:
workflow_dispatch:
branches:
- "main"

env:
DBT_PROFILES_DIR: ./

Expand All @@ -23,5 +23,5 @@ concurrency:

jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main
secrets: inherit
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7
secrets: inherit
10 changes: 5 additions & 5 deletions dbt_project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ name: "bitcoin_models"
version: "1.0.0"
config-version: 2

require-dbt-version: ">=1.4.0"
require-dbt-version: ">=1.7.0"

# This setting configures which "profile" dbt uses for this project.
profile: "bitcoin"
Expand Down Expand Up @@ -54,11 +54,11 @@ tests:
+store_failures: true # all tests

on-run-start:
- '{{create_sps()}}'
- '{{create_udfs()}}'
- "{{create_sps()}}"
- "{{create_udfs()}}"

on-run-end:
- '{{ apply_meta_as_tags(results) }}'
- "{{ apply_meta_as_tags(results) }}"

dispatch:
- macro_namespace: dbt
Expand All @@ -68,5 +68,5 @@ dispatch:
- dbt

query-comment:
comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}'
comment: "{{ dbt_snowflake_query_tags.get_query_comment(node) }}"
append: true # Snowflake removes prefixed comments.
16 changes: 16 additions & 0 deletions package-lock.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
packages:
- package: calogica/dbt_expectations
version: 0.8.2
- package: dbt-labs/dbt_external_tables
version: 0.8.2
- package: dbt-labs/dbt_utils
version: 1.0.0
- git: https://github.com/FlipsideCrypto/fsc-utils.git
revision: c84d623aa09ce2acb9451e6aedf5fa70c19b0b95
- package: get-select/dbt_snowflake_query_tags
version: 2.3.2
- package: calogica/dbt_date
version: 0.7.2
- git: https://github.com/FlipsideCrypto/livequery-models.git
revision: bca494102fbd2d621d32746e9a7fe780678044f8
sha1_hash: ce25373a314499826f1bef1d1fca885141e945e9
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
dbt-snowflake~=1.4
dbt-snowflake>=1.7,<1.8