diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index ef27d59f..500263c6 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -28,7 +28,7 @@ jobs: uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main with: dbt_command: > - dbt run -s "bitcoin_models,tag:load" "bitcoin_models,tag:core" + dbt run -s "bitcoin_models,tag:scheduled_core" environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} secrets: inherit diff --git a/.github/workflows/dbt_run_scheduled_non_core.yml b/.github/workflows/dbt_run_scheduled_non_core.yml new file mode 100644 index 00000000..bdb0cc8a --- /dev/null +++ b/.github/workflows/dbt_run_scheduled_non_core.yml @@ -0,0 +1,34 @@ +name: dbt_run_scheduled +run-name: dbt_run_scheduled + +on: + workflow_dispatch: + # schedule: + # # Runs 8 minutes after each dbt job kicks off (which is set to 15 min interval) (see https://crontab.guru) + # - cron: '8,23,38,53 * * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: > + dbt run -s "bitcoin_models,tag:scheduled_non_core" + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 06ef6ba6..2c9a084d 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,5 +1,6 @@ workflow_name,workflow_schedule -dbt_run_scheduled,"8,23,38,53 * * * *" -dbt_run_streamline_blocks_hash_realtime,"*/10 * * * *" -dbt_run_streamline_realtime,"3,13,23,33,43,53 * * * *" -dbt_test_tasks,"0,30 * * * *" \ No newline at end of file +dbt_run_scheduled,"*/30 * * * *" +dbt_run_scheduled_non_core,"7 * * * *" +dbt_run_streamline_blocks_hash_realtime,"0,15,30,45 * * * *" +dbt_run_streamline_realtime,"5,20,35,50 * * * *" +dbt_test_tasks,"0,30 * * * *" diff --git a/models/bronze/core/bronze__blocks.sql b/models/bronze/core/bronze__blocks.sql index fbf7531a..580818ae 100644 --- a/models/bronze/core/bronze__blocks.sql +++ b/models/bronze/core/bronze__blocks.sql @@ -2,7 +2,7 @@ materialized = 'incremental', cluster_by = ["_inserted_timestamp::DATE"], unique_key = 'block_number', - tags = ["load"], + tags = ["load", "scheduled_core"], incremental_strategy = 'delete+insert' ) }} -- depends on {{ref('bronze__streamline_blocks')}} diff --git a/models/bronze/core/bronze__transactions.sql b/models/bronze/core/bronze__transactions.sql index 5c08201e..45e19a60 100644 --- a/models/bronze/core/bronze__transactions.sql +++ b/models/bronze/core/bronze__transactions.sql @@ -2,7 +2,7 @@ materialized = 'incremental', cluster_by = ["_inserted_timestamp::DATE"], unique_key = "tx_id", - tags = ["load"], + tags = ["load", "scheduled_core"], incremental_strategy = 'delete+insert', incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], ) }} diff --git a/models/bronze/labels/bronze__labels.sql b/models/bronze/labels/bronze__labels.sql index d8525ce0..9c3c1a1d 100644 --- a/models/bronze/labels/bronze__labels.sql +++ b/models/bronze/labels/bronze__labels.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - tags = ['core'] + tags = ['core', 'scheduled_non_core'] ) }} diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index 24e2f295..2154eeaf 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ["_inserted_timestamp::DATE", "block_number"], - tags = ["core"] + tags = ["core", "scheduled_core"] ) }} WITH bronze_blocks AS ( diff --git a/models/silver/core/silver__inputs.sql b/models/silver/core/silver__inputs.sql index 39a75285..7c6f893a 100644 --- a/models/silver/core/silver__inputs.sql +++ b/models/silver/core/silver__inputs.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], unique_key = 'input_id', - tags = ["core"], + tags = ["core", "scheduled_core"], cluster_by = ["_inserted_timestamp"], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION" ) }} diff --git a/models/silver/core/silver__inputs_final.sql b/models/silver/core/silver__inputs_final.sql index fe89b63e..14d1bbee 100644 --- a/models/silver/core/silver__inputs_final.sql +++ b/models/silver/core/silver__inputs_final.sql @@ -4,7 +4,7 @@ incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], unique_key = 'input_id', cluster_by = ["block_number", "tx_id"], - tags = ["core"], + tags = ["core", "scheduled_core"], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION" ) }} diff --git a/models/silver/core/silver__outputs.sql b/models/silver/core/silver__outputs.sql index ef10de84..a4452fc0 100644 --- a/models/silver/core/silver__outputs.sql +++ b/models/silver/core/silver__outputs.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], unique_key = 'output_id', - tags = ["core"], + tags = ["core", "scheduled_core"], cluster_by = ["_partition_by_block_id", "tx_id"], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION" ) }} diff --git a/models/silver/core/silver__transaction_index.sql b/models/silver/core/silver__transaction_index.sql index 5b750af1..55da639a 100644 --- a/models/silver/core/silver__transaction_index.sql +++ b/models/silver/core/silver__transaction_index.sql @@ -3,7 +3,7 @@ unique_key = 'tx_id', incremental_strategy = 'delete+insert', cluster_by = ["_partition_by_block_id", "tx_id"], - tags = ["core"] + tags = ["core", "scheduled_core"] ) }} WITH blocks AS ( diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index a81e8950..ee0415d0 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -4,7 +4,7 @@ incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], unique_key = 'tx_id', cluster_by = ["_inserted_timestamp::DATE", "block_number"], - tags = ["core"] + tags = ["core", "scheduled_core"] ) }} -- depends_on: {{ ref('silver__blocks') }} WITH bronze_transactions AS ( diff --git a/models/silver/core/silver__transactions_final.sql b/models/silver/core/silver__transactions_final.sql index 2c52adfd..51c739e0 100644 --- a/models/silver/core/silver__transactions_final.sql +++ b/models/silver/core/silver__transactions_final.sql @@ -4,7 +4,7 @@ incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], unique_key = 'tx_id', cluster_by = ["block_number", "tx_id"], - tags = ["core"], + tags = ["core", "scheduled_core"], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION" ) }} diff --git a/models/silver/ez/silver__block_miner_rewards.sql b/models/silver/ez/silver__block_miner_rewards.sql index 6d474ba7..098f04a6 100644 --- a/models/silver/ez/silver__block_miner_rewards.sql +++ b/models/silver/ez/silver__block_miner_rewards.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ["block_number", "block_timestamp::DATE"], - tags = ["core", "ez"], + tags = ["core", "ez", "scheduled_non_core" ], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION" ) }} diff --git a/models/silver/labels/silver__labels.sql b/models/silver/labels/silver__labels.sql index db5725a8..b276c882 100644 --- a/models/silver/labels/silver__labels.sql +++ b/models/silver/labels/silver__labels.sql @@ -1,7 +1,7 @@ {{ config( materialized = 'incremental', unique_key = 'address', - tags = ['core'] + tags = ["core", "scheduled_non_core"] ) }} SELECT diff --git a/models/silver/price/silver__price_all_providers_hourly.sql b/models/silver/price/silver__price_all_providers_hourly.sql index 19719513..51741e69 100644 --- a/models/silver/price/silver__price_all_providers_hourly.sql +++ b/models/silver/price/silver__price_all_providers_hourly.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = 'id', cluster_by = ['hour::DATE'], - tags = ['prices', 'core'] + tags = ["prices", "core", "scheduled_non_core"] ) }} WITH coinmarketcap AS ( diff --git a/models/silver/price/silver__price_coingecko_hourly.sql b/models/silver/price/silver__price_coingecko_hourly.sql index 291515f7..f8ce2632 100644 --- a/models/silver/price/silver__price_coingecko_hourly.sql +++ b/models/silver/price/silver__price_coingecko_hourly.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = 'recorded_hour', cluster_by = ['recorded_hour ::DATE'], - tags = ['prices', 'core'] + tags = ["prices", "core", "scheduled_non_core"] ) }} WITH prices AS ( diff --git a/models/silver/price/silver__price_coinmarketcap_hourly.sql b/models/silver/price/silver__price_coinmarketcap_hourly.sql index 9864ce81..4dbb8a83 100644 --- a/models/silver/price/silver__price_coinmarketcap_hourly.sql +++ b/models/silver/price/silver__price_coinmarketcap_hourly.sql @@ -2,7 +2,7 @@ materialized = 'incremental', unique_key = 'recorded_hour', cluster_by = ['recorded_hour ::DATE'], - tags = ['prices', 'core'] + tags = ["prices", "core", "scheduled_non_core"] ) }} WITH prices AS ( diff --git a/models/silver/price/silver__price_coinpaprika_hourly.sql b/models/silver/price/silver__price_coinpaprika_hourly.sql index 024f6d72..5cb52a67 100644 --- a/models/silver/price/silver__price_coinpaprika_hourly.sql +++ b/models/silver/price/silver__price_coinpaprika_hourly.sql @@ -3,7 +3,7 @@ unique_key = 'recorded_hour', incremental_strategy='delete+insert', cluster_by = ['recorded_hour ::DATE'], - tags = ['prices', 'core'] + tags = ["prices", "core", "scheduled_non_core"] ) }} WITH prices AS (