upd tags core vs non core and workflow schedule csv

.github/workflows/dbt_run_scheduled.yml

@@ -28,7 +28,7 @@ jobs:
     uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
     with:
       dbt_command: >
-        dbt run -s "bitcoin_models,tag:load" "bitcoin_models,tag:core"
+        dbt run -s "bitcoin_models,tag:scheduled_core"
       environment: workflow_prod
       warehouse: ${{ vars.WAREHOUSE }}
     secrets: inherit 

.github/workflows/dbt_run_scheduled_non_core.yml

@@ -0,0 +1,34 @@
+name: dbt_run_scheduled
+run-name: dbt_run_scheduled
+
+on:
+  workflow_dispatch:
+  # schedule:
+  #   # Runs 8 minutes after each dbt job kicks off (which is set to 15 min interval) (see https://crontab.guru)
+  #   - cron: '8,23,38,53 * * * *'
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
+    with:
+      dbt_command: >
+        dbt run -s "bitcoin_models,tag:scheduled_non_core"
+      environment: workflow_prod
+      warehouse: ${{ vars.WAREHOUSE }}
+    secrets: inherit 

data/github_actions__workflows.csv

@@ -1,5 +1,6 @@
 workflow_name,workflow_schedule
-dbt_run_scheduled,"8,23,38,53 * * * *"
-dbt_run_streamline_blocks_hash_realtime,"*/10 * * * *"
-dbt_run_streamline_realtime,"3,13,23,33,43,53 * * * *"
-dbt_test_tasks,"0,30 * * * *"
+dbt_run_scheduled,"*/30 * * * *"
+dbt_run_scheduled_non_core,"0 * * * *"
+dbt_run_streamline_blocks_hash_realtime,"0,15,30,45 * * * *"
+dbt_run_streamline_realtime,"5,20,35,50 * * * *"
+dbt_test_tasks,"0,30 * * * *"

models/bronze/core/bronze__blocks.sql

@@ -2,7 +2,7 @@
     materialized = 'incremental',
     cluster_by = ["_inserted_timestamp::DATE"],
     unique_key = 'block_number',
-    tags = ["load"],
+    tags = ["load", "scheduled_core"],
     incremental_strategy = 'delete+insert'
 ) }}
 -- depends on {{ref('bronze__streamline_blocks')}}

models/bronze/core/bronze__transactions.sql

@@ -2,7 +2,7 @@
     materialized = 'incremental',
     cluster_by = ["_inserted_timestamp::DATE"],
     unique_key = "tx_id",
-    tags = ["load"],
+    tags = ["load", "scheduled_core"],
     incremental_strategy = 'delete+insert',
     incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
 ) }}

models/bronze/labels/bronze__labels.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'view',
-    tags = ['core']
+    tags = ['core', 'scheduled_non_core']
 
 ) }}
 

models/silver/core/silver__blocks.sql

@@ -3,7 +3,7 @@
     incremental_strategy = 'delete+insert',
     unique_key = 'block_number',
     cluster_by = ["_inserted_timestamp::DATE", "block_number"],
-    tags = ["core"]
+    tags = ["core", "scheduled_core"]
 ) }}
 
 WITH bronze_blocks AS (

models/silver/core/silver__inputs.sql

@@ -3,7 +3,7 @@
     incremental_strategy = 'delete+insert',
     incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
     unique_key = 'input_id',
-    tags = ["core"],
+    tags = ["core", "scheduled_core"],
     cluster_by = ["_inserted_timestamp"],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}

models/silver/core/silver__inputs_final.sql

@@ -4,7 +4,7 @@
     incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
     unique_key = 'input_id',
     cluster_by = ["block_number", "tx_id"],
-    tags = ["core"],
+    tags = ["core", "scheduled_core"],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}
 

models/silver/core/silver__outputs.sql

@@ -3,7 +3,7 @@
     incremental_strategy = 'delete+insert',
     incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
     unique_key = 'output_id',
-    tags = ["core"],
+    tags = ["core", "scheduled_core"],
     cluster_by = ["_partition_by_block_id", "tx_id"],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}

models/silver/core/silver__transaction_index.sql

@@ -3,7 +3,7 @@
     unique_key = 'tx_id',
     incremental_strategy = 'delete+insert',
     cluster_by = ["_partition_by_block_id", "tx_id"],
-    tags = ["core"]
+    tags = ["core", "scheduled_core"]
 ) }}
 
 WITH blocks AS (

models/silver/core/silver__transactions.sql

@@ -4,7 +4,7 @@
     incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
     unique_key = 'tx_id',
     cluster_by = ["_inserted_timestamp::DATE", "block_number"],
-    tags = ["core"]
+    tags = ["core", "scheduled_core"]
 ) }}
 -- depends_on: {{ ref('silver__blocks') }}
 WITH bronze_transactions AS (

models/silver/core/silver__transactions_final.sql

@@ -4,7 +4,7 @@
     incremental_predicates = ['block_number >= (select min(block_number) from ' ~ generate_tmp_view_name(this) ~ ')'],
     unique_key = 'tx_id',
     cluster_by = ["block_number", "tx_id"],
-    tags = ["core"],
+    tags = ["core", "scheduled_core"],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}
 

models/silver/ez/silver__block_miner_rewards.sql

@@ -3,7 +3,7 @@
     incremental_strategy = 'delete+insert',
     unique_key = 'block_number',
     cluster_by = ["block_number", "block_timestamp::DATE"],
-    tags = ["core", "ez"],
+    tags = ["core", "ez", "scheduled_non_core" ],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}
 

models/silver/labels/silver__labels.sql

@@ -1,7 +1,7 @@
 {{ config(
     materialized = 'incremental',
     unique_key = 'address',
-    tags = ['core']
+    tags = ["core", "scheduled_non_core"]
 ) }}
 
 SELECT

models/silver/price/silver__price_all_providers_hourly.sql

@@ -2,7 +2,7 @@
     materialized = 'incremental',
     unique_key = 'id',
     cluster_by = ['hour::DATE'],
-    tags = ['prices', 'core']
+    tags = ["prices", "core", "scheduled_non_core"]
 ) }}
 
 WITH coinmarketcap AS (

models/silver/price/silver__price_coingecko_hourly.sql

@@ -2,7 +2,7 @@
     materialized = 'incremental',
     unique_key = 'recorded_hour',
     cluster_by = ['recorded_hour ::DATE'],
-    tags = ['prices', 'core']
+    tags = ["prices", "core", "scheduled_non_core"]
 ) }}
 
 WITH prices AS (

models/silver/price/silver__price_coinmarketcap_hourly.sql

@@ -2,7 +2,7 @@
     materialized = 'incremental',
     unique_key = 'recorded_hour',
     cluster_by = ['recorded_hour ::DATE'],
-    tags = ['prices', 'core']
+    tags = ["prices", "core", "scheduled_non_core"]
 ) }}
 
 WITH prices AS (

models/silver/price/silver__price_coinpaprika_hourly.sql

@@ -3,7 +3,7 @@
     unique_key = 'recorded_hour',
     incremental_strategy='delete+insert',
     cluster_by = ['recorded_hour ::DATE'],
-    tags = ['prices', 'core']
+    tags = ["prices", "core", "scheduled_non_core"]
 ) }}
 
 WITH prices AS (