Remove unique key from vals

.github/workflows/dbt_docs_update.yml

@@ -22,48 +22,6 @@ concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  scheduled_run:
-    name: docs_update
-    runs-on: ubuntu-latest
-    environment: 
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v1
-        with:
-          python-version: "3.7.x"
-
-      - name: install dependencies
-        run: |
-          pip install dbt-snowflake==${{ vars.DBT_VERSION }}
-          dbt deps
-      - name: checkout docs branch
-        run: |
-          git checkout -b docs origin/main
-
-      - name: generate dbt docs
-        run: dbt docs generate -t prod
-
-      - name: move files to docs directory
-        run: |
-          mkdir -p ./docs
-          cp target/{catalog.json,manifest.json,index.html} docs/
-      - name: clean up target directory
-        run: dbt clean
-
-      - name: check for changes
-        run: git status
-
-      - name: stage changed files
-        run: git add .
-
-      - name: commit changed files
-        run: |
-          git config user.email "abc@xyz"
-          git config user.name "github-actions"
-          git commit -am "Auto-update docs"
-      - name: push changes to docs
-        run: |
-          git push -f --set-upstream origin docs
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@main
+    secrets: inherit 

.github/workflows/dbt_run_deployment.yml

@@ -0,0 +1,47 @@
+name: dbt_run_deployment
+run-name: dbt_run_deployment
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    inputs:
+      warehouse:
+        type: choice 
+        description: Snowflake warehouse
+        required: true 
+        options:
+          - DBT
+          - DBT_CLOUD
+          - DBT_EMERGENCY
+        default: DBT
+      dbt_command:
+        type: string
+        description: 'DBT Run Command'
+        required: true
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ inputs.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@main
+    with:
+      dbt_command: |
+         ${{ inputs.dbt_command }}
+      environment: workflow_prod
+      warehouse: ${{ inputs.WAREHOUSE }}
+    secrets: inherit 

.github/workflows/dbt_run_incremental.yml

@@ -25,7 +25,7 @@ jobs:
     uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
     with:
       dbt_command: >
-        dbt run -s ./models
+        dbt run -s ./models --exclude models/streamline*
       environment: workflow_prod
       warehouse: ${{ vars.WAREHOUSE }}
     secrets: inherit 

models/descriptions/msgs.md

@@ -0,0 +1,5 @@
+{% docs msgs %}
+
+The underlying json from the messages or events within the transactions.
+
+{% enddocs %}

models/gold/core__fact_transactions.sql

@@ -0,0 +1,18 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+SELECT
+    t.block_id,
+    t.block_timestamp,
+    t.tx_id,
+    t.tx_from,
+    t.tx_succeeded,
+    t.codespace,
+    t.fee,
+    t.gas_used,
+    t.gas_wanted,
+    t.msgs
+FROM
+    {{ ref('silver__transactions_final') }}
+    t

models/core/core__fact_transactions.yml → models/gold/core__fact_transactions.yml

@@ -44,15 +44,7 @@ models:
         description: "{{ doc('gas_wanted') }}"
         tests: 
           - dbt_expectations.expect_column_to_exist  
-      - name: TX_CODE
-        description: "{{ doc('tx_code') }}"
-        tests: 
-          - dbt_expectations.expect_column_to_exist 
-      - name: TX_TYPE 
-        description: "The type of transaction"
-        tests: 
-          - dbt_expectations.expect_column_to_exist 
       - name: MSGS
-        description: "The underlying json from the messages or events within the transactions"
+        description: "{{ doc('msgs') }}" 
         tests: 
           - dbt_expectations.expect_column_to_exist

models/core/core__fact_transfers.yml → models/gold/core__fact_transfers.yml

@@ -37,9 +37,5 @@ models:
           - dbt_expectations.expect_column_to_exist
       - name: RECEIVER
         description: "{{ doc('receiver') }}"
-        tests: 
-          - dbt_expectations.expect_column_to_exist
-      - name: UNIQUE_KEY
-        description: "{{ doc('_unique_key') }}"
         tests: 
           - dbt_expectations.expect_column_to_exist

models/core/core__fact_validators.yml → models/gold/core__fact_validators.yml

@@ -61,9 +61,5 @@ models:
           - dbt_expectations.expect_column_to_exist
       - name: RAW_METADATA
         description: "{{ doc('raw_metadata') }}"
-        tests: 
-          - dbt_expectations.expect_column_to_exist
-      - name: UNIQUE_KEY
-        description: The unique key of the table
         tests: 
           - dbt_expectations.expect_column_to_exist

models/silver/silver__blocks.sql

@@ -1,12 +1,13 @@
 {{ config(
   materialized = 'incremental',
-  unique_key = "CONCAT_WS('-', chain_id, block_id)",
-  incremental_strategy = 'delete+insert',
+  unique_key = ["chain_id", "block_id"],
+  incremental_strategy = 'merge',
   cluster_by = ['block_timestamp::DATE'],
 ) }}
+-- depends_on: {{ ref('bronze__streamline_tendermint_blocks') }}
 
 SELECT
-   block_id,
+  block_number AS block_id,
   COALESCE(
     DATA [0] :result :block :header :time :: TIMESTAMP,
     DATA :result :block :header :time :: TIMESTAMP
@@ -17,30 +18,43 @@ SELECT
   ) AS chain_id,
   ARRAY_SIZE(
     COALESCE(
-      data [0] :result :block :data :txs,
-      data :result :block :data :txs
+      DATA [0] :result :block :data :txs,
+      DATA :result :block :data :txs
     )
   ) AS tx_count,
   COALESCE(
-    data [0] :result :block :header :proposer_address :: STRING,
-    data :result :block :header :proposer_address :: STRING
+    DATA [0] :result :block :header :proposer_address :: STRING,
+    DATA :result :block :header :proposer_address :: STRING
   ) AS proposer_address,
   COALESCE(
     DATA [0] :result :block :header :validators_hash :: STRING,
     DATA :result :block :header :validators_hash :: STRING
   ) AS validator_hash,
   _inserted_timestamp :: TIMESTAMP AS _inserted_timestamp
 FROM
-  {{ ref('bronze__tendermint_blocks') }}
+
+{% if is_incremental() %}
+{{ ref('bronze__streamline_tendermint_blocks') }}
+{% else %}
+  {{ ref('bronze__streamline_FR_tendermint_blocks') }}
+{% endif %}
 WHERE
-  data [0] :error IS NULL
+  DATA [0] :error IS NULL
   AND DATA :error IS NULL
+  AND (
+    DATA :result :block :header :chain_id :: STRING IS NOT NULL
+    OR DATA [0] :result :block :header :chain_id :: STRING IS NOT NULL
+  )
 
 {% if is_incremental() %}
-AND _inserted_timestamp :: DATE >= (
+AND _inserted_timestamp >= (
   SELECT
-    MAX(_inserted_timestamp) :: DATE - 2
+    MAX(_inserted_timestamp) _inserted_timestamp
   FROM
     {{ this }}
 )
 {% endif %}
+
+qualify(ROW_NUMBER() over (PARTITION BY block_number
+ORDER BY
+  _inserted_timestamp DESC)) = 1

models/silver/silver__blocks.yml

@@ -8,7 +8,7 @@ models:
             - CHAIN_ID
             - BLOCK_ID
       - sequence_gaps:
-          column_name: block_ID
+          column_name: BLOCK_ID
           where: BLOCK_TIMESTAMP <= CURRENT_DATE - 1 
     columns:
       - name: BLOCK_ID