Update node add methods

.github/workflows/.dbt/profiles.yml

@@ -0,0 +1,29 @@
+aurora:
+  target: dev
+  outputs:
+    dev:
+      type: snowflake
+      account: "{{ env_var('ACCOUNT') }}"
+      user: "{{ env_var('USER') }}"
+      password: "{{ env_var('PASSWORD') }}"
+      role: "{{ env_var('ROLE') }}"
+      schema: "{{ env_var('SCHEMA') }}"
+      region: "{{ env_var('REGION') }}"
+      database: "{{ env_var('DATABASE') }}"
+      warehouse: "{{ env_var('WAREHOUSE') }}"
+      threads: 8
+      client_session_keep_alive: False
+    prod:
+      type: snowflake
+      account: "{{ env_var('ACCOUNT') }}"
+      user: "{{ env_var('USER') }}"
+      password: "{{ env_var('PASSWORD') }}"
+      role: "{{ env_var('ROLE') }}"
+      schema: "{{ env_var('SCHEMA') }}"
+      region: "{{ env_var('REGION') }}"
+      database: "{{ env_var('DATABASE') }}"
+      warehouse: "{{ env_var('WAREHOUSE') }}"
+      threads: 8
+      client_session_keep_alive: False
+  config:
+    send_anonymous_usage_stats: False

.github/workflows/dbt_docs_update.yml

@@ -22,48 +22,6 @@ concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  scheduled_run:
-    name: docs_update
-    runs-on: ubuntu-latest
-    environment: 
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v1
-        with:
-          python-version: "3.7.x"
-
-      - name: install dependencies
-        run: |
-          pip install dbt-snowflake==${{ vars.DBT_VERSION }}
-          dbt deps
-      - name: checkout docs branch
-        run: |
-          git checkout -b docs origin/main
-
-      - name: generate dbt docs
-        run: dbt docs generate -t prod
-
-      - name: move files to docs directory
-        run: |
-          mkdir -p ./docs
-          cp target/{catalog.json,manifest.json,index.html} docs/
-      - name: clean up target directory
-        run: dbt clean
-
-      - name: check for changes
-        run: git status
-
-      - name: stage changed files
-        run: git add .
-
-      - name: commit changed files
-        run: |
-          git config user.email "abc@xyz"
-          git config user.name "github-actions"
-          git commit -am "Auto-update docs"
-      - name: push changes to docs
-        run: |
-          git push -f --set-upstream origin docs
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@main
+    secrets: inherit 

.github/workflows/dbt_run_adhoc.yml

@@ -63,4 +63,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          ${{ inputs.dbt_command }}
+          ${{ inputs.dbt_command }}

.github/workflows/dbt_run_dev_refresh.yml

@@ -4,8 +4,9 @@ run-name: dbt_run_dev_refresh
 on:
   workflow_dispatch:
   # schedule:
-  #   # Runs "at 9:00 UTC" (see https://crontab.guru)
-  #   - cron: '0 9 * * *'
+    # TODO enable schedule when ready
+    # Runs "at 9:00 UTC" (see https://crontab.guru)
+    # - cron: '0 9 * * *'
 
 env:
   USE_VARS: "${{ vars.USE_VARS }}"

.github/workflows/dbt_run_scheduled.yml

@@ -0,0 +1,34 @@
+name: dbt_run_scheduled
+run-name: dbt_run_scheduled
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs "every hour" (see https://crontab.guru)
+    - cron: '0 */1 * * *'
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
+    with:
+      dbt_command: >
+        dbt run
+      environment: workflow_prod
+      warehouse: ${{ vars.WAREHOUSE }}
+    secrets: inherit 

.github/workflows/dbt_run_streamline_realtime.yml

@@ -0,0 +1,34 @@
+name: dbt_run_streamline_realtime
+run-name: dbt_run_streamline_realtime
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs "every 2 hours" (see https://crontab.guru)
+    - cron: '0 */2 * * *'
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
+    with:
+      dbt_command: >
+        dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/ 
+      environment: workflow_prod
+      warehouse: ${{ vars.WAREHOUSE }}
+    secrets: inherit 

.github/workflows/dbt_test_daily.yml

@@ -0,0 +1,50 @@
+name: dbt_test_scheduled
+run-name: dbt_test_scheduled
+
+on:
+  workflow_dispatch:
+  #schedule:
+  # Run at 1600 UTC daily (see https://crontab.guru)
+  #- cron: "0 9 * * *"
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+  SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt test --exclude tag:full_test --models tag:recent_test
+        continue-on-error: true
+
+      - name: Log test results
+        run: |
+          python python_scripts/test_alert/dbt_test_alert.py

.github/workflows/dbt_test_weekly.yml

@@ -0,0 +1,50 @@
+name: dbt_test_scheduled
+run-name: dbt_test_scheduled
+
+on:
+  workflow_dispatch:
+  #schedule:
+  # Run at 1600 UTC Monday (see https://crontab.guru)
+  #- cron: "0 9 * * 1"
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+  SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt test --exclude tag:recent_test --models tag:full_test
+        continue-on-error: true
+
+      - name: Log test results
+        run: |
+          python python_scripts/test_alert/dbt_test_alert.py

.gitignore

@@ -15,4 +15,9 @@ logs/
 .vscode/
 .env
 .DS_Store
-.user.yml
+.user.yml
+
+.dbt/.user.yml
+.github/workflows/.dbt/.user.yml
+
+local*

README.md

@@ -1,13 +1,12 @@
-
-# Please find and replace all instances of `xyz` with your project name.
+# Aurora Models
 
 ## Profile Set Up
 
 #### Use the following within profiles.yml 
 ----
 
 ```yml
-xyz:
+aurora:
   target: dev
   outputs:
     dev:
@@ -17,7 +16,7 @@ xyz:
       user: <USERNAME>
       password: <PASSWORD>
       region: <REGION>
-      database: xyz_DEV
+      database: AURORA_DEV
       warehouse: <WAREHOUSE>
       schema: silver
       threads: 4
@@ -73,5 +72,5 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.s
 
 ```
 select *
-from table(xyz.information_schema.tag_references('xyz.core.fact_blocks', 'table'));
+from table(auruora.information_schema.tag_references('aurora.core.fact_blocks', 'table'));
 ```

dbt_project.yml

@@ -1,14 +1,14 @@
 # Name your project! Project names should contain only lowercase characters
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
-name: "xyz_models"
+name: "aurora_models"
 version: "1.0.0"
 config-version: 2
 
 require-dbt-version: ">=1.4.0"
 
 # This setting configures which "profile" dbt uses for this project.
-profile: "xyz"
+profile: "aurora"
 
 # These configurations specify where dbt should look for different types of files.
 # The `model-paths` config, for example, states that models in this project can be
@@ -41,14 +41,19 @@ models:
 
 vars:
   "dbt_date:time_zone": GMT
-  "UPDATE_SNOWFLAKE_TAGS": TRUE
+  STREAMLINE_INVOKE_STREAMS: False
+  STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False
+  STREAMLINE_RUN_HISTORY: False
+  UPDATE_UDFS_AND_SPS: False
+  UPDATE_SNOWFLAKE_TAGS: True
+  WAIT: 0
 
 tests:
   +store_failures: true # all tests
 
 on-run-start:
-  - '{{create_sps()}}'
-  - '{{create_udfs()}}'
+  - "{{create_sps()}}"
+  - "{{create_udfs()}}"
 
 on-run-end:
-  - '{{ apply_meta_as_tags(results) }}'
+  - "{{ apply_meta_as_tags(results) }}"

macros/create_sps.sql

@@ -1,6 +1,6 @@
 {% macro create_sps() %}
-    {% if target.database == 'xyz' %}
+    {% if target.database == 'AURORA' %}
         CREATE SCHEMA IF NOT EXISTS _internal;
         {{ sp_create_prod_clone('_internal') }};
     {% endif %}
-{% endmacro %}
+{% endmacro %}

macros/create_udfs.sql

@@ -1,2 +1,5 @@
 {% macro create_udfs() %}
+    {% if var("UPDATE_UDFS_AND_SPS") %}
+            {{- fsc_utils.create_udfs() -}}
+    {% endif %}
 {% endmacro %}

macros/js_hextoint.sql

@@ -0,0 +1,6 @@
+{% macro create_js_hex_to_int() %}
+    CREATE
+    OR REPLACE FUNCTION {{ target.schema }}.js_hex_to_int (
+        s STRING
+    ) returns DOUBLE LANGUAGE javascript AS 'if (S !== null) { yourNumber = parseInt(S, 16); } return yourNumber'
+{% endmacro %}