From a0fc417f399c07fd248b78d60e3ba46de5620ad8 Mon Sep 17 00:00:00 2001 From: Jack Forgash Date: Wed, 7 Jun 2023 12:17:41 -0600 Subject: [PATCH 1/3] init setup --- .github/workflows/dbt_docs_update.yml | 48 ++--------------------- .github/workflows/dbt_run_adhoc.yml | 2 +- .github/workflows/dbt_run_dev_refresh.yml | 6 +-- .github/workflows/dbt_run_scheduled.yml | 35 +++++++++++++++++ .gitignore | 4 +- README.md | 9 ++--- dbt_project.yml | 7 ++-- macros/create_sps.sql | 4 +- macros/create_udfs.sql | 4 ++ packages.yml | 2 +- profiles.yml | 4 +- 11 files changed, 62 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/dbt_run_scheduled.yml diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index ffeb9ee7..51f2d88e 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -22,48 +22,6 @@ concurrency: group: ${{ github.workflow }} jobs: - scheduled_run: - name: docs_update - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip install dbt-snowflake==${{ vars.DBT_VERSION }} - dbt deps - - name: checkout docs branch - run: | - git checkout -b docs origin/main - - - name: generate dbt docs - run: dbt docs generate -t prod - - - name: move files to docs directory - run: | - mkdir -p ./docs - cp target/{catalog.json,manifest.json,index.html} docs/ - - name: clean up target directory - run: dbt clean - - - name: check for changes - run: git status - - - name: stage changed files - run: git add . - - - name: commit changed files - run: | - git config user.email "abc@xyz" - git config user.name "github-actions" - git commit -am "Auto-update docs" - - name: push changes to docs - run: | - git push -f --set-upstream origin docs \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@main + secrets: inherit diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 02ec9e39..58a4df77 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -63,4 +63,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - ${{ inputs.dbt_command }} \ No newline at end of file + ${{ inputs.dbt_command }} diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 2302c1ae..9362e7a4 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -3,9 +3,9 @@ run-name: dbt_run_dev_refresh on: workflow_dispatch: - # schedule: - # # Runs "at 9:00 UTC" (see https://crontab.guru) - # - cron: '0 9 * * *' + schedule: + # Runs "at 9:00 UTC" (see https://crontab.guru) + - cron: '0 9 * * *' env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml new file mode 100644 index 00000000..5541aa3f --- /dev/null +++ b/.github/workflows/dbt_run_scheduled.yml @@ -0,0 +1,35 @@ +name: dbt_run_scheduled +run-name: dbt_run_scheduled + +on: + workflow_dispatch: + # schedule: + # TODO - enable once pipeline created + # Runs "every hour" (see https://crontab.guru) + # - cron: '0 */1 * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: > + dbt run + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit diff --git a/.gitignore b/.gitignore index e7f63fa6..8bbd04eb 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,6 @@ logs/ .vscode/ .env .DS_Store -.user.yml \ No newline at end of file +.user.yml + +local/ diff --git a/README.md b/README.md index 1528017f..f685b64a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ - -# Please find and replace all instances of `xyz` with your project name. +# Aurora Models ## Profile Set Up @@ -7,7 +6,7 @@ ---- ```yml -xyz: +aurora: target: dev outputs: dev: @@ -17,7 +16,7 @@ xyz: user: password: region: - database: xyz_DEV + database: AURORA_DEV warehouse: schema: silver threads: 4 @@ -73,5 +72,5 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.s ``` select * -from table(xyz.information_schema.tag_references('xyz.core.fact_blocks', 'table')); +from table(auruora.information_schema.tag_references('aurora.core.fact_blocks', 'table')); ``` \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 3a5b5f49..630cd9b9 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,14 +1,14 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: "xyz_models" +name: "aurora_models" version: "1.0.0" config-version: 2 require-dbt-version: ">=1.4.0" # This setting configures which "profile" dbt uses for this project. -profile: "xyz" +profile: "aurora" # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be @@ -42,6 +42,7 @@ models: vars: "dbt_date:time_zone": GMT "UPDATE_SNOWFLAKE_TAGS": TRUE + "UPDATE_UDFS_AND_SPS": FALSE tests: +store_failures: true # all tests @@ -51,4 +52,4 @@ on-run-start: - '{{create_udfs()}}' on-run-end: - - '{{ apply_meta_as_tags(results) }}' \ No newline at end of file + - '{{ apply_meta_as_tags(results) }}' diff --git a/macros/create_sps.sql b/macros/create_sps.sql index 8577716b..9ed1647a 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -1,6 +1,6 @@ {% macro create_sps() %} - {% if target.database == 'xyz' %} + {% if target.database == 'AURORA' %} CREATE SCHEMA IF NOT EXISTS _internal; {{ sp_create_prod_clone('_internal') }}; {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 56fe0ffe..f3359702 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -1,2 +1,6 @@ {% macro create_udfs() %} + {% if var("UPDATE_UDFS_AND_SPS") %} + {{- fsc_utils.create_udfs() -}} + {% endif %} + {% endif %} {% endmacro %} diff --git a/packages.yml b/packages.yml index e9570789..f44f45e8 100644 --- a/packages.yml +++ b/packages.yml @@ -2,4 +2,4 @@ packages: - package: calogica/dbt_expectations version: [">=0.4.0", "<0.9.0"] - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.3.0 \ No newline at end of file + revision: v1.3.0 diff --git a/profiles.yml b/profiles.yml index 0661f982..ba8ba73c 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,4 +1,4 @@ -xyz: +aurora: target: dev outputs: dev: @@ -26,4 +26,4 @@ xyz: threads: 8 client_session_keep_alive: False config: - send_anonymous_usage_stats: False \ No newline at end of file + send_anonymous_usage_stats: False From d0dba4929438db7f6bb7fc8153fe99ef01dab0b5 Mon Sep 17 00:00:00 2001 From: Jack Forgash Date: Wed, 7 Jun 2023 12:28:31 -0600 Subject: [PATCH 2/3] disable dev refresh schedula --- .github/workflows/dbt_run_dev_refresh.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 9362e7a4..7049fb84 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -3,9 +3,10 @@ run-name: dbt_run_dev_refresh on: workflow_dispatch: - schedule: + # schedule: + # TODO enable schedule when ready # Runs "at 9:00 UTC" (see https://crontab.guru) - - cron: '0 9 * * *' + # - cron: '0 9 * * *' env: USE_VARS: "${{ vars.USE_VARS }}" From e56ed8baaa59e061ee5d9050107186418b3872ee Mon Sep 17 00:00:00 2001 From: Jack Forgash Date: Wed, 7 Jun 2023 12:38:00 -0600 Subject: [PATCH 3/3] other xyz references --- macros/run_sp_create_prod_clone.sql | 2 +- macros/tags/add_database_or_schema_tags.sql | 4 ++-- models/descriptions/__overview__.md | 13 ++++++------- models/sources.yml | 2 +- profiles.yml | 4 ++-- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql index 18686830..89e94d32 100644 --- a/macros/run_sp_create_prod_clone.sql +++ b/macros/run_sp_create_prod_clone.sql @@ -1,6 +1,6 @@ {% macro run_sp_create_prod_clone() %} {% set clone_query %} -call xyz._internal.create_prod_clone('xyz', 'xyz_dev', 'internal_dev'); +call aurora._internal.create_prod_clone('aurora', 'aurora_dev', 'internal_dev'); {% endset %} {% do run_query(clone_query) %} diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql index 184e0bdb..2c066cd0 100644 --- a/macros/tags/add_database_or_schema_tags.sql +++ b/macros/tags/add_database_or_schema_tags.sql @@ -1,10 +1,10 @@ {% macro add_database_or_schema_tags() %} {{ set_database_tag_value( 'BLOCKCHAIN_NAME', - 'xyz' + 'AURORA' ) }} {{ set_database_tag_value( 'BLOCKCHAIN_TYPE', - 'IBC' + 'EVM' ) }} {% endmacro %} diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 588ffbd6..71d6c92f 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -1,10 +1,10 @@ {% docs __overview__ %} -# Welcome to the Flipside Crypto xyz Models Documentation +# Welcome to the Flipside Crypto Aurora Models Documentation ## **What does this documentation cover?** -The documentation included here details the design of the xyz - tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/xyz-models/) +The documentation included here details the design of the Aurora + tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/aurora-models/) ## **How do I use these docs?** The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. @@ -17,7 +17,7 @@ There is more information on how to use dbt docs in the last section of this doc **Click on the links below to jump to the documentation for each schema.** -### Core Tables (`xyz`.`CORE`.``) +### Core Tables (`AURORA`.`CORE`.``) **Dimension Tables:** @@ -33,8 +33,7 @@ There is more information on how to use dbt docs in the last section of this doc ## **Data Model Overview** -The xyz - models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** +The Aurora models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** - Bronze: Data is loaded in from the source as a view - Silver: All necessary parsing, filtering, de-duping, and other transformations are done here @@ -68,7 +67,7 @@ Note that you can also right-click on models to interactively filter and explore - [Flipside](https://flipsidecrypto.xyz/) - [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) -- [Github](https://github.com/FlipsideCrypto/xyz-models) +- [Github](https://github.com/FlipsideCrypto/aurora-models) - [What is dbt?](https://docs.getdbt.com/docs/introduction) {% enddocs %} \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index e31d5cc1..9c8bd602 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: crosschain - database: "{{ 'crosschain' if target.database == 'xyz' else 'crosschain_dev' }}" + database: "{{ 'crosschain' if target.database == 'AURORA' else 'crosschain_dev' }}" schema: core tables: - name: dim_date_hours diff --git a/profiles.yml b/profiles.yml index ba8ba73c..5925a8d3 100644 --- a/profiles.yml +++ b/profiles.yml @@ -5,7 +5,7 @@ aurora: type: snowflake account: "{{ env_var('ACCOUNT') }}" user: "{{ env_var('USER') }}" - password: "{{ env_var('PASSWORD') }}" + authenticator: externalbrowser role: "{{ env_var('ROLE') }}" schema: "{{ env_var('SCHEMA') }}" region: "{{ env_var('REGION') }}" @@ -17,7 +17,7 @@ aurora: type: snowflake account: "{{ env_var('ACCOUNT') }}" user: "{{ env_var('USER') }}" - password: "{{ env_var('PASSWORD') }}" + authenticator: externalbrowser role: "{{ env_var('ROLE') }}" schema: "{{ env_var('SCHEMA') }}" region: "{{ env_var('REGION') }}"