diff --git a/.github/workflows/.dbt/profiles.yml b/.github/workflows/.dbt/profiles.yml new file mode 100644 index 00000000..ba8ba73c --- /dev/null +++ b/.github/workflows/.dbt/profiles.yml @@ -0,0 +1,29 @@ +aurora: + target: dev + outputs: + dev: + type: snowflake + account: "{{ env_var('ACCOUNT') }}" + user: "{{ env_var('USER') }}" + password: "{{ env_var('PASSWORD') }}" + role: "{{ env_var('ROLE') }}" + schema: "{{ env_var('SCHEMA') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + threads: 8 + client_session_keep_alive: False + prod: + type: snowflake + account: "{{ env_var('ACCOUNT') }}" + user: "{{ env_var('USER') }}" + password: "{{ env_var('PASSWORD') }}" + role: "{{ env_var('ROLE') }}" + schema: "{{ env_var('SCHEMA') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + threads: 8 + client_session_keep_alive: False + config: + send_anonymous_usage_stats: False diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index ffeb9ee7..51f2d88e 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -22,48 +22,6 @@ concurrency: group: ${{ github.workflow }} jobs: - scheduled_run: - name: docs_update - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip install dbt-snowflake==${{ vars.DBT_VERSION }} - dbt deps - - name: checkout docs branch - run: | - git checkout -b docs origin/main - - - name: generate dbt docs - run: dbt docs generate -t prod - - - name: move files to docs directory - run: | - mkdir -p ./docs - cp target/{catalog.json,manifest.json,index.html} docs/ - - name: clean up target directory - run: dbt clean - - - name: check for changes - run: git status - - - name: stage changed files - run: git add . - - - name: commit changed files - run: | - git config user.email "abc@xyz" - git config user.name "github-actions" - git commit -am "Auto-update docs" - - name: push changes to docs - run: | - git push -f --set-upstream origin docs \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@main + secrets: inherit diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 02ec9e39..58a4df77 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -63,4 +63,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - ${{ inputs.dbt_command }} \ No newline at end of file + ${{ inputs.dbt_command }} diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 2302c1ae..7049fb84 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -4,8 +4,9 @@ run-name: dbt_run_dev_refresh on: workflow_dispatch: # schedule: - # # Runs "at 9:00 UTC" (see https://crontab.guru) - # - cron: '0 9 * * *' + # TODO enable schedule when ready + # Runs "at 9:00 UTC" (see https://crontab.guru) + # - cron: '0 9 * * *' env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml new file mode 100644 index 00000000..c114d27d --- /dev/null +++ b/.github/workflows/dbt_run_scheduled.yml @@ -0,0 +1,34 @@ +name: dbt_run_scheduled +run-name: dbt_run_scheduled + +on: + workflow_dispatch: + schedule: + # Runs "every hour" (see https://crontab.guru) + - cron: '0 */1 * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: > + dbt run + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml new file mode 100644 index 00000000..3227803d --- /dev/null +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -0,0 +1,34 @@ +name: dbt_run_streamline_realtime +run-name: dbt_run_streamline_realtime + +on: + workflow_dispatch: + schedule: + # Runs "every 2 hours" (see https://crontab.guru) + - cron: '0 */2 * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: > + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/ + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml new file mode 100644 index 00000000..8e0be6d6 --- /dev/null +++ b/.github/workflows/dbt_test_daily.yml @@ -0,0 +1,50 @@ +name: dbt_test_scheduled +run-name: dbt_test_scheduled + +on: + workflow_dispatch: + #schedule: + # Run at 1600 UTC daily (see https://crontab.guru) + #- cron: "0 9 * * *" +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt test --exclude tag:full_test --models tag:recent_test + continue-on-error: true + + - name: Log test results + run: | + python python_scripts/test_alert/dbt_test_alert.py diff --git a/.github/workflows/dbt_test_weekly.yml b/.github/workflows/dbt_test_weekly.yml new file mode 100644 index 00000000..18ab14c9 --- /dev/null +++ b/.github/workflows/dbt_test_weekly.yml @@ -0,0 +1,50 @@ +name: dbt_test_scheduled +run-name: dbt_test_scheduled + +on: + workflow_dispatch: + #schedule: + # Run at 1600 UTC Monday (see https://crontab.guru) + #- cron: "0 9 * * 1" +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt test --exclude tag:recent_test --models tag:full_test + continue-on-error: true + + - name: Log test results + run: | + python python_scripts/test_alert/dbt_test_alert.py diff --git a/.gitignore b/.gitignore index e7f63fa6..0e9ff346 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,9 @@ logs/ .vscode/ .env .DS_Store -.user.yml \ No newline at end of file +.user.yml + +.dbt/.user.yml +.github/workflows/.dbt/.user.yml + +local* diff --git a/README.md b/README.md index 1528017f..f685b64a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ - -# Please find and replace all instances of `xyz` with your project name. +# Aurora Models ## Profile Set Up @@ -7,7 +6,7 @@ ---- ```yml -xyz: +aurora: target: dev outputs: dev: @@ -17,7 +16,7 @@ xyz: user: password: region: - database: xyz_DEV + database: AURORA_DEV warehouse: schema: silver threads: 4 @@ -73,5 +72,5 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.s ``` select * -from table(xyz.information_schema.tag_references('xyz.core.fact_blocks', 'table')); +from table(auruora.information_schema.tag_references('aurora.core.fact_blocks', 'table')); ``` \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 3a5b5f49..58ae6269 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,14 +1,14 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: "xyz_models" +name: "aurora_models" version: "1.0.0" config-version: 2 require-dbt-version: ">=1.4.0" # This setting configures which "profile" dbt uses for this project. -profile: "xyz" +profile: "aurora" # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be @@ -41,14 +41,19 @@ models: vars: "dbt_date:time_zone": GMT - "UPDATE_SNOWFLAKE_TAGS": TRUE + STREAMLINE_INVOKE_STREAMS: False + STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False + STREAMLINE_RUN_HISTORY: False + UPDATE_UDFS_AND_SPS: False + UPDATE_SNOWFLAKE_TAGS: True + WAIT: 0 tests: +store_failures: true # all tests on-run-start: - - '{{create_sps()}}' - - '{{create_udfs()}}' + - "{{create_sps()}}" + - "{{create_udfs()}}" on-run-end: - - '{{ apply_meta_as_tags(results) }}' \ No newline at end of file + - "{{ apply_meta_as_tags(results) }}" diff --git a/macros/create_sps.sql b/macros/create_sps.sql index 8577716b..9ed1647a 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -1,6 +1,6 @@ {% macro create_sps() %} - {% if target.database == 'xyz' %} + {% if target.database == 'AURORA' %} CREATE SCHEMA IF NOT EXISTS _internal; {{ sp_create_prod_clone('_internal') }}; {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 56fe0ffe..a1d20d3a 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -1,2 +1,5 @@ {% macro create_udfs() %} + {% if var("UPDATE_UDFS_AND_SPS") %} + {{- fsc_utils.create_udfs() -}} + {% endif %} {% endmacro %} diff --git a/macros/js_hextoint.sql b/macros/js_hextoint.sql new file mode 100644 index 00000000..8742ac3c --- /dev/null +++ b/macros/js_hextoint.sql @@ -0,0 +1,6 @@ +{% macro create_js_hex_to_int() %} + CREATE + OR REPLACE FUNCTION {{ target.schema }}.js_hex_to_int ( + s STRING + ) returns DOUBLE LANGUAGE javascript AS 'if (S !== null) { yourNumber = parseInt(S, 16); } return yourNumber' +{% endmacro %} diff --git a/macros/python/udfs.sql b/macros/python/udfs.sql new file mode 100644 index 00000000..df0489e8 --- /dev/null +++ b/macros/python/udfs.sql @@ -0,0 +1,21 @@ +{% macro create_udf_hex_to_int(schema) %} +create or replace function {{ schema }}.udf_hex_to_int(hex string) +returns string +language python +runtime_version = '3.8' +handler = 'hex_to_int' +as +$$ +def hex_to_int(hex) -> str: + """ + Converts hex (of any size) to int (as a string). Snowflake and java script can only handle up to 64-bit (38 digits of precision) + select hex_to_int('200000000000000000000000000000211'); + >> 680564733841876926926749214863536423441 + select hex_to_int('0x200000000000000000000000000000211'); + >> 680564733841876926926749214863536423441 + select hex_to_int(NULL); + >> NULL + """ + return str(int(hex, 16)) if hex else None +$$; +{% endmacro %} \ No newline at end of file diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql index 18686830..89e94d32 100644 --- a/macros/run_sp_create_prod_clone.sql +++ b/macros/run_sp_create_prod_clone.sql @@ -1,6 +1,6 @@ {% macro run_sp_create_prod_clone() %} {% set clone_query %} -call xyz._internal.create_prod_clone('xyz', 'xyz_dev', 'internal_dev'); +call aurora._internal.create_prod_clone('aurora', 'aurora_dev', 'internal_dev'); {% endset %} {% do run_query(clone_query) %} diff --git a/macros/streamline/api_integrations.sql b/macros/streamline/api_integrations.sql new file mode 100644 index 00000000..92f44a23 --- /dev/null +++ b/macros/streamline/api_integrations.sql @@ -0,0 +1,17 @@ +{% macro create_aws_aurora_api() %} + {% if target.name == "prod" %} + {% set sql %} + CREATE api integration IF NOT EXISTS aws_aurora_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/aurora-api-prod-rolesnowflakeudfsAF733095-3WVDCVO54NPX' api_allowed_prefixes = ( + 'https://sl2f5beopl.execute-api.us-east-1.amazonaws.com/prod/' + ) enabled = TRUE; + {% endset %} + {% do run_query(sql) %} + {% else %} + {% set sql %} + CREATE api integration IF NOT EXISTS aws_aurora_dev_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/aurora-api-dev-rolesnowflakeudfsAF733095-1MX4LOX4UFE9M' api_allowed_prefixes = ( + 'https://66lx4fxkui.execute-api.us-east-1.amazonaws.com/dev/' + ) enabled = TRUE; + {% endset %} + {% do run_query(sql) %} + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/get_base_table_udtf.sql b/macros/streamline/get_base_table_udtf.sql new file mode 100644 index 00000000..c4ee9ab0 --- /dev/null +++ b/macros/streamline/get_base_table_udtf.sql @@ -0,0 +1,22 @@ +{% macro create_udtf_get_base_table(schema) %} + CREATE + OR REPLACE FUNCTION {{ schema }}.udtf_get_base_table( + max_height INTEGER + ) returns TABLE ( + height NUMBER + ) AS $$ WITH base AS ( + SELECT + ROW_NUMBER() over ( + ORDER BY + SEQ4() + ) AS id + FROM + TABLE(GENERATOR(rowcount => 100000000)) + ) +SELECT + id AS height +FROM + base +WHERE + id <= max_height $$; +{% endmacro %} diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql new file mode 100644 index 00000000..e17327d3 --- /dev/null +++ b/macros/streamline/models.sql @@ -0,0 +1,179 @@ +{% macro decode_logs_history( + start, + stop + ) %} + WITH look_back AS ( + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 1 + ) +SELECT + l.block_number, + l._log_id, + A.abi AS abi, + OBJECT_CONSTRUCT( + 'topics', + l.topics, + 'data', + l.data, + 'address', + l.contract_address + ) AS DATA +FROM + {{ ref("silver__logs") }} + l + INNER JOIN {{ ref("silver__complete_event_abis") }} A + ON A.parent_contract_address = l.contract_address + AND A.event_signature = l.topics[0]:: STRING + AND l.block_number BETWEEN A.start_block + AND A.end_block +WHERE + ( + l.block_number BETWEEN {{ start }} + AND {{ stop }} + ) + AND l.block_number <= ( + SELECT + block_number + FROM + look_back + ) + AND _log_id NOT IN ( + SELECT + _log_id + FROM + {{ ref("streamline__complete_decode_logs") }} + WHERE + ( + block_number BETWEEN {{ start }} + AND {{ stop }} + ) + AND block_number <= ( + SELECT + block_number + FROM + look_back + ) + ) +{% endmacro %} + +{% macro streamline_external_table_query( + model, + partition_function, + partition_name, + unique_key + ) %} + WITH meta AS ( + SELECT + job_created_time AS _inserted_timestamp, + file_name, + {{ partition_function }} AS {{ partition_name }} + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -7, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", model) }}') + ) A + ) + SELECT + {{ unique_key }}, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text + ) + ) AS id, + s.{{ partition_name }}, + s.value AS VALUE + FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.{{ partition_name }} = s.{{ partition_name }} + WHERE + b.{{ partition_name }} = s.{{ partition_name }} + AND ( + DATA :error :code IS NULL + OR DATA :error :code NOT IN ( + '-32000', + '-32001', + '-32002', + '-32003', + '-32004', + '-32005', + '-32006', + '-32007', + '-32008', + '-32009', + '-32010' + ) + ) +{% endmacro %} + +{% macro streamline_external_table_FR_query( + model, + partition_function, + partition_name, + unique_key + ) %} + WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + {{ partition_function }} AS {{ partition_name }} + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", model) }}' + ) + ) A + ) +SELECT + {{ unique_key }}, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text + ) + ) AS id, + s.{{ partition_name }}, + s.value AS VALUE +FROM + {{ source( + "bronze_streamline", + model + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b.{{ partition_name }} = s.{{ partition_name }} +WHERE + b.{{ partition_name }} = s.{{ partition_name }} + AND ( + DATA :error :code IS NULL + OR DATA :error :code NOT IN ( + '-32000', + '-32001', + '-32002', + '-32003', + '-32004', + '-32005', + '-32006', + '-32007', + '-32008', + '-32009', + '-32010' + ) + ) +{% endmacro %} diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql new file mode 100644 index 00000000..767ac101 --- /dev/null +++ b/macros/streamline/streamline_udfs.sql @@ -0,0 +1,23 @@ +{% macro create_udf_get_chainhead() %} + {% if target.name == "prod" %} + CREATE OR REPLACE EXTERNAL FUNCTION streamline.udf_get_chainhead() returns variant api_integration = aws_aurora_api AS + 'https://sl2f5beopl.execute-api.us-east-1.amazonaws.com/prod/get_chainhead' + {% else %} + CREATE OR REPLACE EXTERNAL FUNCTION streamline.udf_get_chainhead() returns variant api_integration = aws_aurora_dev_api AS + 'https://66lx4fxkui.execute-api.us-east-1.amazonaws.com/dev/get_chainhead' + {%- endif %}; +{% endmacro %} + +{% macro create_udf_json_rpc() %} + {% if target.name == "prod" %} + CREATE OR REPLACE EXTERNAL FUNCTION streamline.udf_json_rpc( + json OBJECT + ) returns ARRAY api_integration = aws_aurora_api AS + 'https://sl2f5beopl.execute-api.us-east-1.amazonaws.com/prod/bulk_get_json_rpc' + {% else %} + CREATE OR REPLACE EXTERNAL FUNCTION streamline.udf_json_rpc( + json OBJECT + ) returns ARRAY api_integration = aws_aurora_dev_api AS + 'https://66lx4fxkui.execute-api.us-east-1.amazonaws.com/dev/bulk_get_json_rpc' + {%- endif %}; +{% endmacro %} \ No newline at end of file diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql index 184e0bdb..2c066cd0 100644 --- a/macros/tags/add_database_or_schema_tags.sql +++ b/macros/tags/add_database_or_schema_tags.sql @@ -1,10 +1,10 @@ {% macro add_database_or_schema_tags() %} {{ set_database_tag_value( 'BLOCKCHAIN_NAME', - 'xyz' + 'AURORA' ) }} {{ set_database_tag_value( 'BLOCKCHAIN_TYPE', - 'IBC' + 'EVM' ) }} {% endmacro %} diff --git a/macros/tests/sequence_gaps.sql b/macros/tests/sequence_gaps.sql index 84a9aa94..b34e7390 100644 --- a/macros/tests/sequence_gaps.sql +++ b/macros/tests/sequence_gaps.sql @@ -1,37 +1,34 @@ -{% macro sequence_gaps( - table, - partition_by, - column - ) %} - {%- set partition_sql = partition_by | join(", ") -%} - {%- set previous_column = "prev_" ~ column -%} - WITH source AS ( - SELECT - {{ partition_sql + "," if partition_sql }} - {{ column }}, - LAG( - {{ column }}, - 1 - ) over ( - {{ "PARTITION BY " ~ partition_sql if partition_sql }} - ORDER BY - {{ column }} ASC - ) AS {{ previous_column }} - FROM - {{ table }} - WHERE - block_timestamp::date <= current_date - 1 - ) +{% test sequence_gaps( + model, + partition_by, + column_name +) %} +{%- set partition_sql = partition_by | join(", ") -%} +{%- set previous_column = "prev_" ~ column_name -%} +WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column_name }}, + LAG( + {{ column_name }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column_name }} ASC + ) AS {{ previous_column }} + FROM + {{ model }} +) SELECT {{ partition_sql + "," if partition_sql }} {{ previous_column }}, - {{ column }}, - {{ column }} - {{ previous_column }} + {{ column_name }}, + {{ column_name }} - {{ previous_column }} - 1 AS gap FROM source WHERE - {{ column }} - {{ previous_column }} <> 1 + {{ column_name }} - {{ previous_column }} <> 1 ORDER BY - gap DESC -{% endmacro %} + gap DESC {% endtest %} \ No newline at end of file diff --git a/macros/tests/tx_block_count.sql b/macros/tests/tx_block_count.sql new file mode 100644 index 00000000..fac6ce1b --- /dev/null +++ b/macros/tests/tx_block_count.sql @@ -0,0 +1,14 @@ +{% test tx_block_count( + model, + column_name +) %} + +SELECT + {{ column_name }}, + COUNT(DISTINCT block_number) AS num_blocks +FROM + {{ model }} +GROUP BY {{ column_name }} +HAVING num_blocks > 1 + +{% endtest %} \ No newline at end of file diff --git a/macros/tests/tx_gaps.sql b/macros/tests/tx_gaps.sql index 82b449f9..399c5781 100644 --- a/macros/tests/tx_gaps.sql +++ b/macros/tests/tx_gaps.sql @@ -1,4 +1,4 @@ -{% macro tx_gaps( +{% test tx_gaps( model ) %} WITH block_base AS ( @@ -30,4 +30,4 @@ FROM ON block_base.block_id = model_name.block_id WHERE tx_count <> model_tx_count -{% endmacro %} +{% endtest %} diff --git a/macros/utils.sql b/macros/utils.sql new file mode 100644 index 00000000..0b800b82 --- /dev/null +++ b/macros/utils.sql @@ -0,0 +1,78 @@ +{% macro if_data_call_function( + func, + target + ) %} + {% if var( + "STREAMLINE_INVOKE_STREAMS" + ) %} + {% if execute %} + {{ log( + "Running macro `if_data_call_function`: Calling udf " ~ func ~ " on " ~ target, + True + ) }} + {% endif %} + SELECT + {{ func }} + WHERE + EXISTS( + SELECT + 1 + FROM + {{ target }} + LIMIT + 1 + ) + {% else %} + {% if execute %} + {{ log( + "Running macro `if_data_call_function`: NOOP", + False + ) }} + {% endif %} + SELECT + NULL + {% endif %} +{% endmacro %} + +{% macro if_data_call_wait() %} + {% if var( + "STREAMLINE_INVOKE_STREAMS" + ) %} + {% set query %} + SELECT + 1 + WHERE + EXISTS( + SELECT + 1 + FROM + {{ model.schema ~ "." ~ model.alias }} + LIMIT + 1 + ) {% endset %} + {% if execute %} + {% set results = run_query( + query + ) %} + {% if results %} + {{ log( + "Waiting...", + info = True + ) }} + + {% set wait_query %} + SELECT + system$wait( + {{ var( + "WAIT", + 600 + ) }} + ) {% endset %} + {% do run_query(wait_query) %} + {% else %} + SELECT + NULL; + {% endif %} + {% endif %} + {% endif %} +{% endmacro %} diff --git a/models/bronze/core/bronze__streamline_FR_blocks.sql b/models/bronze/core/bronze__streamline_FR_blocks.sql new file mode 100644 index 00000000..e0a16c6a --- /dev/null +++ b/models/bronze/core/bronze__streamline_FR_blocks.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_FR_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/bronze/core/bronze__streamline_FR_transactions.sql b/models/bronze/core/bronze__streamline_FR_transactions.sql new file mode 100644 index 00000000..e0a16c6a --- /dev/null +++ b/models/bronze/core/bronze__streamline_FR_transactions.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_FR_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/bronze/core/bronze__streamline_FR_tx_receipts.sql b/models/bronze/core/bronze__streamline_FR_tx_receipts.sql new file mode 100644 index 00000000..d7c17400 --- /dev/null +++ b/models/bronze/core/bronze__streamline_FR_tx_receipts.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-2] + '_' + this.identifier.split("_") [-1] %} +{{ streamline_external_table_FR_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/bronze/core/bronze__streamline_blocks.sql b/models/bronze/core/bronze__streamline_blocks.sql new file mode 100644 index 00000000..de332253 --- /dev/null +++ b/models/bronze/core/bronze__streamline_blocks.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/bronze/core/bronze__streamline_transactions.sql b/models/bronze/core/bronze__streamline_transactions.sql new file mode 100644 index 00000000..de332253 --- /dev/null +++ b/models/bronze/core/bronze__streamline_transactions.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/bronze/core/bronze__streamline_tx_receipts.sql b/models/bronze/core/bronze__streamline_tx_receipts.sql new file mode 100644 index 00000000..b845227c --- /dev/null +++ b/models/bronze/core/bronze__streamline_tx_receipts.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-2] + '_' + this.identifier.split("_") [-1] %} +{{ streamline_external_table_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", + partition_name = "_partition_by_block_id", + unique_key = "block_number" +) }} diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 588ffbd6..71d6c92f 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -1,10 +1,10 @@ {% docs __overview__ %} -# Welcome to the Flipside Crypto xyz Models Documentation +# Welcome to the Flipside Crypto Aurora Models Documentation ## **What does this documentation cover?** -The documentation included here details the design of the xyz - tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/xyz-models/) +The documentation included here details the design of the Aurora + tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/aurora-models/) ## **How do I use these docs?** The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. @@ -17,7 +17,7 @@ There is more information on how to use dbt docs in the last section of this doc **Click on the links below to jump to the documentation for each schema.** -### Core Tables (`xyz`.`CORE`.``) +### Core Tables (`AURORA`.`CORE`.``) **Dimension Tables:** @@ -33,8 +33,7 @@ There is more information on how to use dbt docs in the last section of this doc ## **Data Model Overview** -The xyz - models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** +The Aurora models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** - Bronze: Data is loaded in from the source as a view - Silver: All necessary parsing, filtering, de-duping, and other transformations are done here @@ -68,7 +67,7 @@ Note that you can also right-click on models to interactively filter and explore - [Flipside](https://flipsidecrypto.xyz/) - [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) -- [Github](https://github.com/FlipsideCrypto/xyz-models) +- [Github](https://github.com/FlipsideCrypto/aurora-models) - [What is dbt?](https://docs.getdbt.com/docs/introduction) {% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_header_json.md b/models/descriptions/block_header_json.md new file mode 100644 index 00000000..9a10f326 --- /dev/null +++ b/models/descriptions/block_header_json.md @@ -0,0 +1,3 @@ +{% docs block_header_json %} +This JSON column contains the block header details. +{% enddocs %} diff --git a/models/descriptions/block_number.md b/models/descriptions/block_number.md new file mode 100644 index 00000000..cfaba736 --- /dev/null +++ b/models/descriptions/block_number.md @@ -0,0 +1,3 @@ +{% docs block_number %} +Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. +{% enddocs %} diff --git a/models/descriptions/block_timestamp.md b/models/descriptions/block_timestamp.md new file mode 100644 index 00000000..486beea2 --- /dev/null +++ b/models/descriptions/block_timestamp.md @@ -0,0 +1,5 @@ +{% docs block_timestamp %} + +The date and time at which the block was produced. + +{% enddocs %} diff --git a/models/descriptions/blockchain.md b/models/descriptions/blockchain.md new file mode 100644 index 00000000..1e77675c --- /dev/null +++ b/models/descriptions/blockchain.md @@ -0,0 +1,5 @@ +{% docs blockchain %} + +The blockchain on which transactions are being confirmed. + +{% enddocs %} diff --git a/models/descriptions/blocks_hash.md b/models/descriptions/blocks_hash.md new file mode 100644 index 00000000..f234a148 --- /dev/null +++ b/models/descriptions/blocks_hash.md @@ -0,0 +1,5 @@ +{% docs blocks_hash %} + +Block hash is a unique 66-character identifier that is generate when a block is produced. + +{% enddocs %} diff --git a/models/descriptions/blocks_nonce.md b/models/descriptions/blocks_nonce.md new file mode 100644 index 00000000..24b9d488 --- /dev/null +++ b/models/descriptions/blocks_nonce.md @@ -0,0 +1,5 @@ +{% docs blocks_nonce %} + +Block nonce is a value used during mining to demonstrate proof of work for a given block. + +{% enddocs %} diff --git a/models/descriptions/blocks_table_doc.md b/models/descriptions/blocks_table_doc.md new file mode 100644 index 00000000..d2388b75 --- /dev/null +++ b/models/descriptions/blocks_table_doc.md @@ -0,0 +1,5 @@ +{% docs blocks_table_doc %} + +This table contains block level data for the Aurora Blockchain. This table can be used to analyze trends at a block level, for example gas fees vs. total transactions over time. For more information, please see Aurora scan Resources. + +{% enddocs %} diff --git a/models/descriptions/contract_address.md b/models/descriptions/contract_address.md new file mode 100644 index 00000000..6531d141 --- /dev/null +++ b/models/descriptions/contract_address.md @@ -0,0 +1,5 @@ +{% docs logs_contract_address %} + +The address interacted with for a given event. + +{% enddocs %} diff --git a/models/descriptions/cumulative_gas_used.md b/models/descriptions/cumulative_gas_used.md new file mode 100644 index 00000000..60c3a0b3 --- /dev/null +++ b/models/descriptions/cumulative_gas_used.md @@ -0,0 +1,3 @@ +{% docs cumulative_gas_used %} +The total amount of gas used when this transaction was executed in the block. +{% enddocs %} diff --git a/models/descriptions/difficulty.md b/models/descriptions/difficulty.md new file mode 100644 index 00000000..49050649 --- /dev/null +++ b/models/descriptions/difficulty.md @@ -0,0 +1,5 @@ +{% docs difficulty %} + +The effort required to mine the block. + +{% enddocs %} diff --git a/models/descriptions/event_index.md b/models/descriptions/event_index.md new file mode 100644 index 00000000..6a21d59f --- /dev/null +++ b/models/descriptions/event_index.md @@ -0,0 +1,5 @@ +{% docs event_index %} + +Event number within a transaction. + +{% enddocs %} diff --git a/models/descriptions/event_removed.md b/models/descriptions/event_removed.md new file mode 100644 index 00000000..2a8ecd37 --- /dev/null +++ b/models/descriptions/event_removed.md @@ -0,0 +1,5 @@ +{% docs event_removed %} + +Whether the event has been removed from the transaction. + +{% enddocs %} diff --git a/models/descriptions/extra_data.md b/models/descriptions/extra_data.md new file mode 100644 index 00000000..2be3d447 --- /dev/null +++ b/models/descriptions/extra_data.md @@ -0,0 +1,5 @@ +{% docs extra_data %} + +Any data included by the miner for a given block. + +{% enddocs %} diff --git a/models/descriptions/gas_limit.md b/models/descriptions/gas_limit.md new file mode 100644 index 00000000..10402da4 --- /dev/null +++ b/models/descriptions/gas_limit.md @@ -0,0 +1,5 @@ +{% docs gas_limit %} + +Total gas limit provided by all transactions in the block. + +{% enddocs %} diff --git a/models/descriptions/gas_used.md b/models/descriptions/gas_used.md new file mode 100644 index 00000000..5b7f51fc --- /dev/null +++ b/models/descriptions/gas_used.md @@ -0,0 +1,5 @@ +{% docs gas_used %} + +Total gas used in the block. + +{% enddocs %} diff --git a/models/descriptions/id_events.md b/models/descriptions/id_events.md new file mode 100644 index 00000000..b7290760 --- /dev/null +++ b/models/descriptions/id_events.md @@ -0,0 +1,5 @@ +{% docs id_events %} + +This is the primary key for this table. This is a concatenation of the transaction hash and the event index at which the event occurred. + +{% enddocs %} diff --git a/models/descriptions/logs_bloom.md b/models/descriptions/logs_bloom.md new file mode 100644 index 00000000..c2395e81 --- /dev/null +++ b/models/descriptions/logs_bloom.md @@ -0,0 +1,5 @@ +{% docs logs_bloom %} + +Bloom filter for light clients to quickly retrieve related logs. + +{% enddocs %} diff --git a/models/descriptions/logs_data.md b/models/descriptions/logs_data.md new file mode 100644 index 00000000..2d11a533 --- /dev/null +++ b/models/descriptions/logs_data.md @@ -0,0 +1,5 @@ +{% docs logs_data %} + +The un-decoded event data. + +{% enddocs %} diff --git a/models/descriptions/logs_table_doc.md b/models/descriptions/logs_table_doc.md new file mode 100644 index 00000000..934df8bd --- /dev/null +++ b/models/descriptions/logs_table_doc.md @@ -0,0 +1,5 @@ +{% docs logs_table_doc %} + +This table contains log level data for the Aurora Blockchain. This table can be used to analyze trends at a log level, for example gas fees vs. total transactions over time. For more information, please see Aurora scan Resources. + +{% enddocs %} diff --git a/models/descriptions/miner.md b/models/descriptions/miner.md new file mode 100644 index 00000000..f6c5052d --- /dev/null +++ b/models/descriptions/miner.md @@ -0,0 +1,5 @@ +{% docs miner %} + +Miner who successfully added a given block to the blockchain. + +{% enddocs %} diff --git a/models/descriptions/network.md b/models/descriptions/network.md new file mode 100644 index 00000000..aa6f467b --- /dev/null +++ b/models/descriptions/network.md @@ -0,0 +1,5 @@ +{% docs network %} + +The network on the blockchain used by a transaction. + +{% enddocs %} diff --git a/models/descriptions/nft_origin_sig.md b/models/descriptions/nft_origin_sig.md new file mode 100644 index 00000000..b5623650 --- /dev/null +++ b/models/descriptions/nft_origin_sig.md @@ -0,0 +1,5 @@ +{% docs nft_origin_sig %} + +The function signature of this transaction. + +{% enddocs %} diff --git a/models/descriptions/parent_hash.md b/models/descriptions/parent_hash.md new file mode 100644 index 00000000..8f8a04ef --- /dev/null +++ b/models/descriptions/parent_hash.md @@ -0,0 +1,5 @@ +{% docs parent_hash %} + +The hash of the block from which a given block is generated. Also known as the parent block. + +{% enddocs %} diff --git a/models/descriptions/receipts_logs.md b/models/descriptions/receipts_logs.md new file mode 100644 index 00000000..6fe1c018 --- /dev/null +++ b/models/descriptions/receipts_logs.md @@ -0,0 +1,5 @@ +{% docs receipts_log %} + +Event data. + +{% enddocs %} diff --git a/models/descriptions/receipts_root.md b/models/descriptions/receipts_root.md new file mode 100644 index 00000000..72673781 --- /dev/null +++ b/models/descriptions/receipts_root.md @@ -0,0 +1,5 @@ +{% docs receipts_root %} + +The root of the state trie. + +{% enddocs %} diff --git a/models/descriptions/receipts_table_doc.md b/models/descriptions/receipts_table_doc.md new file mode 100644 index 00000000..8cd44a98 --- /dev/null +++ b/models/descriptions/receipts_table_doc.md @@ -0,0 +1,5 @@ +{% docs receipts_table_doc %} + +This table contains receipt level data for the Aurora Blockchain. + +{% enddocs %} diff --git a/models/descriptions/sha3_uncles.md b/models/descriptions/sha3_uncles.md new file mode 100644 index 00000000..055ae3cc --- /dev/null +++ b/models/descriptions/sha3_uncles.md @@ -0,0 +1,5 @@ +{% docs sha3_uncles %} + +The mechanism which Aurora encodes an empty string. + +{% enddocs %} diff --git a/models/descriptions/size.md b/models/descriptions/size.md new file mode 100644 index 00000000..8059958c --- /dev/null +++ b/models/descriptions/size.md @@ -0,0 +1,5 @@ +{% docs size %} + +Block size, which is determined by a given block's gas limit. + +{% enddocs %} diff --git a/models/descriptions/to_address.md b/models/descriptions/to_address.md new file mode 100644 index 00000000..5a9918b2 --- /dev/null +++ b/models/descriptions/to_address.md @@ -0,0 +1,5 @@ +{% docs to_address %} + +To address is a unique 42-character identifier that is generated when a transaction is produced. + +{% enddocs %} diff --git a/models/descriptions/topics.md b/models/descriptions/topics.md new file mode 100644 index 00000000..94744e5a --- /dev/null +++ b/models/descriptions/topics.md @@ -0,0 +1,5 @@ +{% docs topics %} + +The un-decoded event input topics. + +{% enddocs %} diff --git a/models/descriptions/total_difficulty.md b/models/descriptions/total_difficulty.md new file mode 100644 index 00000000..514dab30 --- /dev/null +++ b/models/descriptions/total_difficulty.md @@ -0,0 +1,5 @@ +{% docs total_difficulty %} + +Total difficulty of the chain at a given block. + +{% enddocs %} diff --git a/models/descriptions/tx_block_hash.md b/models/descriptions/tx_block_hash.md new file mode 100644 index 00000000..85828ff5 --- /dev/null +++ b/models/descriptions/tx_block_hash.md @@ -0,0 +1,5 @@ +{% docs tx_block_hash %} + +Block hash is a unique 66-character identifier that is generate when a block is produced. + +{% enddocs %} diff --git a/models/descriptions/tx_count.md b/models/descriptions/tx_count.md new file mode 100644 index 00000000..c4570214 --- /dev/null +++ b/models/descriptions/tx_count.md @@ -0,0 +1,5 @@ +{% docs tx_count %} + +Total number of transactions within a block. + +{% enddocs %} diff --git a/models/descriptions/tx_fee.md b/models/descriptions/tx_fee.md new file mode 100644 index 00000000..5623c621 --- /dev/null +++ b/models/descriptions/tx_fee.md @@ -0,0 +1,3 @@ +{% docs tx_fee %} +Transaction fee is. +{% enddocs %} diff --git a/models/descriptions/tx_from_address.md b/models/descriptions/tx_from_address.md new file mode 100644 index 00000000..8dd3a060 --- /dev/null +++ b/models/descriptions/tx_from_address.md @@ -0,0 +1,5 @@ +{% docs from_address %} + +Address from where the transaction was sent + +{% enddocs %} diff --git a/models/descriptions/tx_gas_limit.md b/models/descriptions/tx_gas_limit.md new file mode 100644 index 00000000..24edbad2 --- /dev/null +++ b/models/descriptions/tx_gas_limit.md @@ -0,0 +1,5 @@ +{% docs tx_gas_limit %} + +Total gas limit provided by all transactions in the block. + +{% enddocs %} diff --git a/models/descriptions/tx_gas_price.md b/models/descriptions/tx_gas_price.md new file mode 100644 index 00000000..4658ec8d --- /dev/null +++ b/models/descriptions/tx_gas_price.md @@ -0,0 +1,5 @@ +{% docs tx_gas_price %} + +Cost per unit of gas in Gwei. + +{% enddocs %} diff --git a/models/descriptions/tx_gas_used.md b/models/descriptions/tx_gas_used.md new file mode 100644 index 00000000..44a33268 --- /dev/null +++ b/models/descriptions/tx_gas_used.md @@ -0,0 +1,5 @@ +{% docs tx_gas_used %} + +Gas used by the transaction. + +{% enddocs %} diff --git a/models/descriptions/tx_hash.md b/models/descriptions/tx_hash.md new file mode 100644 index 00000000..03de8263 --- /dev/null +++ b/models/descriptions/tx_hash.md @@ -0,0 +1,5 @@ +{% docs tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. + +{% enddocs %} diff --git a/models/descriptions/tx_input_data.md b/models/descriptions/tx_input_data.md new file mode 100644 index 00000000..b9e6728b --- /dev/null +++ b/models/descriptions/tx_input_data.md @@ -0,0 +1,5 @@ +{% docs tx_input_data %} + +This column contains additional data for this transaction, and is commonly used as part of a contract interaction or as a message to the recipient. + +{% enddocs %} diff --git a/models/descriptions/tx_nonce.md b/models/descriptions/tx_nonce.md new file mode 100644 index 00000000..f5d54062 --- /dev/null +++ b/models/descriptions/tx_nonce.md @@ -0,0 +1,5 @@ +{% docs tx_nonce %} + +The number of transactions sent from a given address. + +{% enddocs %} diff --git a/models/descriptions/tx_origin_sig.md b/models/descriptions/tx_origin_sig.md new file mode 100644 index 00000000..79c83bf0 --- /dev/null +++ b/models/descriptions/tx_origin_sig.md @@ -0,0 +1,5 @@ +{% docs tx_origin_sig %} + +The function signature of the contract call. + +{% enddocs %} diff --git a/models/descriptions/tx_position.md b/models/descriptions/tx_position.md new file mode 100644 index 00000000..89194de2 --- /dev/null +++ b/models/descriptions/tx_position.md @@ -0,0 +1,5 @@ +{% docs tx_position %} + +The position of the transaction within the block. + +{% enddocs %} diff --git a/models/descriptions/tx_status.md b/models/descriptions/tx_status.md new file mode 100644 index 00000000..0b68f136 --- /dev/null +++ b/models/descriptions/tx_status.md @@ -0,0 +1,5 @@ +{% docs tx_status %} + +Status of the transaction. + +{% enddocs %} diff --git a/models/descriptions/tx_table_doc.md b/models/descriptions/tx_table_doc.md new file mode 100644 index 00000000..c9732541 --- /dev/null +++ b/models/descriptions/tx_table_doc.md @@ -0,0 +1,5 @@ +{% docs tx_table_doc %} + +This table contains transaction level data for the Aurora. + +{% enddocs %} diff --git a/models/descriptions/uncle_blocks.md b/models/descriptions/uncle_blocks.md new file mode 100644 index 00000000..ad4ce060 --- /dev/null +++ b/models/descriptions/uncle_blocks.md @@ -0,0 +1,5 @@ +{% docs uncle_blocks %} + +Uncle blocks occur when two blocks are mined and broadcasted at the same time, with the same block number. The block validated across the most nodes will be added to the primary chain, and the other one becomes an uncle block. Miners do receive rewards for uncle blocks. + +{% enddocs %} diff --git a/models/descriptions/value.md b/models/descriptions/value.md new file mode 100644 index 00000000..f8bfead4 --- /dev/null +++ b/models/descriptions/value.md @@ -0,0 +1,5 @@ +{% docs value %} + +Value of the transaction. + +{% enddocs %} diff --git a/models/gold/core__fact_blocks.sql b/models/gold/core__fact_blocks.sql new file mode 100644 index 00000000..5df19b56 --- /dev/null +++ b/models/gold/core__fact_blocks.sql @@ -0,0 +1,66 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + tags = ['core'] +) }} + +SELECT + block_number, + block_timestamp, + 'mainnet' AS network, + 'aurora' AS blockchain, + tx_count, + difficulty, + total_difficulty, + extra_data, + gas_limit, + gas_used, + miner, + nonce, + parent_hash, + HASH, + receipts_root, + sha3_uncles, + SIZE, + uncles AS uncle_blocks, + OBJECT_CONSTRUCT( + 'difficulty', + difficulty, + 'extraData', + extra_data, + 'gasLimit', + gas_limit, + 'gasUsed', + gas_used, + 'hash', + HASH, + 'logsBloom', + logs_bloom, + 'miner', + miner, + 'nonce', + nonce, + 'number', + NUMBER, + 'parentHash', + parent_hash, + 'receiptsRoot', + receipts_root, + 'sha3Uncles', + sha3_uncles, + 'size', + SIZE, + 'stateRoot', + state_root, + 'timestamp', + block_timestamp, + 'totalDifficulty', + total_difficulty, + 'transactionsRoot', + transactions_root, + 'uncles', + uncles + ) AS block_header_json +FROM + {{ ref('silver__blocks') }} \ No newline at end of file diff --git a/models/gold/core__fact_blocks.yml b/models/gold/core__fact_blocks.yml new file mode 100644 index 00000000..742d1b98 --- /dev/null +++ b/models/gold/core__fact_blocks.yml @@ -0,0 +1,44 @@ +version: 2 +models: + - name: core__fact_blocks + description: '{{ doc("blocks_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: NETWORK + description: '{{ doc("network") }}' + - name: BLOCKCHAIN + description: '{{ doc("blockchain") }}' + - name: TX_COUNT + description: '{{ doc("tx_count") }}' + - name: DIFFICULTY + description: '{{ doc("difficulty") }}' + - name: TOTAL_DIFFICULTY + description: '{{ doc("total_difficulty") }}' + - name: EXTRA_DATA + description: '{{ doc("extra_data") }}' + - name: GAS_LIMIT + description: '{{ doc("gas_limit") }}' + - name: GAS_USED + description: '{{ doc("gas_used") }}' + - name: HASH + description: '{{ doc("blocks_hash") }}' + - name: PARENT_HASH + description: '{{ doc("parent_hash") }}' + - name: MINER + description: '{{ doc("miner") }}' + - name: NONCE + description: '{{ doc("blocks_nonce") }}' + - name: RECEIPTS_ROOT + description: '{{ doc("receipts_root") }}' + - name: SHA3_UNCLES + description: '{{ doc("sha3_uncles") }}' + - name: SIZE + description: '{{ doc("size") }}' + - name: UNCLE_BLOCKS + description: '{{ doc("uncle_blocks") }}' + - name: BLOCK_HEADER_JSON + description: '{{ doc("block_header_json") }}' diff --git a/models/gold/core__fact_logs.sql b/models/gold/core__fact_logs.sql new file mode 100644 index 00000000..9c262cae --- /dev/null +++ b/models/gold/core__fact_logs.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + tags = ['core'] +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + event_index, + contract_address, + topics, + DATA, + event_removed, + tx_status, + _log_id +FROM + {{ ref('silver__logs') }} diff --git a/models/gold/core__fact_logs.yml b/models/gold/core__fact_logs.yml new file mode 100644 index 00000000..81d46fe8 --- /dev/null +++ b/models/gold/core__fact_logs.yml @@ -0,0 +1,32 @@ +version: 2 +models: + - name: core__fact_logs + description: '{{ doc("logs_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: EVENT_INDEX + description: '{{ doc("event_index") }}' + - name: CONTRACT_ADDRESS + description: '{{ doc("logs_contract_address") }}' + - name: TOPICS + description: '{{ doc("topics") }}' + - name: DATA + description: '{{ doc("logs_data") }}' + - name: EVENT_REMOVED + description: '{{ doc("event_removed") }}' + - name: _LOG_ID + description: '{{ doc("id_events") }}' + - name: TX_STATUS + description: '{{ doc("tx_status") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("nft_origin_sig") }}' + - name: ORIGIN_FROM_ADDRESS + description: "The from address at the transaction level." + - name: ORIGIN_TO_ADDRESS + description: "The to address at the transaction level." diff --git a/models/gold/core__fact_receipts.sql b/models/gold/core__fact_receipts.sql new file mode 100644 index 00000000..523f71fb --- /dev/null +++ b/models/gold/core__fact_receipts.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + tags = ['core'] +) }} + +SELECT + block_number, + block_hash, + cumulative_gas_used, + from_address, + to_address, + gas_used, + logs, + logs_bloom, + status, + tx_success, + tx_status, + tx_hash, + position, + type, + near_receipt_hash, + near_transaction_hash +FROM + {{ ref('silver__receipts') }} diff --git a/models/gold/core__fact_receipts.yml b/models/gold/core__fact_receipts.yml new file mode 100644 index 00000000..edf8ca02 --- /dev/null +++ b/models/gold/core__fact_receipts.yml @@ -0,0 +1,36 @@ +version: 2 +models: + - name: core__fact_receipts + description: '{{ doc("receipts_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_HASH + description: '{{ doc("blocks_hash") }}' + - name: CUMULATIVE_GAS_USED + description: '{{ doc("cumulative_gas_used") }}' + - name: FROM_ADDRESS + description: '{{ doc("from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("to_address") }}' + - name: GAS_USED + description: '{{ doc("gas_used") }}' + - name: LOGS + description: '{{ doc("receipts_log") }}' + - name: LOGS_BLOOM + description: '{{ doc("logs_bloom") }}' + - name: STATUS + description: '{{ doc("tx_status") }}' + - name: TX_STATUS + description: '{{ doc("tx_status") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: POSITION + description: '{{ doc("tx_position") }}' + - name: TYPE + description: "Type of transaction" + - name: NEAR_RECEIPT_HASH + description: "Receipt hash on NEAR blockchain" + - name: NEAR_TRANSACTION_HASH + description: "Transaction hash on NEAR blockchain" diff --git a/models/gold/core__fact_transactions.sql b/models/gold/core__fact_transactions.sql new file mode 100644 index 00000000..0dce8d93 --- /dev/null +++ b/models/gold/core__fact_transactions.sql @@ -0,0 +1,33 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true }, + tags = ['core'] +) }} + +SELECT + block_number, + block_hash, + block_timestamp, + tx_hash, + nonce, + POSITION, + origin_function_signature, + from_address, + to_address, + VALUE, + tx_fee, + gas_price, + gas AS gas_limit, + gas_used, + cumulative_Gas_Used, + input_data, + tx_status AS status, + max_fee_per_gas, + max_priority_fee_per_gas, + r, + s, + v, + tx_type +FROM + {{ ref('silver__transactions') }} diff --git a/models/gold/core__fact_transactions.yml b/models/gold/core__fact_transactions.yml new file mode 100644 index 00000000..bd823e67 --- /dev/null +++ b/models/gold/core__fact_transactions.yml @@ -0,0 +1,54 @@ +version: 2 +models: + - name: core__fact_transactions + description: '{{ doc("tx_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: BLOCK_HASH + description: '{{ doc("tx_block_hash") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: NONCE + description: '{{ doc("tx_nonce") }}' + - name: POSITION + description: '{{ doc("tx_position") }}' + - name: FROM_ADDRESS + description: '{{ doc("from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("to_address") }}' + - name: VALUE + description: '{{ doc("value") }}' + - name: TX_FEE + description: '{{ doc("tx_fee") }}' + - name: GAS_PRICE + description: '{{ doc("tx_gas_price") }}' + - name: GAS_LIMIT + description: '{{ doc("tx_gas_limit") }}' + - name: GAS_USED + description: '{{ doc("tx_gas_used") }}' + - name: CUMULATIVE_GAS_USED + description: '{{ doc("cumulative_gas_used") }}' + - name: STATUS + description: '{{ doc("tx_status") }}' + - name: INPUT_DATA + description: '{{ doc("tx_input_data") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("tx_origin_sig") }}' + - name: EFFECTIVE_GAS_PRICE + description: The effective gas price of the transaction, in wei. + - name: MAX_FEE_PER_GAS + description: The maximum fee per gas of the transaction, in wei. + - name: MAX_PRIORITY_FEE_PER_GAS + description: The maximum priority fee per gas of the transaction, in wei. + - name: TX_TYPE + description: The type of the transaction + - name: r + description: The r value of the transaction signature. + - name: s + description: The s value of the transaction signature. + - name: v + description: The v value of the transaction signature. diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql new file mode 100644 index 00000000..8eb39e2d --- /dev/null +++ b/models/silver/silver__blocks.sql @@ -0,0 +1,67 @@ +-- depends_on: {{ ref('bronze__streamline_blocks') }} +{{ config( + materialized = 'incremental', + unique_key = "block_number", + cluster_by = "block_timestamp::date", + tags = ['core'] +) }} + +SELECT + block_number, + utils.udf_hex_to_int( + DATA :result :timestamp :: STRING + ) :: TIMESTAMP AS block_timestamp, + ARRAY_SIZE( + DATA :result :transactions + ) AS tx_count, + utils.udf_hex_to_int( + DATA :result :difficulty :: STRING + ) :: INT AS difficulty, + utils.udf_hex_to_int( + DATA :result :totalDifficulty :: STRING + ) :: INT AS total_difficulty, + DATA :result :extraData :: STRING AS extra_data, + utils.udf_hex_to_int( + DATA :result :gasLimit :: STRING + ) :: INT AS gas_limit, + utils.udf_hex_to_int( + DATA :result :gasUsed :: STRING + ) :: INT AS gas_used, + DATA :result :miner :: STRING AS miner, + utils.udf_hex_to_int( + DATA :result :nonce :: STRING + ) :: INT AS nonce, + DATA :result :parentHash :: STRING AS parent_hash, + DATA :result :hash :: STRING AS HASH, + DATA :result :receiptsRoot :: STRING AS receipts_root, + utils.udf_hex_to_int( + DATA :result :number :: STRING + ) :: INT AS NUMBER, + DATA :result :sha3Uncles :: STRING AS sha3_uncles, + utils.udf_hex_to_int( + DATA :result :size :: STRING + ) :: INT AS SIZE, + DATA :result :uncles AS uncles, + DATA :result :logsBloom :: STRING AS logs_bloom, + DATA :result :stateRoot :: STRING AS state_root, + DATA :result :transactionsRoot :: STRING AS transactions_root, + _partition_by_block_id, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_blocks') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_blocks') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/silver/silver__logs.sql b/models/silver/silver__logs.sql new file mode 100644 index 00000000..e34f24e8 --- /dev/null +++ b/models/silver/silver__logs.sql @@ -0,0 +1,183 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "block_timestamp::date, _inserted_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + full_refresh = false, + tags = ['core'] +) }} + +WITH base AS ( + + SELECT + block_number, + tx_hash, + from_address AS origin_from_address, + to_address AS origin_to_address, + tx_status, + logs, + _inserted_timestamp + FROM + {{ ref('silver__receipts') }} + WHERE + ARRAY_SIZE(logs) > 0 + +{% if is_incremental() %} +AND _INSERTED_TIMESTAMP >= ( + SELECT + MAX(_INSERTED_TIMESTAMP) _INSERTED_TIMESTAMP + FROM + {{ this }} +) +{% endif %} +), +flat_logs AS ( + SELECT + block_number, + tx_hash, + origin_from_address, + origin_to_address, + tx_status, + VALUE :address :: STRING AS contract_address, + VALUE :blockHash :: STRING AS block_hash, + VALUE :data :: STRING AS DATA, + utils.udf_hex_to_int( + VALUE :logIndex :: STRING + ) :: INT AS event_index, + VALUE :removed :: BOOLEAN AS event_removed, + VALUE :topics AS topics, + _inserted_timestamp + FROM + base, + LATERAL FLATTEN( + input => logs + ) +), +new_records AS ( + SELECT + l.block_number, + txs.block_timestamp, + l.tx_hash, + l.origin_from_address, + l.origin_to_address, + txs.origin_function_signature, + l.tx_status, + l.contract_address, + l.block_hash, + l.data, + l.event_index, + l.event_removed, + l.topics, + l._inserted_timestamp, + CASE + WHEN txs.block_timestamp IS NULL + OR txs.origin_function_signature IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + CONCAT( + l.tx_hash :: STRING, + '-', + l.event_index :: STRING + ) AS _log_id + FROM + flat_logs l + LEFT OUTER JOIN {{ ref('silver__transactions') }} + txs + ON l.block_number = txs.block_number + AND l.tx_hash = txs.tx_hash + +{% if is_incremental() %} +AND txs._INSERTED_TIMESTAMP >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + txs.block_timestamp, + t.tx_hash, + t.origin_from_address, + t.origin_to_address, + txs.origin_function_signature, + t.tx_status, + t.contract_address, + t.block_hash, + t.data, + t.event_index, + t.event_removed, + t.topics, + GREATEST( + t._inserted_timestamp, + txs._inserted_timestamp + ) AS _inserted_timestamp, + _log_id, + FALSE AS is_pending + FROM + {{ this }} + t + INNER JOIN {{ ref('silver__transactions') }} + txs USING ( + block_number, + tx_hash + ) + WHERE + t.is_pending +) +{% endif %}, +FINAL AS ( + SELECT + block_number, + block_timestamp, + tx_hash, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_status, + contract_address, + block_hash, + DATA, + event_index, + event_removed, + topics, + _inserted_timestamp, + _log_id, + is_pending + FROM + new_records + +{% if is_incremental() %} +UNION +SELECT + block_number, + block_timestamp, + tx_hash, + origin_from_address, + origin_to_address, + origin_function_signature, + tx_status, + contract_address, + block_hash, + DATA, + event_index, + event_removed, + topics, + _inserted_timestamp, + _log_id, + is_pending +FROM + missing_data +{% endif %} +) +SELECT + * +FROM + FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, event_index +ORDER BY + _inserted_timestamp DESC, is_pending ASC)) = 1 diff --git a/models/silver/silver__receipts.sql b/models/silver/silver__receipts.sql new file mode 100644 index 00000000..568973ef --- /dev/null +++ b/models/silver/silver__receipts.sql @@ -0,0 +1,72 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_hash)", + full_refresh = false, + tags = ['core'] +) }} + +WITH base AS ( + SELECT + block_number, + DATA, + _inserted_timestamp + FROM + {{ ref('bronze__streamline_tx_receipts') }} +), +FINAL AS ( + SELECT + block_number, + DATA :blockHash :: STRING AS block_hash, + utils.udf_hex_to_int( + DATA :blockNumber :: STRING + ) :: INT AS blockNumber, + utils.udf_hex_to_int( + DATA :cumulativeGasUsed :: STRING + ) :: INT AS cumulative_gas_used, + DATA :from :: STRING AS from_address, + utils.udf_hex_to_int( + DATA :gasUsed :: STRING + ) :: INT AS gas_used, + DATA :logs AS logs, + DATA :logsBloom :: STRING AS logs_bloom, + utils.udf_hex_to_int( + DATA :status :: STRING + ) :: INT AS status, + CASE + WHEN status = 1 THEN TRUE + ELSE FALSE + END AS tx_success, + CASE + WHEN status = 1 THEN 'SUCCESS' + ELSE 'FAIL' + END AS tx_status, + DATA :to :: STRING AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + DATA :transactionHash :: STRING AS tx_hash, + utils.udf_hex_to_int( + DATA :transactionIndex :: STRING + ) :: INT AS POSITION, + utils.udf_hex_to_int( + DATA :type :: STRING + ) :: INT AS TYPE, + DATA :nearReceiptHash :: STRING AS near_receipt_hash, + DATA :nearTransactionHash :: STRING AS near_transaction_hash, + _inserted_timestamp + FROM + base +) + +SELECT + * +FROM + FINAL +WHERE + tx_hash IS NOT NULL qualify(ROW_NUMBER() over (PARTITION BY block_number, POSITION +ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/silver/silver__relevant_contracts.sql b/models/silver/silver__relevant_contracts.sql new file mode 100644 index 00000000..a1c13f81 --- /dev/null +++ b/models/silver/silver__relevant_contracts.sql @@ -0,0 +1,17 @@ +{{ config ( + materialized = "table", + unique_key = "contract_address" +) }} + +SELECT + contract_address, + 'Aurora' AS blockchain, + COUNT(*) AS events, + MAX(block_number) AS latest_block +FROM + {{ ref('silver__logs') }} +GROUP BY + 1, + 2 +HAVING + COUNT(*) >= 25 \ No newline at end of file diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql new file mode 100644 index 00000000..6f6c2cd0 --- /dev/null +++ b/models/silver/silver__transactions.sql @@ -0,0 +1,284 @@ +-- depends_on: {{ ref('bronze__streamline_transactions') }} +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_number", + cluster_by = "block_timestamp::date, _inserted_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core'] +) }} + +WITH base AS ( + + SELECT + block_number, + DATA, + _inserted_timestamp + FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_transactions') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) + AND IS_OBJECT(DATA) +{% else %} + {{ ref('bronze__streamline_FR_transactions') }} +WHERE + IS_OBJECT(DATA) +{% endif %} +), +base_tx AS ( + SELECT + A.block_number AS block_number, + A.data :blockHash :: STRING AS block_hash, + utils.udf_hex_to_int( + A.data :blockNumber :: STRING + ) :: INT AS blockNumber, + A.data :from :: STRING AS from_address, + utils.udf_hex_to_int( + A.data :gas :: STRING + ) :: INT AS gas, + utils.udf_hex_to_int( + A.data :gasPrice :: STRING + ) :: INT / pow( + 10, + 9 + ) AS gas_price, + A.data :hash :: STRING AS tx_hash, + A.data :input :: STRING AS input_data, + SUBSTR( + input_data, + 1, + 10 + ) AS origin_function_signature, + COALESCE(utils.udf_hex_to_int( + A.data :maxFeePerGas :: STRING + ) :: INT / pow( + 10, + 9 + ), 0) AS max_fee_per_gas, + COALESCE( utils.udf_hex_to_int( + A.data :maxPriorityFeePerGas :: STRING + ) :: INT / pow( + 10, + 9 + ), 0) AS max_priority_fee_per_gas, + utils.udf_hex_to_int( + A.data :nonce :: STRING + ) :: INT AS nonce, + A.data :r :: STRING AS r, + A.data :s :: STRING AS s, + A.data :to :: STRING AS to_address1, + CASE + WHEN to_address1 = '' THEN NULL + ELSE to_address1 + END AS to_address, + utils.udf_hex_to_int( + A.data :transactionIndex :: STRING + ) :: INT AS POSITION, + A.data :type :: STRING AS TYPE, + A.data :v :: STRING AS v, + utils.udf_hex_to_int( + A.data :value :: STRING + ) / pow( + 10, + 18 + ) :: FLOAT AS VALUE, + A.data :accessList AS access_list, + A._INSERTED_TIMESTAMP + FROM + base A +), +new_records AS ( + SELECT + t.block_number, + t.block_hash, + t.from_address, + t.gas, + t.gas_price, + t.tx_hash, + t.input_data, + t.origin_function_signature, + t.max_fee_per_gas, + t.max_priority_fee_per_gas, + t.nonce, + t.r, + t.s, + t.to_address1, + t.to_address, + t.position, + t.type, + t.v, + t.value, + block_timestamp, + CASE + WHEN block_timestamp IS NULL + OR tx_status IS NULL THEN TRUE + ELSE FALSE + END AS is_pending, + r.gas_used, + tx_success, + tx_status, + cumulative_gas_used, + ( + gas_price * r.gas_used + ) / pow( + 10, + 9 + ) AS tx_fee, + r.type AS tx_type, + t.access_list, + t._inserted_timestamp + FROM + base_tx t + LEFT OUTER JOIN {{ ref('silver__blocks') }} + b + ON t.block_number = b.block_number + LEFT OUTER JOIN {{ ref('silver__receipts') }} + r + ON t.block_number = r.block_number + AND t.tx_hash = r.tx_hash + +{% if is_incremental() %} +AND r._INSERTED_TIMESTAMP >= ( + SELECT + MAX(_inserted_timestamp) :: DATE - 1 + FROM + {{ this }} +) +{% endif %} +) + +{% if is_incremental() %}, +missing_data AS ( + SELECT + t.block_number, + t.block_hash, + t.from_address, + t.gas, + t.gas_price, + t.tx_hash, + t.input_data, + t.origin_function_signature, + t.max_fee_per_gas, + t.max_priority_fee_per_gas, + t.nonce, + t.r, + t.s, + t.to_address, + t.position, + t.type, + t.v, + t.value, + b.block_timestamp, + FALSE AS is_pending, + r.gas_used, + r.tx_success, + r.tx_status, + r.cumulative_gas_used, + ( + t.gas_price * r.gas_used + ) / pow( + 10, + 9 + ) AS tx_fee, + r.type AS tx_type, + t.access_list, + GREATEST( + t._inserted_timestamp, + b._inserted_timestamp, + r._inserted_timestamp + ) AS _inserted_timestamp + FROM + {{ this }} + t + INNER JOIN {{ ref('silver__blocks') }} + b + ON t.block_number = b.block_number + INNER JOIN {{ ref('silver__receipts') }} + r + ON t.tx_hash = r.tx_hash + AND t.block_number = r.block_number + WHERE + t.is_pending +) +{% endif %}, +FINAL AS ( + SELECT + block_number, + block_hash, + from_address, + gas, + gas_price, + tx_hash, + input_data, + origin_function_signature, + max_fee_per_gas, + max_priority_fee_per_gas, + nonce, + r, + s, + to_address, + POSITION, + TYPE, + v, + VALUE, + block_timestamp, + is_pending, + gas_used, + tx_success, + tx_status, + cumulative_gas_used, + tx_fee, + tx_type, + access_list, + _inserted_timestamp + FROM + new_records + +{% if is_incremental() %} +UNION +SELECT + block_number, + block_hash, + from_address, + gas, + gas_price, + tx_hash, + input_data, + origin_function_signature, + max_fee_per_gas, + max_priority_fee_per_gas, + nonce, + r, + s, + to_address, + POSITION, + TYPE, + v, + VALUE, + block_timestamp, + is_pending, + gas_used, + tx_success, + tx_status, + cumulative_gas_used, + tx_fee, + tx_type, + access_list, + _inserted_timestamp +FROM + missing_data +{% endif %} +) +SELECT + * +FROM + FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, POSITION +ORDER BY + _inserted_timestamp DESC, is_pending ASC)) = 1 diff --git a/models/silver/streamline/_max_block_by_date.sql b/models/silver/streamline/_max_block_by_date.sql new file mode 100644 index 00000000..a56cf82d --- /dev/null +++ b/models/silver/streamline/_max_block_by_date.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = "ephemeral", + unique_key = "block_number", +) }} + +WITH base AS ( + + SELECT + block_timestamp :: DATE AS block_date, + MAX(block_number) block_number + FROM + {{ ref("silver__blocks") }} + GROUP BY + block_timestamp :: DATE +) +SELECT + block_date, + block_number +FROM + base +WHERE + block_date <> ( + SELECT + MAX(block_date) + FROM + base + ) \ No newline at end of file diff --git a/models/silver/streamline/complete/streamline__complete_blocks.sql b/models/silver/streamline/complete/streamline__complete_blocks.sql new file mode 100644 index 00000000..f66b0fcb --- /dev/null +++ b/models/silver/streamline/complete/streamline__complete_blocks.sql @@ -0,0 +1,30 @@ +-- depends_on: {{ ref('bronze__streamline_blocks') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_blocks') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_blocks') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/complete/streamline__complete_transactions.sql b/models/silver/streamline/complete/streamline__complete_transactions.sql new file mode 100644 index 00000000..61cd3459 --- /dev/null +++ b/models/silver/streamline/complete/streamline__complete_transactions.sql @@ -0,0 +1,30 @@ +-- depends_on: {{ ref('bronze__streamline_transactions') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" +) }} + +SELECT + id, + block_number, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_transactions') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) +{% else %} + {{ ref('bronze__streamline_FR_transactions') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/complete/streamline__complete_tx_receipts.sql b/models/silver/streamline/complete/streamline__complete_tx_receipts.sql new file mode 100644 index 00000000..ccddbae9 --- /dev/null +++ b/models/silver/streamline/complete/streamline__complete_tx_receipts.sql @@ -0,0 +1,34 @@ +-- depends_on: {{ ref('bronze__streamline_tx_receipts') }} + +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" +) }} + +SELECT + id, + block_number, + data:result:transactionHash::STRING AS tx_hash, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_tx_receipts') }} +WHERE + _inserted_timestamp >= ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ) + AND tx_hash IS NOT NULL +{% else %} + {{ ref('bronze__streamline_FR_tx_receipts') }} + WHERE tx_hash IS NOT NULL +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/realtime/streamline__blocks_realtime.sql b/models/silver/streamline/realtime/streamline__blocks_realtime.sql new file mode 100644 index 00000000..d1c51485 --- /dev/null +++ b/models/silver/streamline/realtime/streamline__blocks_realtime.sql @@ -0,0 +1,55 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table','blocks', 'producer_batch_size',10000, 'producer_limit_size', 10000000, 'worker_batch_size',200))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} + + SELECT + 0 AS block_number + {% else %} + SELECT + MAX(block_number) - 50000 AS block_number --aprox 3 days + FROM + {{ ref("streamline__blocks") }} + {% endif %}), + tbl AS ( + SELECT + block_number, + block_number_hex + FROM + {{ ref("streamline__blocks") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number, + REPLACE( + concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex + FROM + {{ ref("streamline__complete_blocks") }} + ) +SELECT + block_number, + 'eth_getBlockByNumber' AS method, + CONCAT( + block_number_hex, + '_-_', + 'false' + ) AS params +FROM + tbl diff --git a/models/silver/streamline/realtime/streamline__transactions_realtime.sql b/models/silver/streamline/realtime/streamline__transactions_realtime.sql new file mode 100644 index 00000000..e2062028 --- /dev/null +++ b/models/silver/streamline/realtime/streamline__transactions_realtime.sql @@ -0,0 +1,55 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'transactions', 'exploded_key','[\"result\", \"transactions\"]', 'producer_batch_size',10000, 'producer_limit_size', 10000000, 'worker_batch_size',200))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} + + SELECT + 0 AS block_number + {% else %} + SELECT + MAX(block_number) - 50000 AS block_number --aprox 3 days + FROM + {{ ref("streamline__blocks") }} + {% endif %}), + tbl AS ( + SELECT + block_number, + block_number_hex + FROM + {{ ref("streamline__blocks") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + EXCEPT + SELECT + block_number, + REPLACE( + concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex + FROM + {{ ref("streamline__complete_blocks") }} + ) +SELECT + block_number, + 'eth_getBlockByNumber' AS method, + CONCAT( + block_number_hex, + '_-_', + 'true' + ) AS params +FROM + tbl \ No newline at end of file diff --git a/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql b/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql new file mode 100644 index 00000000..f18d1015 --- /dev/null +++ b/models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql @@ -0,0 +1,49 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'tx_receipts', 'producer_batch_size',10000, 'producer_limit_size', 10000000, 'worker_batch_size',200))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + + +WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} + + SELECT + 0 AS block_number + {% else %} + SELECT + MAX(block_number) - 10000 AS block_number --aprox 3 days + FROM + {{ ref("bronze__streamline_FR_transactions") }} + {% endif %}), + tbl AS ( + SELECT + block_number, + data:hash::STRING AS tx_hash + FROM + {{ ref("bronze__streamline_FR_transactions") }} + WHERE + ( + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) + ) + AND block_number IS NOT NULL + AND tx_hash IS NOT NULL + AND tx_hash NOT IN ( + SELECT + tx_hash + FROM + {{ ref("streamline__complete_tx_receipts") }} + ) + ) +SELECT + block_number, + 'eth_getTransactionReceipt' AS method, + tx_hash AS params +FROM + tbl \ No newline at end of file diff --git a/models/silver/streamline/streamline__blocks.sql b/models/silver/streamline/streamline__blocks.sql new file mode 100644 index 00000000..bad8e4d5 --- /dev/null +++ b/models/silver/streamline/streamline__blocks.sql @@ -0,0 +1,21 @@ +{{ config ( + materialized = "view", + tags = ['streamline_view'] +) }} + +{% if execute %} + {% set height = run_query('SELECT streamline.udf_get_chainhead()') %} + {% set block_height = height.columns [0].values() [0] %} +{% else %} + {% set block_height = 0 %} +{% endif %} + +SELECT + height AS block_number, + REPLACE( + concat_ws('', '0x', to_char(height, 'XXXXXXXX')), + ' ', + '' + ) AS block_number_hex +FROM + TABLE(streamline.udtf_get_base_table({{ block_height }})) diff --git a/models/silver/test/blocks/test_silver__blocks_full.sql b/models/silver/test/blocks/test_silver__blocks_full.sql new file mode 100644 index 00000000..a8e43767 --- /dev/null +++ b/models/silver/test/blocks/test_silver__blocks_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__blocks') }} \ No newline at end of file diff --git a/models/silver/test/blocks/test_silver__blocks_full.yml b/models/silver/test/blocks/test_silver__blocks_full.yml new file mode 100644 index 00000000..254c47bb --- /dev/null +++ b/models/silver/test/blocks/test_silver__blocks_full.yml @@ -0,0 +1,93 @@ +version: 2 +models: + - name: test_silver__blocks_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TOTAL_DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EXTRA_DATA + tests: + - not_null + - name: GAS_LIMIT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: PARENT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: MINER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: RECEIPTS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SHA3_UNCLES + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SIZE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER diff --git a/models/silver/test/blocks/test_silver__blocks_recent.sql b/models/silver/test/blocks/test_silver__blocks_recent.sql new file mode 100644 index 00000000..c27a8a93 --- /dev/null +++ b/models/silver/test/blocks/test_silver__blocks_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__blocks') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/test/blocks/test_silver__blocks_recent.yml b/models/silver/test/blocks/test_silver__blocks_recent.yml new file mode 100644 index 00000000..eaa4b4e8 --- /dev/null +++ b/models/silver/test/blocks/test_silver__blocks_recent.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: test_silver__blocks_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + - sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">10" + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ diff --git a/models/silver/test/event_logs/test_silver__logs_full.sql b/models/silver/test/event_logs/test_silver__logs_full.sql new file mode 100644 index 00000000..e1172988 --- /dev/null +++ b/models/silver/test/event_logs/test_silver__logs_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__logs') }} diff --git a/models/silver/test/event_logs/test_silver__logs_full.yml b/models/silver/test/event_logs/test_silver__logs_full.yml new file mode 100644 index 00000000..1b230cc8 --- /dev/null +++ b/models/silver/test/event_logs/test_silver__logs_full.yml @@ -0,0 +1,77 @@ +version: 2 +models: + - name: test_silver__logs_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - tx_block_count: + config: + severity: error + error_if: "!=0" + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TOPICS + tests: + - not_null + - name: DATA + tests: + - not_null + - name: EVENT_REMOVED + tests: + - not_null + - name: _LOG_ID + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null: + where: NOT IS_PENDING + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ diff --git a/models/silver/test/event_logs/test_silver__logs_recent.sql b/models/silver/test/event_logs/test_silver__logs_recent.sql new file mode 100644 index 00000000..faa40cf0 --- /dev/null +++ b/models/silver/test/event_logs/test_silver__logs_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__logs') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/test/event_logs/test_silver__logs_recent.yml b/models/silver/test/event_logs/test_silver__logs_recent.yml new file mode 100644 index 00000000..39615ebb --- /dev/null +++ b/models/silver/test/event_logs/test_silver__logs_recent.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: test_silver__logs_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null diff --git a/models/silver/test/receipts/test_silver__receipts_full.sql b/models/silver/test/receipts/test_silver__receipts_full.sql new file mode 100644 index 00000000..a51ab58f --- /dev/null +++ b/models/silver/test/receipts/test_silver__receipts_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__receipts') }} \ No newline at end of file diff --git a/models/silver/test/receipts/test_silver__receipts_full.yml b/models/silver/test/receipts/test_silver__receipts_full.yml new file mode 100644 index 00000000..122db41c --- /dev/null +++ b/models/silver/test/receipts/test_silver__receipts_full.yml @@ -0,0 +1,73 @@ +version: 2 +models: + - name: test_silver__receipts_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: CUMULATIVE_GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_STATUS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ["SUCCESS", "FAIL"] + - name: TYPE + tests: + - not_null + # - name: _INSERTED_TIMESTAMP + # tests: + # - not_null + # - dbt_expectations.expect_row_values_to_have_recent_data: + # datepart: day + # interval: 1 diff --git a/models/silver/test/receipts/test_silver__receipts_recent.sql b/models/silver/test/receipts/test_silver__receipts_recent.sql new file mode 100644 index 00000000..0b5a9017 --- /dev/null +++ b/models/silver/test/receipts/test_silver__receipts_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__receipts') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/test/receipts/test_silver__receipts_recent.yml b/models/silver/test/receipts/test_silver__receipts_recent.yml new file mode 100644 index 00000000..e5167784 --- /dev/null +++ b/models/silver/test/receipts/test_silver__receipts_recent.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: test_silver__receipts_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 diff --git a/models/silver/test/transactions/test_silver__transactions_full.sql b/models/silver/test/transactions/test_silver__transactions_full.sql new file mode 100644 index 00000000..df58d177 --- /dev/null +++ b/models/silver/test/transactions/test_silver__transactions_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('silver__transactions') }} diff --git a/models/silver/test/transactions/test_silver__transactions_full.yml b/models/silver/test/transactions/test_silver__transactions_full.yml new file mode 100644 index 00000000..987eafd4 --- /dev/null +++ b/models/silver/test/transactions/test_silver__transactions_full.yml @@ -0,0 +1,113 @@ +version: 2 +models: + - name: test_silver__transactions_full + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: VALUE + tests: + - not_null + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS + tests: + - not_null + - name: INPUT_DATA + tests: + - not_null + - name: TX_STATUS + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ["SUCCESS", "FAIL"] + where: NOT IS_PENDING + - name: GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CUMULATIVE_GAS_USED + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_FEE + tests: + - not_null: + where: NOT IS_PENDING + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null diff --git a/models/silver/test/transactions/test_silver__transactions_recent.sql b/models/silver/test/transactions/test_silver__transactions_recent.sql new file mode 100644 index 00000000..a7b08022 --- /dev/null +++ b/models/silver/test/transactions/test_silver__transactions_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__transactions') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/test/transactions/test_silver__transactions_recent.yml b/models/silver/test/transactions/test_silver__transactions_recent.yml new file mode 100644 index 00000000..26ed3d05 --- /dev/null +++ b/models/silver/test/transactions/test_silver__transactions_recent.yml @@ -0,0 +1,21 @@ +version: 2 +models: + - name: test_silver__transactions_recent + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 diff --git a/models/sources.yml b/models/sources.yml index e31d5cc1..fa4952da 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -2,9 +2,26 @@ version: 2 sources: - name: crosschain - database: "{{ 'crosschain' if target.database == 'xyz' else 'crosschain_dev' }}" + database: "{{ 'crosschain' if target.database == 'AURORA' else 'crosschain_dev' }}" schema: core tables: - name: dim_date_hours - name: address_tags - name: dim_dates + + - name: bronze_streamline + database: streamline + + schema: | + {{ "AURORA_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "AURORA" }} + tables: + - name: blocks + - name: transactions + - name: tx_receipts + + - name: streamline + database: streamline + schema: | + {{ "AURORA_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "AURORA" }} + tables: + - name: tx_receipts diff --git a/packages.yml b/packages.yml index e9570789..f44f45e8 100644 --- a/packages.yml +++ b/packages.yml @@ -2,4 +2,4 @@ packages: - package: calogica/dbt_expectations version: [">=0.4.0", "<0.9.0"] - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.3.0 \ No newline at end of file + revision: v1.3.0 diff --git a/profiles.yml b/profiles.yml index 0661f982..5925a8d3 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,11 +1,11 @@ -xyz: +aurora: target: dev outputs: dev: type: snowflake account: "{{ env_var('ACCOUNT') }}" user: "{{ env_var('USER') }}" - password: "{{ env_var('PASSWORD') }}" + authenticator: externalbrowser role: "{{ env_var('ROLE') }}" schema: "{{ env_var('SCHEMA') }}" region: "{{ env_var('REGION') }}" @@ -17,7 +17,7 @@ xyz: type: snowflake account: "{{ env_var('ACCOUNT') }}" user: "{{ env_var('USER') }}" - password: "{{ env_var('PASSWORD') }}" + authenticator: externalbrowser role: "{{ env_var('ROLE') }}" schema: "{{ env_var('SCHEMA') }}" region: "{{ env_var('REGION') }}" @@ -26,4 +26,4 @@ xyz: threads: 8 client_session_keep_alive: False config: - send_anonymous_usage_stats: False \ No newline at end of file + send_anonymous_usage_stats: False diff --git a/python_scripts/test_alert/dbt_test_alert.py b/python_scripts/test_alert/dbt_test_alert.py new file mode 100644 index 00000000..e76e358b --- /dev/null +++ b/python_scripts/test_alert/dbt_test_alert.py @@ -0,0 +1,124 @@ +import datetime +import requests +import json +import sys +import os + + +def log_test_result(): + """Reads the run_results.json file and returns a dictionary of targeted test results""" + + filepath = "target/run_results.json" + + with open(filepath) as f: + run = json.load(f) + + logs = [] + messages = {"fail": [], "warn": []} + test_count = 0 + warn_count = 0 + fail_count = 0 + + for test in run["results"]: + test_count += 1 + if test["status"] != "pass": + logs.append(test) + + message = f"{test['failures']} record failure(s) in {test['unique_id']}" + + if test["status"] == "warn": + messages["warn"].append(message) + warn_count += 1 + elif test["status"] == "fail": + messages["fail"].append(message) + fail_count += 1 + + dbt_test_result = { + "logs": logs, + "messages": messages, + "test_count": test_count, + "warn_count": warn_count, + "fail_count": fail_count, + "elapsed_time": str(datetime.timedelta(seconds=run["elapsed_time"])), + } + + return dbt_test_result + + +def create_message(**kwargs): + messageBody = { + "text": f"Hey{' <@U037AH6DS8P>' if len(kwargs['messages']['fail']) > 0 else ''}, new DBT test results for :{os.environ.get('DATABASE').split('_DEV')[0]}: {os.environ.get('DATABASE')}", + "attachments": [ + { + "color": kwargs["color"], + "fields": [ + { + "title": "Total Tests Run", + "value": kwargs["test_count"], + "short": True, + }, + { + "title": "Total Time Elapsed", + "value": kwargs["elapsed_time"], + "short": True, + }, + { + "title": "Number of Unsuccessful Tests", + "value": f"Fail: {kwargs['fail_count']}, Warn: {kwargs['warn_count']}", + "short": True, + }, + { + "title": "Failed Tests:", + "value": "\n".join(kwargs["messages"]["fail"]) + if len(kwargs["messages"]["fail"]) > 0 + else "None :)", + "short": False, + }, + ], + "actions": [ + { + "type": "button", + "text": "View the full run results", + "style": "primary", + "url": "https://github.com/FlipsideCrypto/aurora-models/actions/workflows/", + "confirm": { + "title": "I haven't figured this part out yet", + "text": "I'm not sure how to get the run id from the workflow run, see all tests by clicking Ok", + "ok_text": "Ok", + "dismiss_text": "Dismiss", + }, + } + ], + } + ], + } + + return messageBody + + +def send_alert(webhook_url): + """Sends a message to a slack channel""" + + url = webhook_url + + data = log_test_result() + + send_message = create_message( + fail_count=data["fail_count"], + warn_count=data["warn_count"], + test_count=data["test_count"], + messages=data["messages"], + elapsed_time=data["elapsed_time"], + color="#f44336" if data["fail_count"] > 0 else "#4CAF50", + ) + + x = requests.post(url, json=send_message) + + # test config to continue on error in workflow, so we want to exit with a non-zero code if there are any failures + if data["fail_count"] > 0: + sys.exit(1) + + +if __name__ == "__main__": + webhook_url = os.environ.get("SLACK_WEBHOOK_URL") + send_alert(webhook_url)