From abaad349ce4f3213e5458466dbd2970993c43e80 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:28:00 -0700 Subject: [PATCH 01/17] actions model v0 --- models/silver/actions/silver__actions.sql | 144 ++++++++++++++++++ .../silver__streamline_receipts.sql | 3 +- .../silver__streamline_receipts_final.sql | 2 +- .../silver__streamline_transactions_final.sql | 4 +- 4 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 models/silver/actions/silver__actions.sql diff --git a/models/silver/actions/silver__actions.sql b/models/silver/actions/silver__actions.sql new file mode 100644 index 00000000..755bc8b1 --- /dev/null +++ b/models/silver/actions/silver__actions.sql @@ -0,0 +1,144 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + unique_key = 'actions_id' +) }} +-- TODO: add back cluster, SO, incremental predicates, tags +WITH transactions AS ( + + SELECT + tx_hash, + tx_signer, + tx_receiver, + gas_used AS tx_gas_used, + tx_succeeded + FROM + {{ ref('silver__streamline_transactions_final') }} + WHERE + block_timestamp > CURRENT_DATE - 30 + -- TODO add incremental filter +), +receipts AS ( + SELECT + tx_hash, + block_id, + block_timestamp, + receipt_object_id AS receipt_id, + receiver_id AS receipt_receiver_id, + signer_id AS receipt_signer_id, + receipt_actions :predecessor_id :: STRING AS receipt_predecessor_id, + receipt_succeeded, + gas_burnt AS receipt_gas_burnt, + receipt_actions, + logs AS receipt_logs, + _partition_by_block_number, + _inserted_timestamp + FROM + {{ ref('silver__streamline_receipts_final') }} + WHERE + block_timestamp > CURRENT_DATE - 30 + -- TODO add incremental filter +), +join_data AS ( + SELECT + r.block_id, + r.block_timestamp, + t.tx_hash, + t.tx_signer, + t.tx_receiver, + t.tx_gas_used, + t.tx_succeeded, + r.receipt_id, + r.receipt_receiver_id, + r.receipt_signer_id, + r.receipt_predecessor_id, + r.receipt_succeeded, + r.receipt_gas_burnt, + r.receipt_actions, + r.receipt_logs, + r._partition_by_block_number, + r._inserted_timestamp + FROM + transactions t + LEFT JOIN receipts r + ON t.tx_hash = r.tx_hash +), +flatten_actions AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + tx_signer, + tx_receiver, + tx_gas_used, + tx_succeeded, + receipt_id, + receipt_receiver_id, + receipt_signer_id, + receipt_predecessor_id, + receipt_succeeded, + receipt_gas_burnt, + receipt_logs, -- TODO review logs, clean logs? + INDEX AS action_index, + receipt_actions :receipt :Action :gas_price :: NUMBER AS action_gas_price, + IFF(VALUE = 'CreateAccount', VALUE, object_keys(VALUE) [0] :: STRING) AS action_name, + IFF( + VALUE = 'CreateAccount', + {}, + GET(VALUE, object_keys(VALUE) [0] :: STRING) + ) AS action_data, + IFF( + action_name = 'FunctionCall', + OBJECT_INSERT( + action_data, + 'args', + COALESCE( + TRY_PARSE_JSON( + TRY_BASE64_DECODE_STRING( + action_data :args :: STRING + ) + ), + action_data :args + ), + TRUE + ), + action_data + ) AS action_data_parsed, + _partition_by_block_number, + _inserted_timestamp + FROM + join_data, + LATERAL FLATTEN( + receipt_actions :receipt :Action :actions :: ARRAY + ) +) +SELECT + block_id, + block_timestamp, + tx_hash, + tx_signer, + tx_receiver, + tx_gas_used, + tx_succeeded, + receipt_id, + receipt_receiver_id, + receipt_signer_id, + receipt_predecessor_id, + receipt_succeeded, + receipt_gas_burnt, + receipt_logs, + action_index, + action_gas_price, + action_name, + action_data_parsed AS action_data, + _partition_by_block_number, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['receipt_id', 'action_index'] + ) }} AS actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + flatten_actions diff --git a/models/silver/streamline/silver__streamline_receipts.sql b/models/silver/streamline/silver__streamline_receipts.sql index 55030976..3182a5ae 100644 --- a/models/silver/streamline/silver__streamline_receipts.sql +++ b/models/silver/streamline/silver__streamline_receipts.sql @@ -37,7 +37,7 @@ WITH shards AS ( {% endif %} ), flatten_receipts AS ( - + -- TODO review new keys like priority, input_data_ids, output_data_receivers SELECT concat_ws( '-', @@ -95,6 +95,7 @@ FINAL AS ( ) AS error_type_2, failure_message [error_type_0] :kind [error_type_1] [error_type_2] :: STRING AS error_message, execution_outcome :outcome :receipt_ids :: ARRAY AS outcome_receipts, + -- TODO extract predecessor_id receipt :receiver_id :: STRING AS receiver_id, receipt :receipt :Action :signer_id :: STRING AS signer_id, LOWER( diff --git a/models/silver/streamline/silver__streamline_receipts_final.sql b/models/silver/streamline/silver__streamline_receipts_final.sql index db79add2..f54356ee 100644 --- a/models/silver/streamline/silver__streamline_receipts_final.sql +++ b/models/silver/streamline/silver__streamline_receipts_final.sql @@ -138,7 +138,7 @@ append_tx_hash AS ( FINAL AS ( SELECT tx_hash, - receipt_id AS receipt_object_id, + receipt_id AS receipt_object_id, -- TODO upd this r.block_id, b.block_timestamp, receipt_index, diff --git a/models/silver/streamline/silver__streamline_transactions_final.sql b/models/silver/streamline/silver__streamline_transactions_final.sql index 150ccf97..2b4f8c29 100644 --- a/models/silver/streamline/silver__streamline_transactions_final.sql +++ b/models/silver/streamline/silver__streamline_transactions_final.sql @@ -200,6 +200,8 @@ determine_tx_status AS ( ORDER BY block_id ASC ) AS tx_succeeded + -- TODO review + -- Dojw9TnLbTLTxeJAuGtiSTZGdruCAmdu1KsLgjLHwkkb incorrect FROM int_receipts ), @@ -250,7 +252,7 @@ SELECT signature, tx_receiver, tx_signer, - tx, + tx, -- TODO dropping this gas_used, transaction_fee, attached_gas, From 79b3ba424dae427bfbc13aaac7f9d13343210152 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Fri, 6 Dec 2024 00:47:17 +0700 Subject: [PATCH 02/17] actions model v1 --- models/silver/actions/silver__actions.sql | 87 ++++++++++++++++++++--- 1 file changed, 77 insertions(+), 10 deletions(-) diff --git a/models/silver/actions/silver__actions.sql b/models/silver/actions/silver__actions.sql index 755bc8b1..66160a90 100644 --- a/models/silver/actions/silver__actions.sql +++ b/models/silver/actions/silver__actions.sql @@ -2,9 +2,11 @@ materialized = 'incremental', incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], - unique_key = 'actions_id' + cluster_by = ['block_timestamp::DATE', '_modified_timestamp::DATE'], + unique_key = 'actions_id', + tags = ['actions', 'curated', 'scheduled_core', 'grail'] ) }} --- TODO: add back cluster, SO, incremental predicates, tags +-- TODO: add back SO, incremental predicates WITH transactions AS ( SELECT @@ -12,12 +14,26 @@ WITH transactions AS ( tx_signer, tx_receiver, gas_used AS tx_gas_used, - tx_succeeded + tx_succeeded, + _modified_timestamp FROM {{ ref('silver__streamline_transactions_final') }} WHERE - block_timestamp > CURRENT_DATE - 30 - -- TODO add incremental filter + block_timestamp > CURRENT_DATE - 3 + + {% if var("MANUAL_FIX") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + {% if is_incremental() %} + WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + {% endif %} ), receipts AS ( SELECT @@ -30,15 +46,30 @@ receipts AS ( receipt_actions :predecessor_id :: STRING AS receipt_predecessor_id, receipt_succeeded, gas_burnt AS receipt_gas_burnt, + status_value, receipt_actions, logs AS receipt_logs, _partition_by_block_number, - _inserted_timestamp + _inserted_timestamp, + modified_timestamp AS _modified_timestamp FROM {{ ref('silver__streamline_receipts_final') }} WHERE block_timestamp > CURRENT_DATE - 30 - -- TODO add incremental filter + + {% if var("MANUAL_FIX") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + {% if is_incremental() %} + WHERE _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + {% endif %} ), join_data AS ( SELECT @@ -55,6 +86,7 @@ join_data AS ( r.receipt_predecessor_id, r.receipt_succeeded, r.receipt_gas_burnt, + r.status_value, r.receipt_actions, r.receipt_logs, r._partition_by_block_number, @@ -79,7 +111,24 @@ flatten_actions AS ( receipt_predecessor_id, receipt_succeeded, receipt_gas_burnt, + IFF( + object_keys(status_value)[0] :: STRING = 'SuccessValue', + OBJECT_INSERT( + status_value, + 'SuccessValue', + COALESCE( + TRY_PARSE_JSON( + TRY_BASE64_DECODE_STRING( + GET(status_value, 'SuccessValue') + ) + ), + GET(status_value, 'SuccessValue') + ), + TRUE + ), + status_value) as receipt_status_value, receipt_logs, -- TODO review logs, clean logs? + False AS is_delegated, INDEX AS action_index, receipt_actions :receipt :Action :gas_price :: NUMBER AS action_gas_price, IFF(VALUE = 'CreateAccount', VALUE, object_keys(VALUE) [0] :: STRING) AS action_name, @@ -112,21 +161,35 @@ flatten_actions AS ( LATERAL FLATTEN( receipt_actions :receipt :Action :actions :: ARRAY ) +), +flatten_delegated_actions AS ( + SELECT + tx_hash, + True AS is_delegated, + object_keys(VALUE)[0] ::STRING AS delegated_action_name, + IFF( + VALUE = 'CreateAccount', + {}, + GET(VALUE, object_keys(VALUE) [0] :: STRING) + ) AS delegated_action_data + FROM flatten_actions, LATERAL FLATTEN(action_data :delegate_action :actions :: ARRAY) + WHERE action_name = 'Delegate' ) SELECT block_id, block_timestamp, - tx_hash, + fa.tx_hash, tx_signer, tx_receiver, tx_gas_used, tx_succeeded, - receipt_id, + fa.receipt_id, receipt_receiver_id, receipt_signer_id, receipt_predecessor_id, receipt_succeeded, receipt_gas_burnt, + receipt_status_value, receipt_logs, action_index, action_gas_price, @@ -141,4 +204,8 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - flatten_actions + flatten_actions fa + LEFT JOIN flatten_delegated_actions da + ON fa.tx_hash = da.tx_hash + AND fa.action_name = da.delegated_action_name + AND fa.action_data :: STRING = da.delegated_action_data :: STRING From 81e3c7189b24b3ddb63c04ccf3431da0d9e965d2 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Fri, 6 Dec 2024 18:14:19 +0700 Subject: [PATCH 03/17] added yml file and completed the ez_actions silver model --- models/silver/actions/silver__actions.sql | 12 ++- models/silver/actions/silver__actions.yml | 109 ++++++++++++++++++++++ 2 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 models/silver/actions/silver__actions.yml diff --git a/models/silver/actions/silver__actions.sql b/models/silver/actions/silver__actions.sql index 66160a90..2298a8fe 100644 --- a/models/silver/actions/silver__actions.sql +++ b/models/silver/actions/silver__actions.sql @@ -2,8 +2,10 @@ materialized = 'incremental', incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], cluster_by = ['block_timestamp::DATE', '_modified_timestamp::DATE'], unique_key = 'actions_id', + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receipt_receiver_id,receipt_signer_id,receipt_predecessor_id,actions_id);", tags = ['actions', 'curated', 'scheduled_core', 'grail'] ) }} -- TODO: add back SO, incremental predicates @@ -179,22 +181,22 @@ SELECT block_id, block_timestamp, fa.tx_hash, - tx_signer, + tx_succeeded, tx_receiver, + tx_signer, tx_gas_used, - tx_succeeded, fa.receipt_id, + receipt_predecessor_id, receipt_receiver_id, receipt_signer_id, - receipt_predecessor_id, receipt_succeeded, receipt_gas_burnt, receipt_status_value, - receipt_logs, action_index, - action_gas_price, action_name, action_data_parsed AS action_data, + action_gas_price, + receipt_logs, _partition_by_block_number, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( diff --git a/models/silver/actions/silver__actions.yml b/models/silver/actions/silver__actions.yml new file mode 100644 index 00000000..26aefd2a --- /dev/null +++ b/models/silver/actions/silver__actions.yml @@ -0,0 +1,109 @@ +version: 2 + +models: + - name: silver__actions + description: |- + This table extracts and processes all action events from NEAR trransactions and receipts - storing the argument data under action_data. It combines transaction and receipt information to provide a comprehensive view on each of the actions events. + + columns: + - name: BLOCK_ID + description: "{{ doc('block_id')}}" + tests: + - not_null: + where: _inserted_timestamp <= current_timestamp - interval '1 hour + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp')}}" + tests: + - not_null: + where: _inserted_timestamp <= current_timestamp - interval '1 hour' + + - name: TX_HASH + description: "{{ doc('tx_hash')}}" + tests: + - not_null: + where: _inserted_timestamp <= current_timestamp - interval '1 hour' + + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - not_null: + where: _inserted_timestamp <= current_timestamp - interval '1 hour' + + - name: TX_RECEIVER + description: "{{ doc('tx_receiver')}}" + + - name: TX_SIGNER + description: "{{ doc('tx_signer')}}" + + - name: TX_GAS_USED + description: "{{ doc('gas_used')}}" + tests: + - not_null: + where: _inserted_timestamp <= current_timestamp - interval '1 hour' + + - name: RECEIPT_ID + description: "{{ doc('receipt_object_id')}}" + + - name: RECEIPT_PREDECESSOR_ID + description: "{{ doc('predecessor_id')}}" + + - name: RECEIPT_RECEIVER_ID + description: "{{ doc('receiver_id')}}" + + - name: RECEIPT_SIGNER_ID + description: "{{ doc('signer_id')}}" + + - name: RECEIPT_SUCCEEDED + description: "{{ doc('receipt_succeeded')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - not_null: + where: _inserted_timestamp <= current_timestamp - interval '1 hour' + + - name: RECEIPT_GAS_BURNT + description: "{{ doc('gas_burnt')}}" + + - name: RECEIPT_STATUS_VALUE + description: "{{ doc('status_value')}}" + + - name: ACTION_INDEX + description: "{{ doc('action_index')}}" + + - name: ACTION_NAME + description: "{{ doc('action_name')}}" + + - name: ACTION_DATA + description: "{{ doc('action_data')}}" + + - name: ACTION_GAS_PRICE + description: "{{ doc('action_gas_price')}}" + + - name: RECEIPT_LOGS + description: "{{ doc('logs')}}" + + - name: _PARTITION_BY_BLOCK_NUMBER + description: "{{ doc('_partition_by_block_number')}}" + + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp')}}" + + - name: ACTIONS_ID + description: "{{ doc('action_id')}}" + tests: + - unique + - not_null + + - name: INSERTED_TIMESTAMP + description: "{{doc('inserted_timestamp')}}" + + - name: MODIFIED_TIMESTAMP + description: "{{doc('modified_timestamp')}}" + + - name: _INVOCATION_ID + description: "{{doc('invocation_id')}}" \ No newline at end of file From eed9a0e0c175a5c02fa2da69f24110babd7bc160 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Sat, 7 Dec 2024 01:52:30 +0700 Subject: [PATCH 04/17] added the gold model --- models/descriptions/action_gas_price.md | 5 + models/gold/core/core__fact_actions.sql | 38 +++++ models/gold/core/core__fact_actions.yml | 191 ++++++++++++++++++++++ models/silver/actions/silver__actions.sql | 38 ++--- models/silver/actions/silver__actions.yml | 5 +- 5 files changed, 256 insertions(+), 21 deletions(-) create mode 100644 models/descriptions/action_gas_price.md create mode 100644 models/gold/core/core__fact_actions.sql create mode 100644 models/gold/core/core__fact_actions.yml diff --git a/models/descriptions/action_gas_price.md b/models/descriptions/action_gas_price.md new file mode 100644 index 00000000..6541d273 --- /dev/null +++ b/models/descriptions/action_gas_price.md @@ -0,0 +1,5 @@ +{% docs action_gas_price %} + +Gas price parsed from the receipt_actions JSON object, specifically from the nested path receipt:Action:gas_price. Used to determine the cost per unit of gas for the specific action. + +{% enddocs %} diff --git a/models/gold/core/core__fact_actions.sql b/models/gold/core/core__fact_actions.sql new file mode 100644 index 00000000..ccd6f617 --- /dev/null +++ b/models/gold/core/core__fact_actions.sql @@ -0,0 +1,38 @@ +{{ config( + materialized = 'view', + secure = false, + tags = ['core'] +) }} + +WITH actions AS ( + + SELECT + * + FROM + {{ ref('silver__actions') }} +) +SELECT + block_id, + block_timestamp, + fa.tx_hash, + tx_succeeded, + tx_receiver, + tx_signer, + tx_gas_used, + fa.receipt_id, + receipt_predecessor_id, + receipt_receiver_id, + receipt_signer_id, + receipt_succeeded, + receipt_gas_burnt, + receipt_status_value, + action_index, + action_name, + action_data_parsed AS action_data, + action_gas_price, + receipt_logs, + actions_id AS fact_actions_id, + inserted_timestamp, + modified_timestamp +FROM + actions \ No newline at end of file diff --git a/models/gold/core/core__fact_actions.yml b/models/gold/core/core__fact_actions.yml new file mode 100644 index 00000000..da3f3e68 --- /dev/null +++ b/models/gold/core/core__fact_actions.yml @@ -0,0 +1,191 @@ +version: 2 + +models: + - name: core__fact_actions + description: |- + This table extracts all actions from a transaction and stores the argument data under action_data. + tests: + - dbt_utils.recency: + datepart: hours + field: block_timestamp + interval: 2 + + columns: + - name: BLOCK_ID + description: "{{ doc('block_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - not_null: + where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + + - name: TX_HASH + description: "{{ doc('tx_hash')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - not_null: + where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + + - name: TX_SUCCEEDED + description: "{{ doc('tx_succeeded')}}" + tests: + - not_null: + where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + + - name: TX_RECEIVER + description: "{{ doc('tx_receiver')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: TX_SIGNER + description: "{{ doc('tx_signer')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: TX_GAS_USED + description: "{{ doc('gas_used')}}" + tests: + - not_null: + where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: RECEIPT_ID + description: "{{ doc('receipt_object_id')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: RECEIPT_PREDECESSOR_ID + description: "{{ doc('predecessor_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: RECEIPT_RECEIVER_ID + description: "{{ doc('receiver_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: RECEIPT_SIGNER_ID + description: "{{ doc('signer_id')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: RECEIPT_SUCCEEDED + description: "{{ doc('receipt_succeeded')}}" + tests: + - not_null: + where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + + - name: RECEIPT_GAS_BURNT + description: "{{ doc('gas_burnt')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - DOUBLE + - FLOAT + + - name: RECEIPT_STATUS_VALUE + description: "{{ doc('status_value')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT + + - name: ACTION_INDEX + description: "{{ doc('action_index')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + + - name: ACTION_NAME + description: "{{ doc('action_name')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: ACTION_DATA + description: "{{ doc('action_data')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - OBJECT + - VARIANT + + - name: ACTION_GAS_PRICE + description: "{{ doc('action_gas_price')}}" + tests: + - not_null: + where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: RECEIPT_LOGS + description: "{{ doc('logs')}}" + + - name: FACT_ACTIONS_ID + description: "{{ doc('id')}}" + tests: + - unique: + where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' AND FACT_ACTIONS_EVENTS_ID != 'cf646ad92e6df243ffabf07c47c0f2c1' + - not_null: + where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp')}}" + + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp')}}" \ No newline at end of file diff --git a/models/silver/actions/silver__actions.sql b/models/silver/actions/silver__actions.sql index 2298a8fe..123fe541 100644 --- a/models/silver/actions/silver__actions.sql +++ b/models/silver/actions/silver__actions.sql @@ -17,7 +17,7 @@ WITH transactions AS ( tx_receiver, gas_used AS tx_gas_used, tx_succeeded, - _modified_timestamp + modified_timestamp AS _modified_timestamp FROM {{ ref('silver__streamline_transactions_final') }} WHERE @@ -27,15 +27,15 @@ WITH transactions AS ( WHERE {{ partition_load_manual('no_buffer') }} {% else %} - {% if is_incremental() %} - WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} +{% if is_incremental() %} +WHERE _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) +{% endif %} +{% endif %} ), receipts AS ( SELECT @@ -63,15 +63,15 @@ receipts AS ( WHERE {{ partition_load_manual('no_buffer') }} {% else %} - {% if is_incremental() %} - WHERE _modified_timestamp >= ( - SELECT - MAX(_modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} +{% if is_incremental() %} +WHERE _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) +{% endif %} +{% endif %} ), join_data AS ( SELECT diff --git a/models/silver/actions/silver__actions.yml b/models/silver/actions/silver__actions.yml index 26aefd2a..1866331b 100644 --- a/models/silver/actions/silver__actions.yml +++ b/models/silver/actions/silver__actions.yml @@ -3,7 +3,7 @@ version: 2 models: - name: silver__actions description: |- - This table extracts and processes all action events from NEAR trransactions and receipts - storing the argument data under action_data. It combines transaction and receipt information to provide a comprehensive view on each of the actions events. + This table extracts and processes all action events from NEAR transactions and receipts - storing the argument data under action_data. It combines transaction and receipt information to provide a comprehensive view on each of the actions events. columns: - name: BLOCK_ID @@ -96,7 +96,8 @@ models: - name: ACTIONS_ID description: "{{ doc('action_id')}}" tests: - - unique + - unique: + where: tx_hash != 'J4CZZQrZK6kYPVLkrdbTEpcqhUNZiRxktbMzHviqeGgf' - not_null - name: INSERTED_TIMESTAMP From 8c442c61fdb73b200828c4c89c3de4ffc619c748 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Mon, 9 Dec 2024 18:02:37 -0700 Subject: [PATCH 05/17] rm comments --- models/silver/streamline/silver__streamline_receipts.sql | 3 +-- .../streamline/silver__streamline_transactions_final.sql | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/models/silver/streamline/silver__streamline_receipts.sql b/models/silver/streamline/silver__streamline_receipts.sql index 3182a5ae..55030976 100644 --- a/models/silver/streamline/silver__streamline_receipts.sql +++ b/models/silver/streamline/silver__streamline_receipts.sql @@ -37,7 +37,7 @@ WITH shards AS ( {% endif %} ), flatten_receipts AS ( - -- TODO review new keys like priority, input_data_ids, output_data_receivers + SELECT concat_ws( '-', @@ -95,7 +95,6 @@ FINAL AS ( ) AS error_type_2, failure_message [error_type_0] :kind [error_type_1] [error_type_2] :: STRING AS error_message, execution_outcome :outcome :receipt_ids :: ARRAY AS outcome_receipts, - -- TODO extract predecessor_id receipt :receiver_id :: STRING AS receiver_id, receipt :receipt :Action :signer_id :: STRING AS signer_id, LOWER( diff --git a/models/silver/streamline/silver__streamline_transactions_final.sql b/models/silver/streamline/silver__streamline_transactions_final.sql index 2b4f8c29..150ccf97 100644 --- a/models/silver/streamline/silver__streamline_transactions_final.sql +++ b/models/silver/streamline/silver__streamline_transactions_final.sql @@ -200,8 +200,6 @@ determine_tx_status AS ( ORDER BY block_id ASC ) AS tx_succeeded - -- TODO review - -- Dojw9TnLbTLTxeJAuGtiSTZGdruCAmdu1KsLgjLHwkkb incorrect FROM int_receipts ), @@ -252,7 +250,7 @@ SELECT signature, tx_receiver, tx_signer, - tx, -- TODO dropping this + tx, gas_used, transaction_fee, attached_gas, From 8e505d8aa4b9ab1e0d20cb29deb05771c393d0c7 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Mon, 9 Dec 2024 18:03:05 -0700 Subject: [PATCH 06/17] rm comment --- models/silver/streamline/silver__streamline_receipts_final.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/streamline/silver__streamline_receipts_final.sql b/models/silver/streamline/silver__streamline_receipts_final.sql index f54356ee..db79add2 100644 --- a/models/silver/streamline/silver__streamline_receipts_final.sql +++ b/models/silver/streamline/silver__streamline_receipts_final.sql @@ -138,7 +138,7 @@ append_tx_hash AS ( FINAL AS ( SELECT tx_hash, - receipt_id AS receipt_object_id, -- TODO upd this + receipt_id AS receipt_object_id, r.block_id, b.block_timestamp, receipt_index, From b9b3a1ee2763b6f21349bfa00e6bce415cf1c566 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Mon, 9 Dec 2024 18:13:11 -0700 Subject: [PATCH 07/17] upd silver actions model --- models/silver/actions/silver__actions.sql | 34 ++++++++--------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/models/silver/actions/silver__actions.sql b/models/silver/actions/silver__actions.sql index 123fe541..e85ef444 100644 --- a/models/silver/actions/silver__actions.sql +++ b/models/silver/actions/silver__actions.sql @@ -3,12 +3,12 @@ incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], - cluster_by = ['block_timestamp::DATE', '_modified_timestamp::DATE'], + cluster_by = ['block_timestamp::DATE', 'block_id'], unique_key = 'actions_id', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receipt_receiver_id,receipt_signer_id,receipt_predecessor_id,actions_id);", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receipt_receiver_id,receipt_signer_id,receipt_predecessor_id);", tags = ['actions', 'curated', 'scheduled_core', 'grail'] ) }} --- TODO: add back SO, incremental predicates + WITH transactions AS ( SELECT @@ -16,21 +16,18 @@ WITH transactions AS ( tx_signer, tx_receiver, gas_used AS tx_gas_used, - tx_succeeded, - modified_timestamp AS _modified_timestamp + tx_succeeded FROM {{ ref('silver__streamline_transactions_final') }} - WHERE - block_timestamp > CURRENT_DATE - 3 {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} {% if is_incremental() %} -WHERE _modified_timestamp >= ( +WHERE modified_timestamp >= ( SELECT - MAX(_modified_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -50,23 +47,19 @@ receipts AS ( gas_burnt AS receipt_gas_burnt, status_value, receipt_actions, - logs AS receipt_logs, _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp + _inserted_timestamp FROM {{ ref('silver__streamline_receipts_final') }} - WHERE - block_timestamp > CURRENT_DATE - 30 {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} {% if is_incremental() %} -WHERE _modified_timestamp >= ( +WHERE modified_timestamp >= ( SELECT - MAX(_modified_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -90,13 +83,12 @@ join_data AS ( r.receipt_gas_burnt, r.status_value, r.receipt_actions, - r.receipt_logs, r._partition_by_block_number, r._inserted_timestamp FROM - transactions t - LEFT JOIN receipts r - ON t.tx_hash = r.tx_hash + receipts r + LEFT JOIN transactions t + ON r.tx_hash = t.tx_hash ), flatten_actions AS ( SELECT @@ -129,7 +121,6 @@ flatten_actions AS ( TRUE ), status_value) as receipt_status_value, - receipt_logs, -- TODO review logs, clean logs? False AS is_delegated, INDEX AS action_index, receipt_actions :receipt :Action :gas_price :: NUMBER AS action_gas_price, @@ -196,7 +187,6 @@ SELECT action_name, action_data_parsed AS action_data, action_gas_price, - receipt_logs, _partition_by_block_number, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( From b03a5076956157d2d0d0871e58bc5c1ea17c9797 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Mon, 9 Dec 2024 21:30:46 -0700 Subject: [PATCH 08/17] add missing is_delegated --- models/silver/actions/silver__actions.sql | 90 ++++++++++++++--------- 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/models/silver/actions/silver__actions.sql b/models/silver/actions/silver__actions.sql index e85ef444..cb08947d 100644 --- a/models/silver/actions/silver__actions.sql +++ b/models/silver/actions/silver__actions.sql @@ -24,15 +24,15 @@ WITH transactions AS ( WHERE {{ partition_load_manual('no_buffer') }} {% else %} -{% if is_incremental() %} -WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) -{% endif %} -{% endif %} + {% if is_incremental() %} + WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + {% endif %} ), receipts AS ( SELECT @@ -56,15 +56,15 @@ receipts AS ( WHERE {{ partition_load_manual('no_buffer') }} {% else %} -{% if is_incremental() %} -WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) -{% endif %} -{% endif %} + {% if is_incremental() %} + WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + {% endif %} ), join_data AS ( SELECT @@ -108,28 +108,41 @@ flatten_actions AS ( IFF( object_keys(status_value)[0] :: STRING = 'SuccessValue', OBJECT_INSERT( - status_value, - 'SuccessValue', - COALESCE( - TRY_PARSE_JSON( - TRY_BASE64_DECODE_STRING( - GET(status_value, 'SuccessValue') - ) - ), - GET(status_value, 'SuccessValue') - ), - TRUE - ), - status_value) as receipt_status_value, + status_value, + 'SuccessValue', + COALESCE( + TRY_PARSE_JSON( + TRY_BASE64_DECODE_STRING( + GET(status_value, 'SuccessValue') + ) + ), + GET(status_value, 'SuccessValue') + ), + TRUE + ), + status_value + ) as receipt_status_value, False AS is_delegated, INDEX AS action_index, receipt_actions :receipt :Action :gas_price :: NUMBER AS action_gas_price, - IFF(VALUE = 'CreateAccount', VALUE, object_keys(VALUE) [0] :: STRING) AS action_name, + IFF( + VALUE = 'CreateAccount', + VALUE, + object_keys(VALUE) [0] :: STRING + ) AS action_name, IFF( VALUE = 'CreateAccount', {}, GET(VALUE, object_keys(VALUE) [0] :: STRING) ) AS action_data, + MD5( + CONCAT( + action_data:args::STRING, ',', + action_data:deposit::STRING, ',', + action_data:gas::STRING, ',', + action_data:method_name::STRING + ) + ) AS action_hash, IFF( action_name = 'FunctionCall', OBJECT_INSERT( @@ -164,7 +177,15 @@ flatten_delegated_actions AS ( VALUE = 'CreateAccount', {}, GET(VALUE, object_keys(VALUE) [0] :: STRING) - ) AS delegated_action_data + ) AS delegated_action_data, + MD5( + CONCAT( + delegated_action_data:args::STRING, ',', + delegated_action_data:deposit::STRING, ',', + delegated_action_data:gas::STRING, ',', + delegated_action_data:method_name::STRING + ) + ) AS delegated_action_hash FROM flatten_actions, LATERAL FLATTEN(action_data :delegate_action :actions :: ARRAY) WHERE action_name = 'Delegate' ) @@ -184,6 +205,7 @@ SELECT receipt_gas_burnt, receipt_status_value, action_index, + COALESCE(da.is_delegated, fa.is_delegated) AS is_delegated, action_name, action_data_parsed AS action_data, action_gas_price, @@ -200,4 +222,4 @@ FROM LEFT JOIN flatten_delegated_actions da ON fa.tx_hash = da.tx_hash AND fa.action_name = da.delegated_action_name - AND fa.action_data :: STRING = da.delegated_action_data :: STRING + AND fa.action_hash = da.delegated_action_hash From 1ab58ef06d0867e2c90aa19ccd4c7c004a648000 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Mon, 9 Dec 2024 22:19:38 -0700 Subject: [PATCH 09/17] add more tests to silver --- models/silver/actions/silver__actions.sql | 7 ++- models/silver/actions/silver__actions.yml | 73 ++++++++++++++++------- 2 files changed, 58 insertions(+), 22 deletions(-) diff --git a/models/silver/actions/silver__actions.sql b/models/silver/actions/silver__actions.sql index cb08947d..6737adf9 100644 --- a/models/silver/actions/silver__actions.sql +++ b/models/silver/actions/silver__actions.sql @@ -70,7 +70,7 @@ join_data AS ( SELECT r.block_id, r.block_timestamp, - t.tx_hash, + r.tx_hash, t.tx_signer, t.tx_receiver, t.tx_gas_used, @@ -205,7 +205,10 @@ SELECT receipt_gas_burnt, receipt_status_value, action_index, - COALESCE(da.is_delegated, fa.is_delegated) AS is_delegated, + COALESCE( + da.is_delegated, + fa.is_delegated + ) AS is_delegated, action_name, action_data_parsed AS action_data, action_gas_price, diff --git a/models/silver/actions/silver__actions.yml b/models/silver/actions/silver__actions.yml index 1866331b..947a1b78 100644 --- a/models/silver/actions/silver__actions.yml +++ b/models/silver/actions/silver__actions.yml @@ -4,25 +4,27 @@ models: - name: silver__actions description: |- This table extracts and processes all action events from NEAR transactions and receipts - storing the argument data under action_data. It combines transaction and receipt information to provide a comprehensive view on each of the actions events. + tests: + - dbt_utils.recency: + datepart: hours + field: block_timestamp + interval: 2 columns: - name: BLOCK_ID description: "{{ doc('block_id')}}" tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour + - not_null - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp')}}" tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' + - not_null - name: TX_HASH description: "{{ doc('tx_hash')}}" tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' + - not_null - name: TX_SUCCEEDED description: "{{ doc('tx_succeeded')}}" @@ -30,32 +32,45 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - BOOLEAN - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - name: TX_RECEIVER description: "{{ doc('tx_receiver')}}" - + tests: + - not_null + - name: TX_SIGNER description: "{{ doc('tx_signer')}}" - + tests: + - not_null + - name: TX_GAS_USED description: "{{ doc('gas_used')}}" tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - INTEGER - name: RECEIPT_ID description: "{{ doc('receipt_object_id')}}" + tests: + - not_null - name: RECEIPT_PREDECESSOR_ID description: "{{ doc('predecessor_id')}}" + tests: + - not_null - name: RECEIPT_RECEIVER_ID description: "{{ doc('receiver_id')}}" + tests: + - not_null - name: RECEIPT_SIGNER_ID description: "{{ doc('signer_id')}}" + tests: + - not_null - name: RECEIPT_SUCCEEDED description: "{{ doc('receipt_succeeded')}}" @@ -63,29 +78,47 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - BOOLEAN - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' + - not_null - name: RECEIPT_GAS_BURNT description: "{{ doc('gas_burnt')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - INTEGER - name: RECEIPT_STATUS_VALUE description: "{{ doc('status_value')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT - name: ACTION_INDEX description: "{{ doc('action_index')}}" - + tests: + - not_null + - name: ACTION_NAME description: "{{ doc('action_name')}}" - + tests: + - not_null + - name: ACTION_DATA description: "{{ doc('action_data')}}" + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - OBJECT - name: ACTION_GAS_PRICE description: "{{ doc('action_gas_price')}}" - - - name: RECEIPT_LOGS - description: "{{ doc('logs')}}" + tests: + - not_null - name: _PARTITION_BY_BLOCK_NUMBER description: "{{ doc('_partition_by_block_number')}}" @@ -107,4 +140,4 @@ models: description: "{{doc('modified_timestamp')}}" - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" \ No newline at end of file + description: "{{doc('invocation_id')}}" From dd0210190d4515aed659d81a4d54404c8151acbb Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Wed, 11 Dec 2024 01:25:09 +0700 Subject: [PATCH 10/17] proposed non-unique actions_id fix --- models/silver/actions/silver__actions.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/silver/actions/silver__actions.sql b/models/silver/actions/silver__actions.sql index 6737adf9..7f9f08dd 100644 --- a/models/silver/actions/silver__actions.sql +++ b/models/silver/actions/silver__actions.sql @@ -172,6 +172,7 @@ flatten_delegated_actions AS ( SELECT tx_hash, True AS is_delegated, + INDEX AS delegated_action_index, object_keys(VALUE)[0] ::STRING AS delegated_action_name, IFF( VALUE = 'CreateAccount', @@ -226,3 +227,4 @@ FROM ON fa.tx_hash = da.tx_hash AND fa.action_name = da.delegated_action_name AND fa.action_hash = da.delegated_action_hash + AND fa.action_index = da.delegated_action_index \ No newline at end of file From 6b58b6a6eb6dfdd1c85049532d2ee45884b8e703 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:23:46 -0700 Subject: [PATCH 11/17] rn fact to ez, expand action hash to case stmt --- models/descriptions/is_delegated.md | 5 ++ ..._fact_actions.sql => core__ez_actions.sql} | 17 ++-- ..._fact_actions.yml => core__ez_actions.yml} | 31 ++++--- models/silver/actions/silver__actions.sql | 86 ++++++++++++++++--- models/silver/actions/silver__actions.yml | 8 ++ 5 files changed, 108 insertions(+), 39 deletions(-) create mode 100644 models/descriptions/is_delegated.md rename models/gold/core/{core__fact_actions.sql => core__ez_actions.sql} (71%) rename models/gold/core/{core__fact_actions.yml => core__ez_actions.yml} (82%) diff --git a/models/descriptions/is_delegated.md b/models/descriptions/is_delegated.md new file mode 100644 index 00000000..e06a73ad --- /dev/null +++ b/models/descriptions/is_delegated.md @@ -0,0 +1,5 @@ +{% docs is_delegated %} + +This column is a boolean value that indicates whether the executed action is a delegated action. + +{% enddocs %} diff --git a/models/gold/core/core__fact_actions.sql b/models/gold/core/core__ez_actions.sql similarity index 71% rename from models/gold/core/core__fact_actions.sql rename to models/gold/core/core__ez_actions.sql index ccd6f617..0763538b 100644 --- a/models/gold/core/core__fact_actions.sql +++ b/models/gold/core/core__ez_actions.sql @@ -4,22 +4,15 @@ tags = ['core'] ) }} -WITH actions AS ( - - SELECT - * - FROM - {{ ref('silver__actions') }} -) SELECT block_id, block_timestamp, - fa.tx_hash, + tx_hash, tx_succeeded, tx_receiver, tx_signer, tx_gas_used, - fa.receipt_id, + receipt_id, receipt_predecessor_id, receipt_receiver_id, receipt_signer_id, @@ -27,12 +20,12 @@ SELECT receipt_gas_burnt, receipt_status_value, action_index, + is_delegated, action_name, - action_data_parsed AS action_data, + action_data, action_gas_price, - receipt_logs, actions_id AS fact_actions_id, inserted_timestamp, modified_timestamp FROM - actions \ No newline at end of file + {{ ref('silver__actions') }} diff --git a/models/gold/core/core__fact_actions.yml b/models/gold/core/core__ez_actions.yml similarity index 82% rename from models/gold/core/core__fact_actions.yml rename to models/gold/core/core__ez_actions.yml index da3f3e68..fea5bd99 100644 --- a/models/gold/core/core__fact_actions.yml +++ b/models/gold/core/core__ez_actions.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: core__fact_actions + - name: core__ez_actions description: |- This table extracts all actions from a transaction and stores the argument data under action_data. tests: @@ -27,7 +27,7 @@ models: column_type_list: - TIMESTAMP_NTZ - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: TX_HASH description: "{{ doc('tx_hash')}}" @@ -37,13 +37,13 @@ models: - STRING - VARCHAR - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: TX_SUCCEEDED description: "{{ doc('tx_succeeded')}}" tests: - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - BOOLEAN @@ -70,7 +70,7 @@ models: description: "{{ doc('gas_used')}}" tests: - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER @@ -115,7 +115,7 @@ models: description: "{{ doc('receipt_succeeded')}}" tests: - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - BOOLEAN @@ -146,6 +146,14 @@ models: column_type_list: - NUMBER + - name: IS_DELEGATED + description: "{{ doc('is_delegated')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: ACTION_NAME description: "{{ doc('action_name')}}" tests: @@ -167,25 +175,22 @@ models: description: "{{ doc('action_gas_price')}}" tests: - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - FLOAT - - name: RECEIPT_LOGS - description: "{{ doc('logs')}}" - - name: FACT_ACTIONS_ID description: "{{ doc('id')}}" tests: - unique: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' AND FACT_ACTIONS_EVENTS_ID != 'cf646ad92e6df243ffabf07c47c0f2c1' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' AND FACT_ACTIONS_EVENTS_ID != 'cf646ad92e6df243ffabf07c47c0f2c1' - not_null: - where: inserted_timestamp BETWEEN SYSDATE() - INTERVAL '7 days' AND SYSDATE() - INTERVAL '2 hours' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: INSERTED_TIMESTAMP description: "{{ doc('inserted_timestamp')}}" - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp')}}" \ No newline at end of file + description: "{{ doc('modified_timestamp')}}" diff --git a/models/silver/actions/silver__actions.sql b/models/silver/actions/silver__actions.sql index 7f9f08dd..5e16413e 100644 --- a/models/silver/actions/silver__actions.sql +++ b/models/silver/actions/silver__actions.sql @@ -136,12 +136,39 @@ flatten_actions AS ( GET(VALUE, object_keys(VALUE) [0] :: STRING) ) AS action_data, MD5( - CONCAT( - action_data:args::STRING, ',', - action_data:deposit::STRING, ',', - action_data:gas::STRING, ',', - action_data:method_name::STRING - ) + CASE action_name + WHEN 'FunctionCall' THEN + CONCAT_WS(',', + action_data :args :: STRING, + action_data :deposit :: STRING, + action_data :gas :: STRING, + action_data :method_name :: STRING + ) + WHEN 'AddKey' THEN + CONCAT_WS(',', + action_data :access_key :nonce :: STRING, + action_data :access_key :permission :: STRING, + action_data :public_key :: STRING + ) + WHEN 'DeleteKey' THEN + action_data :public_key :: STRING + WHEN 'CreateAccount' THEN + 'empty' -- consistent hash for empty objects + WHEN 'DeleteAccount' THEN + action_data :beneficiary_id :: STRING + WHEN 'DeployContract' THEN + action_data :code :: STRING + WHEN 'Transfer' THEN + action_data :deposit :: STRING + WHEN 'Stake' THEN + CONCAT_WS(',', + action_data :public_key :: STRING, + action_data :stake :: STRING + ) + ELSE + -- Fallback: convert entire variant to string + action_data :: STRING + END ) AS action_hash, IFF( action_name = 'FunctionCall', @@ -173,19 +200,50 @@ flatten_delegated_actions AS ( tx_hash, True AS is_delegated, INDEX AS delegated_action_index, - object_keys(VALUE)[0] ::STRING AS delegated_action_name, + IFF( + VALUE = 'CreateAccount', + VALUE, + object_keys(VALUE) [0] :: STRING + ) AS delegated_action_name, IFF( VALUE = 'CreateAccount', {}, GET(VALUE, object_keys(VALUE) [0] :: STRING) ) AS delegated_action_data, MD5( - CONCAT( - delegated_action_data:args::STRING, ',', - delegated_action_data:deposit::STRING, ',', - delegated_action_data:gas::STRING, ',', - delegated_action_data:method_name::STRING - ) + CASE delegated_action_name + WHEN 'FunctionCall' THEN + CONCAT_WS(',', + delegated_action_data :args :: STRING, + delegated_action_data :deposit :: STRING, + delegated_action_data :gas :: STRING, + delegated_action_data :method_name :: STRING + ) + WHEN 'AddKey' THEN + CONCAT_WS(',', + delegated_action_data :access_key :nonce :: STRING, + delegated_action_data :access_key :permission :: STRING, + delegated_action_data :public_key :: STRING + ) + WHEN 'DeleteKey' THEN + delegated_action_data :public_key :: STRING + WHEN 'CreateAccount' THEN + 'empty' + WHEN 'DeleteAccount' THEN + delegated_action_data :beneficiary_id :: STRING + WHEN 'DeployContract' THEN + delegated_action_data :code :: STRING + WHEN 'Transfer' THEN + delegated_action_data :deposit :: STRING + WHEN 'Stake' THEN + CONCAT_WS(',', + delegated_action_data :public_key :: STRING, + delegated_action_data :stake :: STRING + ) + ELSE + -- Fallback: convert entire variant to string + delegated_action_data :: STRING + END ) AS delegated_action_hash FROM flatten_actions, LATERAL FLATTEN(action_data :delegate_action :actions :: ARRAY) WHERE action_name = 'Delegate' @@ -227,4 +285,4 @@ FROM ON fa.tx_hash = da.tx_hash AND fa.action_name = da.delegated_action_name AND fa.action_hash = da.delegated_action_hash - AND fa.action_index = da.delegated_action_index \ No newline at end of file + AND fa.action_index = da.delegated_action_index diff --git a/models/silver/actions/silver__actions.yml b/models/silver/actions/silver__actions.yml index 947a1b78..1d9e1100 100644 --- a/models/silver/actions/silver__actions.yml +++ b/models/silver/actions/silver__actions.yml @@ -102,6 +102,14 @@ models: tests: - not_null + - name: IS_DELEGATED + description: "{{ doc('is_delegated')}}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: ACTION_NAME description: "{{ doc('action_name')}}" tests: From 8b9e1de24232eb33a70deb0e7e9f2f8d331f9487 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Wed, 11 Dec 2024 11:02:48 +0700 Subject: [PATCH 12/17] add a short note to docs for clarity --- models/descriptions/is_delegated.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/descriptions/is_delegated.md b/models/descriptions/is_delegated.md index e06a73ad..f49c940f 100644 --- a/models/descriptions/is_delegated.md +++ b/models/descriptions/is_delegated.md @@ -1,5 +1,5 @@ {% docs is_delegated %} -This column is a boolean value that indicates whether the executed action is a delegated action. +This column is a boolean value that indicates whether the executed action is a delegated action. Note: Actions with action_name = 'Delegate' themselves are marked as FALSE, while the actions within them are marked as TRUE. {% enddocs %} From ebb22ba24989a5fb4fde151669c90c8eb4ea3c68 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:39:41 -0700 Subject: [PATCH 13/17] actions -> gold --- models/gold/core/core__ez_actions.sql | 279 ++++++++++++++++++++- models/gold/core/core__ez_actions.yml | 98 +++----- models/silver/actions/silver__actions.sql | 288 ---------------------- models/silver/actions/silver__actions.yml | 151 ------------ 4 files changed, 300 insertions(+), 516 deletions(-) delete mode 100644 models/silver/actions/silver__actions.sql delete mode 100644 models/silver/actions/silver__actions.yml diff --git a/models/gold/core/core__ez_actions.sql b/models/gold/core/core__ez_actions.sql index 0763538b..5e16413e 100644 --- a/models/gold/core/core__ez_actions.sql +++ b/models/gold/core/core__ez_actions.sql @@ -1,18 +1,262 @@ {{ config( - materialized = 'view', - secure = false, - tags = ['core'] + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + cluster_by = ['block_timestamp::DATE', 'block_id'], + unique_key = 'actions_id', + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receipt_receiver_id,receipt_signer_id,receipt_predecessor_id);", + tags = ['actions', 'curated', 'scheduled_core', 'grail'] ) }} +WITH transactions AS ( + + SELECT + tx_hash, + tx_signer, + tx_receiver, + gas_used AS tx_gas_used, + tx_succeeded + FROM + {{ ref('silver__streamline_transactions_final') }} + + {% if var("MANUAL_FIX") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + {% if is_incremental() %} + WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + {% endif %} +), +receipts AS ( + SELECT + tx_hash, + block_id, + block_timestamp, + receipt_object_id AS receipt_id, + receiver_id AS receipt_receiver_id, + signer_id AS receipt_signer_id, + receipt_actions :predecessor_id :: STRING AS receipt_predecessor_id, + receipt_succeeded, + gas_burnt AS receipt_gas_burnt, + status_value, + receipt_actions, + _partition_by_block_number, + _inserted_timestamp + FROM + {{ ref('silver__streamline_receipts_final') }} + + {% if var("MANUAL_FIX") %} + WHERE + {{ partition_load_manual('no_buffer') }} + {% else %} + {% if is_incremental() %} + WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + {% endif %} +), +join_data AS ( + SELECT + r.block_id, + r.block_timestamp, + r.tx_hash, + t.tx_signer, + t.tx_receiver, + t.tx_gas_used, + t.tx_succeeded, + r.receipt_id, + r.receipt_receiver_id, + r.receipt_signer_id, + r.receipt_predecessor_id, + r.receipt_succeeded, + r.receipt_gas_burnt, + r.status_value, + r.receipt_actions, + r._partition_by_block_number, + r._inserted_timestamp + FROM + receipts r + LEFT JOIN transactions t + ON r.tx_hash = t.tx_hash +), +flatten_actions AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + tx_signer, + tx_receiver, + tx_gas_used, + tx_succeeded, + receipt_id, + receipt_receiver_id, + receipt_signer_id, + receipt_predecessor_id, + receipt_succeeded, + receipt_gas_burnt, + IFF( + object_keys(status_value)[0] :: STRING = 'SuccessValue', + OBJECT_INSERT( + status_value, + 'SuccessValue', + COALESCE( + TRY_PARSE_JSON( + TRY_BASE64_DECODE_STRING( + GET(status_value, 'SuccessValue') + ) + ), + GET(status_value, 'SuccessValue') + ), + TRUE + ), + status_value + ) as receipt_status_value, + False AS is_delegated, + INDEX AS action_index, + receipt_actions :receipt :Action :gas_price :: NUMBER AS action_gas_price, + IFF( + VALUE = 'CreateAccount', + VALUE, + object_keys(VALUE) [0] :: STRING + ) AS action_name, + IFF( + VALUE = 'CreateAccount', + {}, + GET(VALUE, object_keys(VALUE) [0] :: STRING) + ) AS action_data, + MD5( + CASE action_name + WHEN 'FunctionCall' THEN + CONCAT_WS(',', + action_data :args :: STRING, + action_data :deposit :: STRING, + action_data :gas :: STRING, + action_data :method_name :: STRING + ) + WHEN 'AddKey' THEN + CONCAT_WS(',', + action_data :access_key :nonce :: STRING, + action_data :access_key :permission :: STRING, + action_data :public_key :: STRING + ) + WHEN 'DeleteKey' THEN + action_data :public_key :: STRING + WHEN 'CreateAccount' THEN + 'empty' -- consistent hash for empty objects + WHEN 'DeleteAccount' THEN + action_data :beneficiary_id :: STRING + WHEN 'DeployContract' THEN + action_data :code :: STRING + WHEN 'Transfer' THEN + action_data :deposit :: STRING + WHEN 'Stake' THEN + CONCAT_WS(',', + action_data :public_key :: STRING, + action_data :stake :: STRING + ) + ELSE + -- Fallback: convert entire variant to string + action_data :: STRING + END + ) AS action_hash, + IFF( + action_name = 'FunctionCall', + OBJECT_INSERT( + action_data, + 'args', + COALESCE( + TRY_PARSE_JSON( + TRY_BASE64_DECODE_STRING( + action_data :args :: STRING + ) + ), + action_data :args + ), + TRUE + ), + action_data + ) AS action_data_parsed, + _partition_by_block_number, + _inserted_timestamp + FROM + join_data, + LATERAL FLATTEN( + receipt_actions :receipt :Action :actions :: ARRAY + ) +), +flatten_delegated_actions AS ( + SELECT + tx_hash, + True AS is_delegated, + INDEX AS delegated_action_index, + IFF( + VALUE = 'CreateAccount', + VALUE, + object_keys(VALUE) [0] :: STRING + ) AS delegated_action_name, + IFF( + VALUE = 'CreateAccount', + {}, + GET(VALUE, object_keys(VALUE) [0] :: STRING) + ) AS delegated_action_data, + MD5( + CASE delegated_action_name + WHEN 'FunctionCall' THEN + CONCAT_WS(',', + delegated_action_data :args :: STRING, + delegated_action_data :deposit :: STRING, + delegated_action_data :gas :: STRING, + delegated_action_data :method_name :: STRING + ) + WHEN 'AddKey' THEN + CONCAT_WS(',', + delegated_action_data :access_key :nonce :: STRING, + delegated_action_data :access_key :permission :: STRING, + delegated_action_data :public_key :: STRING + ) + WHEN 'DeleteKey' THEN + delegated_action_data :public_key :: STRING + WHEN 'CreateAccount' THEN + 'empty' + WHEN 'DeleteAccount' THEN + delegated_action_data :beneficiary_id :: STRING + WHEN 'DeployContract' THEN + delegated_action_data :code :: STRING + WHEN 'Transfer' THEN + delegated_action_data :deposit :: STRING + WHEN 'Stake' THEN + CONCAT_WS(',', + delegated_action_data :public_key :: STRING, + delegated_action_data :stake :: STRING + ) + ELSE + -- Fallback: convert entire variant to string + delegated_action_data :: STRING + END + ) AS delegated_action_hash + FROM flatten_actions, LATERAL FLATTEN(action_data :delegate_action :actions :: ARRAY) + WHERE action_name = 'Delegate' +) SELECT block_id, block_timestamp, - tx_hash, + fa.tx_hash, tx_succeeded, tx_receiver, tx_signer, tx_gas_used, - receipt_id, + fa.receipt_id, receipt_predecessor_id, receipt_receiver_id, receipt_signer_id, @@ -20,12 +264,25 @@ SELECT receipt_gas_burnt, receipt_status_value, action_index, - is_delegated, + COALESCE( + da.is_delegated, + fa.is_delegated + ) AS is_delegated, action_name, - action_data, + action_data_parsed AS action_data, action_gas_price, - actions_id AS fact_actions_id, - inserted_timestamp, - modified_timestamp + _partition_by_block_number, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['receipt_id', 'action_index'] + ) }} AS actions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__actions') }} + flatten_actions fa + LEFT JOIN flatten_delegated_actions da + ON fa.tx_hash = da.tx_hash + AND fa.action_name = da.delegated_action_name + AND fa.action_hash = da.delegated_action_hash + AND fa.action_index = da.delegated_action_index diff --git a/models/gold/core/core__ez_actions.yml b/models/gold/core/core__ez_actions.yml index fea5bd99..64f8251e 100644 --- a/models/gold/core/core__ez_actions.yml +++ b/models/gold/core/core__ez_actions.yml @@ -1,9 +1,11 @@ version: 2 models: - - name: core__ez_actions + - name: silver__actions description: |- - This table extracts all actions from a transaction and stores the argument data under action_data. + This table extracts and processes executed Actions from NEAR receipts - storing the argument data under action_data. FunctionCall data is decoded, where possible, and returned to the action_data object under args. + It combines transaction and receipt information to provide a comprehensive view on each of the actions events. + tests: - dbt_utils.recency: datepart: hours @@ -14,57 +16,40 @@ models: - name: BLOCK_ID description: "{{ doc('block_id')}}" tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT + - not_null: + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp')}}" tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - not_null: where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: TX_HASH description: "{{ doc('tx_hash')}}" tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - not_null: where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: TX_SUCCEEDED description: "{{ doc('tx_succeeded')}}" tests: - - not_null: - where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - BOOLEAN + - not_null: + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: TX_RECEIVER description: "{{ doc('tx_receiver')}}" tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR + - not_null: + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: TX_SIGNER description: "{{ doc('tx_signer')}}" tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - name: TX_GAS_USED description: "{{ doc('gas_used')}}" @@ -74,51 +59,36 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - - FLOAT + - INTEGER - name: RECEIPT_ID description: "{{ doc('receipt_object_id')}}" tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR + - not_null: + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: RECEIPT_PREDECESSOR_ID description: "{{ doc('predecessor_id')}}" tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - name: RECEIPT_RECEIVER_ID description: "{{ doc('receiver_id')}}" tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - name: RECEIPT_SIGNER_ID description: "{{ doc('signer_id')}}" tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - name: RECEIPT_SUCCEEDED description: "{{ doc('receipt_succeeded')}}" tests: - - not_null: - where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - BOOLEAN + - not_null - name: RECEIPT_GAS_BURNT description: "{{ doc('gas_burnt')}}" @@ -127,8 +97,7 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - - DOUBLE - - FLOAT + - INTEGER - name: RECEIPT_STATUS_VALUE description: "{{ doc('status_value')}}" @@ -137,14 +106,11 @@ models: column_type_list: - VARIANT - OBJECT - + - name: ACTION_INDEX description: "{{ doc('action_index')}}" tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - name: IS_DELEGATED description: "{{ doc('is_delegated')}}" @@ -158,39 +124,39 @@ models: description: "{{ doc('action_name')}}" tests: - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - name: ACTION_DATA description: "{{ doc('action_data')}}" tests: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - - OBJECT - VARIANT + - OBJECT - name: ACTION_GAS_PRICE description: "{{ doc('action_gas_price')}}" tests: - - not_null: - where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT + - not_null - - name: FACT_ACTIONS_ID - description: "{{ doc('id')}}" + - name: _PARTITION_BY_BLOCK_NUMBER + description: "{{ doc('_partition_by_block_number')}}" + + - name: _INSERTED_TIMESTAMP + description: "{{ doc('_inserted_timestamp')}}" + + - name: ACTIONS_ID + description: "{{ doc('action_id')}}" tests: - unique: - where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' AND FACT_ACTIONS_EVENTS_ID != 'cf646ad92e6df243ffabf07c47c0f2c1' + where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - not_null: where: inserted_timestamp >= SYSDATE() - INTERVAL '7 days' - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp')}}" + description: "{{doc('inserted_timestamp')}}" - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp')}}" + description: "{{doc('modified_timestamp')}}" + + - name: _INVOCATION_ID + description: "{{doc('invocation_id')}}" diff --git a/models/silver/actions/silver__actions.sql b/models/silver/actions/silver__actions.sql deleted file mode 100644 index 5e16413e..00000000 --- a/models/silver/actions/silver__actions.sql +++ /dev/null @@ -1,288 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'merge', - merge_exclude_columns = ["inserted_timestamp"], - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], - cluster_by = ['block_timestamp::DATE', 'block_id'], - unique_key = 'actions_id', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receipt_receiver_id,receipt_signer_id,receipt_predecessor_id);", - tags = ['actions', 'curated', 'scheduled_core', 'grail'] -) }} - -WITH transactions AS ( - - SELECT - tx_hash, - tx_signer, - tx_receiver, - gas_used AS tx_gas_used, - tx_succeeded - FROM - {{ ref('silver__streamline_transactions_final') }} - - {% if var("MANUAL_FIX") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} -), -receipts AS ( - SELECT - tx_hash, - block_id, - block_timestamp, - receipt_object_id AS receipt_id, - receiver_id AS receipt_receiver_id, - signer_id AS receipt_signer_id, - receipt_actions :predecessor_id :: STRING AS receipt_predecessor_id, - receipt_succeeded, - gas_burnt AS receipt_gas_burnt, - status_value, - receipt_actions, - _partition_by_block_number, - _inserted_timestamp - FROM - {{ ref('silver__streamline_receipts_final') }} - - {% if var("MANUAL_FIX") %} - WHERE - {{ partition_load_manual('no_buffer') }} - {% else %} - {% if is_incremental() %} - WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} - {% endif %} -), -join_data AS ( - SELECT - r.block_id, - r.block_timestamp, - r.tx_hash, - t.tx_signer, - t.tx_receiver, - t.tx_gas_used, - t.tx_succeeded, - r.receipt_id, - r.receipt_receiver_id, - r.receipt_signer_id, - r.receipt_predecessor_id, - r.receipt_succeeded, - r.receipt_gas_burnt, - r.status_value, - r.receipt_actions, - r._partition_by_block_number, - r._inserted_timestamp - FROM - receipts r - LEFT JOIN transactions t - ON r.tx_hash = t.tx_hash -), -flatten_actions AS ( - SELECT - block_id, - block_timestamp, - tx_hash, - tx_signer, - tx_receiver, - tx_gas_used, - tx_succeeded, - receipt_id, - receipt_receiver_id, - receipt_signer_id, - receipt_predecessor_id, - receipt_succeeded, - receipt_gas_burnt, - IFF( - object_keys(status_value)[0] :: STRING = 'SuccessValue', - OBJECT_INSERT( - status_value, - 'SuccessValue', - COALESCE( - TRY_PARSE_JSON( - TRY_BASE64_DECODE_STRING( - GET(status_value, 'SuccessValue') - ) - ), - GET(status_value, 'SuccessValue') - ), - TRUE - ), - status_value - ) as receipt_status_value, - False AS is_delegated, - INDEX AS action_index, - receipt_actions :receipt :Action :gas_price :: NUMBER AS action_gas_price, - IFF( - VALUE = 'CreateAccount', - VALUE, - object_keys(VALUE) [0] :: STRING - ) AS action_name, - IFF( - VALUE = 'CreateAccount', - {}, - GET(VALUE, object_keys(VALUE) [0] :: STRING) - ) AS action_data, - MD5( - CASE action_name - WHEN 'FunctionCall' THEN - CONCAT_WS(',', - action_data :args :: STRING, - action_data :deposit :: STRING, - action_data :gas :: STRING, - action_data :method_name :: STRING - ) - WHEN 'AddKey' THEN - CONCAT_WS(',', - action_data :access_key :nonce :: STRING, - action_data :access_key :permission :: STRING, - action_data :public_key :: STRING - ) - WHEN 'DeleteKey' THEN - action_data :public_key :: STRING - WHEN 'CreateAccount' THEN - 'empty' -- consistent hash for empty objects - WHEN 'DeleteAccount' THEN - action_data :beneficiary_id :: STRING - WHEN 'DeployContract' THEN - action_data :code :: STRING - WHEN 'Transfer' THEN - action_data :deposit :: STRING - WHEN 'Stake' THEN - CONCAT_WS(',', - action_data :public_key :: STRING, - action_data :stake :: STRING - ) - ELSE - -- Fallback: convert entire variant to string - action_data :: STRING - END - ) AS action_hash, - IFF( - action_name = 'FunctionCall', - OBJECT_INSERT( - action_data, - 'args', - COALESCE( - TRY_PARSE_JSON( - TRY_BASE64_DECODE_STRING( - action_data :args :: STRING - ) - ), - action_data :args - ), - TRUE - ), - action_data - ) AS action_data_parsed, - _partition_by_block_number, - _inserted_timestamp - FROM - join_data, - LATERAL FLATTEN( - receipt_actions :receipt :Action :actions :: ARRAY - ) -), -flatten_delegated_actions AS ( - SELECT - tx_hash, - True AS is_delegated, - INDEX AS delegated_action_index, - IFF( - VALUE = 'CreateAccount', - VALUE, - object_keys(VALUE) [0] :: STRING - ) AS delegated_action_name, - IFF( - VALUE = 'CreateAccount', - {}, - GET(VALUE, object_keys(VALUE) [0] :: STRING) - ) AS delegated_action_data, - MD5( - CASE delegated_action_name - WHEN 'FunctionCall' THEN - CONCAT_WS(',', - delegated_action_data :args :: STRING, - delegated_action_data :deposit :: STRING, - delegated_action_data :gas :: STRING, - delegated_action_data :method_name :: STRING - ) - WHEN 'AddKey' THEN - CONCAT_WS(',', - delegated_action_data :access_key :nonce :: STRING, - delegated_action_data :access_key :permission :: STRING, - delegated_action_data :public_key :: STRING - ) - WHEN 'DeleteKey' THEN - delegated_action_data :public_key :: STRING - WHEN 'CreateAccount' THEN - 'empty' - WHEN 'DeleteAccount' THEN - delegated_action_data :beneficiary_id :: STRING - WHEN 'DeployContract' THEN - delegated_action_data :code :: STRING - WHEN 'Transfer' THEN - delegated_action_data :deposit :: STRING - WHEN 'Stake' THEN - CONCAT_WS(',', - delegated_action_data :public_key :: STRING, - delegated_action_data :stake :: STRING - ) - ELSE - -- Fallback: convert entire variant to string - delegated_action_data :: STRING - END - ) AS delegated_action_hash - FROM flatten_actions, LATERAL FLATTEN(action_data :delegate_action :actions :: ARRAY) - WHERE action_name = 'Delegate' -) -SELECT - block_id, - block_timestamp, - fa.tx_hash, - tx_succeeded, - tx_receiver, - tx_signer, - tx_gas_used, - fa.receipt_id, - receipt_predecessor_id, - receipt_receiver_id, - receipt_signer_id, - receipt_succeeded, - receipt_gas_burnt, - receipt_status_value, - action_index, - COALESCE( - da.is_delegated, - fa.is_delegated - ) AS is_delegated, - action_name, - action_data_parsed AS action_data, - action_gas_price, - _partition_by_block_number, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['receipt_id', 'action_index'] - ) }} AS actions_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - flatten_actions fa - LEFT JOIN flatten_delegated_actions da - ON fa.tx_hash = da.tx_hash - AND fa.action_name = da.delegated_action_name - AND fa.action_hash = da.delegated_action_hash - AND fa.action_index = da.delegated_action_index diff --git a/models/silver/actions/silver__actions.yml b/models/silver/actions/silver__actions.yml deleted file mode 100644 index 1d9e1100..00000000 --- a/models/silver/actions/silver__actions.yml +++ /dev/null @@ -1,151 +0,0 @@ -version: 2 - -models: - - name: silver__actions - description: |- - This table extracts and processes all action events from NEAR transactions and receipts - storing the argument data under action_data. It combines transaction and receipt information to provide a comprehensive view on each of the actions events. - tests: - - dbt_utils.recency: - datepart: hours - field: block_timestamp - interval: 2 - - columns: - - name: BLOCK_ID - description: "{{ doc('block_id')}}" - tests: - - not_null - - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp')}}" - tests: - - not_null - - - name: TX_HASH - description: "{{ doc('tx_hash')}}" - tests: - - not_null - - - name: TX_SUCCEEDED - description: "{{ doc('tx_succeeded')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: TX_RECEIVER - description: "{{ doc('tx_receiver')}}" - tests: - - not_null - - - name: TX_SIGNER - description: "{{ doc('tx_signer')}}" - tests: - - not_null - - - name: TX_GAS_USED - description: "{{ doc('gas_used')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: RECEIPT_ID - description: "{{ doc('receipt_object_id')}}" - tests: - - not_null - - - name: RECEIPT_PREDECESSOR_ID - description: "{{ doc('predecessor_id')}}" - tests: - - not_null - - - name: RECEIPT_RECEIVER_ID - description: "{{ doc('receiver_id')}}" - tests: - - not_null - - - name: RECEIPT_SIGNER_ID - description: "{{ doc('signer_id')}}" - tests: - - not_null - - - name: RECEIPT_SUCCEEDED - description: "{{ doc('receipt_succeeded')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - not_null - - - name: RECEIPT_GAS_BURNT - description: "{{ doc('gas_burnt')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - INTEGER - - - name: RECEIPT_STATUS_VALUE - description: "{{ doc('status_value')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - - OBJECT - - - name: ACTION_INDEX - description: "{{ doc('action_index')}}" - tests: - - not_null - - - name: IS_DELEGATED - description: "{{ doc('is_delegated')}}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: ACTION_NAME - description: "{{ doc('action_name')}}" - tests: - - not_null - - - name: ACTION_DATA - description: "{{ doc('action_data')}}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - - OBJECT - - - name: ACTION_GAS_PRICE - description: "{{ doc('action_gas_price')}}" - tests: - - not_null - - - name: _PARTITION_BY_BLOCK_NUMBER - description: "{{ doc('_partition_by_block_number')}}" - - - name: _INSERTED_TIMESTAMP - description: "{{ doc('_inserted_timestamp')}}" - - - name: ACTIONS_ID - description: "{{ doc('action_id')}}" - tests: - - unique: - where: tx_hash != 'J4CZZQrZK6kYPVLkrdbTEpcqhUNZiRxktbMzHviqeGgf' - - not_null - - - name: INSERTED_TIMESTAMP - description: "{{doc('inserted_timestamp')}}" - - - name: MODIFIED_TIMESTAMP - description: "{{doc('modified_timestamp')}}" - - - name: _INVOCATION_ID - description: "{{doc('invocation_id')}}" From 89b9ebe30565a0a031feadc15d229089d82d1444 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Sat, 14 Dec 2024 09:17:27 -0700 Subject: [PATCH 14/17] add incremental if execute block --- models/gold/core/core__ez_actions.sql | 91 ++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 14 deletions(-) diff --git a/models/gold/core/core__ez_actions.sql b/models/gold/core/core__ez_actions.sql index 5e16413e..830b16c4 100644 --- a/models/gold/core/core__ez_actions.sql +++ b/models/gold/core/core__ez_actions.sql @@ -8,6 +8,62 @@ post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receipt_receiver_id,receipt_signer_id,receipt_predecessor_id);", tags = ['actions', 'curated', 'scheduled_core', 'grail'] ) }} +-- depends_on: {{ ref('silver__streamline_transactions_final') }} +-- depends_on: {{ ref('silver__streamline_receipts_final') }} + +{% if execute %} + + {% if is_incremental() and not var("MANUAL_FIX") %} + {% do log("Incremental and not MANUAL_FIX", info=True) %} + {% set max_mod_query %} + + SELECT + MAX(modified_timestamp) modified_timestamp + FROM + {{ this }} + + {% endset %} + + {% set max_mod = run_query(max_mod_query) [0] [0] %} + {% if not max_mod or max_mod == 'None' %} + {% set max_mod = '2099-01-01' %} + {% endif %} + + {% do log("max_mod: " ~ max_mod, info=True) %} + + {% set min_block_date_query %} + SELECT + MIN( + block_timestamp :: DATE + ) + FROM + ( + SELECT + MIN(block_timestamp) block_timestamp + FROM + {{ ref('silver__streamline_transactions_final') }} A + WHERE + modified_timestamp >= '{{max_mod}}' + UNION ALL + SELECT + MIN(block_timestamp) block_timestamp + FROM + {{ ref('silver__streamline_receipts_final') }} A + WHERE + modified_timestamp >= '{{max_mod}}' + ) + {% endset %} + + {% set min_bd = run_query(min_block_date_query) [0] [0] %} + {% if not min_bd or min_bd == 'None' %} + {% set min_bd = '2099-01-01' %} + {% endif %} + + {% do log("min_bd: " ~ min_bd, info=True) %} + + {% endif %} + +{% endif %} WITH transactions AS ( @@ -16,21 +72,20 @@ WITH transactions AS ( tx_signer, tx_receiver, gas_used AS tx_gas_used, - tx_succeeded + tx_succeeded, + modified_timestamp FROM {{ ref('silver__streamline_transactions_final') }} + -- temp for dev + -- where modified_timestamp :: date >= current_date - 14 + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} {% if is_incremental() %} - WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) + WHERE block_timestamp :: DATE >= '{{min_bd}}' {% endif %} {% endif %} ), @@ -48,21 +103,20 @@ receipts AS ( status_value, receipt_actions, _partition_by_block_number, - _inserted_timestamp + _inserted_timestamp, + modified_timestamp FROM {{ ref('silver__streamline_receipts_final') }} + -- temp for dev + -- where modified_timestamp :: date >= current_date - 14 + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} {% if is_incremental() %} - WHERE modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) + WHERE block_timestamp :: DATE >= '{{min_bd}}' {% endif %} {% endif %} ), @@ -89,6 +143,15 @@ join_data AS ( receipts r LEFT JOIN transactions t ON r.tx_hash = t.tx_hash + + {% if is_incremental() and not var("MANUAL_FIX") %} + WHERE + GREATEST( + r.modified_timestamp, + t.modified_timestamp + ) >= '{{max_mod}}' + {% endif %} + ), flatten_actions AS ( SELECT From 5f75cf809c611c34dd55385977238417531c183c Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:16:32 -0500 Subject: [PATCH 15/17] add coalesce to modts --- models/gold/core/core__ez_actions.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/gold/core/core__ez_actions.sql b/models/gold/core/core__ez_actions.sql index 830b16c4..faf5cfd6 100644 --- a/models/gold/core/core__ez_actions.sql +++ b/models/gold/core/core__ez_actions.sql @@ -146,9 +146,9 @@ join_data AS ( {% if is_incremental() and not var("MANUAL_FIX") %} WHERE - GREATEST( - r.modified_timestamp, - t.modified_timestamp + GREATEST( + COALESCE(r.modified_timestamp, '1970-01-01'), + COALESCE(t.modified_timestamp, '1970-01-01') ) >= '{{max_mod}}' {% endif %} From 4f7284149140bde953266e4e10d0fea9fd58b978 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:03:30 -0500 Subject: [PATCH 16/17] chg cluster --- models/gold/core/core__ez_actions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/gold/core/core__ez_actions.sql b/models/gold/core/core__ez_actions.sql index faf5cfd6..af3bb527 100644 --- a/models/gold/core/core__ez_actions.sql +++ b/models/gold/core/core__ez_actions.sql @@ -3,7 +3,7 @@ incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], - cluster_by = ['block_timestamp::DATE', 'block_id'], + cluster_by = ['block_timestamp::DATE', 'modified_timestamp::DATE'], unique_key = 'actions_id', post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,receipt_id,receipt_receiver_id,receipt_signer_id,receipt_predecessor_id);", tags = ['actions', 'curated', 'scheduled_core', 'grail'] From 95b9cb8cd6fc61b566a88222933cda87d15165b4 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:17:30 -0500 Subject: [PATCH 17/17] rm dev limit, upd model name in yml --- models/gold/core/core__ez_actions.sql | 10 ++++------ models/gold/core/core__ez_actions.yml | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/models/gold/core/core__ez_actions.sql b/models/gold/core/core__ez_actions.sql index af3bb527..6f9d96d1 100644 --- a/models/gold/core/core__ez_actions.sql +++ b/models/gold/core/core__ez_actions.sql @@ -73,12 +73,10 @@ WITH transactions AS ( tx_receiver, gas_used AS tx_gas_used, tx_succeeded, + transaction_fee AS tx_fee, modified_timestamp FROM {{ ref('silver__streamline_transactions_final') }} - - -- temp for dev - -- where modified_timestamp :: date >= current_date - 14 {% if var("MANUAL_FIX") %} WHERE @@ -108,9 +106,6 @@ receipts AS ( FROM {{ ref('silver__streamline_receipts_final') }} - -- temp for dev - -- where modified_timestamp :: date >= current_date - 14 - {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} @@ -129,6 +124,7 @@ join_data AS ( t.tx_receiver, t.tx_gas_used, t.tx_succeeded, + t.tx_fee, r.receipt_id, r.receipt_receiver_id, r.receipt_signer_id, @@ -162,6 +158,7 @@ flatten_actions AS ( tx_receiver, tx_gas_used, tx_succeeded, + tx_fee, receipt_id, receipt_receiver_id, receipt_signer_id, @@ -319,6 +316,7 @@ SELECT tx_receiver, tx_signer, tx_gas_used, + tx_fee, fa.receipt_id, receipt_predecessor_id, receipt_receiver_id, diff --git a/models/gold/core/core__ez_actions.yml b/models/gold/core/core__ez_actions.yml index 64f8251e..da9d3135 100644 --- a/models/gold/core/core__ez_actions.yml +++ b/models/gold/core/core__ez_actions.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: silver__actions + - name: core__ez_actions description: |- This table extracts and processes executed Actions from NEAR receipts - storing the argument data under action_data. FunctionCall data is decoded, where possible, and returned to the action_data object under args. It combines transaction and receipt information to provide a comprehensive view on each of the actions events.