From cd92e0ae244e2faaf7d698ec1023903f8c6d2763 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Tue, 24 Sep 2024 16:59:07 -0600 Subject: [PATCH 1/2] parse fee from logs --- models/gold/defi/defi__ez_dex_swaps.sql | 7 +++- models/gold/defi/defi__fact_dex_swaps.sql | 3 ++ .../curated/defi/silver__dex_swaps_v2.sql | 39 ++++++++++++------- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql index e6608c15..39b1bd12 100644 --- a/models/gold/defi/defi__ez_dex_swaps.sql +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -13,6 +13,7 @@ WITH dex_swaps AS ( block_id, block_timestamp, receiver_id, + predecessor_id, signer_id, swap_index, amount_out_raw, @@ -21,6 +22,8 @@ WITH dex_swaps AS ( token_in, swap_input_data, LOG, + total_fee, + admin_fee, dex_swaps_v2_id AS ez_dex_swaps_id, inserted_timestamp, modified_timestamp @@ -58,7 +61,7 @@ FINAL AS ( s.block_timestamp, s.receiver_id AS platform, s.swap_input_data :pool_id :: INT AS pool_id, - s.signer_id AS trader, + s.predecessor_id AS trader, s.swap_index, s.amount_out_raw, s.amount_out_raw / pow( @@ -78,6 +81,8 @@ FINAL AS ( l2.symbol AS symbol_in, s.swap_input_data, s.log, + s.total_fee, + s.admin_fee, s.ez_dex_swaps_id, s.inserted_timestamp, s.modified_timestamp diff --git a/models/gold/defi/defi__fact_dex_swaps.sql b/models/gold/defi/defi__fact_dex_swaps.sql index f79b61d1..1c9f389f 100644 --- a/models/gold/defi/defi__fact_dex_swaps.sql +++ b/models/gold/defi/defi__fact_dex_swaps.sql @@ -11,6 +11,7 @@ SELECT block_id, block_timestamp, receiver_id, + predecessor_id, signer_id, swap_index, amount_out_raw, @@ -19,6 +20,8 @@ SELECT token_in, swap_input_data, LOG, + total_fee, + admin_fee, dex_swaps_v2_id AS fact_dex_swaps_id, inserted_timestamp, modified_timestamp diff --git a/models/silver/curated/defi/silver__dex_swaps_v2.sql b/models/silver/curated/defi/silver__dex_swaps_v2.sql index 289d82b0..1bff5247 100644 --- a/models/silver/curated/defi/silver__dex_swaps_v2.sql +++ b/models/silver/curated/defi/silver__dex_swaps_v2.sql @@ -17,6 +17,7 @@ WITH swap_logs AS ( block_id, block_timestamp, receiver_id, + predecessor_id, signer_id, log_index, clean_log, @@ -28,11 +29,10 @@ WITH swap_logs AS ( WHERE receipt_succeeded AND clean_log LIKE 'Swapped%' - AND receiver_id NOT LIKE '%dragon_bot.near' - - {% if var("MANUAL_FIX") %} + AND receiver_id NOT LIKE '%dragon_bot.near' {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} + {% if is_incremental() %} AND _modified_timestamp >= ( SELECT @@ -48,6 +48,7 @@ receipts AS ( receipt_object_id, receipt_actions, receiver_id, + receipt_actions :predecessor_id :: STRING AS predecessor_id, signer_id, _partition_by_block_number, _inserted_timestamp, @@ -60,10 +61,10 @@ receipts AS ( receipt_object_id FROM swap_logs - ) - {% if var("MANUAL_FIX") %} + ) {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} + {% if is_incremental() %} AND _modified_timestamp >= ( SELECT @@ -81,30 +82,34 @@ swap_outcome AS ( block_id, block_timestamp, receiver_id, + predecessor_id, signer_id, ROW_NUMBER() over ( PARTITION BY receipt_object_id ORDER BY log_index ASC ) - 1 AS swap_index, - COALESCE(SPLIT(clean_log, ',') [0], clean_log) AS LOG, + clean_log AS LOG, + COALESCE(SPLIT(clean_log, ',') [0], clean_log) AS primary_log, + REGEXP_SUBSTR(SPLIT(clean_log, ', ') [1], 'total fee ([0-9]+)', 1, 1, 'e', 1) AS total_fee, + REGEXP_SUBSTR(SPLIT(clean_log, ', ') [2], 'admin fee ([0-9]+)', 1, 1, 'e', 1) AS admin_fee, REGEXP_REPLACE( - LOG, + primary_log, '.*Swapped (\\d+) (.*) for (\\d+) (.*)', '\\1' ) :: INT AS amount_in_raw, REGEXP_REPLACE( - LOG, + primary_log, '.*Swapped \\d+ (\\S+) for (\\d+) (.*)', '\\1' ) :: STRING AS token_in, REGEXP_REPLACE( - LOG, + primary_log, '.*Swapped \\d+ \\S+ for (\\d+) (.*)', '\\1' ) :: INT AS amount_out_raw, REGEXP_REPLACE( - LOG, + primary_log, '.*Swapped \\d+ \\S+ for \\d+ (.*)', '\\1' ) :: STRING AS token_out, @@ -120,10 +125,14 @@ parse_actions AS ( o.receipt_object_id, block_id, block_timestamp, - receiver_id, - signer_id, + o.receiver_id, + o.predecessor_id, + o.signer_id, swap_index, LOG, + primary_log, + total_fee, + admin_fee, amount_out_raw, token_out, amount_in_raw, @@ -152,7 +161,7 @@ parse_actions AS ( -- for multi-swaps, there is (often) one action with an array of input dicts that correspond with the swap index decoded_action :msg, -- Swap must be capitalized! Autoformat may change to "swap" - decoded_action :operation: Swap, + decoded_action :operation: swap, decoded_action ) ) :actions [swap_index], @@ -178,6 +187,7 @@ FINAL AS ( block_id, block_timestamp, receiver_id, + predecessor_id, signer_id, swap_index, amount_out_raw, @@ -186,6 +196,9 @@ FINAL AS ( token_in, swap_input_data, LOG, + primary_log, + total_fee, + admin_fee, _partition_by_block_number, _inserted_timestamp, _modified_timestamp From 0bf9e213bb66f72070712718d56bab4bc683ce48 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:10:15 -0600 Subject: [PATCH 2/2] fix swap->Swap --- .../curated/defi/silver__dex_swaps_v2.sql | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/models/silver/curated/defi/silver__dex_swaps_v2.sql b/models/silver/curated/defi/silver__dex_swaps_v2.sql index 1bff5247..ad8d77b6 100644 --- a/models/silver/curated/defi/silver__dex_swaps_v2.sql +++ b/models/silver/curated/defi/silver__dex_swaps_v2.sql @@ -29,19 +29,20 @@ WITH swap_logs AS ( WHERE receipt_succeeded AND clean_log LIKE 'Swapped%' - AND receiver_id NOT LIKE '%dragon_bot.near' {% if var("MANUAL_FIX") %} + AND receiver_id NOT LIKE '%dragon_bot.near' + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} -{% if is_incremental() %} -AND _modified_timestamp >= ( - SELECT - MAX(_modified_timestamp) - FROM - {{ this }} -) -{% endif %} -{% endif %} + {% if is_incremental() %} + AND _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + {% endif %} ), receipts AS ( SELECT @@ -61,19 +62,20 @@ receipts AS ( receipt_object_id FROM swap_logs - ) {% if var("MANUAL_FIX") %} + ) + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} -{% if is_incremental() %} -AND _modified_timestamp >= ( - SELECT - MAX(_modified_timestamp) - FROM - {{ this }} -) -{% endif %} -{% endif %} + {% if is_incremental() %} + AND _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} + ) + {% endif %} + {% endif %} ), swap_outcome AS ( SELECT @@ -161,7 +163,7 @@ parse_actions AS ( -- for multi-swaps, there is (often) one action with an array of input dicts that correspond with the swap index decoded_action :msg, -- Swap must be capitalized! Autoformat may change to "swap" - decoded_action :operation: swap, + decoded_action :operation :Swap, decoded_action ) ) :actions [swap_index],