From 47b2aa9e0d2d8185807d4f534a72933c073b84d6 Mon Sep 17 00:00:00 2001 From: Christina Ovezik <20790332+LadyChristina@users.noreply.github.com> Date: Fri, 21 Jul 2023 13:11:41 +0100 Subject: [PATCH] Rename coinbase param and addresses (#82) * Rename coinbase param and addresses * Add forgotten files --- docs/data.md | 20 ++--- docs/mappings.md | 18 ++-- docs/parsers.md | 10 +-- input/sample_bitcoin_raw_data.json | 26 +++--- input/sample_cardano_raw_data.json | 8 +- input/sample_ethereum_raw_data.json | 20 ++--- input/sample_tezos_raw_data.json | 10 +-- queries.yaml | 90 +++++++++---------- src/helpers/helper.py | 16 ++-- src/helpers/pool_information/README.md | 8 +- .../bitcoin.json | 0 .../bitcoin_cash.json | 0 .../cardano.json | 0 .../{coinbase_tags => identifiers}/dash.json | 0 .../dogecoin.json | 0 .../ethereum.json | 0 .../litecoin.json | 0 .../{coinbase_tags => identifiers}/test.json | 0 .../unidentified.json | 0 .../{coinbase_tags => identifiers}/zcash.json | 0 src/map.py | 8 +- .../{bitcoin.py => bitcoin_mapping.py} | 23 ++--- .../{cardano.py => cardano_mapping.py} | 4 +- src/mappings/{dummy.py => dummy_mapping.py} | 8 +- .../{ethereum.py => ethereum_mapping.py} | 24 ++--- src/mappings/{tezos.py => tezos_mapping.py} | 12 +-- src/parse.py | 3 +- src/parsers/cardano_parser.py | 30 ------- src/parsers/default_parser.py | 4 +- src/parsers/dummy_parser.py | 10 ++- tests/test_3_parsers.py | 72 ++++++--------- tests/test_4_mappings.py | 43 +++++---- tests/test_6_end_to_end.py | 12 +-- 33 files changed, 216 insertions(+), 263 deletions(-) rename src/helpers/pool_information/{coinbase_tags => identifiers}/bitcoin.json (100%) rename src/helpers/pool_information/{coinbase_tags => identifiers}/bitcoin_cash.json (100%) rename src/helpers/pool_information/{coinbase_tags => identifiers}/cardano.json (100%) rename src/helpers/pool_information/{coinbase_tags => identifiers}/dash.json (100%) rename src/helpers/pool_information/{coinbase_tags => identifiers}/dogecoin.json (100%) rename src/helpers/pool_information/{coinbase_tags => identifiers}/ethereum.json (100%) rename src/helpers/pool_information/{coinbase_tags => identifiers}/litecoin.json (100%) rename src/helpers/pool_information/{coinbase_tags => identifiers}/test.json (100%) rename src/helpers/pool_information/{coinbase_tags => identifiers}/unidentified.json (100%) rename src/helpers/pool_information/{coinbase_tags => identifiers}/zcash.json (100%) rename src/mappings/{bitcoin.py => bitcoin_mapping.py} (83%) rename src/mappings/{cardano.py => cardano_mapping.py} (95%) rename src/mappings/{dummy.py => dummy_mapping.py} (63%) rename src/mappings/{ethereum.py => ethereum_mapping.py} (77%) rename src/mappings/{tezos.py => tezos_mapping.py} (83%) delete mode 100644 src/parsers/cardano_parser.py diff --git a/docs/data.md b/docs/data.md index c851f0a..3844dc5 100644 --- a/docs/data.md +++ b/docs/data.md @@ -15,7 +15,7 @@ Sample raw Bitcoin data are available They can be retrieved using [Google BigQuery](https://console.cloud.google.com/bigquery) with the following query: ``` -SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_bitcoin.transactions`.outputs +SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_bitcoin.transactions`.outputs FROM `bigquery-public-data.crypto_bitcoin.transactions` JOIN `bigquery-public-data.crypto_bitcoin.blocks` ON `bigquery-public-data.crypto_bitcoin.transactions`.block_number = `bigquery-public-data.crypto_bitcoin.blocks`.number WHERE is_coinbase is TRUE @@ -30,7 +30,7 @@ Sample raw Bitcoin Cash data are available They can be retrieved using [Google BigQuery](https://console.cloud.google.com/bigquery) with the following query: ``` -SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_bitcoin_cash.transactions`.outputs +SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_bitcoin_cash.transactions`.outputs FROM `bigquery-public-data.crypto_bitcoin_cash.transactions` JOIN `bigquery-public-data.crypto_bitcoin_cash.blocks` ON `bigquery-public-data.crypto_bitcoin_cash.transactions`.block_number = `bigquery-public-data.crypto_bitcoin_cash.blocks`.number WHERE is_coinbase is TRUE @@ -45,8 +45,8 @@ Sample raw Cardano data are available They can be retrieved using [Google BigQuery](https://console.cloud.google.com/bigquery) with the following query: ``` -SELECT `iog-data-analytics.cardano_mainnet.block`.slot_no as number, `iog-data-analytics.cardano_mainnet.pool_offline_data`.ticker_name as coinbase_param, `iog-data-analytics.cardano_mainnet.block`.block_time as timestamp, `iog-data-analytics.cardano_mainnet.block`.pool_hash -FROM `iog-data-analytics.cardano_mainnet.block` +SELECT `iog-data-analytics.cardano_mainnet.block`.slot_no as number, `iog-data-analytics.cardano_mainnet.pool_offline_data`.ticker_name as identifiers, `iog-data-analytics.cardano_mainnet.block`.block_time as timestamp,`iog-data-analytics.cardano_mainnet.block`.pool_hash as reward_addresses +FROM `iog-data-analytics.cardano_mainnet.block` LEFT JOIN `iog-data-analytics.cardano_mainnet.pool_offline_data` ON `iog-data-analytics.cardano_mainnet.block`.pool_hash = `iog-data-analytics.cardano_mainnet.pool_offline_data`.pool_hash WHERE `iog-data-analytics.cardano_mainnet.block`.block_time > '2020-12-31' ``` @@ -59,7 +59,7 @@ Sample raw Dash data are available They can be retrieved using [Google BigQuery](https://console.cloud.google.com/bigquery) with the following query: ``` -SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_dash.transactions`.outputs +SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_dash.transactions`.outputs FROM `bigquery-public-data.crypto_dash.transactions` JOIN `bigquery-public-data.crypto_dash.blocks` ON `bigquery-public-data.crypto_dash.transactions`.block_number = `bigquery-public-data.crypto_dash.blocks`.number WHERE is_coinbase is TRUE @@ -74,7 +74,7 @@ Sample raw Dogecoin data are available They can be retrieved using [Google BigQuery](https://console.cloud.google.com/bigquery) with the following query: ``` -SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_dogecoin.transactions`.outputs +SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_dogecoin.transactions`.outputs FROM `bigquery-public-data.crypto_dogecoin.transactions` JOIN `bigquery-public-data.crypto_dogecoin.blocks` ON `bigquery-public-data.crypto_dogecoin.transactions`.block_number = `bigquery-public-data.crypto_dogecoin.blocks`.number WHERE is_coinbase is TRUE @@ -89,7 +89,7 @@ Sample raw Ethereum data are available They can be retrieved using [Google BigQuery](https://console.cloud.google.com/bigquery) with the following query: ``` -SELECT number, timestamp, miner as coinbase_addresses, extra_data as coinbase_param +SELECT number, timestamp, miner as reward_addresses, extra_data as identifiers FROM `bigquery-public-data.crypto_ethereum.blocks` WHERE timestamp > '2018-12-31' ``` @@ -102,7 +102,7 @@ Sample raw Litecoin data are available They can be retrieved using [Google BigQuery](https://console.cloud.google.com/bigquery) with the following query: ``` -SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_litecoin.transactions`.outputs +SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_litecoin.transactions`.outputs FROM `bigquery-public-data.crypto_litecoin.transactions` JOIN `bigquery-public-data.crypto_litecoin.blocks` ON `bigquery-public-data.crypto_litecoin.transactions`.block_number = `bigquery-public-data.crypto_litecoin.blocks`.number WHERE is_coinbase is TRUE @@ -117,7 +117,7 @@ Sample raw Tezos data are available They can be retrieved using [Google BigQuery](https://console.cloud.google.com/bigquery) with the following query: ``` -SELECT level as number, timestamp, baker as coinbase_addresses +SELECT level as number, timestamp, baker as reward_addresses FROM `public-data-finance.crypto_tezos.blocks` WHERE timestamp > '2020-12-31' ``` @@ -130,7 +130,7 @@ Sample raw Zcash data are available They can be retrieved using [Google BigQuery](https://console.cloud.google.com/bigquery) with the following query: ``` -SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_zcash.transactions`.outputs +SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_zcash.transactions`.outputs FROM `bigquery-public-data.crypto_zcash.transactions` JOIN `bigquery-public-data.crypto_zcash.blocks` ON `bigquery-public-data.crypto_zcash.transactions`.block_number = `bigquery-public-data.crypto_zcash.blocks`.number WHERE is_coinbase is TRUE diff --git a/docs/mappings.md b/docs/mappings.md index f3a699d..9c645e0 100644 --- a/docs/mappings.md +++ b/docs/mappings.md @@ -12,18 +12,18 @@ The name of the `csv` file is the timeframe, over which the mapping was executed project's output directory (`output//`). The logic of the mapping depends on the type of clustering we want to achieve. So, different mappings will output -different results, even if applied on the same data. An exception to this is the "no-cluster" mapping, which maps blocks to -coinbase addresses, so it doesn't perform any extra processing on the raw data. +different results, even if applied on the same data. An exception to this is the "no-cluster" mapping (DummyMapping +in the code), which maps blocks to reward addresses, so it doesn't perform any extra processing on the raw data. ## Pool Information -To assist the mapping process, the directory `helpers/pool_information/` contains +To assist the mapping process, the directory `helpers/pool_information/` contains pool information about the supported projects. There exist three subdirectories. In each subdirectory there exists a file for the corresponding ledger data, if such data exists. -`coinbase_tags` defines information about block creators. Each key +`identifiers` defines information about block creators. Each key corresponds to a tag or ticker, by which the pool is identifiable in its produced blocks. The value for each key is a dictionary of pool-related information, specifically its name, a url to its homepage, etc. Each file's @@ -89,19 +89,19 @@ The values for each entry are the same as `clusters` in the above pool informati In our implementation, the mapping of a block uses the auxiliary information as follows. -First, it iterates over all known tags and compares each one with the block's coinbase parameter. If the tag is a +First, it iterates over all known tags and compares each one with the block's identifiers. If the tag is a substring of the parameter, then we have a match. -Second, if the first step fails, we compare the block's coinbase addresses with known pool addresses and again look for +Second, if the first step fails, we compare the block's reward addresses with known pool addresses and again look for a match. In both cases, if there is a match, then: (i) we map the block to the matched pool; (ii) we associate all of the block's -coinbase addresses (that is, the addresses that receive fees from the block) with the matched pool. +reward addresses (that is, the addresses that receive fees from the block) with the matched pool. -In essence, the coinbase parameter is the principal element for mapping a block to an entity and the known addresses is +In essence, the identifiers are the principal element for mapping a block to an entity and the known addresses are the fallback mechanism. If there is a match, we also parse the auxiliary information, such as pool ownership or clusters, in order to assign the block to the top level entity, e.g., the pool's parent company or cluster. -If both mechanisms fail, then no match is found. In this case, we assign the coinbase addresses as the block's entity. +If both mechanisms fail, then no match is found. In this case, we assign the reward addresses as the block's entity. diff --git a/docs/parsers.md b/docs/parsers.md index 1e3e3ae..2f1d39c 100644 --- a/docs/parsers.md +++ b/docs/parsers.md @@ -12,23 +12,23 @@ The output file is stored under `output//parsed_data.json` and is { "number": "", "timestamp": "", - "coinbase_addresses": "," - "coinbase_param": "" + "reward_addresses": "," + "identifiers": "" } ] ``` `number` and `timestamp` are consistent among different blockchains. -`coinbase_addresses` and `coinbase_param` vary, depending on each ledger. +`reward_addresses` and `identifiers` vary, depending on each ledger. -Specifically, `coinbase_addresses` corresponds to: +Specifically, `reward_addresses` corresponds to: - `Bitcoin`, `Bitcoin Cash`, `Dogecoin`, `Litecoin`, `Zcash`, `Dash`: a string of comma-separated addresses which appear in the block's coinbase transaction with non-negative value (i.e., which are given part of the block's fees) - `Ethereum`: the block's `miner` field - `Cardano`: the hash of the pool that created the data, if defined, otherwise the empty string - `Tezos`: the block's `baker` field -The field `coinbase_param` corresponds to: +The field `identifiers` corresponds to: - `Bitcoin`, `Bitcoin Cash`, `Dogecoin`, `Litecoin`, `Zcash`, `Dash`: the field `coinbase_param` of the block's coinbase transaction - `Ethereum`: the block's `extra_data` field diff --git a/input/sample_bitcoin_raw_data.json b/input/sample_bitcoin_raw_data.json index a2026bd..a677b09 100644 --- a/input/sample_bitcoin_raw_data.json +++ b/input/sample_bitcoin_raw_data.json @@ -1,13 +1,13 @@ -{"number":"507715","timestamp":"2018-02-05 04:54:34 UTC","coinbase_param":"0343bf07132f6d696e65642062792067626d696e6572732f2cfabe6d6d94976ecebbc73b3d4214b3d7ab330dca2129ebfcc863fa75623c6f95891e7346010000000000000010af8b66002da910ef408c776852840100","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 bba87b8e503100c5cb8e308222a86eedb1fc1859 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a914bba87b8e503100c5cb8e308222a86eedb1fc185988ac","required_signatures":"1","type":"pubkeyhash","addresses":["1J7FCFaafPRxqu4X9VsaiMZr1XMemx69GR"],"value":13250677},{"index":"1","script_asm":"OP_DUP OP_HASH160 1604af30867311fdaf8cc0124ab9dc439b96e50f OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9141604af30867311fdaf8cc0124ab9dc439b96e50f88ac","required_signatures":"1","type":"pubkeyhash","addresses":["131RUhDyyjxXSbSPxGRCm3t6vcei1TB6MB"],"value":1311817091},{"index":"2","script_asm":"OP_RETURN aa21a9ed74968dffdb1b5ba08799029837ca1431814fa626c096d1f480832b6d61ce1954","script_hex":"6a24aa21a9ed74968dffdb1b5ba08799029837ca1431814fa626c096d1f480832b6d61ce1954","type":"nonstandard","addresses":["nonstandard6fb740799c3cba97aad17499544692e97b02072d"],"value":0}]} -{"number":"508242","timestamp":"2018-03-08 10:57:02 UTC","coinbase_param":"0352c10704ff2c7c5a2f087cf540446c010000000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 6682d73fc35667607ab4a510a57bedd0ef17aa01 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9146682d73fc35667607ab4a510a57bedd0ef17aa0188ac","required_signatures":"1","type":"pubkeyhash","addresses":["1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"],"value":1246373424}]} -{"number":"511342","timestamp":"2018-02-28 16:12:07 UTC","coinbase_param":"036ecd07132f6d696e65642062792067626d696e6572732f2cfabe6d6d5945c2c64cd6a760e01bc586cee9124629c0b0d6cfa37efcf3d1b912ed7c9467010000000000000010925765006fa22002c89d0875748fe338","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 bba87b8e503100c5cb8e308222a86eedb1fc1859 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a914bba87b8e503100c5cb8e308222a86eedb1fc185988ac","required_signatures":"1","type":"pubkeyhash","addresses":["1J7FCFaafPRxqu4X9VsaiMZr1XMemx69GR"],"value":13052036},{"index":"1","script_asm":"OP_DUP OP_HASH160 1604af30867311fdaf8cc0124ab9dc439b96e50f OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9141604af30867311fdaf8cc0124ab9dc439b96e50f88ac","required_signatures":"1","type":"pubkeyhash","addresses":["131RUhDyyjxXSbSPxGRCm3t6vcei1TB6MB"],"value":1292151656},{"index":"2","script_asm":"OP_RETURN aa21a9ed849e08b69ddbb5f8b28827611fda7c23fbb90615cd3ce9865da6ec42bbc0c7ad","script_hex":"6a24aa21a9ed849e08b69ddbb5f8b28827611fda7c23fbb90615cd3ce9865da6ec42bbc0c7ad","type":"nonstandard","addresses":["nonstandardd2cbea75dd1c981d46c3aeaeb6325b1433fc96e3"],"value":0}]} -{"number":"509373","timestamp":"2018-02-15 23:50:04 UTC","coinbase_param":"03bdc50704ae1c865a2f07a6e2831b1e010000000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 6682d73fc35667607ab4a510a57bedd0ef17aa01 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9146682d73fc35667607ab4a510a57bedd0ef17aa0188ac","required_signatures":"1","type":"pubkeyhash","addresses":["1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"],"value":1251989883},{"index":"1","script_asm":"OP_RETURN aa21a9edfbc1a4edb07a5e138717d2c05959f4190d732d3f09a8e29401514bc633589fae","script_hex":"6a24aa21a9edfbc1a4edb07a5e138717d2c05959f4190d732d3f09a8e29401514bc633589fae","type":"nonstandard","addresses":["nonstandard5b1e3b705fcf8819fcec1f83a390121be2d0a819"],"value":0},{"index":"2","script_asm":"OP_HASH160 9e49b87ea71cf1c0e6ea843ecf23608875d1ddbc OP_EQUAL","script_hex":"a9149e49b87ea71cf1c0e6ea843ecf23608875d1ddbc87","required_signatures":"1","type":"scripthash","addresses":["3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"],"value":12646362}]} -{"number":"507516","timestamp":"2018-02-04 02:36:23 UTC","coinbase_param":"037cbe0741d69d9c6acce4d141d69d9c69f9bef52f4254432e544f502ffabe6d6d143120f7b3da918f12ffb328ab935fbfe2d1cd9bb4707265d7fee23fd6cf372780000000000000005a44cacf0000f8a441200000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 9eecda14fe314502197be7401f4bc3000d73335a OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9149eecda14fe314502197be7401f4bc3000d73335a88ac","required_signatures":"1","type":"pubkeyhash","addresses":["1FVKW4rp5rN23dqFVk2tYGY4niAXMB8eZC"],"value":1263090149},{"index":"1","script_asm":"OP_RETURN aa21a9ed21b64b93e1d779f26a8813bb44995f05817020a500cb151ac38bb0209cb395ab","script_hex":"6a24aa21a9ed21b64b93e1d779f26a8813bb44995f05817020a500cb151ac38bb0209cb395ab","type":"nonstandard","addresses":["nonstandard80e57796416b73c445153a69c1086dbfe27ddeba"],"value":0},{"index":"2","script_asm":"OP_DUP OP_HASH160 172cc5187962a7255beddfbd99b547ff7a793527 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a914172cc5187962a7255beddfbd99b547ff7a79352788ac","required_signatures":"1","type":"pubkeyhash","addresses":["137YB5cpBLxLKvy8T6qXsycJ699iJjWCHH"],"value":10186210}]} -{"number":"508434","timestamp":"2018-02-09 22:17:45 UTC","coinbase_param":"0312c207040a1e7e5a2f0879deffabf9010000000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 6682d73fc35667607ab4a510a57bedd0ef17aa01 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9146682d73fc35667607ab4a510a57bedd0ef17aa0188ac","required_signatures":"1","type":"pubkeyhash","addresses":["1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"],"value":1352072429},{"index":"1","script_asm":"OP_RETURN aa21a9edf652372474827ced77917a03262cab63227c1c11b19b7a86a41c48c841551a7f","script_hex":"6a24aa21a9edf652372474827ced77917a03262cab63227c1c11b19b7a86a41c48c841551a7f","type":"nonstandard","addresses":["nonstandardc8b35d1cd90fcf75e7f222abebdf19e8d6a4ff9f"],"value":0},{"index":"2","script_asm":"OP_HASH160 9e49b87ea71cf1c0e6ea843ecf23608875d1ddbc OP_EQUAL","script_hex":"a9149e49b87ea71cf1c0e6ea843ecf23608875d1ddbc87","required_signatures":"1","type":"scripthash","addresses":["3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"],"value":13657297}]} -{"number":"510199","timestamp":"2018-02-21 06:43:35 UTC","coinbase_param":"03f7c8070418158d5a2f08b033a0a15b020000000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 6682d73fc35667607ab4a510a57bedd0ef17aa01 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9146682d73fc35667607ab4a510a57bedd0ef17aa0188ac","required_signatures":"1","type":"pubkeyhash","addresses":["1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"],"value":1241641757},{"index":"1","script_asm":"OP_RETURN aa21a9eda256aa178bf6b87d7a34f0d35292f9a27257c9b4ef0501cf1532c766da3f07e1","script_hex":"6a24aa21a9eda256aa178bf6b87d7a34f0d35292f9a27257c9b4ef0501cf1532c766da3f07e1","type":"nonstandard","addresses":["nonstandard4fcb01f18ea0ddf77636d152a45ad2a2d31b1f8e"],"value":0},{"index":"2","script_asm":"OP_HASH160 9e49b87ea71cf1c0e6ea843ecf23608875d1ddbc OP_EQUAL","script_hex":"a9149e49b87ea71cf1c0e6ea843ecf23608875d1ddbc87","required_signatures":"1","type":"scripthash","addresses":["3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"],"value":12541835}]} -{"number":"509432","timestamp":"2018-02-16 09:25:03 UTC","coinbase_param":"03f8c5070470a3865a2f08a35ce4804a010000000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 6682d73fc35667607ab4a510a57bedd0ef17aa01 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9146682d73fc35667607ab4a510a57bedd0ef17aa0188ac","required_signatures":"1","type":"pubkeyhash","addresses":["1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"],"value":1263586803},{"index":"1","script_asm":"OP_RETURN aa21a9edf919c95cb3dbb30fb7687b0d8074408c39527cbcb31c4710cfbdd94ad622d6df","script_hex":"6a24aa21a9edf919c95cb3dbb30fb7687b0d8074408c39527cbcb31c4710cfbdd94ad622d6df","type":"nonstandard","addresses":["nonstandard05ca20e125a84bb875ff3d02312773e372c86b6e"],"value":0},{"index":"2","script_asm":"OP_HASH160 9e49b87ea71cf1c0e6ea843ecf23608875d1ddbc OP_EQUAL","script_hex":"a9149e49b87ea71cf1c0e6ea843ecf23608875d1ddbc87","required_signatures":"1","type":"scripthash","addresses":["3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"],"value":12763503}]} -{"number":"682736","timestamp":"2021-05-09 11:12:32 UTC","coinbase_param":"03f06a0a202f5669614254432f4d696e6564206279206a617669647361656964373037332f2cfabe6d6d6e43ef2e06f7137b897180388403ee5019b8ff0ca4a045ea3cd82e3e41620fe91000000000000000105462a20fc21591f70e691905660b0000","outputs":[{"index":"0","script_asm":"OP_RETURN 52534b424c4f434b3a221107dbfb1f541bf709bbe399892abcdeff334445d767189d861e210032e61e","script_hex":"6a2952534b424c4f434b3a221107dbfb1f541bf709bbe399892abcdeff334445d767189d861e210032e61e","type":"nonstandard","addresses":["nonstandard6208e13509178a441fd4310ed231d7e5a97adf44"],"value":0},{"index":"1","script_asm":"OP_DUP OP_HASH160 536ffa992491508dca0354e52f32a3a7a679a53a OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a914536ffa992491508dca0354e52f32a3a7a679a53a88ac","required_signatures":"1","type":"pubkeyhash","addresses":["18cBEMRxXHqzWWCxZNtU91F5sbUNKhL5PX"],"value":625000000}]} -{"number":"510888","timestamp":"2018-02-25 18:02:53 UTC","coinbase_param":"03a8cb0741d6a4be8b5ec50141d6a4be8aedceda2f45324d2026204254432e544f502ffabe6d6dfb9300ef260402c08b34b5d1dd57904edf0e9e167e99c51d7dbe5f0c06d6b31d8000000000000000e600589dbdabaaca00000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 9eecda14fe314502197be7401f4bc3000d73335a OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9149eecda14fe314502197be7401f4bc3000d73335a88ac","required_signatures":"1","type":"pubkeyhash","addresses":["1FVKW4rp5rN23dqFVk2tYGY4niAXMB8eZC"],"value":1256971835},{"index":"1","script_asm":"OP_RETURN aa21a9edb6b01edce8b01235bcb035cd086a015d2b1414921a394b6e29210b95992eb914","script_hex":"6a24aa21a9edb6b01edce8b01235bcb035cd086a015d2b1414921a394b6e29210b95992eb914","type":"nonstandard","addresses":["nonstandard1335abedb69554ea085f6b3987f4cb28cc3edd81"],"value":0},{"index":"2","script_asm":"OP_DUP OP_HASH160 172cc5187962a7255beddfbd99b547ff7a793527 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a914172cc5187962a7255beddfbd99b547ff7a79352788ac","required_signatures":"1","type":"pubkeyhash","addresses":["137YB5cpBLxLKvy8T6qXsycJ699iJjWCHH"],"value":10136869}]} -{"number":"649061","timestamp":"2020-09-19 11:17:15 UTC","coinbase_param":"0365e7091c4d696e656420627920416e74506f6f6c36313170000b0320e01b0d80093c0000b7020000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 11dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb59 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a91411dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb5988ac","required_signatures":"1","type":"pubkeyhash","addresses":["12dRugNcdxK39288NjcDV4GX7rMsKCGn6B"],"value":632651061},{"index":"1","script_asm":"OP_RETURN aa21a9ed3b446f87f6488d8c6211db6cc1df53f39c6a963229baf7cf795e921ec2d326df","script_hex":"6a24aa21a9ed3b446f87f6488d8c6211db6cc1df53f39c6a963229baf7cf795e921ec2d326df","type":"nonstandard","addresses":["nonstandard7977ec384ce35760ed5361af730490aef279d8c9"],"value":0}]} -{"number":"649062","timestamp":"2020-09-19 11:17:00 UTC","coinbase_param":"","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 11dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb59 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a91411dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb5988ac","required_signatures":"1","type":"pubkeyhash","addresses":["0000000000000000000000000000000000000000"],"value":632651061}]} -{"number":"649064","timestamp":"2020-09-20 11:17:00 UTC","coinbase_param":"","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 11dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb59 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a91411dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb5988ac","required_signatures":"1","type":"pubkeyhash","addresses":["0000000000000000000000000000000000000000"],"value":632651061}]} +{"number":"507715","timestamp":"2018-02-05 04:54:34 UTC","identifiers":"0343bf07132f6d696e65642062792067626d696e6572732f2cfabe6d6d94976ecebbc73b3d4214b3d7ab330dca2129ebfcc863fa75623c6f95891e7346010000000000000010af8b66002da910ef408c776852840100","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 bba87b8e503100c5cb8e308222a86eedb1fc1859 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a914bba87b8e503100c5cb8e308222a86eedb1fc185988ac","required_signatures":"1","type":"pubkeyhash","addresses":["1J7FCFaafPRxqu4X9VsaiMZr1XMemx69GR"],"value":13250677},{"index":"1","script_asm":"OP_DUP OP_HASH160 1604af30867311fdaf8cc0124ab9dc439b96e50f OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9141604af30867311fdaf8cc0124ab9dc439b96e50f88ac","required_signatures":"1","type":"pubkeyhash","addresses":["131RUhDyyjxXSbSPxGRCm3t6vcei1TB6MB"],"value":1311817091},{"index":"2","script_asm":"OP_RETURN aa21a9ed74968dffdb1b5ba08799029837ca1431814fa626c096d1f480832b6d61ce1954","script_hex":"6a24aa21a9ed74968dffdb1b5ba08799029837ca1431814fa626c096d1f480832b6d61ce1954","type":"nonstandard","addresses":["nonstandard6fb740799c3cba97aad17499544692e97b02072d"],"value":0}]} +{"number":"508242","timestamp":"2018-03-08 10:57:02 UTC","identifiers":"0352c10704ff2c7c5a2f087cf540446c010000000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 6682d73fc35667607ab4a510a57bedd0ef17aa01 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9146682d73fc35667607ab4a510a57bedd0ef17aa0188ac","required_signatures":"1","type":"pubkeyhash","addresses":["1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"],"value":1246373424}]} +{"number":"511342","timestamp":"2018-02-28 16:12:07 UTC","identifiers":"036ecd07132f6d696e65642062792067626d696e6572732f2cfabe6d6d5945c2c64cd6a760e01bc586cee9124629c0b0d6cfa37efcf3d1b912ed7c9467010000000000000010925765006fa22002c89d0875748fe338","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 bba87b8e503100c5cb8e308222a86eedb1fc1859 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a914bba87b8e503100c5cb8e308222a86eedb1fc185988ac","required_signatures":"1","type":"pubkeyhash","addresses":["1J7FCFaafPRxqu4X9VsaiMZr1XMemx69GR"],"value":13052036},{"index":"1","script_asm":"OP_DUP OP_HASH160 1604af30867311fdaf8cc0124ab9dc439b96e50f OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9141604af30867311fdaf8cc0124ab9dc439b96e50f88ac","required_signatures":"1","type":"pubkeyhash","addresses":["131RUhDyyjxXSbSPxGRCm3t6vcei1TB6MB"],"value":1292151656},{"index":"2","script_asm":"OP_RETURN aa21a9ed849e08b69ddbb5f8b28827611fda7c23fbb90615cd3ce9865da6ec42bbc0c7ad","script_hex":"6a24aa21a9ed849e08b69ddbb5f8b28827611fda7c23fbb90615cd3ce9865da6ec42bbc0c7ad","type":"nonstandard","addresses":["nonstandardd2cbea75dd1c981d46c3aeaeb6325b1433fc96e3"],"value":0}]} +{"number":"509373","timestamp":"2018-02-15 23:50:04 UTC","identifiers":"03bdc50704ae1c865a2f07a6e2831b1e010000000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 6682d73fc35667607ab4a510a57bedd0ef17aa01 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9146682d73fc35667607ab4a510a57bedd0ef17aa0188ac","required_signatures":"1","type":"pubkeyhash","addresses":["1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"],"value":1251989883},{"index":"1","script_asm":"OP_RETURN aa21a9edfbc1a4edb07a5e138717d2c05959f4190d732d3f09a8e29401514bc633589fae","script_hex":"6a24aa21a9edfbc1a4edb07a5e138717d2c05959f4190d732d3f09a8e29401514bc633589fae","type":"nonstandard","addresses":["nonstandard5b1e3b705fcf8819fcec1f83a390121be2d0a819"],"value":0},{"index":"2","script_asm":"OP_HASH160 9e49b87ea71cf1c0e6ea843ecf23608875d1ddbc OP_EQUAL","script_hex":"a9149e49b87ea71cf1c0e6ea843ecf23608875d1ddbc87","required_signatures":"1","type":"scripthash","addresses":["3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"],"value":12646362}]} +{"number":"507516","timestamp":"2018-02-04 02:36:23 UTC","identifiers":"037cbe0741d69d9c6acce4d141d69d9c69f9bef52f4254432e544f502ffabe6d6d143120f7b3da918f12ffb328ab935fbfe2d1cd9bb4707265d7fee23fd6cf372780000000000000005a44cacf0000f8a441200000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 9eecda14fe314502197be7401f4bc3000d73335a OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9149eecda14fe314502197be7401f4bc3000d73335a88ac","required_signatures":"1","type":"pubkeyhash","addresses":["1FVKW4rp5rN23dqFVk2tYGY4niAXMB8eZC"],"value":1263090149},{"index":"1","script_asm":"OP_RETURN aa21a9ed21b64b93e1d779f26a8813bb44995f05817020a500cb151ac38bb0209cb395ab","script_hex":"6a24aa21a9ed21b64b93e1d779f26a8813bb44995f05817020a500cb151ac38bb0209cb395ab","type":"nonstandard","addresses":["nonstandard80e57796416b73c445153a69c1086dbfe27ddeba"],"value":0},{"index":"2","script_asm":"OP_DUP OP_HASH160 172cc5187962a7255beddfbd99b547ff7a793527 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a914172cc5187962a7255beddfbd99b547ff7a79352788ac","required_signatures":"1","type":"pubkeyhash","addresses":["137YB5cpBLxLKvy8T6qXsycJ699iJjWCHH"],"value":10186210}]} +{"number":"508434","timestamp":"2018-02-09 22:17:45 UTC","identifiers":"0312c207040a1e7e5a2f0879deffabf9010000000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 6682d73fc35667607ab4a510a57bedd0ef17aa01 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9146682d73fc35667607ab4a510a57bedd0ef17aa0188ac","required_signatures":"1","type":"pubkeyhash","addresses":["1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"],"value":1352072429},{"index":"1","script_asm":"OP_RETURN aa21a9edf652372474827ced77917a03262cab63227c1c11b19b7a86a41c48c841551a7f","script_hex":"6a24aa21a9edf652372474827ced77917a03262cab63227c1c11b19b7a86a41c48c841551a7f","type":"nonstandard","addresses":["nonstandardc8b35d1cd90fcf75e7f222abebdf19e8d6a4ff9f"],"value":0},{"index":"2","script_asm":"OP_HASH160 9e49b87ea71cf1c0e6ea843ecf23608875d1ddbc OP_EQUAL","script_hex":"a9149e49b87ea71cf1c0e6ea843ecf23608875d1ddbc87","required_signatures":"1","type":"scripthash","addresses":["3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"],"value":13657297}]} +{"number":"510199","timestamp":"2018-02-21 06:43:35 UTC","identifiers":"03f7c8070418158d5a2f08b033a0a15b020000000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 6682d73fc35667607ab4a510a57bedd0ef17aa01 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9146682d73fc35667607ab4a510a57bedd0ef17aa0188ac","required_signatures":"1","type":"pubkeyhash","addresses":["1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"],"value":1241641757},{"index":"1","script_asm":"OP_RETURN aa21a9eda256aa178bf6b87d7a34f0d35292f9a27257c9b4ef0501cf1532c766da3f07e1","script_hex":"6a24aa21a9eda256aa178bf6b87d7a34f0d35292f9a27257c9b4ef0501cf1532c766da3f07e1","type":"nonstandard","addresses":["nonstandard4fcb01f18ea0ddf77636d152a45ad2a2d31b1f8e"],"value":0},{"index":"2","script_asm":"OP_HASH160 9e49b87ea71cf1c0e6ea843ecf23608875d1ddbc OP_EQUAL","script_hex":"a9149e49b87ea71cf1c0e6ea843ecf23608875d1ddbc87","required_signatures":"1","type":"scripthash","addresses":["3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"],"value":12541835}]} +{"number":"509432","timestamp":"2018-02-16 09:25:03 UTC","identifiers":"03f8c5070470a3865a2f08a35ce4804a010000000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 6682d73fc35667607ab4a510a57bedd0ef17aa01 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9146682d73fc35667607ab4a510a57bedd0ef17aa0188ac","required_signatures":"1","type":"pubkeyhash","addresses":["1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"],"value":1263586803},{"index":"1","script_asm":"OP_RETURN aa21a9edf919c95cb3dbb30fb7687b0d8074408c39527cbcb31c4710cfbdd94ad622d6df","script_hex":"6a24aa21a9edf919c95cb3dbb30fb7687b0d8074408c39527cbcb31c4710cfbdd94ad622d6df","type":"nonstandard","addresses":["nonstandard05ca20e125a84bb875ff3d02312773e372c86b6e"],"value":0},{"index":"2","script_asm":"OP_HASH160 9e49b87ea71cf1c0e6ea843ecf23608875d1ddbc OP_EQUAL","script_hex":"a9149e49b87ea71cf1c0e6ea843ecf23608875d1ddbc87","required_signatures":"1","type":"scripthash","addresses":["3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"],"value":12763503}]} +{"number":"682736","timestamp":"2021-05-09 11:12:32 UTC","identifiers":"03f06a0a202f5669614254432f4d696e6564206279206a617669647361656964373037332f2cfabe6d6d6e43ef2e06f7137b897180388403ee5019b8ff0ca4a045ea3cd82e3e41620fe91000000000000000105462a20fc21591f70e691905660b0000","outputs":[{"index":"0","script_asm":"OP_RETURN 52534b424c4f434b3a221107dbfb1f541bf709bbe399892abcdeff334445d767189d861e210032e61e","script_hex":"6a2952534b424c4f434b3a221107dbfb1f541bf709bbe399892abcdeff334445d767189d861e210032e61e","type":"nonstandard","addresses":["nonstandard6208e13509178a441fd4310ed231d7e5a97adf44"],"value":0},{"index":"1","script_asm":"OP_DUP OP_HASH160 536ffa992491508dca0354e52f32a3a7a679a53a OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a914536ffa992491508dca0354e52f32a3a7a679a53a88ac","required_signatures":"1","type":"pubkeyhash","addresses":["18cBEMRxXHqzWWCxZNtU91F5sbUNKhL5PX"],"value":625000000}]} +{"number":"510888","timestamp":"2018-02-25 18:02:53 UTC","identifiers":"03a8cb0741d6a4be8b5ec50141d6a4be8aedceda2f45324d2026204254432e544f502ffabe6d6dfb9300ef260402c08b34b5d1dd57904edf0e9e167e99c51d7dbe5f0c06d6b31d8000000000000000e600589dbdabaaca00000000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 9eecda14fe314502197be7401f4bc3000d73335a OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a9149eecda14fe314502197be7401f4bc3000d73335a88ac","required_signatures":"1","type":"pubkeyhash","addresses":["1FVKW4rp5rN23dqFVk2tYGY4niAXMB8eZC"],"value":1256971835},{"index":"1","script_asm":"OP_RETURN aa21a9edb6b01edce8b01235bcb035cd086a015d2b1414921a394b6e29210b95992eb914","script_hex":"6a24aa21a9edb6b01edce8b01235bcb035cd086a015d2b1414921a394b6e29210b95992eb914","type":"nonstandard","addresses":["nonstandard1335abedb69554ea085f6b3987f4cb28cc3edd81"],"value":0},{"index":"2","script_asm":"OP_DUP OP_HASH160 172cc5187962a7255beddfbd99b547ff7a793527 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a914172cc5187962a7255beddfbd99b547ff7a79352788ac","required_signatures":"1","type":"pubkeyhash","addresses":["137YB5cpBLxLKvy8T6qXsycJ699iJjWCHH"],"value":10136869}]} +{"number":"649061","timestamp":"2020-09-19 11:17:15 UTC","identifiers":"0365e7091c4d696e656420627920416e74506f6f6c36313170000b0320e01b0d80093c0000b7020000","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 11dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb59 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a91411dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb5988ac","required_signatures":"1","type":"pubkeyhash","addresses":["12dRugNcdxK39288NjcDV4GX7rMsKCGn6B"],"value":632651061},{"index":"1","script_asm":"OP_RETURN aa21a9ed3b446f87f6488d8c6211db6cc1df53f39c6a963229baf7cf795e921ec2d326df","script_hex":"6a24aa21a9ed3b446f87f6488d8c6211db6cc1df53f39c6a963229baf7cf795e921ec2d326df","type":"nonstandard","addresses":["nonstandard7977ec384ce35760ed5361af730490aef279d8c9"],"value":0}]} +{"number":"649062","timestamp":"2020-09-19 11:17:00 UTC","identifiers":"","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 11dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb59 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a91411dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb5988ac","required_signatures":"1","type":"pubkeyhash","addresses":["0000000000000000000000000000000000000000"],"value":632651061}]} +{"number":"649064","timestamp":"2020-09-20 11:17:00 UTC","identifiers":"","outputs":[{"index":"0","script_asm":"OP_DUP OP_HASH160 11dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb59 OP_EQUALVERIFY OP_CHECKSIG","script_hex":"76a91411dbe48cc6b617f9c6adaf4d9ed5f625b1c7cb5988ac","required_signatures":"1","type":"pubkeyhash","addresses":["0000000000000000000000000000000000000000"],"value":632651061}]} diff --git a/input/sample_cardano_raw_data.json b/input/sample_cardano_raw_data.json index e24b8d0..048c6a6 100644 --- a/input/sample_cardano_raw_data.json +++ b/input/sample_cardano_raw_data.json @@ -1,5 +1,5 @@ -{"number":"17809932","coinbase_param":"CFLOW","timestamp":"2020-12-31T00:57:03","pool_hash":"e7b605b72af41d6e8e6894274dedd18114f1759fea500b6d07031535"} -{"number":"55555555555","coinbase_param":"1PCT5","timestamp":"2020-12-31T06:42:01"} -{"number":"66666666666","timestamp":"2020-12-31T04:42:01","pool_hash":"1d8988c2057d6efd6a094e468840a51942ab03b5b69b07a2bca71b53"} -{"number":"00000000000","coinbase_param":"ARRA","timestamp":"2020-12-31T06:42:00"} +{"number":"17809932","identifiers":"CFLOW","timestamp":"2020-12-31T00:57:03","reward_addresses":"e7b605b72af41d6e8e6894274dedd18114f1759fea500b6d07031535"} +{"number":"55555555555","identifiers":"1PCT5","timestamp":"2020-12-31T06:42:01"} +{"number":"66666666666","timestamp":"2020-12-31T04:42:01","reward_addresses":"1d8988c2057d6efd6a094e468840a51942ab03b5b69b07a2bca71b53"} +{"number":"00000000000","identifiers":"ARRA","timestamp":"2020-12-31T06:42:00"} {"number":"00000000001","timestamp":"2020-12-31T06:00:00"} diff --git a/input/sample_ethereum_raw_data.json b/input/sample_ethereum_raw_data.json index c63a981..37ad1e0 100644 --- a/input/sample_ethereum_raw_data.json +++ b/input/sample_ethereum_raw_data.json @@ -1,10 +1,10 @@ -{"number":"16382083","timestamp":"2023-01-11 07:29:47 UTC","coinbase_addresses":"0x3bee5122e2a2fbe11287aafb0cb918e22abb5436","coinbase_param":"0x"} -{"number":"11184490","timestamp":"2020-11-03 13:33:18 UTC","coinbase_addresses":"0x45133a7e1cc7e18555ae8a4ee632a8a61de90df6","coinbase_param":"0x20"} -{"number":"11183739","timestamp":"2020-11-03 10:44:14 UTC","coinbase_addresses":"0xe9b54a47e3f401d37798fc4e22f14b78475c2afc","coinbase_param":"0x3"} -{"number":"11185799","timestamp":"2020-11-03 18:34:11 UTC","coinbase_addresses":"0xe9b54a47e3f401d37798fc4e22f14b78475c2afc","coinbase_param":"0x36"} -{"number":"11186643","timestamp":"2020-11-03 21:33:27 UTC","coinbase_addresses":"0xe9b54a47e3f401d37798fc4e22f14b78475c2afc","coinbase_param":"0x36"} -{"number":"11183702","timestamp":"2020-11-03 10:37:32 UTC","coinbase_addresses":"0xe9b54a47e3f401d37798fc4e22f14b78475c2afc","coinbase_param":"0x36"} -{"number":"11181062","timestamp":"2020-11-03 00:56:48 UTC","coinbase_addresses":"0xe9b54a47e3f401d37798fc4e22f14b78475c2afc","coinbase_param":"0x36"} -{"number":"11183969","timestamp":"2020-11-03 11:33:44 UTC","coinbase_addresses":"0x8595dd9e0438640b5e1254f9df579ac12a86865f","coinbase_param":"0x657a696c2e6d65"} -{"number":"11184329","timestamp":"2020-11-03 12:56:41 UTC","coinbase_addresses":"0x8595dd9e0438640b5e1254f9df579ac12a86865f","coinbase_param":"0x657a696c2e6d65"} -{"number":"11183793","timestamp":"2020-11-03 10:56:07 UTC","coinbase_addresses":"0x8595dd9e0438640b5e1254f9df579ac12a86865f","coinbase_param":"0x657a696c2e6d65"} +{"number":"16382083","timestamp":"2023-01-11 07:29:47 UTC","reward_addresses":"0x3bee5122e2a2fbe11287aafb0cb918e22abb5436","identifiers":"0x"} +{"number":"11184490","timestamp":"2020-11-03 13:33:18 UTC","reward_addresses":"0x45133a7e1cc7e18555ae8a4ee632a8a61de90df6","identifiers":"0x20"} +{"number":"11183739","timestamp":"2020-11-03 10:44:14 UTC","reward_addresses":"0xe9b54a47e3f401d37798fc4e22f14b78475c2afc","identifiers":"0x3"} +{"number":"11185799","timestamp":"2020-11-03 18:34:11 UTC","reward_addresses":"0xe9b54a47e3f401d37798fc4e22f14b78475c2afc","identifiers":"0x36"} +{"number":"11186643","timestamp":"2020-11-03 21:33:27 UTC","reward_addresses":"0xe9b54a47e3f401d37798fc4e22f14b78475c2afc","identifiers":"0x36"} +{"number":"11183702","timestamp":"2020-11-03 10:37:32 UTC","reward_addresses":"0xe9b54a47e3f401d37798fc4e22f14b78475c2afc","identifiers":"0x36"} +{"number":"11181062","timestamp":"2020-11-03 00:56:48 UTC","reward_addresses":"0xe9b54a47e3f401d37798fc4e22f14b78475c2afc","identifiers":"0x36"} +{"number":"11183969","timestamp":"2020-11-03 11:33:44 UTC","reward_addresses":"0x8595dd9e0438640b5e1254f9df579ac12a86865f","identifiers":"0x657a696c2e6d65"} +{"number":"11184329","timestamp":"2020-11-03 12:56:41 UTC","reward_addresses":"0x8595dd9e0438640b5e1254f9df579ac12a86865f","identifiers":"0x657a696c2e6d65"} +{"number":"11183793","timestamp":"2020-11-03 10:56:07 UTC","reward_addresses":"0x8595dd9e0438640b5e1254f9df579ac12a86865f","identifiers":"0x657a696c2e6d65"} diff --git a/input/sample_tezos_raw_data.json b/input/sample_tezos_raw_data.json index 65de3d3..2e5fde0 100644 --- a/input/sample_tezos_raw_data.json +++ b/input/sample_tezos_raw_data.json @@ -1,6 +1,6 @@ -{"number":"1649812","timestamp":"2021-08-30 00:36:18 UTC","coinbase_addresses":"tz1Kf25fX1VdmYGSEzwFy1wNmkbSEZ2V83sY"} -{"number":"1650474","timestamp":"2021-08-30 06:11:58 UTC","coinbase_addresses":"tz1Vd1rXpV8hTHbFXCXN3c3qzCsgcU5BZw1e"} -{"number":"1650309","timestamp":"2021-08-30 04:49:28 UTC","coinbase_addresses":"tz1Kf25fX1VdmYGSEzwFy1wNmkbSEZ2V83sY"} +{"number":"1649812","timestamp":"2021-08-30 00:36:18 UTC","reward_addresses":"tz1Kf25fX1VdmYGSEzwFy1wNmkbSEZ2V83sY"} +{"number":"1650474","timestamp":"2021-08-30 06:11:58 UTC","reward_addresses":"tz1Vd1rXpV8hTHbFXCXN3c3qzCsgcU5BZw1e"} +{"number":"1650309","timestamp":"2021-08-30 04:49:28 UTC","reward_addresses":"tz1Kf25fX1VdmYGSEzwFy1wNmkbSEZ2V83sY"} {"number":"1651794","timestamp":"2021-08-30 17:41:08 UTC"} -{"number":"1649839","timestamp":"2021-08-30 00:49:48 UTC","coinbase_addresses":"tz1Kt4P8BCaP93AEV4eA7gmpRryWt5hznjCP"} -{"number":"0000000","timestamp":"2018-08-30 00:36:18 UTC","coinbase_addresses":"tz0000000000000000000000000000000000"} +{"number":"1649839","timestamp":"2021-08-30 00:49:48 UTC","reward_addresses":"tz1Kt4P8BCaP93AEV4eA7gmpRryWt5hznjCP"} +{"number":"0000000","timestamp":"2018-08-30 00:36:18 UTC","reward_addresses":"tz0000000000000000000000000000000000"} diff --git a/queries.yaml b/queries.yaml index 3d0e31f..47c5b9c 100644 --- a/queries.yaml +++ b/queries.yaml @@ -1,59 +1,59 @@ -bitcoin: - SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_bitcoin.transactions`.outputs - FROM `bigquery-public-data.crypto_bitcoin.transactions` - JOIN `bigquery-public-data.crypto_bitcoin.blocks` ON `bigquery-public-data.crypto_bitcoin.transactions`.block_number = `bigquery-public-data.crypto_bitcoin.blocks`.number - WHERE is_coinbase is TRUE - AND timestamp > '2017-12-31' - -bitcoin_cash: - SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_bitcoin_cash.transactions`.outputs - FROM `bigquery-public-data.crypto_bitcoin_cash.transactions` - JOIN `bigquery-public-data.crypto_bitcoin_cash.blocks` ON `bigquery-public-data.crypto_bitcoin_cash.transactions`.block_number = `bigquery-public-data.crypto_bitcoin_cash.blocks`.number - WHERE is_coinbase is TRUE - AND timestamp > '2018-12-31' +#bitcoin: +# SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_bitcoin.transactions`.outputs +# FROM `bigquery-public-data.crypto_bitcoin.transactions` +# JOIN `bigquery-public-data.crypto_bitcoin.blocks` ON `bigquery-public-data.crypto_bitcoin.transactions`.block_number = `bigquery-public-data.crypto_bitcoin.blocks`.number +# WHERE is_coinbase is TRUE +# AND timestamp > '2017-12-31' + +#bitcoin_cash: +# SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_bitcoin_cash.transactions`.outputs +# FROM `bigquery-public-data.crypto_bitcoin_cash.transactions` +# JOIN `bigquery-public-data.crypto_bitcoin_cash.blocks` ON `bigquery-public-data.crypto_bitcoin_cash.transactions`.block_number = `bigquery-public-data.crypto_bitcoin_cash.blocks`.number +# WHERE is_coinbase is TRUE +# AND timestamp > '2018-12-31' cardano: - SELECT `iog-data-analytics.cardano_mainnet.block`.slot_no as number, `iog-data-analytics.cardano_mainnet.pool_offline_data`.ticker_name as coinbase_param, `iog-data-analytics.cardano_mainnet.block`.block_time as timestamp, `iog-data-analytics.cardano_mainnet.block`.pool_hash - FROM `iog-data-analytics.cardano_mainnet.block` - LEFT JOIN `iog-data-analytics.cardano_mainnet.pool_offline_data` ON `iog-data-analytics.cardano_mainnet.block`.pool_hash = `iog-data-analytics.cardano_mainnet.pool_offline_data`.pool_hash - WHERE `iog-data-analytics.cardano_mainnet.block`.block_time > '2020-12-31' - -dash: - SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_dash.transactions`.outputs - FROM `bigquery-public-data.crypto_dash.transactions` - JOIN `bigquery-public-data.crypto_dash.blocks` ON `bigquery-public-data.crypto_dash.transactions`.block_number = `bigquery-public-data.crypto_dash.blocks`.number - WHERE is_coinbase is TRUE - AND timestamp > '2018-12-31' + SELECT `iog-data-analytics.cardano_mainnet.block`.slot_no as number, `iog-data-analytics.cardano_mainnet.pool_offline_data`.ticker_name as identifiers, `iog-data-analytics.cardano_mainnet.block`.block_time as timestamp,`iog-data-analytics.cardano_mainnet.block`.pool_hash as reward_addresses + FROM `iog-data-analytics.cardano_mainnet.block` + LEFT JOIN `iog-data-analytics.cardano_mainnet.pool_offline_data` ON `iog-data-analytics.cardano_mainnet.block`.pool_hash = `iog-data-analytics.cardano_mainnet.pool_offline_data`.pool_hash + WHERE `iog-data-analytics.cardano_mainnet.block`.block_time > '2020-12-31' + +#dash: +# SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_dash.transactions`.outputs +# FROM `bigquery-public-data.crypto_dash.transactions` +# JOIN `bigquery-public-data.crypto_dash.blocks` ON `bigquery-public-data.crypto_dash.transactions`.block_number = `bigquery-public-data.crypto_dash.blocks`.number +# WHERE is_coinbase is TRUE +# AND timestamp > '2018-12-31' dogecoin: - SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_dogecoin.transactions`.outputs - FROM `bigquery-public-data.crypto_dogecoin.transactions` - JOIN `bigquery-public-data.crypto_dogecoin.blocks` ON `bigquery-public-data.crypto_dogecoin.transactions`.block_number = `bigquery-public-data.crypto_dogecoin.blocks`.number - WHERE is_coinbase is TRUE - AND timestamp > '2019-12-31' + SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_dogecoin.transactions`.outputs + FROM `bigquery-public-data.crypto_dogecoin.transactions` + JOIN `bigquery-public-data.crypto_dogecoin.blocks` ON `bigquery-public-data.crypto_dogecoin.transactions`.block_number = `bigquery-public-data.crypto_dogecoin.blocks`.number + WHERE is_coinbase is TRUE + AND timestamp > '2019-12-31' ethereum: - SELECT number, timestamp, miner as coinbase_addresses, extra_data as coinbase_param - FROM `bigquery-public-data.crypto_ethereum.blocks` - WHERE timestamp > '2018-12-31' + SELECT number, timestamp, miner as reward_addresses, extra_data as identifiers + FROM `bigquery-public-data.crypto_ethereum.blocks` + WHERE timestamp > '2018-12-31' litecoin: - SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_litecoin.transactions`.outputs - FROM `bigquery-public-data.crypto_litecoin.transactions` - JOIN `bigquery-public-data.crypto_litecoin.blocks` ON `bigquery-public-data.crypto_litecoin.transactions`.block_number = `bigquery-public-data.crypto_litecoin.blocks`.number - WHERE is_coinbase is TRUE - AND timestamp > '2018-12-31' + SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_litecoin.transactions`.outputs + FROM `bigquery-public-data.crypto_litecoin.transactions` + JOIN `bigquery-public-data.crypto_litecoin.blocks` ON `bigquery-public-data.crypto_litecoin.transactions`.block_number = `bigquery-public-data.crypto_litecoin.blocks`.number + WHERE is_coinbase is TRUE + AND timestamp > '2018-12-31' tezos: - SELECT level as number, timestamp, baker as coinbase_addresses - FROM `public-data-finance.crypto_tezos.blocks` - WHERE timestamp > '2020-12-31' + SELECT level as number, timestamp, baker as reward_addresses + FROM `public-data-finance.crypto_tezos.blocks` + WHERE timestamp > '2020-12-31' zcash: - SELECT block_number as number, block_timestamp as timestamp, coinbase_param, `bigquery-public-data.crypto_zcash.transactions`.outputs - FROM `bigquery-public-data.crypto_zcash.transactions` - JOIN `bigquery-public-data.crypto_zcash.blocks` ON `bigquery-public-data.crypto_zcash.transactions`.block_number = `bigquery-public-data.crypto_zcash.blocks`.number - WHERE is_coinbase is TRUE - AND timestamp > '2018-12-31' + SELECT block_number as number, block_timestamp as timestamp, coinbase_param as identifiers, `bigquery-public-data.crypto_zcash.transactions`.outputs + FROM `bigquery-public-data.crypto_zcash.transactions` + JOIN `bigquery-public-data.crypto_zcash.blocks` ON `bigquery-public-data.crypto_zcash.transactions`.block_number = `bigquery-public-data.crypto_zcash.blocks`.number + WHERE is_coinbase is TRUE + AND timestamp > '2018-12-31' diff --git a/src/helpers/helper.py b/src/helpers/helper.py index a1d0b09..e8254dd 100644 --- a/src/helpers/helper.py +++ b/src/helpers/helper.py @@ -70,9 +70,9 @@ def get_time_period(frm, to): def get_known_entities(ledger): known_entities = set() try: - with open(HELPERS_DIR / f'pool_information/coinbase_tags/{ledger}.json') as f: - coinbase_tags = json.load(f) - for info in coinbase_tags.values(): + with open(HELPERS_DIR / f'pool_information/identifiers/{ledger}.json') as f: + identifiers = json.load(f) + for info in identifiers.values(): known_entities.add(info['name']) except FileNotFoundError: pass @@ -105,17 +105,17 @@ def get_known_entities(ledger): def get_pool_tags(project_name): """ - Retrieves coinbase tag data regarding the pools of a project. + Retrieves tag data (identifiers) about the pools of a project. :param project_name: string that corresponds to the project under consideration :returns: pool_tags, a dictionary with the tags information of each pool """ try: - with open(HELPERS_DIR / f'pool_information/coinbase_tags/{project_name}.json') as f: - coinbase_tags = json.load(f) + with open(HELPERS_DIR / f'pool_information/identifiers/{project_name}.json') as f: + tags = json.load(f) except FileNotFoundError: - coinbase_tags = {} + tags = {} - return coinbase_tags + return tags def get_pool_links(project_name, timeframe): diff --git a/src/helpers/pool_information/README.md b/src/helpers/pool_information/README.md index a545ce2..cb0e375 100644 --- a/src/helpers/pool_information/README.md +++ b/src/helpers/pool_information/README.md @@ -5,7 +5,7 @@ This directory contains pool information about the supported projects. There exist three subdirectories. In each subdirectory there exists a file for the corresponding ledger data, if such data exists. -`coinbase_tags` defines information about pools and miners. Each key +`identifiers` defines information about pools and miners. Each key corresponds to a tag or ticker, by which the pool is identifiable in its produced blocks. The value for each key is a dictionary of pool-related information, specifically its name, a url to its homepage, etc. Each file's @@ -65,7 +65,7 @@ clusters: ] } -coinbase_tags: +identifiers: { "P1": { "name": "Pool P1", @@ -94,7 +94,9 @@ address "address1" is known to be controlled by P2 since the beginning of 2019. To update or add information, you should open a Pull Request that changes one of the files. To do this: -- Open the file (e.g., for Bitcoin, follow [this link](https://github.com/Blockchain-Technology-Lab/pooling-analysis/blob/main/src/helpers/pool_information/coinbase_tags/bitcoin.json)) on your browser. +- Open the file (e.g., for Bitcoin, follow [this link](https://github. + com/Blockchain-Technology-Lab/pooling-analysis/blob/main/src/helpers/pool_information/identifiers/bitcoin.json)) on + your browser. - Click `Edit this file`. - Make your changes in the file. - On the bottom, initiate a Pull Request. diff --git a/src/helpers/pool_information/coinbase_tags/bitcoin.json b/src/helpers/pool_information/identifiers/bitcoin.json similarity index 100% rename from src/helpers/pool_information/coinbase_tags/bitcoin.json rename to src/helpers/pool_information/identifiers/bitcoin.json diff --git a/src/helpers/pool_information/coinbase_tags/bitcoin_cash.json b/src/helpers/pool_information/identifiers/bitcoin_cash.json similarity index 100% rename from src/helpers/pool_information/coinbase_tags/bitcoin_cash.json rename to src/helpers/pool_information/identifiers/bitcoin_cash.json diff --git a/src/helpers/pool_information/coinbase_tags/cardano.json b/src/helpers/pool_information/identifiers/cardano.json similarity index 100% rename from src/helpers/pool_information/coinbase_tags/cardano.json rename to src/helpers/pool_information/identifiers/cardano.json diff --git a/src/helpers/pool_information/coinbase_tags/dash.json b/src/helpers/pool_information/identifiers/dash.json similarity index 100% rename from src/helpers/pool_information/coinbase_tags/dash.json rename to src/helpers/pool_information/identifiers/dash.json diff --git a/src/helpers/pool_information/coinbase_tags/dogecoin.json b/src/helpers/pool_information/identifiers/dogecoin.json similarity index 100% rename from src/helpers/pool_information/coinbase_tags/dogecoin.json rename to src/helpers/pool_information/identifiers/dogecoin.json diff --git a/src/helpers/pool_information/coinbase_tags/ethereum.json b/src/helpers/pool_information/identifiers/ethereum.json similarity index 100% rename from src/helpers/pool_information/coinbase_tags/ethereum.json rename to src/helpers/pool_information/identifiers/ethereum.json diff --git a/src/helpers/pool_information/coinbase_tags/litecoin.json b/src/helpers/pool_information/identifiers/litecoin.json similarity index 100% rename from src/helpers/pool_information/coinbase_tags/litecoin.json rename to src/helpers/pool_information/identifiers/litecoin.json diff --git a/src/helpers/pool_information/coinbase_tags/test.json b/src/helpers/pool_information/identifiers/test.json similarity index 100% rename from src/helpers/pool_information/coinbase_tags/test.json rename to src/helpers/pool_information/identifiers/test.json diff --git a/src/helpers/pool_information/coinbase_tags/unidentified.json b/src/helpers/pool_information/identifiers/unidentified.json similarity index 100% rename from src/helpers/pool_information/coinbase_tags/unidentified.json rename to src/helpers/pool_information/identifiers/unidentified.json diff --git a/src/helpers/pool_information/coinbase_tags/zcash.json b/src/helpers/pool_information/identifiers/zcash.json similarity index 100% rename from src/helpers/pool_information/coinbase_tags/zcash.json rename to src/helpers/pool_information/identifiers/zcash.json diff --git a/src/map.py b/src/map.py index c3f66bc..41084e8 100644 --- a/src/map.py +++ b/src/map.py @@ -1,8 +1,8 @@ import argparse -from src.mappings.bitcoin import BitcoinMapping -from src.mappings.ethereum import EthereumMapping -from src.mappings.cardano import CardanoMapping -from src.mappings.tezos import TezosMapping +from src.mappings.bitcoin_mapping import BitcoinMapping +from src.mappings.ethereum_mapping import EthereumMapping +from src.mappings.cardano_mapping import CardanoMapping +from src.mappings.tezos_mapping import TezosMapping from src.helpers.helper import OUTPUT_DIR START_YEAR = 2018 diff --git a/src/mappings/bitcoin.py b/src/mappings/bitcoin_mapping.py similarity index 83% rename from src/mappings/bitcoin.py rename to src/mappings/bitcoin_mapping.py index f420bca..86aafda 100644 --- a/src/mappings/bitcoin.py +++ b/src/mappings/bitcoin_mapping.py @@ -1,6 +1,7 @@ from collections import defaultdict import codecs -from src.helpers.helper import write_blocks_per_entity_to_file, get_pool_tags, get_pool_links, get_pool_addresses, get_special_addresses +from src.helpers.helper import write_blocks_per_entity_to_file, get_pool_tags, get_pool_links, get_pool_addresses, \ + get_special_addresses from src.mappings.mapping import Mapping YEAR_DIGITS = 4 @@ -41,15 +42,15 @@ def process(self, timeframe): pool_links = get_pool_links(self.project_name, day) daily_links[day] = pool_links - coinbase_param = codecs.decode(tx['coinbase_param'], 'hex') - coinbase_addresses = list(set(tx['coinbase_addresses'].split(',')) - special_addresses) + identifiers = codecs.decode(tx['identifiers'], 'hex') + reward_addresses = list(set(tx['reward_addresses'].split(',')) - special_addresses) pool_match = False - for (tag, info) in pool_tags.items(): # Check if coinbase param contains known pool tag - if tag in str(coinbase_param): + for (tag, info) in pool_tags.items(): # Check if identifiers contain known pool tag + if tag in str(identifiers): entity = info['name'] pool_match = True - for addr in coinbase_addresses: + for addr in reward_addresses: if addr in pool_addresses.keys() and pool_addresses[addr] != entity: multi_pool_addresses.append(f'{tx["number"]},{tx["timestamp"]},{addr},{entity}') pool_addresses[addr] = entity @@ -57,7 +58,7 @@ def process(self, timeframe): if not pool_match: block_pools = set() - for addr in coinbase_addresses: # Check if address is associated with pool + for addr in reward_addresses: # Check if address is associated with pool if addr in pool_addresses.keys(): block_pools.add((addr, pool_addresses[addr])) if block_pools: @@ -67,13 +68,13 @@ def process(self, timeframe): multi_pool_info = '/'.join([f'{i[0]}({i[1]})' for i in block_pools]) multi_pool_blocks.append(f'{tx["number"]},{tx["timestamp"]},{multi_pool_info}') else: - if len(coinbase_addresses) == 1: - entity = coinbase_addresses[0] - elif len(coinbase_addresses) == 0: + if len(reward_addresses) == 1: + entity = reward_addresses[0] + elif len(reward_addresses) == 0: entity = '----- UNDEFINED MINER -----' else: entity = '/'.join([ - addr[:5] + '...' + addr[-5:] for addr in sorted(coinbase_addresses) + addr[:5] + '...' + addr[-5:] for addr in sorted(reward_addresses) ]) if entity in pool_links.keys(): diff --git a/src/mappings/cardano.py b/src/mappings/cardano_mapping.py similarity index 95% rename from src/mappings/cardano.py rename to src/mappings/cardano_mapping.py index 9e75073..41f744e 100644 --- a/src/mappings/cardano.py +++ b/src/mappings/cardano_mapping.py @@ -33,14 +33,14 @@ def process(self, timeframe): pool_links = get_pool_links(self.project_name, day) daily_links[day] = pool_links - entity = tx['coinbase_param'] + entity = tx['identifiers'] if entity: if entity in pool_links.keys(): entity = pool_links[entity] elif entity in pool_tags.keys(): entity = pool_tags[entity]['name'] else: - pool = tx['coinbase_addresses'] + pool = tx['reward_addresses'] if pool: entity = pool else: diff --git a/src/mappings/dummy.py b/src/mappings/dummy_mapping.py similarity index 63% rename from src/mappings/dummy.py rename to src/mappings/dummy_mapping.py index 69be88d..1c6f04d 100644 --- a/src/mappings/dummy.py +++ b/src/mappings/dummy_mapping.py @@ -1,5 +1,5 @@ from collections import defaultdict -from src.helpers.helper import write_csv_file +from src.helpers.helper import write_blocks_per_entity_to_file from src.mappings.mapping import Mapping @@ -13,11 +13,11 @@ def process(self, timeframe): blocks_per_entity = defaultdict(int) for tx in data: - coinbase_addresses = tx['coinbase_addresses'].split(',') - entity = coinbase_addresses[0] + reward_addresses = tx['reward_addresses'].split(',') + entity = reward_addresses[0] blocks_per_entity[entity] += 1 - write_csv_file(self.io_dir, blocks_per_entity, timeframe) + write_blocks_per_entity_to_file(self.io_dir, blocks_per_entity, blocks_per_entity.keys, timeframe) return blocks_per_entity diff --git a/src/mappings/ethereum.py b/src/mappings/ethereum_mapping.py similarity index 77% rename from src/mappings/ethereum.py rename to src/mappings/ethereum_mapping.py index 9bf0f71..cb8e07a 100644 --- a/src/mappings/ethereum.py +++ b/src/mappings/ethereum_mapping.py @@ -38,29 +38,29 @@ def process(self, timeframe): daily_links[day] = pool_links try: - coinbase_param = bytes.fromhex(tx['coinbase_param'][2:]).decode('utf-8') + identifiers = bytes.fromhex(tx['identifiers'][2:]).decode('utf-8') except (UnicodeDecodeError, ValueError): - coinbase_param = tx['coinbase_param'] + identifiers = tx['identifiers'] - coinbase_addresses = tx['coinbase_addresses'] - if coinbase_addresses in special_addresses: + reward_addresses = tx['reward_addresses'] + if reward_addresses in special_addresses: continue pool_match = False - for (tag, info) in pool_tags.items(): # Check if coinbase param contains known pool tag - if tag in str(coinbase_param): + for (tag, info) in pool_tags.items(): # Check if identifiers contain known pool tag + if tag in str(identifiers): entity = info['name'] - pool_addresses[coinbase_addresses] = entity + pool_addresses[reward_addresses] = entity pool_match = True - if coinbase_addresses in pool_addresses.keys() and pool_addresses[coinbase_addresses] != entity: - multi_pool_addresses.append(f'{tx["number"]},{tx["timestamp"]},{coinbase_addresses},{entity}') + if reward_addresses in pool_addresses.keys() and pool_addresses[reward_addresses] != entity: + multi_pool_addresses.append(f'{tx["number"]},{tx["timestamp"]},{reward_addresses},{entity}') break if not pool_match: - if coinbase_addresses in pool_addresses.keys(): - entity = pool_addresses[coinbase_addresses] + if reward_addresses in pool_addresses.keys(): + entity = pool_addresses[reward_addresses] else: - entity = coinbase_addresses + entity = reward_addresses if entity in pool_links.keys(): entity = pool_links[entity] diff --git a/src/mappings/tezos.py b/src/mappings/tezos_mapping.py similarity index 83% rename from src/mappings/tezos.py rename to src/mappings/tezos_mapping.py index c9f200f..1e0f764 100644 --- a/src/mappings/tezos.py +++ b/src/mappings/tezos_mapping.py @@ -33,14 +33,14 @@ def process(self, timeframe): pool_links = get_pool_links(self.project_name, day) daily_links[day] = pool_links - coinbase_addresses = tx['coinbase_addresses'] - if coinbase_addresses is None: - coinbase_addresses = '----- UNDEFINED MINER -----' + reward_addresses = tx['reward_addresses'] + if reward_addresses is None: + reward_addresses = '----- UNDEFINED MINER -----' - if coinbase_addresses in pool_addresses.keys(): - entity = pool_addresses[coinbase_addresses] + if reward_addresses in pool_addresses.keys(): + entity = pool_addresses[reward_addresses] else: - entity = coinbase_addresses + entity = reward_addresses if entity in pool_links.keys(): entity = pool_links[entity] diff --git a/src/parse.py b/src/parse.py index 5ed2869..0dbcd1c 100644 --- a/src/parse.py +++ b/src/parse.py @@ -1,6 +1,5 @@ import argparse from src.parsers.default_parser import DefaultParser -from src.parsers.cardano_parser import CardanoParser from src.parsers.dummy_parser import DummyParser from src.helpers.helper import INPUT_DIR, OUTPUT_DIR @@ -10,7 +9,7 @@ 'ethereum': DummyParser, 'bitcoin_cash': DefaultParser, 'dogecoin': DefaultParser, - 'cardano': CardanoParser, + 'cardano': DummyParser, 'litecoin': DefaultParser, 'zcash': DefaultParser, 'tezos': DummyParser, diff --git a/src/parsers/cardano_parser.py b/src/parsers/cardano_parser.py deleted file mode 100644 index 13a99e9..0000000 --- a/src/parsers/cardano_parser.py +++ /dev/null @@ -1,30 +0,0 @@ -from src.parsers.default_parser import DefaultParser - - -class CardanoParser(DefaultParser): - """ - Parser used for Cardano. Inherits from DefaultParser. - """ - - def __init__(self, project_name, input_dir, output_dir): - super().__init__(project_name, input_dir, output_dir) - - def parse(self): - """ - Parses the data and writes the results into a file in a directory associated with the parser instance - (specifically in /) - """ - data = self.read_and_sort_data() - for tx in data: - try: - tx['coinbase_addresses'] = tx['pool_hash'] - del tx['pool_hash'] - except KeyError: - tx['coinbase_addresses'] = '' - - try: - tx['coinbase_param'] - except KeyError: - tx['coinbase_param'] = '' - - self.write_parsed_data(data) diff --git a/src/parsers/default_parser.py b/src/parsers/default_parser.py index e9cf662..5d548ee 100644 --- a/src/parsers/default_parser.py +++ b/src/parsers/default_parser.py @@ -37,8 +37,8 @@ def parse(self): data = self.read_and_sort_data() for block in data: - block['coinbase_addresses'] = ','.join(set([tx['addresses'][0] for tx in block['outputs'] - if (tx['addresses'] and int(tx['value']) > MIN_TX_VALUE)])) + block['reward_addresses'] = ','.join(set([tx['addresses'][0] for tx in block['outputs'] + if (tx['addresses'] and int(tx['value']) > MIN_TX_VALUE)])) del block['outputs'] self.write_parsed_data(data) diff --git a/src/parsers/dummy_parser.py b/src/parsers/dummy_parser.py index 0c95e02..ba47ee6 100644 --- a/src/parsers/dummy_parser.py +++ b/src/parsers/dummy_parser.py @@ -11,11 +11,13 @@ def __init__(self, project_name, input_dir, output_dir): def parse(self): """ - Sorts the data and writes the results into a file in a directory associated with the parser instance - (specifically in /) + Sorts the data, makes sure that each entry includes all required fields and writes the results into a file in a + directory associated with the parser instance (specifically in /) """ data = self.read_and_sort_data() for block in data: - if 'coinbase_addresses' not in block.keys(): - block['coinbase_addresses'] = None + if 'identifiers' not in block.keys(): + block['identifiers'] = None + if 'reward_addresses' not in block.keys(): + block['reward_addresses'] = None self.write_parsed_data(data) diff --git a/tests/test_3_parsers.py b/tests/test_3_parsers.py index 13b3a79..710e710 100644 --- a/tests/test_3_parsers.py +++ b/tests/test_3_parsers.py @@ -4,7 +4,6 @@ from src.parse import parse, ledger_parser from src.parsers.default_parser import DefaultParser from src.parsers.dummy_parser import DummyParser -from src.parsers.cardano_parser import CardanoParser from src.helpers.helper import INPUT_DIR, OUTPUT_DIR @@ -33,24 +32,24 @@ def compare_parsed_samples(correct_data, parsed_file): if sample['number'] == item['number']: assert all([ sample['timestamp'] == item['timestamp'], - sample['coinbase_param'] == item['coinbase_param'], - set(sample['coinbase_addresses'].split(',')) == set(item['coinbase_addresses'].split(',')) + sample['identifiers'] == item['identifiers'], + set(sample['reward_addresses'].split(',')) == set(item['reward_addresses'].split(',')) ]) def test_default_parser(setup_and_cleanup): test_input_dir, test_output_dir = setup_and_cleanup sample_parsed_data = [ - {"number": "507516", "timestamp": "2018-02-04 02:36:23 UTC", "coinbase_param": "037cbe0741d69d9c6acce4d141d69d9c69f9bef52f4254432e544f502ffabe6d6d143120f7b3da918f12ffb328ab935fbfe2d1cd9bb4707265d7fee23fd6cf372780000000000000005a44cacf0000f8a441200000", "coinbase_addresses": "137YB5cpBLxLKvy8T6qXsycJ699iJjWCHH,1FVKW4rp5rN23dqFVk2tYGY4niAXMB8eZC"}, - {"number": "507715", "timestamp": "2018-02-05 04:54:34 UTC", "coinbase_param": "0343bf07132f6d696e65642062792067626d696e6572732f2cfabe6d6d94976ecebbc73b3d4214b3d7ab330dca2129ebfcc863fa75623c6f95891e7346010000000000000010af8b66002da910ef408c776852840100", "coinbase_addresses": "1J7FCFaafPRxqu4X9VsaiMZr1XMemx69GR,131RUhDyyjxXSbSPxGRCm3t6vcei1TB6MB"}, - {"number": "508242", "timestamp": "2018-03-08 10:57:02 UTC", "coinbase_param": "0352c10704ff2c7c5a2f087cf540446c010000000000", "coinbase_addresses": "1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"}, - {"number": "508434", "timestamp": "2018-02-09 22:17:45 UTC", "coinbase_param": "0312c207040a1e7e5a2f0879deffabf9010000000000", "coinbase_addresses": "1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx,3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"}, - {"number": "509373", "timestamp": "2018-02-15 23:50:04 UTC", "coinbase_param": "03bdc50704ae1c865a2f07a6e2831b1e010000000000", "coinbase_addresses": "1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx,3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"}, - {"number": "509432", "timestamp": "2018-02-16 09:25:03 UTC", "coinbase_param": "03f8c5070470a3865a2f08a35ce4804a010000000000", "coinbase_addresses": "1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx,3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"}, - {"number": "510199", "timestamp": "2018-02-21 06:43:35 UTC", "coinbase_param": "03f7c8070418158d5a2f08b033a0a15b020000000000", "coinbase_addresses": "1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx,3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"}, - {"number": "510888", "timestamp": "2018-02-25 18:02:53 UTC", "coinbase_param": "03a8cb0741d6a4be8b5ec50141d6a4be8aedceda2f45324d2026204254432e544f502ffabe6d6dfb9300ef260402c08b34b5d1dd57904edf0e9e167e99c51d7dbe5f0c06d6b31d8000000000000000e600589dbdabaaca00000000", "coinbase_addresses": "137YB5cpBLxLKvy8T6qXsycJ699iJjWCHH,1FVKW4rp5rN23dqFVk2tYGY4niAXMB8eZC"}, - {"number": "511342", "timestamp": "2018-02-28 16:12:07 UTC", "coinbase_param": "036ecd07132f6d696e65642062792067626d696e6572732f2cfabe6d6d5945c2c64cd6a760e01bc586cee9124629c0b0d6cfa37efcf3d1b912ed7c9467010000000000000010925765006fa22002c89d0875748fe338", "coinbase_addresses": "1J7FCFaafPRxqu4X9VsaiMZr1XMemx69GR,131RUhDyyjxXSbSPxGRCm3t6vcei1TB6MB"}, - {"number": "682736", "timestamp": "2021-05-09 11:12:32 UTC", "coinbase_param": "03f06a0a202f5669614254432f4d696e6564206279206a617669647361656964373037332f2cfabe6d6d6e43ef2e06f7137b897180388403ee5019b8ff0ca4a045ea3cd82e3e41620fe91000000000000000105462a20fc21591f70e691905660b0000", "coinbase_addresses": "18cBEMRxXHqzWWCxZNtU91F5sbUNKhL5PX"} + {"number": "507516", "timestamp": "2018-02-04 02:36:23 UTC", "identifiers": "037cbe0741d69d9c6acce4d141d69d9c69f9bef52f4254432e544f502ffabe6d6d143120f7b3da918f12ffb328ab935fbfe2d1cd9bb4707265d7fee23fd6cf372780000000000000005a44cacf0000f8a441200000", "reward_addresses": "137YB5cpBLxLKvy8T6qXsycJ699iJjWCHH,1FVKW4rp5rN23dqFVk2tYGY4niAXMB8eZC"}, + {"number": "507715", "timestamp": "2018-02-05 04:54:34 UTC", "identifiers": "0343bf07132f6d696e65642062792067626d696e6572732f2cfabe6d6d94976ecebbc73b3d4214b3d7ab330dca2129ebfcc863fa75623c6f95891e7346010000000000000010af8b66002da910ef408c776852840100", "reward_addresses": "1J7FCFaafPRxqu4X9VsaiMZr1XMemx69GR,131RUhDyyjxXSbSPxGRCm3t6vcei1TB6MB"}, + {"number": "508242", "timestamp": "2018-03-08 10:57:02 UTC", "identifiers": "0352c10704ff2c7c5a2f087cf540446c010000000000", "reward_addresses": "1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx"}, + {"number": "508434", "timestamp": "2018-02-09 22:17:45 UTC", "identifiers": "0312c207040a1e7e5a2f0879deffabf9010000000000", "reward_addresses": "1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx,3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"}, + {"number": "509373", "timestamp": "2018-02-15 23:50:04 UTC", "identifiers": "03bdc50704ae1c865a2f07a6e2831b1e010000000000", "reward_addresses": "1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx,3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"}, + {"number": "509432", "timestamp": "2018-02-16 09:25:03 UTC", "identifiers": "03f8c5070470a3865a2f08a35ce4804a010000000000", "reward_addresses": "1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx,3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"}, + {"number": "510199", "timestamp": "2018-02-21 06:43:35 UTC", "identifiers": "03f7c8070418158d5a2f08b033a0a15b020000000000", "reward_addresses": "1AM2fYfpY3ZeMeCKXmN66haoWxvB89pJUx,3G7y14BudP2a4kjPAuecg4iUKM84GgPPWb"}, + {"number": "510888", "timestamp": "2018-02-25 18:02:53 UTC", "identifiers": "03a8cb0741d6a4be8b5ec50141d6a4be8aedceda2f45324d2026204254432e544f502ffabe6d6dfb9300ef260402c08b34b5d1dd57904edf0e9e167e99c51d7dbe5f0c06d6b31d8000000000000000e600589dbdabaaca00000000", "reward_addresses": "137YB5cpBLxLKvy8T6qXsycJ699iJjWCHH,1FVKW4rp5rN23dqFVk2tYGY4niAXMB8eZC"}, + {"number": "511342", "timestamp": "2018-02-28 16:12:07 UTC", "identifiers": "036ecd07132f6d696e65642062792067626d696e6572732f2cfabe6d6d5945c2c64cd6a760e01bc586cee9124629c0b0d6cfa37efcf3d1b912ed7c9467010000000000000010925765006fa22002c89d0875748fe338", "reward_addresses": "1J7FCFaafPRxqu4X9VsaiMZr1XMemx69GR,131RUhDyyjxXSbSPxGRCm3t6vcei1TB6MB"}, + {"number": "682736", "timestamp": "2021-05-09 11:12:32 UTC", "identifiers": "03f06a0a202f5669614254432f4d696e6564206279206a617669647361656964373037332f2cfabe6d6d6e43ef2e06f7137b897180388403ee5019b8ff0ca4a045ea3cd82e3e41620fe91000000000000000105462a20fc21591f70e691905660b0000", "reward_addresses": "18cBEMRxXHqzWWCxZNtU91F5sbUNKhL5PX"} ] project_name = 'sample_bitcoin' @@ -67,16 +66,16 @@ def test_default_parser(setup_and_cleanup): def test_dummy_parser(setup_and_cleanup): test_input_dir, test_output_dir = setup_and_cleanup sample_parsed_data = [ - {"number": "11181062", "timestamp": "2020-11-03 00:56:48 UTC", "coinbase_addresses": "0xe9b54a47e3f401d37798fc4e22f14b78475c2afc", "coinbase_param": "0x36"}, - {"number": "11183702", "timestamp": "2020-11-03 10:37:32 UTC", "coinbase_addresses": "0xe9b54a47e3f401d37798fc4e22f14b78475c2afc", "coinbase_param": "0x36"}, - {"number": "11183739", "timestamp": "2020-11-03 10:44:14 UTC", "coinbase_addresses": "0xe9b54a47e3f401d37798fc4e22f14b78475c2afc", "coinbase_param": "0x3"}, - {"number": "11183793", "timestamp": "2020-11-03 10:56:07 UTC", "coinbase_addresses": "0x8595dd9e0438640b5e1254f9df579ac12a86865f", "coinbase_param": "0x657a696c2e6d65"}, - {"number": "11183969", "timestamp": "2020-11-03 11:33:44 UTC", "coinbase_addresses": "0x8595dd9e0438640b5e1254f9df579ac12a86865f", "coinbase_param": "0x657a696c2e6d65"}, - {"number": "11184329", "timestamp": "2020-11-03 12:56:41 UTC", "coinbase_addresses": "0x8595dd9e0438640b5e1254f9df579ac12a86865f", "coinbase_param": "0x657a696c2e6d65"}, - {"number": "11184490", "timestamp": "2020-11-03 13:33:18 UTC", "coinbase_addresses": "0x45133a7e1cc7e18555ae8a4ee632a8a61de90df6", "coinbase_param": "0x20"}, - {"number": "11185799", "timestamp": "2020-11-03 18:34:11 UTC", "coinbase_addresses": "0xe9b54a47e3f401d37798fc4e22f14b78475c2afc", "coinbase_param": "0x36"}, - {"number": "11186643", "timestamp": "2020-11-03 21:33:27 UTC", "coinbase_addresses": "0xe9b54a47e3f401d37798fc4e22f14b78475c2afc", "coinbase_param": "0x36"}, - {"number": "16382083", "timestamp": "2023-01-11 07:29:47 UTC", "coinbase_addresses": "0x3bee5122e2a2fbe11287aafb0cb918e22abb5436", "coinbase_param": "0x"} + {"number": "11181062", "timestamp": "2020-11-03 00:56:48 UTC", "reward_addresses": "0xe9b54a47e3f401d37798fc4e22f14b78475c2afc", "identifiers": "0x36"}, + {"number": "11183702", "timestamp": "2020-11-03 10:37:32 UTC", "reward_addresses": "0xe9b54a47e3f401d37798fc4e22f14b78475c2afc", "identifiers": "0x36"}, + {"number": "11183739", "timestamp": "2020-11-03 10:44:14 UTC", "reward_addresses": "0xe9b54a47e3f401d37798fc4e22f14b78475c2afc", "identifiers": "0x3"}, + {"number": "11183793", "timestamp": "2020-11-03 10:56:07 UTC", "reward_addresses": "0x8595dd9e0438640b5e1254f9df579ac12a86865f", "identifiers": "0x657a696c2e6d65"}, + {"number": "11183969", "timestamp": "2020-11-03 11:33:44 UTC", "reward_addresses": "0x8595dd9e0438640b5e1254f9df579ac12a86865f", "identifiers": "0x657a696c2e6d65"}, + {"number": "11184329", "timestamp": "2020-11-03 12:56:41 UTC", "reward_addresses": "0x8595dd9e0438640b5e1254f9df579ac12a86865f", "identifiers": "0x657a696c2e6d65"}, + {"number": "11184490", "timestamp": "2020-11-03 13:33:18 UTC", "reward_addresses": "0x45133a7e1cc7e18555ae8a4ee632a8a61de90df6", "identifiers": "0x20"}, + {"number": "11185799", "timestamp": "2020-11-03 18:34:11 UTC", "reward_addresses": "0xe9b54a47e3f401d37798fc4e22f14b78475c2afc", "identifiers": "0x36"}, + {"number": "11186643", "timestamp": "2020-11-03 21:33:27 UTC", "reward_addresses": "0xe9b54a47e3f401d37798fc4e22f14b78475c2afc", "identifiers": "0x36"}, + {"number": "16382083", "timestamp": "2023-01-11 07:29:47 UTC", "reward_addresses": "0x3bee5122e2a2fbe11287aafb0cb918e22abb5436", "identifiers": "0x"} ] project_name = 'sample_ethereum' @@ -90,28 +89,9 @@ def test_dummy_parser(setup_and_cleanup): compare_parsed_samples(sample_parsed_data, parsed_file) -def test_cardano_parser(setup_and_cleanup): - test_input_dir, test_output_dir = setup_and_cleanup - sample_parsed_data = [ - {"number": "17809932", "coinbase_param": "CFLOW", "timestamp": "2020-12-31T00:57:03", "coinbase_addresses": "e7b605b72af41d6e8e6894274dedd18114f1759fea500b6d07031535"}, - {"number": "55555555555", "coinbase_param": "1PCT5", "timestamp": "2020-12-31T06:42:01", "coinbase_addresses": ""}, - {"number": "66666666666", "coinbase_param": "", "timestamp": "2020-12-31T04:42:01", "coinbase_addresses": "1d8988c2057d6efd6a094e468840a51942ab03b5b69b07a2bca71b53"} - ] - - project_name = 'sample_cardano' - - parser = CardanoParser(project_name, test_input_dir, test_output_dir) - parser.parse() - - parsed_file = test_output_dir / f'{project_name}/parsed_data.json' - assert parsed_file.is_file() - - compare_parsed_samples(sample_parsed_data, parsed_file) - - def test_parse(setup_and_cleanup): test_input_dir, test_output_dir = setup_and_cleanup - sample_block = {"number": "682736", "timestamp": "2021-05-09 11:12:32 UTC", "coinbase_param": "03f06a0a202f5669614254432f4d696e6564206279206a617669647361656964373037332f2cfabe6d6d6e43ef2e06f7137b897180388403ee5019b8ff0ca4a045ea3cd82e3e41620fe91000000000000000105462a20fc21591f70e691905660b0000", "coinbase_addresses": "18cBEMRxXHqzWWCxZNtU91F5sbUNKhL5PX"} + sample_block = {"number": "682736", "timestamp": "2021-05-09 11:12:32 UTC", "identifiers": "03f06a0a202f5669614254432f4d696e6564206279206a617669647361656964373037332f2cfabe6d6d6e43ef2e06f7137b897180388403ee5019b8ff0ca4a045ea3cd82e3e41620fe91000000000000000105462a20fc21591f70e691905660b0000", "reward_addresses": "18cBEMRxXHqzWWCxZNtU91F5sbUNKhL5PX"} project = 'sample_bitcoin' ledger_parser[project] = DefaultParser @@ -124,7 +104,7 @@ def test_parse(setup_and_cleanup): test_data = json.load(f) for item in test_data: if item['number'] == '682736': - assert item['coinbase_addresses'] == sample_block['coinbase_addresses'] + assert item['reward_addresses'] == sample_block['reward_addresses'] with open(input_file) as f: sample_data = f.read() @@ -137,14 +117,14 @@ def test_parse(setup_and_cleanup): test_data = json.load(f) for item in test_data: if item['number'] == '682736': - assert item['coinbase_addresses'] == sample_block['coinbase_addresses'] + assert item['reward_addresses'] == sample_block['reward_addresses'] parse(project, test_input_dir, test_output_dir, True) with open(parsed_file) as f: test_data = json.load(f) for item in test_data: if item['number'] == '682736': - assert item['coinbase_addresses'] == "----------------------------------" + assert item['reward_addresses'] == "----------------------------------" with open(input_file) as f: sample_data = f.read() diff --git a/tests/test_4_mappings.py b/tests/test_4_mappings.py index 5a7e420..5f54e02 100644 --- a/tests/test_4_mappings.py +++ b/tests/test_4_mappings.py @@ -6,13 +6,12 @@ from src.parse import parse, ledger_parser from src.parsers.default_parser import DefaultParser from src.parsers.dummy_parser import DummyParser -from src.parsers.cardano_parser import CardanoParser from src.map import apply_mapping, ledger_mapping from src.mappings.mapping import Mapping -from src.mappings.bitcoin import BitcoinMapping -from src.mappings.ethereum import EthereumMapping -from src.mappings.cardano import CardanoMapping -from src.mappings.tezos import TezosMapping +from src.mappings.bitcoin_mapping import BitcoinMapping +from src.mappings.ethereum_mapping import EthereumMapping +from src.mappings.cardano_mapping import CardanoMapping +from src.mappings.tezos_mapping import TezosMapping from src.helpers.helper import INPUT_DIR, OUTPUT_DIR @@ -29,7 +28,7 @@ def setup_and_cleanup(): ledger_mapping['sample_ethereum'] = EthereumMapping ledger_parser['sample_ethereum'] = DummyParser ledger_mapping['sample_cardano'] = CardanoMapping - ledger_parser['sample_cardano'] = CardanoParser + ledger_parser['sample_cardano'] = DummyParser ledger_mapping['sample_tezos'] = TezosMapping ledger_parser['sample_tezos'] = DummyParser pool_info_dir = pathlib.Path(__file__).resolve().parent.parent / 'src' / 'helpers' / 'pool_information' # todo maybe better to have separate helper files for the tests s.t. the tests don't break just because the info changes @@ -57,8 +56,8 @@ def test_map(setup_and_cleanup): pass try: - shutil.copy2(str(pool_info_dir / 'coinbase_tags/bitcoin.json'), - str(pool_info_dir / f'coinbase_tags/{project}.json')) # Create a temp pool info file for sample + shutil.copy2(str(pool_info_dir / 'identifiers/bitcoin.json'), + str(pool_info_dir / f'identifiers/{project}.json')) # Create a temp pool info file for sample except FileNotFoundError: pass @@ -83,7 +82,7 @@ def test_map(setup_and_cleanup): except FileNotFoundError: pass try: - os.remove(str(pool_info_dir / f'coinbase_tags/{project}.json')) # Remove temp pool info file + os.remove(str(pool_info_dir / f'identifiers/{project}.json')) # Remove temp pool info file except FileNotFoundError: pass @@ -105,8 +104,8 @@ def test_bitcoin_mapping(setup_and_cleanup): pass try: - shutil.copy2(str(pool_info_dir / 'coinbase_tags/bitcoin.json'), - str(pool_info_dir / f'coinbase_tags/{project}.json')) # Create a temp pool info file for sample + shutil.copy2(str(pool_info_dir / 'identifiers/bitcoin.json'), + str(pool_info_dir / f'identifiers/{project}.json')) # Create a temp pool info file for sample except FileNotFoundError: pass @@ -176,7 +175,7 @@ def test_bitcoin_mapping(setup_and_cleanup): except FileNotFoundError: pass try: - os.remove(str(pool_info_dir / f'coinbase_tags/{project}.json')) # Remove temp pool info file + os.remove(str(pool_info_dir / f'identifiers/{project}.json')) # Remove temp pool info file except FileNotFoundError: pass @@ -198,8 +197,8 @@ def test_ethereum_mapping(setup_and_cleanup): pass try: - shutil.copy2(str(pool_info_dir / 'coinbase_tags/ethereum.json'), - str(pool_info_dir / f'coinbase_tags/{project}.json')) # Create a temp pool info file for sample + shutil.copy2(str(pool_info_dir / 'identifiers/ethereum.json'), + str(pool_info_dir / f'identifiers/{project}.json')) # Create a temp pool info file for sample except FileNotFoundError: pass @@ -269,7 +268,7 @@ def test_ethereum_mapping(setup_and_cleanup): except FileNotFoundError: pass try: - os.remove(str(pool_info_dir / f'coinbase_tags/{project}.json')) # Remove temp pool info file + os.remove(str(pool_info_dir / f'identifiers/{project}.json')) # Remove temp pool info file except FileNotFoundError: pass @@ -291,13 +290,13 @@ def test_cardano_mapping(setup_and_cleanup): pass try: - shutil.copy2(str(pool_info_dir / 'coinbase_tags/cardano.json'), - str(pool_info_dir / f'coinbase_tags/{project}.json')) # Create a temp pool info file for sample + shutil.copy2(str(pool_info_dir / 'identifiers/cardano.json'), + str(pool_info_dir / f'identifiers/{project}.json')) # Create a temp pool info file for sample except FileNotFoundError: pass ledger_mapping[project] = CardanoMapping - ledger_parser[project] = CardanoParser + ledger_parser[project] = DummyParser timeframes = ['2020-12'] force_map = True @@ -333,7 +332,7 @@ def test_cardano_mapping(setup_and_cleanup): except FileNotFoundError: pass try: - os.remove(str(pool_info_dir / f'coinbase_tags/{project}.json')) # Remove temp pool info file + os.remove(str(pool_info_dir / f'identifiers/{project}.json')) # Remove temp pool info file except FileNotFoundError: pass @@ -355,8 +354,8 @@ def test_tezos_mapping(setup_and_cleanup): pass try: - shutil.copy2(str(pool_info_dir / 'coinbase_tags/tezos.json'), - str(pool_info_dir / f'coinbase_tags/{project}.json')) # Create a temp pool info file for sample + shutil.copy2(str(pool_info_dir / 'identifiers/tezos.json'), + str(pool_info_dir / f'identifiers/{project}.json')) # Create a temp pool info file for sample except FileNotFoundError: pass @@ -421,7 +420,7 @@ def test_tezos_mapping(setup_and_cleanup): except FileNotFoundError: pass try: - os.remove(str(pool_info_dir / f'coinbase_tags/{project}.json')) # Remove temp pool info file + os.remove(str(pool_info_dir / f'identifiers/{project}.json')) # Remove temp pool info file except FileNotFoundError: pass diff --git a/tests/test_6_end_to_end.py b/tests/test_6_end_to_end.py index 0fb40c6..c640a4f 100644 --- a/tests/test_6_end_to_end.py +++ b/tests/test_6_end_to_end.py @@ -4,10 +4,10 @@ from run import main from src.parse import ledger_parser from src.parsers.default_parser import DefaultParser -from src.parsers.cardano_parser import CardanoParser +from src.parsers.dummy_parser import DummyParser from src.map import ledger_mapping -from src.mappings.bitcoin import BitcoinMapping -from src.mappings.cardano import CardanoMapping +from src.mappings.bitcoin_mapping import BitcoinMapping +from src.mappings.cardano_mapping import CardanoMapping from src.helpers.helper import OUTPUT_DIR import pytest @@ -24,7 +24,7 @@ def setup_and_cleanup(): ledger_mapping['sample_bitcoin'] = BitcoinMapping ledger_parser['sample_bitcoin'] = DefaultParser ledger_mapping['sample_cardano'] = CardanoMapping - ledger_parser['sample_cardano'] = CardanoParser + ledger_parser['sample_cardano'] = DummyParser yield test_output_dir print("Cleaning up") shutil.rmtree(test_output_dir) @@ -45,7 +45,7 @@ def test_end_to_end(setup_and_cleanup): except FileNotFoundError: pass try: - shutil.copy2(str(pool_info_dir / f'coinbase_tags/{project}.json'), str(pool_info_dir / f'coinbase_tags/sample_{project}.json')) + shutil.copy2(str(pool_info_dir / f'identifiers/{project}.json'), str(pool_info_dir / f'identifiers/sample_{project}.json')) except FileNotFoundError: pass @@ -66,7 +66,7 @@ def test_end_to_end(setup_and_cleanup): except FileNotFoundError: pass try: - os.remove(str(pool_info_dir / f'coinbase_tags/{project}.json')) + os.remove(str(pool_info_dir / f'identifiers/{project}.json')) except FileNotFoundError: pass