From 1dbb4edda009e8fd221e67e767f3faed454764ea Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Fri, 20 Mar 2020 14:23:46 -0400 Subject: [PATCH 01/11] Initial implementation of pruned_transaction --- .../chain/include/eosio/chain/exceptions.hpp | 2 + .../chain/include/eosio/chain/transaction.hpp | 111 +++++++++ libraries/chain/transaction.cpp | 220 ++++++++++++++++-- unittests/misc_tests.cpp | 73 ++++++ 4 files changed, 384 insertions(+), 22 deletions(-) diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index b6ddb73ca7e..b534844ea53 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -277,6 +277,8 @@ namespace eosio { namespace chain { 3040017, "Transaction includes disallowed extensions (invalid block)" ) FC_DECLARE_DERIVED_EXCEPTION( tx_resource_exhaustion, transaction_exception, 3040018, "Transaction exceeded transient resource limit" ) + FC_DECLARE_DERIVED_EXCEPTION( tx_prune_exception, transaction_exception, + 3040019, "Prunable data not found" ) FC_DECLARE_DERIVED_EXCEPTION( action_validate_exception, chain_exception, diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index a9d523a9e0a..fc891962f0e 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -203,6 +203,110 @@ namespace eosio { namespace chain { transaction_id_type trx_id; }; + struct prunable_transaction_data { + using compression_type = packed_transaction::compression_type; + + struct none { + digest_type prunable_digest; + }; + + struct signatures_only { + std::vector signatures; + digest_type context_free_mroot; + }; + + using segment_type = fc::static_variant>; + + struct partial { + std::vector signatures; + std::vector context_free_segments; + }; + + struct full { + std::vector signatures; + std::vector> context_free_segments; + }; + + struct full_legacy { + std::vector signatures; + std::vector packed_context_free_data; + }; + + using prunable_data_type = fc::static_variant< full_legacy, + none, + signatures_only, + partial, + full >; + + prunable_transaction_data prune_all() const; + digest_type digest() const; + + // Returns the maximum pack size of any prunable_transaction_data that is reachable + // by pruning this object. + std::size_t maximum_pruned_pack_size( compression_type segment_compression ) const; + + prunable_data_type prunable_data; + }; + + struct pruned_transaction : fc::reflect_init { + using compression_type = packed_transaction::compression_type; + + pruned_transaction() = default; + pruned_transaction(pruned_transaction&&) = default; + explicit pruned_transaction(const pruned_transaction&) = default; + pruned_transaction& operator=(const pruned_transaction&) = delete; + pruned_transaction& operator=(pruned_transaction&&) = default; + + explicit pruned_transaction(const signed_transaction& t, bool legacy, compression_type _compression = compression_type::none); + explicit pruned_transaction(signed_transaction&& t, bool legacy, compression_type _compression = compression_type::none); + +#if 0 + // used by abi_serializer + packed_transaction( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); + packed_transaction( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ); + packed_transaction( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); +#endif + + uint32_t get_unprunable_size()const; + uint32_t get_prunable_size()const; + + digest_type packed_digest()const; + + const transaction_id_type& id()const { return trx_id; } + + time_point_sec expiration()const { return unpacked_trx.expiration; } + const transaction& get_transaction()const { return unpacked_trx; } + const signed_transaction& get_signed_transaction()const { return unpacked_trx; } + // Returns nullptr if the signatures were pruned + const vector* get_signatures()const; + // Returns nullptr if any context_free_data segment was pruned + const vector* get_context_free_data()const; + // Returns nullptr if the context_free_data segment was pruned segment_ordinal is out of range. + const bytes* get_context_free_data(std::size_t segment_ordinal); + const fc::enum_type& get_compression()const { return compression; } + const bytes& get_packed_transaction()const { return packed_trx; } + + void prune_all(); + + std::size_t maximum_pruned_pack_size( compression_type segment_compression ) const; + + private: + + friend struct fc::reflector; + friend struct fc::reflector_init_visitor; + friend struct fc::has_reflector_init; + void reflector_init(); + private: + fc::enum_type compression; + prunable_transaction_data prunable_data; + bytes packed_trx; + + private: + // cache unpacked trx, for thread safety do not modify after construction + signed_transaction unpacked_trx; + transaction_id_type trx_id; + }; + using packed_transaction_ptr = std::shared_ptr; uint128_t transaction_id_to_sender_id( const transaction_id_type& tid ); @@ -217,3 +321,10 @@ FC_REFLECT_DERIVED( eosio::chain::signed_transaction, (eosio::chain::transaction FC_REFLECT_ENUM( eosio::chain::packed_transaction::compression_type, (none)(zlib)) // @ignore unpacked_trx FC_REFLECT( eosio::chain::packed_transaction, (signatures)(compression)(packed_context_free_data)(packed_trx) ) +FC_REFLECT( eosio::chain::pruned_transaction, (compression)(prunable_data)(packed_trx) ) +FC_REFLECT( eosio::chain::prunable_transaction_data, (prunable_data)); +FC_REFLECT( eosio::chain::prunable_transaction_data::none, (prunable_digest)) +FC_REFLECT( eosio::chain::prunable_transaction_data::signatures_only, (signatures)(context_free_mroot)) +FC_REFLECT( eosio::chain::prunable_transaction_data::partial, (signatures)(context_free_segments)) +FC_REFLECT( eosio::chain::prunable_transaction_data::full, (signatures)(context_free_segments)) +FC_REFLECT( eosio::chain::prunable_transaction_data::full_legacy, (signatures)(packed_context_free_data)) diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index ee51d639304..3383ab6c2bc 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -333,54 +333,230 @@ void packed_transaction::local_unpack_transaction(vector&& context_free_d } FC_CAPTURE_AND_RETHROW( (compression) ) } -void packed_transaction::local_unpack_context_free_data() +static vector unpack_context_free_data(const bytes& packed_context_free_data, packed_transaction::compression_type compression) { try { - EOS_ASSERT(unpacked_trx.context_free_data.empty(), tx_decompression_error, "packed_transaction.context_free_data not empty"); switch( compression ) { - case compression_type::none: - unpacked_trx.context_free_data = unpack_context_free_data( packed_context_free_data ); - break; - case compression_type::zlib: - unpacked_trx.context_free_data = zlib_decompress_context_free_data( packed_context_free_data ); - break; + case packed_transaction::compression_type::none: + return unpack_context_free_data( packed_context_free_data ); + case packed_transaction::compression_type::zlib: + return zlib_decompress_context_free_data( packed_context_free_data ); default: EOS_THROW( unknown_transaction_compression, "Unknown transaction compression algorithm" ); } } FC_CAPTURE_AND_RETHROW( (compression) ) } -void packed_transaction::local_pack_transaction() +void packed_transaction::local_unpack_context_free_data() { + EOS_ASSERT(unpacked_trx.context_free_data.empty(), tx_decompression_error, "packed_transaction.context_free_data not empty"); + unpacked_trx.context_free_data = unpack_context_free_data( packed_context_free_data, compression ); +} + +static bytes pack_transaction(const transaction& trx, packed_transaction::compression_type compression) { try { switch(compression) { - case compression_type::none: - packed_trx = pack_transaction(unpacked_trx); - break; - case compression_type::zlib: - packed_trx = zlib_compress_transaction(unpacked_trx); - break; + case packed_transaction::compression_type::none: + return pack_transaction(trx); + case packed_transaction::compression_type::zlib: + return zlib_compress_transaction(trx); default: EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); } } FC_CAPTURE_AND_RETHROW((compression)) } -void packed_transaction::local_pack_context_free_data() +void packed_transaction::local_pack_transaction() { + packed_trx = pack_transaction(unpacked_trx, compression); +} + +static bytes pack_context_free_data( const vector& cfd, packed_transaction::compression_type compression ) { try { switch(compression) { - case compression_type::none: - packed_context_free_data = pack_context_free_data(unpacked_trx.context_free_data); - break; - case compression_type::zlib: - packed_context_free_data = zlib_compress_context_free_data(unpacked_trx.context_free_data); - break; + case packed_transaction::compression_type::none: + return pack_context_free_data(cfd); + case packed_transaction::compression_type::zlib: + return zlib_compress_context_free_data(cfd); default: EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); } } FC_CAPTURE_AND_RETHROW((compression)) } +void packed_transaction::local_pack_context_free_data() +{ + packed_context_free_data = pack_context_free_data(unpacked_trx.context_free_data, compression); +} + +prunable_transaction_data prunable_transaction_data::prune_all() const { + return { prunable_transaction_data::none{ digest() } }; +} + +static digest_type prunable_digest(const prunable_transaction_data::none& obj) { + return obj.prunable_digest; +} + +static digest_type prunable_digest(const prunable_transaction_data::partial& obj) { + EOS_THROW(tx_prune_exception, "unimplemented"); +} + +static digest_type prunable_digest(const prunable_transaction_data::signatures_only& obj) { + EOS_THROW(tx_prune_exception, "unimplemented"); +} + +static digest_type prunable_digest(const prunable_transaction_data::full& obj) { + EOS_THROW(tx_prune_exception, "unimplemented"); +} + +static digest_type prunable_digest(const prunable_transaction_data::full_legacy& obj) { + digest_type::encoder prunable; + fc::raw::pack( prunable, obj.signatures ); + fc::raw::pack( prunable, obj.packed_context_free_data ); + return prunable.result(); +} + +digest_type prunable_transaction_data::digest() const { + return prunable_data.visit( [](const auto& obj) { return prunable_digest(obj); } ); +} + +static constexpr std::size_t digest_pack_size = 32; + +static std::size_t padded_pack_size(const prunable_transaction_data::none& obj, prunable_transaction_data::compression_type) { + std::size_t result = fc::raw::pack_size(obj); + EOS_ASSERT(result == digest_pack_size, chain_type_exception, "Internal EOSIO error"); + return result; +} + +static std::size_t padded_pack_size(const prunable_transaction_data::signatures_only& obj, prunable_transaction_data::compression_type) { + return fc::raw::pack_size(obj); +} + +static std::size_t padded_pack_size(const prunable_transaction_data::partial& obj, prunable_transaction_data::compression_type segment_compression) { + EOS_THROW(tx_prune_exception, "unimplemented"); +} + +static std::size_t padded_pack_size(const prunable_transaction_data::full& obj, prunable_transaction_data::compression_type segment_compression) { + EOS_THROW(tx_prune_exception, "unimplemented"); +#if 0 + std::size_t context_free_size = fc::raw::pack_size(fc::unsigned_int(obj.context_free_segments.size())); + context_free_size += obj.context_free_segments.size(); + for(const std::vector& v: obj.context_free_segments) { + // TODO: Handle segment_compression + context_free_size += std::max(fc::raw::pack_size(v), digest_pack_size); + } + return fc::raw::pack_size(obj.signatures) + std::max(context_free_size, digest_pack_size); +#endif +} + +static std::size_t padded_pack_size(const prunable_transaction_data::full_legacy& obj, prunable_transaction_data::compression_type) { + return std::max(fc::raw::pack_size(obj), digest_pack_size); +} + +std::size_t prunable_transaction_data::maximum_pruned_pack_size(prunable_transaction_data::compression_type compression) const { + return 1 + prunable_data.visit([&](const auto& t){ return padded_pack_size(t, compression); }); +} + +static prunable_transaction_data make_prunable_transaction_data( bool legacy, const signed_transaction& t, pruned_transaction::compression_type _compression ) { + if(legacy) { + return { prunable_transaction_data::full_legacy{ t.signatures, pack_context_free_data( t.context_free_data, _compression ) } }; + } else { + return { prunable_transaction_data::full{ t.signatures, t.context_free_data } }; + } +} + +pruned_transaction::pruned_transaction(const signed_transaction& t, bool legacy, compression_type _compression) + : compression(_compression), + prunable_data(make_prunable_transaction_data(legacy, t, _compression)), + packed_trx(pack_transaction(t, compression)), + unpacked_trx(t), + trx_id(unpacked_trx.id()) +{} + +pruned_transaction::pruned_transaction(signed_transaction&& t, bool legacy, compression_type _compression) + : compression(_compression), + prunable_data(make_prunable_transaction_data(legacy, t, _compression)), + packed_trx(pack_transaction(t, compression)), + unpacked_trx(std::move(t)), + trx_id(unpacked_trx.id()) +{} + +uint32_t pruned_transaction::get_unprunable_size()const { + uint64_t size = config::fixed_net_overhead_of_packed_trx; + size += packed_trx.size(); + EOS_ASSERT( size <= std::numeric_limits::max(), tx_too_big, "packed_transaction is too big" ); + return static_cast(size); +} + +static uint32_t get_prunable_size_impl(const prunable_transaction_data::full_legacy& obj) { + uint64_t size = fc::raw::pack_size(obj.signatures); + size += obj.packed_context_free_data.size(); + EOS_ASSERT( size <= std::numeric_limits::max(), tx_too_big, "packed_transaction is too big" ); + return static_cast(size); +} + +static uint32_t get_prunable_size_impl(const prunable_transaction_data::full&) { + EOS_THROW(tx_prune_exception, "unimplemented"); +} + +template +static uint32_t get_prunable_size_impl(const T&) { + EOS_THROW(tx_prune_exception, "unknown size: prunable data has been pruned."); +} + +uint32_t pruned_transaction::get_prunable_size()const { + return prunable_data.prunable_data.visit([](const auto& obj) { return get_prunable_size_impl(obj); }); +} + +digest_type pruned_transaction::packed_digest()const { + digest_type::encoder enc; + fc::raw::pack( enc, compression ); + fc::raw::pack( enc, packed_trx ); + fc::raw::pack( enc, prunable_data.digest() ); + return enc.result(); +} + +template +static auto maybe_get_signatures(const T& obj) -> decltype(&obj.signatures) { return &obj.signatures; } +static auto maybe_get_signatures(const prunable_transaction_data::none&) -> const std::vector* { return nullptr; } +const vector* pruned_transaction::get_signatures()const { + return prunable_data.prunable_data.visit([](const auto& obj) { return maybe_get_signatures(obj); }); +} + +const vector* pruned_transaction::get_context_free_data()const { + if(prunable_data.prunable_data.contains() || + prunable_data.prunable_data.contains()) { + return &unpacked_trx.context_free_data; + } else { + return nullptr; + } +} + +const bytes* maybe_get_context_free_data(const prunable_transaction_data::none&, std::size_t) { return nullptr; } +const bytes* maybe_get_context_free_data(const prunable_transaction_data::signatures_only&, std::size_t) { return nullptr; } +const bytes* maybe_get_context_free_data(const prunable_transaction_data::partial&, std::size_t) { + EOS_THROW(tx_prune_exception, "unimplemented"); +} +// full uses the copy in unpacked_trx +const bytes* maybe_get_context_free_data(const prunable_transaction_data::full_legacy&, std::size_t) { return nullptr; } +const bytes* maybe_get_context_free_data(const prunable_transaction_data::full&, std::size_t) { return nullptr; } + +const bytes* pruned_transaction::get_context_free_data(std::size_t segment_ordinal) { + if (segment_ordinal < unpacked_trx.context_free_data.size()) { + return &unpacked_trx.context_free_data[segment_ordinal]; + } else { + return prunable_data.prunable_data.visit([&](const auto& obj) { return maybe_get_context_free_data(obj, segment_ordinal); }); + } +} + +void pruned_transaction::prune_all() { + prunable_data = prunable_data.prune_all(); + unpacked_trx.context_free_data.clear(); + unpacked_trx.signatures.clear(); +} + +std::size_t pruned_transaction::maximum_pruned_pack_size(compression_type segment_compression) const { + return fc::raw::pack_size(compression) + fc::raw::pack_size(packed_trx) + prunable_data.maximum_pruned_pack_size(segment_compression); +} } } // eosio::chain diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index 0dc223879ff..1b0ab4b6fed 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -917,6 +917,79 @@ BOOST_AUTO_TEST_CASE(transaction_metadata_test) { try { } FC_LOG_AND_RETHROW() } +BOOST_AUTO_TEST_CASE(prunable_transaction_data_test) { + { + prunable_transaction_data basic{prunable_transaction_data::full_legacy{{}, {}}}; + prunable_transaction_data pruned = prunable_transaction_data(basic).prune_all(); + BOOST_TEST(basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none) >= + pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + } + + bytes large_bytes(48); + { + prunable_transaction_data basic{prunable_transaction_data::full_legacy{{}, fc::raw::pack(std::vector(4, large_bytes))}}; + prunable_transaction_data pruned = prunable_transaction_data(basic).prune_all(); + BOOST_TEST(basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none) >= + pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + } + + { + prunable_transaction_data basic{prunable_transaction_data::full_legacy{std::vector(4, signature_type()), fc::raw::pack(std::vector(4, large_bytes))}}; + prunable_transaction_data pruned = prunable_transaction_data(basic).prune_all(); + BOOST_TEST(basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none) >= + pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + } + + { + prunable_transaction_data basic{prunable_transaction_data::full_legacy{std::vector(4, signature_type()), {}}}; + prunable_transaction_data pruned = prunable_transaction_data(basic).prune_all(); + BOOST_TEST(basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none) >= + pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + } +} + +BOOST_AUTO_TEST_CASE(pruned_transaction_test) { + tester t; + signed_transaction trx; + trx.context_free_actions.push_back({ {}, N(eosio), N(), bytes() }); + trx.context_free_data.push_back(bytes()); + t.set_transaction_headers(trx); + trx.sign( t.get_private_key( N(eosio), "active" ), t.control->get_chain_id() ); + + packed_transaction packed(trx); + pruned_transaction pruned(trx, true); + BOOST_TEST(packed.packed_digest().str() == pruned.packed_digest().str()); + BOOST_REQUIRE(pruned.get_context_free_data() != nullptr); + BOOST_TEST(*pruned.get_context_free_data() == packed.get_context_free_data()); + BOOST_REQUIRE(pruned.get_context_free_data(0) != nullptr); + BOOST_TEST(*pruned.get_context_free_data(0) == bytes()); + BOOST_TEST(pruned.get_context_free_data(1) == nullptr); + BOOST_REQUIRE(pruned.get_signatures() != nullptr); + BOOST_TEST(*pruned.get_signatures() == packed.get_signatures()); + BOOST_TEST(pruned.get_prunable_size() == packed.get_prunable_size()); + BOOST_TEST(pruned.get_unprunable_size() == packed.get_unprunable_size()); + std::size_t max_size = pruned.maximum_pruned_pack_size(pruned_transaction::compression_type::none); + BOOST_TEST(fc::raw::pack_size(pruned) <= max_size); + + pruned.prune_all(); + BOOST_TEST(packed.packed_digest().str() == pruned.packed_digest().str()); + BOOST_TEST(pruned.get_context_free_data() == nullptr); + BOOST_TEST(pruned.get_context_free_data(0) == nullptr); + BOOST_TEST(pruned.get_context_free_data(1) == nullptr); + BOOST_TEST(pruned.get_signatures() == nullptr); + + BOOST_TEST(fc::raw::pack_size(pruned) <= max_size); + BOOST_CHECK_THROW(pruned.get_prunable_size(), tx_prune_exception); +} + BOOST_AUTO_TEST_CASE(reflector_init_test) { try { From a5f05463b472d9f3f4b02a6b7c6f23b65a446dc2 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Mon, 23 Mar 2020 12:11:31 -0400 Subject: [PATCH 02/11] Add pruned_block. --- libraries/chain/block.cpp | 41 +++++++++- libraries/chain/include/eosio/chain/block.hpp | 76 +++++++++++++++++++ .../chain/include/eosio/chain/transaction.hpp | 1 + libraries/chain/transaction.cpp | 44 ++++++++--- unittests/misc_tests.cpp | 29 +++++++ 5 files changed, 181 insertions(+), 10 deletions(-) diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index 91c93f6c359..7898712c5eb 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -20,6 +20,18 @@ namespace eosio { namespace chain { } } + static fc::static_variant translate_transaction_receipt(const transaction_id_type& tid) { + return tid; + } + static fc::static_variant translate_transaction_receipt(const packed_transaction& ptrx) { + return pruned_transaction(ptrx); + } + + pruned_transaction_receipt::pruned_transaction_receipt(const transaction_receipt& other) + : transaction_receipt_header(static_cast(other)), + trx(other.trx.visit([&](const auto& obj) { return translate_transaction_receipt(obj); })) + {} + flat_multimap signed_block::validate_and_extract_extensions()const { using decompose_t = block_extension_types::decompose_t; @@ -61,4 +73,31 @@ namespace eosio { namespace chain { } -} } /// namespace eosio::chain \ No newline at end of file + pruned_block::pruned_block( const signed_block& other ) + : signed_block_header(static_cast(other)), + prune_state(prune_state_type::complete_legacy), + transactions(other.transactions.begin(), other.transactions.end()), + block_extensions(other.block_extensions) + {} + + static std::size_t pruned_trx_receipt_packed_size(const pruned_transaction& obj, pruned_transaction::compression_type segment_compression) { + return obj.maximum_pruned_pack_size(segment_compression); + } + static std::size_t pruned_trx_receipt_packed_size(const transaction_id_type& obj, pruned_transaction::compression_type) { + return fc::raw::pack_size(obj); + } + + std::size_t pruned_transaction_receipt::maximum_pruned_pack_size( pruned_transaction::compression_type segment_compression ) const { + return fc::raw::pack_size(*static_cast(this)) + 1 + + trx.visit([&](const auto& obj){ return pruned_trx_receipt_packed_size(obj, segment_compression); }); + } + + std::size_t pruned_block::maximum_pruned_pack_size( pruned_transaction::compression_type segment_compression ) const { + std::size_t result = fc::raw::pack_size(fc::unsigned_int(transactions.size())); + for(const pruned_transaction_receipt& r: transactions) { + result += r.maximum_pruned_pack_size( segment_compression ); + } + return fc::raw::pack_size(*static_cast(this)) + fc::raw::pack_size(prune_state) + result + fc::raw::pack_size(block_extensions); + } + +} } /// namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index c2487fd9a76..9d04e549667 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -103,6 +103,79 @@ namespace eosio { namespace chain { }; using signed_block_ptr = std::shared_ptr; + struct pruned_transaction_receipt : public transaction_receipt_header { + + pruned_transaction_receipt():transaction_receipt_header(){} + pruned_transaction_receipt(const transaction_receipt&); + explicit pruned_transaction_receipt( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){} + explicit pruned_transaction_receipt( const pruned_transaction& ptrx ):transaction_receipt_header(executed),trx(ptrx){} + + fc::static_variant trx; + + std::size_t maximum_pruned_pack_size( pruned_transaction::compression_type segment_compression ) const; + + digest_type digest()const { + digest_type::encoder enc; + fc::raw::pack( enc, status ); + fc::raw::pack( enc, cpu_usage_us ); + fc::raw::pack( enc, net_usage_words ); + if( trx.contains() ) + fc::raw::pack( enc, trx.get() ); + else + fc::raw::pack( enc, trx.get().packed_digest() ); + return enc.result(); + } + }; + + struct pruned_block : public signed_block_header{ + private: + pruned_block( const pruned_block& ) = default; + public: + enum class prune_state_type : uint8_t { incomplete, complete, complete_legacy }; + + pruned_block() = default; + explicit pruned_block( const signed_block_header& h ):signed_block_header(h){} + pruned_block( const signed_block& ); + pruned_block( pruned_block&& ) = default; + pruned_block& operator=(const pruned_block&) = delete; + pruned_block clone() const { return *this; } + + fc::enum_type prune_state{prune_state_type::complete_legacy}; + deque transactions; /// new or generated transactions + extensions_type block_extensions; + + std::size_t maximum_pruned_pack_size( pruned_transaction::compression_type segment_compression ) const; + + template + void pack_with_padding(Stream& stream, pruned_transaction::compression_type segment_compression) { + std::size_t padded_size = maximum_pruned_pack_size( segment_compression ); + pack_with_padding(stream, segment_compression, padded_size); + } + template + void pack_with_padding(Stream& stream, pruned_transaction::compression_type segment_compression, std::size_t padded_size) { + fc::raw::pack(stream, fc::unsigned_int(padded_size)); + std::size_t pos = stream.tellp(); + fc::raw::pack(stream, *this); + std::size_t actual_size = stream.tellp() - pos; + for(std::size_t i = 0; i < padded_size - actual_size; ++i) { + stream.put('\0'); + } + } + template + std::size_t unpack_with_padding(Stream& stream, pruned_transaction::compression_type segment_compression) { + fc::unsigned_int padded_size; + fc::raw::unpack(stream, padded_size); + std::size_t pos = stream.tellp(); + fc::raw::unpack(stream, *this); + std::size_t actual_size = stream.tellp() - pos; + stream.skip( padded_size - actual_size ); + return padded_size; + } + + flat_multimap validate_and_extract_extensions()const; + }; + using pruned_block_ptr = std::shared_ptr; + struct producer_confirmation { block_id_type block_id; digest_type block_digest; @@ -119,3 +192,6 @@ FC_REFLECT(eosio::chain::transaction_receipt_header, (status)(cpu_usage_us)(net_ FC_REFLECT_DERIVED(eosio::chain::transaction_receipt, (eosio::chain::transaction_receipt_header), (trx) ) FC_REFLECT(eosio::chain::additional_block_signatures_extension, (signatures)); FC_REFLECT_DERIVED(eosio::chain::signed_block, (eosio::chain::signed_block_header), (transactions)(block_extensions) ) + +FC_REFLECT_DERIVED(eosio::chain::pruned_transaction_receipt, (eosio::chain::transaction_receipt_header), (trx) ) +FC_REFLECT_DERIVED(eosio::chain::pruned_block, (eosio::chain::signed_block_header), (prune_state)(transactions)(block_extensions) ) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index fc891962f0e..4f0a10c0cda 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -257,6 +257,7 @@ namespace eosio { namespace chain { pruned_transaction& operator=(const pruned_transaction&) = delete; pruned_transaction& operator=(pruned_transaction&&) = default; + pruned_transaction(const packed_transaction& other) : pruned_transaction(other.get_signed_transaction(), true, other.get_compression()) {} explicit pruned_transaction(const signed_transaction& t, bool legacy, compression_type _compression = compression_type::none); explicit pruned_transaction(signed_transaction&& t, bool legacy, compression_type _compression = compression_type::none); diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 3383ab6c2bc..8a152421dcc 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -316,23 +316,25 @@ void packed_transaction::reflector_init() local_unpack_context_free_data(); } -void packed_transaction::local_unpack_transaction(vector&& context_free_data) -{ +static transaction unpack_transaction(const bytes& packed_trx, packed_transaction::compression_type compression) { try { switch( compression ) { - case compression_type::none: - unpacked_trx = signed_transaction( unpack_transaction( packed_trx ), signatures, std::move(context_free_data) ); - break; - case compression_type::zlib: - unpacked_trx = signed_transaction( zlib_decompress_transaction( packed_trx ), signatures, std::move(context_free_data) ); - break; + case packed_transaction::compression_type::none: + return unpack_transaction( packed_trx ); + case packed_transaction::compression_type::zlib: + return zlib_decompress_transaction( packed_trx ); default: EOS_THROW( unknown_transaction_compression, "Unknown transaction compression algorithm" ); } - trx_id = unpacked_trx.id(); } FC_CAPTURE_AND_RETHROW( (compression) ) } +void packed_transaction::local_unpack_transaction(vector&& context_free_data) +{ + unpacked_trx = signed_transaction( unpack_transaction( packed_trx, compression ), signatures, std::move(context_free_data) ); + trx_id = unpacked_trx.id(); +} + static vector unpack_context_free_data(const bytes& packed_context_free_data, packed_transaction::compression_type compression) { try { @@ -559,4 +561,28 @@ std::size_t pruned_transaction::maximum_pruned_pack_size(compression_type segmen return fc::raw::pack_size(compression) + fc::raw::pack_size(packed_trx) + prunable_data.maximum_pruned_pack_size(segment_compression); } +static std::vector maybe_unpack_context_free_data(const prunable_transaction_data::full& obj, packed_transaction::compression_type) { + return obj.context_free_segments; +} +static std::vector maybe_unpack_context_free_data(const prunable_transaction_data::full_legacy& obj, packed_transaction::compression_type compression) { + return unpack_context_free_data(obj.packed_context_free_data, compression); +} +template +static std::vector maybe_unpack_context_free_data(const T&, packed_transaction::compression_type) { + return {}; +} + +void pruned_transaction::reflector_init() +{ + // called after construction, but always on the same thread and before packed_transaction passed to any other threads + static_assert(fc::raw::has_feature_reflector_init_on_unpacked_reflected_types, + "FC unpack needs to call reflector_init otherwise unpacked_trx will not be initialized"); + EOS_ASSERT( unpacked_trx.expiration == time_point_sec(), tx_decompression_error, "packed_transaction already unpacked" ); + const auto* signatures = get_signatures(); + unpacked_trx = signed_transaction(unpack_transaction(packed_trx, compression), + signatures ? *signatures : std::vector{}, + prunable_data.prunable_data.visit([&](const auto& obj) { return maybe_unpack_context_free_data(obj, compression); })); + trx_id = unpacked_trx.id(); +} + } } // eosio::chain diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index 1b0ab4b6fed..aae5ec4a184 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -990,6 +990,35 @@ BOOST_AUTO_TEST_CASE(pruned_transaction_test) { BOOST_CHECK_THROW(pruned.get_prunable_size(), tx_prune_exception); } +BOOST_AUTO_TEST_CASE(pruned_block_test) { + tester t; + signed_transaction trx; + trx.actions.push_back({ { permission_level{ N(eosio), N(active) } }, N(eosio), N(), bytes() } ); + trx.context_free_actions.push_back({ {}, N(eosio), N(), bytes() }); + trx.context_free_data.push_back(bytes()); + t.set_transaction_headers(trx); + trx.sign( t.get_private_key( N(eosio), "active" ), t.control->get_chain_id() ); + + t.push_transaction(trx); + signed_block_ptr original = t.produce_block(); + pruned_block basic(*original); + + fc::datastream size_stream; + basic.pack_with_padding(size_stream, pruned_transaction::compression_type::none); + std::vector buffer(size_stream.tellp()); + fc::datastream stream(buffer.data(), buffer.size()); + basic.pack_with_padding(stream, pruned_transaction::compression_type::none); + pruned_block deserialized; + fc::datastream in(buffer.data(), buffer.size()); + std::size_t unpacked_size = deserialized.unpack_with_padding(in, pruned_transaction::compression_type::none); + BOOST_TEST(in.tellp() == buffer.size()); + deserialized.transactions.back().trx.get().prune_all(); + deserialized.prune_state = pruned_block::prune_state_type::incomplete; + fc::datastream out(buffer.data(), buffer.size()); + deserialized.pack_with_padding(out, pruned_transaction::compression_type::none, unpacked_size); + BOOST_TEST(out.tellp() == buffer.size()); +} + BOOST_AUTO_TEST_CASE(reflector_init_test) { try { From 1b4701be104ff38fbac983a9d8ef3ddea98a88ad Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Mon, 23 Mar 2020 14:15:50 -0400 Subject: [PATCH 03/11] Validate the digests of pruned_transactions. --- unittests/misc_tests.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index aae5ec4a184..924263affc3 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -925,6 +925,7 @@ BOOST_AUTO_TEST_CASE(prunable_transaction_data_test) { pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(basic.digest().str() == pruned.digest().str()); } bytes large_bytes(48); @@ -935,6 +936,7 @@ BOOST_AUTO_TEST_CASE(prunable_transaction_data_test) { pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(basic.digest().str() == pruned.digest().str()); } { @@ -944,6 +946,7 @@ BOOST_AUTO_TEST_CASE(prunable_transaction_data_test) { pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(basic.digest().str() == pruned.digest().str()); } { @@ -953,6 +956,7 @@ BOOST_AUTO_TEST_CASE(prunable_transaction_data_test) { pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + BOOST_TEST(basic.digest().str() == pruned.digest().str()); } } @@ -990,6 +994,14 @@ BOOST_AUTO_TEST_CASE(pruned_transaction_test) { BOOST_CHECK_THROW(pruned.get_prunable_size(), tx_prune_exception); } + +static checksum256_type calculate_trx_merkle( const deque& trxs ) { + deque trx_digests; + for( const auto& a : trxs ) + trx_digests.emplace_back( a.digest() ); + return merkle( move( trx_digests ) ); +} + BOOST_AUTO_TEST_CASE(pruned_block_test) { tester t; signed_transaction trx; @@ -1003,6 +1015,9 @@ BOOST_AUTO_TEST_CASE(pruned_block_test) { signed_block_ptr original = t.produce_block(); pruned_block basic(*original); + BOOST_TEST(basic.transaction_mroot.str() == original->transaction_mroot.str()); + BOOST_TEST(basic.transaction_mroot.str() == calculate_trx_merkle(basic.transactions).str()); + fc::datastream size_stream; basic.pack_with_padding(size_stream, pruned_transaction::compression_type::none); std::vector buffer(size_stream.tellp()); @@ -1012,8 +1027,11 @@ BOOST_AUTO_TEST_CASE(pruned_block_test) { fc::datastream in(buffer.data(), buffer.size()); std::size_t unpacked_size = deserialized.unpack_with_padding(in, pruned_transaction::compression_type::none); BOOST_TEST(in.tellp() == buffer.size()); + BOOST_TEST(deserialized.transaction_mroot.str() == original->transaction_mroot.str()); + BOOST_TEST(deserialized.transaction_mroot.str() == calculate_trx_merkle(deserialized.transactions).str()); deserialized.transactions.back().trx.get().prune_all(); deserialized.prune_state = pruned_block::prune_state_type::incomplete; + BOOST_TEST(deserialized.transaction_mroot.str() == calculate_trx_merkle(deserialized.transactions).str()); fc::datastream out(buffer.data(), buffer.size()); deserialized.pack_with_padding(out, pruned_transaction::compression_type::none, unpacked_size); BOOST_TEST(out.tellp() == buffer.size()); From 51dde517b37e096ede05310c690a339497c5df7b Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Mon, 23 Mar 2020 16:23:49 -0400 Subject: [PATCH 04/11] A packed_transaction/transaction_receipt/signed_block is not automatically legacy. --- libraries/chain/block.cpp | 21 +++++++++++-------- libraries/chain/include/eosio/chain/block.hpp | 4 ++-- .../chain/include/eosio/chain/transaction.hpp | 2 +- unittests/misc_tests.cpp | 2 +- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index 7898712c5eb..8d319faaeea 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -20,16 +20,16 @@ namespace eosio { namespace chain { } } - static fc::static_variant translate_transaction_receipt(const transaction_id_type& tid) { + static fc::static_variant translate_transaction_receipt(const transaction_id_type& tid, bool) { return tid; } - static fc::static_variant translate_transaction_receipt(const packed_transaction& ptrx) { - return pruned_transaction(ptrx); + static fc::static_variant translate_transaction_receipt(const packed_transaction& ptrx, bool legacy) { + return pruned_transaction(ptrx, legacy); } - pruned_transaction_receipt::pruned_transaction_receipt(const transaction_receipt& other) + pruned_transaction_receipt::pruned_transaction_receipt(const transaction_receipt& other, bool legacy) : transaction_receipt_header(static_cast(other)), - trx(other.trx.visit([&](const auto& obj) { return translate_transaction_receipt(obj); })) + trx(other.trx.visit([&](const auto& obj) { return translate_transaction_receipt(obj, legacy); })) {} flat_multimap signed_block::validate_and_extract_extensions()const { @@ -73,12 +73,15 @@ namespace eosio { namespace chain { } - pruned_block::pruned_block( const signed_block& other ) + pruned_block::pruned_block( const signed_block& other, bool legacy ) : signed_block_header(static_cast(other)), - prune_state(prune_state_type::complete_legacy), - transactions(other.transactions.begin(), other.transactions.end()), + prune_state(legacy ? prune_state_type::complete_legacy : prune_state_type::complete), block_extensions(other.block_extensions) - {} + { + for(const auto& trx : other.transactions) { + transactions.emplace_back(trx, legacy); + } + } static std::size_t pruned_trx_receipt_packed_size(const pruned_transaction& obj, pruned_transaction::compression_type segment_compression) { return obj.maximum_pruned_pack_size(segment_compression); diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 9d04e549667..bcf71412469 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -106,7 +106,7 @@ namespace eosio { namespace chain { struct pruned_transaction_receipt : public transaction_receipt_header { pruned_transaction_receipt():transaction_receipt_header(){} - pruned_transaction_receipt(const transaction_receipt&); + pruned_transaction_receipt(const transaction_receipt&, bool legacy); explicit pruned_transaction_receipt( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){} explicit pruned_transaction_receipt( const pruned_transaction& ptrx ):transaction_receipt_header(executed),trx(ptrx){} @@ -135,7 +135,7 @@ namespace eosio { namespace chain { pruned_block() = default; explicit pruned_block( const signed_block_header& h ):signed_block_header(h){} - pruned_block( const signed_block& ); + pruned_block( const signed_block&, bool legacy ); pruned_block( pruned_block&& ) = default; pruned_block& operator=(const pruned_block&) = delete; pruned_block clone() const { return *this; } diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 4f0a10c0cda..74d9f99139b 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -257,7 +257,7 @@ namespace eosio { namespace chain { pruned_transaction& operator=(const pruned_transaction&) = delete; pruned_transaction& operator=(pruned_transaction&&) = default; - pruned_transaction(const packed_transaction& other) : pruned_transaction(other.get_signed_transaction(), true, other.get_compression()) {} + pruned_transaction(const packed_transaction& other, bool legacy) : pruned_transaction(other.get_signed_transaction(), legacy, other.get_compression()) {} explicit pruned_transaction(const signed_transaction& t, bool legacy, compression_type _compression = compression_type::none); explicit pruned_transaction(signed_transaction&& t, bool legacy, compression_type _compression = compression_type::none); diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index 924263affc3..1a2d648a957 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -1013,7 +1013,7 @@ BOOST_AUTO_TEST_CASE(pruned_block_test) { t.push_transaction(trx); signed_block_ptr original = t.produce_block(); - pruned_block basic(*original); + pruned_block basic(*original, true); BOOST_TEST(basic.transaction_mroot.str() == original->transaction_mroot.str()); BOOST_TEST(basic.transaction_mroot.str() == calculate_trx_merkle(basic.transactions).str()); From b645604abc8fdc1e667e82061b897fd626b3688d Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Mon, 23 Mar 2020 16:29:43 -0400 Subject: [PATCH 05/11] Better error message. --- libraries/chain/transaction.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 8a152421dcc..68a8f58939f 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -426,7 +426,7 @@ static constexpr std::size_t digest_pack_size = 32; static std::size_t padded_pack_size(const prunable_transaction_data::none& obj, prunable_transaction_data::compression_type) { std::size_t result = fc::raw::pack_size(obj); - EOS_ASSERT(result == digest_pack_size, chain_type_exception, "Internal EOSIO error"); + EOS_ASSERT(result == digest_pack_size, packed_transaction_type_exception, "Unexpected size of packed digest"); return result; } From 98e698e169034b18939bd1d88f335f002ebada76 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Tue, 24 Mar 2020 09:49:29 -0400 Subject: [PATCH 06/11] Separate prunable_transaction_data::compression_type from packed_transaction::compression_type. --- libraries/chain/block.cpp | 8 ++++---- libraries/chain/include/eosio/chain/block.hpp | 10 +++++----- libraries/chain/include/eosio/chain/transaction.hpp | 8 ++++++-- libraries/chain/transaction.cpp | 2 +- unittests/misc_tests.cpp | 10 +++++----- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index 8d319faaeea..13891dc5d68 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -83,19 +83,19 @@ namespace eosio { namespace chain { } } - static std::size_t pruned_trx_receipt_packed_size(const pruned_transaction& obj, pruned_transaction::compression_type segment_compression) { + static std::size_t pruned_trx_receipt_packed_size(const pruned_transaction& obj, pruned_transaction::cf_compression_type segment_compression) { return obj.maximum_pruned_pack_size(segment_compression); } - static std::size_t pruned_trx_receipt_packed_size(const transaction_id_type& obj, pruned_transaction::compression_type) { + static std::size_t pruned_trx_receipt_packed_size(const transaction_id_type& obj, pruned_transaction::cf_compression_type) { return fc::raw::pack_size(obj); } - std::size_t pruned_transaction_receipt::maximum_pruned_pack_size( pruned_transaction::compression_type segment_compression ) const { + std::size_t pruned_transaction_receipt::maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const { return fc::raw::pack_size(*static_cast(this)) + 1 + trx.visit([&](const auto& obj){ return pruned_trx_receipt_packed_size(obj, segment_compression); }); } - std::size_t pruned_block::maximum_pruned_pack_size( pruned_transaction::compression_type segment_compression ) const { + std::size_t pruned_block::maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const { std::size_t result = fc::raw::pack_size(fc::unsigned_int(transactions.size())); for(const pruned_transaction_receipt& r: transactions) { result += r.maximum_pruned_pack_size( segment_compression ); diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index bcf71412469..118362b5ff5 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -112,7 +112,7 @@ namespace eosio { namespace chain { fc::static_variant trx; - std::size_t maximum_pruned_pack_size( pruned_transaction::compression_type segment_compression ) const; + std::size_t maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const; digest_type digest()const { digest_type::encoder enc; @@ -144,15 +144,15 @@ namespace eosio { namespace chain { deque transactions; /// new or generated transactions extensions_type block_extensions; - std::size_t maximum_pruned_pack_size( pruned_transaction::compression_type segment_compression ) const; + std::size_t maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const; template - void pack_with_padding(Stream& stream, pruned_transaction::compression_type segment_compression) { + void pack_with_padding(Stream& stream, pruned_transaction::cf_compression_type segment_compression) { std::size_t padded_size = maximum_pruned_pack_size( segment_compression ); pack_with_padding(stream, segment_compression, padded_size); } template - void pack_with_padding(Stream& stream, pruned_transaction::compression_type segment_compression, std::size_t padded_size) { + void pack_with_padding(Stream& stream, pruned_transaction::cf_compression_type segment_compression, std::size_t padded_size) { fc::raw::pack(stream, fc::unsigned_int(padded_size)); std::size_t pos = stream.tellp(); fc::raw::pack(stream, *this); @@ -162,7 +162,7 @@ namespace eosio { namespace chain { } } template - std::size_t unpack_with_padding(Stream& stream, pruned_transaction::compression_type segment_compression) { + std::size_t unpack_with_padding(Stream& stream, pruned_transaction::cf_compression_type segment_compression) { fc::unsigned_int padded_size; fc::raw::unpack(stream, padded_size); std::size_t pos = stream.tellp(); diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 74d9f99139b..d7cfe44e8bd 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -204,7 +204,10 @@ namespace eosio { namespace chain { }; struct prunable_transaction_data { - using compression_type = packed_transaction::compression_type; + enum class compression_type { + none = 0, + zlib = 1, + }; struct none { digest_type prunable_digest; @@ -250,6 +253,7 @@ namespace eosio { namespace chain { struct pruned_transaction : fc::reflect_init { using compression_type = packed_transaction::compression_type; + using cf_compression_type = prunable_transaction_data::compression_type; pruned_transaction() = default; pruned_transaction(pruned_transaction&&) = default; @@ -289,7 +293,7 @@ namespace eosio { namespace chain { void prune_all(); - std::size_t maximum_pruned_pack_size( compression_type segment_compression ) const; + std::size_t maximum_pruned_pack_size( cf_compression_type segment_compression ) const; private: diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 68a8f58939f..c42e28a4928 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -557,7 +557,7 @@ void pruned_transaction::prune_all() { unpacked_trx.signatures.clear(); } -std::size_t pruned_transaction::maximum_pruned_pack_size(compression_type segment_compression) const { +std::size_t pruned_transaction::maximum_pruned_pack_size(cf_compression_type segment_compression) const { return fc::raw::pack_size(compression) + fc::raw::pack_size(packed_trx) + prunable_data.maximum_pruned_pack_size(segment_compression); } diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index 1a2d648a957..cb968475f2e 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -980,7 +980,7 @@ BOOST_AUTO_TEST_CASE(pruned_transaction_test) { BOOST_TEST(*pruned.get_signatures() == packed.get_signatures()); BOOST_TEST(pruned.get_prunable_size() == packed.get_prunable_size()); BOOST_TEST(pruned.get_unprunable_size() == packed.get_unprunable_size()); - std::size_t max_size = pruned.maximum_pruned_pack_size(pruned_transaction::compression_type::none); + std::size_t max_size = pruned.maximum_pruned_pack_size(pruned_transaction::cf_compression_type::none); BOOST_TEST(fc::raw::pack_size(pruned) <= max_size); pruned.prune_all(); @@ -1019,13 +1019,13 @@ BOOST_AUTO_TEST_CASE(pruned_block_test) { BOOST_TEST(basic.transaction_mroot.str() == calculate_trx_merkle(basic.transactions).str()); fc::datastream size_stream; - basic.pack_with_padding(size_stream, pruned_transaction::compression_type::none); + basic.pack_with_padding(size_stream, pruned_transaction::cf_compression_type::none); std::vector buffer(size_stream.tellp()); fc::datastream stream(buffer.data(), buffer.size()); - basic.pack_with_padding(stream, pruned_transaction::compression_type::none); + basic.pack_with_padding(stream, pruned_transaction::cf_compression_type::none); pruned_block deserialized; fc::datastream in(buffer.data(), buffer.size()); - std::size_t unpacked_size = deserialized.unpack_with_padding(in, pruned_transaction::compression_type::none); + std::size_t unpacked_size = deserialized.unpack_with_padding(in, pruned_transaction::cf_compression_type::none); BOOST_TEST(in.tellp() == buffer.size()); BOOST_TEST(deserialized.transaction_mroot.str() == original->transaction_mroot.str()); BOOST_TEST(deserialized.transaction_mroot.str() == calculate_trx_merkle(deserialized.transactions).str()); @@ -1033,7 +1033,7 @@ BOOST_AUTO_TEST_CASE(pruned_block_test) { deserialized.prune_state = pruned_block::prune_state_type::incomplete; BOOST_TEST(deserialized.transaction_mroot.str() == calculate_trx_merkle(deserialized.transactions).str()); fc::datastream out(buffer.data(), buffer.size()); - deserialized.pack_with_padding(out, pruned_transaction::compression_type::none, unpacked_size); + deserialized.pack_with_padding(out, pruned_transaction::cf_compression_type::none, unpacked_size); BOOST_TEST(out.tellp() == buffer.size()); } From baf7a9329d24ce49b7fd7473f8fc456acf57d3bd Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Tue, 24 Mar 2020 09:50:26 -0400 Subject: [PATCH 07/11] Fix comment. --- libraries/chain/include/eosio/chain/transaction.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index d7cfe44e8bd..cd1c6e0bf43 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -286,7 +286,7 @@ namespace eosio { namespace chain { const vector* get_signatures()const; // Returns nullptr if any context_free_data segment was pruned const vector* get_context_free_data()const; - // Returns nullptr if the context_free_data segment was pruned segment_ordinal is out of range. + // Returns nullptr if the context_free_data segment was pruned or segment_ordinal is out of range. const bytes* get_context_free_data(std::size_t segment_ordinal); const fc::enum_type& get_compression()const { return compression; } const bytes& get_packed_transaction()const { return packed_trx; } From c92ec81cf107dba7f1107f0c18d306093ef04e1d Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Tue, 24 Mar 2020 10:16:17 -0400 Subject: [PATCH 08/11] Add a check that the padded_size is at least the packed size of the block. --- libraries/chain/include/eosio/chain/block.hpp | 6 ++++++ libraries/chain/include/eosio/chain/exceptions.hpp | 2 ++ 2 files changed, 8 insertions(+) diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 118362b5ff5..0cd077b61e7 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -157,6 +157,9 @@ namespace eosio { namespace chain { std::size_t pos = stream.tellp(); fc::raw::pack(stream, *this); std::size_t actual_size = stream.tellp() - pos; + EOS_ASSERT(actual_size <= padded_size, block_padding_exception, + "Provided value of padded_size (${provided}) is less than the packed size of the block (${actual})", + ("actual", actual_size)("provided", padded_size) ); for(std::size_t i = 0; i < padded_size - actual_size; ++i) { stream.put('\0'); } @@ -168,6 +171,9 @@ namespace eosio { namespace chain { std::size_t pos = stream.tellp(); fc::raw::unpack(stream, *this); std::size_t actual_size = stream.tellp() - pos; + EOS_ASSERT(actual_size <= padded_size, block_padding_exception, + "Stored value of padded_size (${provided}) is less than the packed size of the block (${actual})", + ("actual", actual_size)("provided", padded_size) ); stream.skip( padded_size - actual_size ); return padded_size; } diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index b534844ea53..a48f3ac4955 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -236,6 +236,8 @@ namespace eosio { namespace chain { 3030012, "Invalid block extension" ) FC_DECLARE_DERIVED_EXCEPTION( ill_formed_additional_block_signatures_extension, block_validate_exception, 3030013, "Block includes an ill-formed additional block signature extension" ) + FC_DECLARE_DERIVED_EXCEPTION( block_padding_exception, block_validate_exception, + 3030014, "Invalid block padding" ) FC_DECLARE_DERIVED_EXCEPTION( transaction_exception, chain_exception, From 51ccbb49d4a0d59dd1f60bafdf7a57af6e133fe8 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Tue, 24 Mar 2020 11:21:40 -0400 Subject: [PATCH 09/11] pruned_block::pack now only calculates the required padding. It does not attempt to encode it. --- libraries/chain/include/eosio/chain/block.hpp | 30 ++++--------------- unittests/misc_tests.cpp | 16 +++++----- 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 0cd077b61e7..ff8fb821c4c 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -146,36 +146,18 @@ namespace eosio { namespace chain { std::size_t maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const; + // Returns the maximum_pruned_padded_size. It is the caller's responsibility to + // reserve enough space after the end if in-place pruning is desired. template - void pack_with_padding(Stream& stream, pruned_transaction::cf_compression_type segment_compression) { + std::size_t pack(Stream& stream, pruned_transaction::cf_compression_type segment_compression) { std::size_t padded_size = maximum_pruned_pack_size( segment_compression ); - pack_with_padding(stream, segment_compression, padded_size); - } - template - void pack_with_padding(Stream& stream, pruned_transaction::cf_compression_type segment_compression, std::size_t padded_size) { - fc::raw::pack(stream, fc::unsigned_int(padded_size)); - std::size_t pos = stream.tellp(); + // TODO: This only handles legacy transactions. fc::raw::pack(stream, *this); - std::size_t actual_size = stream.tellp() - pos; - EOS_ASSERT(actual_size <= padded_size, block_padding_exception, - "Provided value of padded_size (${provided}) is less than the packed size of the block (${actual})", - ("actual", actual_size)("provided", padded_size) ); - for(std::size_t i = 0; i < padded_size - actual_size; ++i) { - stream.put('\0'); - } + return padded_size; } template - std::size_t unpack_with_padding(Stream& stream, pruned_transaction::cf_compression_type segment_compression) { - fc::unsigned_int padded_size; - fc::raw::unpack(stream, padded_size); - std::size_t pos = stream.tellp(); + void unpack(Stream& stream, pruned_transaction::cf_compression_type segment_compression) { fc::raw::unpack(stream, *this); - std::size_t actual_size = stream.tellp() - pos; - EOS_ASSERT(actual_size <= padded_size, block_padding_exception, - "Stored value of padded_size (${provided}) is less than the packed size of the block (${actual})", - ("actual", actual_size)("provided", padded_size) ); - stream.skip( padded_size - actual_size ); - return padded_size; } flat_multimap validate_and_extract_extensions()const; diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index cb968475f2e..270dfab9e1b 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -1019,22 +1019,24 @@ BOOST_AUTO_TEST_CASE(pruned_block_test) { BOOST_TEST(basic.transaction_mroot.str() == calculate_trx_merkle(basic.transactions).str()); fc::datastream size_stream; - basic.pack_with_padding(size_stream, pruned_transaction::cf_compression_type::none); - std::vector buffer(size_stream.tellp()); + std::size_t padded_size = basic.pack(size_stream, pruned_transaction::cf_compression_type::none); + BOOST_TEST(size_stream.tellp() <= padded_size); + std::vector buffer(padded_size); fc::datastream stream(buffer.data(), buffer.size()); - basic.pack_with_padding(stream, pruned_transaction::cf_compression_type::none); + basic.pack(stream, pruned_transaction::cf_compression_type::none); pruned_block deserialized; fc::datastream in(buffer.data(), buffer.size()); - std::size_t unpacked_size = deserialized.unpack_with_padding(in, pruned_transaction::cf_compression_type::none); - BOOST_TEST(in.tellp() == buffer.size()); + deserialized.unpack(in, pruned_transaction::cf_compression_type::none); + std::size_t unpacked_size = padded_size; + BOOST_TEST(in.tellp() <= buffer.size()); BOOST_TEST(deserialized.transaction_mroot.str() == original->transaction_mroot.str()); BOOST_TEST(deserialized.transaction_mroot.str() == calculate_trx_merkle(deserialized.transactions).str()); deserialized.transactions.back().trx.get().prune_all(); deserialized.prune_state = pruned_block::prune_state_type::incomplete; BOOST_TEST(deserialized.transaction_mroot.str() == calculate_trx_merkle(deserialized.transactions).str()); fc::datastream out(buffer.data(), buffer.size()); - deserialized.pack_with_padding(out, pruned_transaction::cf_compression_type::none, unpacked_size); - BOOST_TEST(out.tellp() == buffer.size()); + deserialized.pack(out, pruned_transaction::cf_compression_type::none); + BOOST_TEST(out.tellp() <= buffer.size()); } BOOST_AUTO_TEST_CASE(reflector_init_test) { From 11131532904ae38fc2e29920c0029ed2f77dd9c8 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Tue, 24 Mar 2020 11:29:23 -0400 Subject: [PATCH 10/11] Implement pruned_block::validate_and_extract_extensions. --- libraries/chain/block.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index 13891dc5d68..5508b92000a 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -32,7 +32,7 @@ namespace eosio { namespace chain { trx(other.trx.visit([&](const auto& obj) { return translate_transaction_receipt(obj, legacy); })) {} - flat_multimap signed_block::validate_and_extract_extensions()const { + static flat_multimap validate_and_extract_block_extensions(const extensions_type& block_extensions) { using decompose_t = block_extension_types::decompose_t; flat_multimap results; @@ -73,6 +73,10 @@ namespace eosio { namespace chain { } + flat_multimap signed_block::validate_and_extract_extensions()const { + return validate_and_extract_block_extensions( block_extensions ); + } + pruned_block::pruned_block( const signed_block& other, bool legacy ) : signed_block_header(static_cast(other)), prune_state(legacy ? prune_state_type::complete_legacy : prune_state_type::complete), @@ -103,4 +107,8 @@ namespace eosio { namespace chain { return fc::raw::pack_size(*static_cast(this)) + fc::raw::pack_size(prune_state) + result + fc::raw::pack_size(block_extensions); } + flat_multimap pruned_block::validate_and_extract_extensions()const { + return validate_and_extract_block_extensions( block_extensions ); + } + } } /// namespace eosio::chain From cd18993a57758a8d4fb5722de19a411d1dc3695b Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Tue, 24 Mar 2020 14:49:21 -0400 Subject: [PATCH 11/11] Remove unused exception --- libraries/chain/include/eosio/chain/exceptions.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index a48f3ac4955..b534844ea53 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -236,8 +236,6 @@ namespace eosio { namespace chain { 3030012, "Invalid block extension" ) FC_DECLARE_DERIVED_EXCEPTION( ill_formed_additional_block_signatures_extension, block_validate_exception, 3030013, "Block includes an ill-formed additional block signature extension" ) - FC_DECLARE_DERIVED_EXCEPTION( block_padding_exception, block_validate_exception, - 3030014, "Invalid block padding" ) FC_DECLARE_DERIVED_EXCEPTION( transaction_exception, chain_exception,