From 2401a98488203d9fa112f051422d4598703ea6f2 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 27 Mar 2020 14:26:34 -0500 Subject: [PATCH 001/142] Added block log v4 --- libraries/chain/block_log.cpp | 136 ++++++++++++++---- libraries/chain/include/eosio/chain/block.hpp | 2 +- .../include/eosio/chain/block_header.hpp | 2 + .../chain/include/eosio/chain/block_log.hpp | 34 +++-- 4 files changed, 130 insertions(+), 44 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 822de1ea8b8..5cf6dbf0741 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -31,15 +31,17 @@ namespace eosio { namespace chain { * this is in the form of an first_block_num that is written immediately after the version * Version 3: improvement on version 2 to not require the genesis state be provided when not starting * from block 1 + * Version 4: changes the block entry from the serialization of signed_block to a tuple of offset to next entry, + * compression_status and pruned_block. */ - const uint32_t block_log::max_supported_version = 3; + const uint32_t block_log::max_supported_version = 4; namespace detail { using unique_file = std::unique_ptr; class block_log_impl { public: - signed_block_ptr head; + signed_block_header_ptr head; block_id_type head_id; fc::cfile block_file; fc::cfile index_file; @@ -64,7 +66,7 @@ namespace eosio { namespace chain { } template - void reset( const T& t, const signed_block_ptr& genesis_block, uint32_t first_block_num ); + void reset( const T& t, const pruned_block_ptr& first_block, pruned_transaction::cf_compression_type segment_compression, uint32_t first_block_num ); void write( const genesis_state& gs ); @@ -72,10 +74,18 @@ namespace eosio { namespace chain { void flush(); - uint64_t append(const signed_block_ptr& b); + uint64_t append(const pruned_block_ptr& b, pruned_transaction::cf_compression_type segment_compression); template static fc::optional extract_chain_context( const fc::path& data_dir, Lambda&& lambda ); + + static std::vector pack(const pruned_block& b, + pruned_transaction::cf_compression_type segment_compression); + static void unpack(fc::cfile_datastream& ds, pruned_block& block); + + uint64_t write_log_entry(const pruned_block& b, + pruned_transaction::cf_compression_type segment_compression); + pruned_block_ptr read_log_entry(); }; void detail::block_log_impl::reopen() { @@ -286,11 +296,61 @@ namespace eosio { namespace chain { } } + std::vector detail::block_log_impl::pack(const pruned_block& b, pruned_transaction::cf_compression_type segment_compression) { + + // In version 4 of the irreversible blocks log format, these log entries consists of the following in order: + // 1. An uint32_t offset from the start of this log entry to the start of the next log entry. + // 2. An uint32_t indicating the compression status for the serialization of the pruned_block following this. + // 3. The serialization of a pruned_block representation of the block for the entry including padding. + + std::size_t padded_size = b.maximum_pruned_pack_size(segment_compression); + std::vector buffer(padded_size + 2* sizeof(uint32_t)); + fc::datastream stream(buffer.data(), buffer.size()); + + uint32_t offset = buffer.size(); + uint32_t compression = static_cast(segment_compression); + stream.write((char*)&offset, sizeof(offset)); + stream.write((char*)&compression, sizeof(compression)); + b.pack(stream, segment_compression); + return buffer; + } + + uint64_t detail::block_log_impl::write_log_entry(const pruned_block& b, pruned_transaction::cf_compression_type segment_compression) { + uint64_t pos = block_file.tellp(); + std::vector buffer; + + if (version >= 4) { + buffer = detail::block_log_impl::pack(b, segment_compression); + } else { + auto block_ptr = b.to_signed_block(); + EOS_ASSERT(block_ptr, block_log_append_fail, "Unable to convert block to legacy format"); + buffer = fc::raw::pack(*block_ptr); + } + block_file.write(buffer.data(), buffer.size()); + block_file.write((char*)&pos, sizeof(pos)); + index_file.write((char*)&pos, sizeof(pos)); + return pos; + } + + void detail::block_log_impl::unpack(fc::cfile_datastream& ds, pruned_block& block) { + uint32_t len; + uint32_t compression; + fc::raw::unpack(ds, len); + fc::raw::unpack(ds, compression); + block.unpack(ds, static_cast(compression)); + } + + + uint64_t block_log::append(const signed_block_ptr& b) { - return my->append(b); + return this->append(std::make_shared(*b, true), pruned_transaction::cf_compression_type::none); } - uint64_t detail::block_log_impl::append(const signed_block_ptr& b) { + uint64_t block_log::append(const pruned_block_ptr& b, pruned_transaction::cf_compression_type segment_compression) { + return my->append(b, segment_compression); + } + + uint64_t detail::block_log_impl::append(const pruned_block_ptr& b, pruned_transaction::cf_compression_type segment_compression) { try { EOS_ASSERT( genesis_written_to_block_log, block_log_append_fail, "Cannot append to block log until the genesis is first written" ); @@ -298,16 +358,14 @@ namespace eosio { namespace chain { block_file.seek_end(0); index_file.seek_end(0); - uint64_t pos = block_file.tellp(); EOS_ASSERT(index_file.tellp() == sizeof(uint64_t) * (b->block_num() - first_block_num), block_log_append_fail, "Append to index file occuring at wrong position.", ("position", (uint64_t) index_file.tellp()) ("expected", (b->block_num() - first_block_num) * sizeof(uint64_t))); - auto data = fc::raw::pack(*b); - block_file.write(data.data(), data.size()); - block_file.write((char*)&pos, sizeof(pos)); - index_file.write((char*)&pos, sizeof(pos)); + + auto pos = write_log_entry(*b, segment_compression); + head = b; head_id = b->id(); @@ -328,7 +386,7 @@ namespace eosio { namespace chain { } template - void detail::block_log_impl::reset( const T& t, const signed_block_ptr& first_block, uint32_t first_bnum ) { + void detail::block_log_impl::reset( const T& t, const pruned_block_ptr& first_block, pruned_transaction::cf_compression_type segment_compression, uint32_t first_bnum ) { close(); fc::remove_all( block_file.get_file_path() ); @@ -351,7 +409,7 @@ namespace eosio { namespace chain { block_file.write((char*)&totem, sizeof(totem)); if (first_block) { - append(first_block); + append(first_block, segment_compression); } else { head.reset(); head_id = {}; @@ -370,13 +428,18 @@ namespace eosio { namespace chain { } void block_log::reset( const genesis_state& gs, const signed_block_ptr& first_block ) { - my->reset(gs, first_block, 1); + auto b = std::make_shared(*first_block,true); + my->reset(gs, b, pruned_transaction::cf_compression_type::none, 1); + } + + void block_log::reset( const genesis_state& gs, const pruned_block_ptr& first_block, pruned_transaction::cf_compression_type segment_compression ) { + my->reset(gs, first_block,segment_compression, 1); } void block_log::reset( const chain_id_type& chain_id, uint32_t first_block_num ) { EOS_ASSERT( first_block_num > 1, block_log_exception, "Block log version ${ver} needs to be created with a genesis state if starting from block number 1." ); - my->reset(chain_id, signed_block_ptr(), first_block_num); + my->reset(chain_id, pruned_block_ptr(), pruned_transaction::cf_compression_type::none, first_block_num); } void detail::block_log_impl::write( const genesis_state& gs ) { @@ -388,27 +451,44 @@ namespace eosio { namespace chain { block_file << chain_id; } - signed_block_ptr block_log::read_block(uint64_t pos)const { - my->check_open_files(); + pruned_block_ptr detail::block_log_impl::read_log_entry() { + auto ds = block_file.create_datastream(); + if (version >= 4) { + pruned_block_ptr result = std::make_shared(); + detail::block_log_impl::unpack(ds, *result); + return result; + } else { + signed_block_ptr result = std::make_shared(); + fc::raw::unpack(ds, *result); + return std::make_shared(*result, true); + } + } + pruned_block_ptr block_log::read_block(uint64_t pos) const { + my->check_open_files(); my->block_file.seek(pos); - signed_block_ptr result = std::make_shared(); - auto ds = my->block_file.create_datastream(); - fc::raw::unpack(ds, *result); - return result; + return my->read_log_entry(); } - void block_log::read_block_header(block_header& bh, uint64_t pos)const { + void block_log::read_block_header(block_header& bh, uint64_t pos) const { my->check_open_files(); + if (my->version == 4) + pos += 2 * sizeof(uint32_t); + my->block_file.seek(pos); auto ds = my->block_file.create_datastream(); fc::raw::unpack(ds, bh); } - signed_block_ptr block_log::read_block_by_num(uint32_t block_num)const { + signed_block_ptr block_log::read_block_by_num(uint32_t block_num) const { + auto r = read_pruned_block_by_num(block_num); + return r ? r->to_signed_block() : signed_block_ptr{}; + } + + pruned_block_ptr block_log::read_pruned_block_by_num(uint32_t block_num) const { try { - signed_block_ptr b; + pruned_block_ptr b; uint64_t pos = get_block_pos(block_num); if (pos != npos) { b = read_block(pos); @@ -443,7 +523,7 @@ namespace eosio { namespace chain { return pos; } - signed_block_ptr block_log::read_head()const { + signed_block_header_ptr block_log::read_head()const { my->check_open_files(); uint64_t pos; @@ -462,11 +542,11 @@ namespace eosio { namespace chain { } } - const signed_block_ptr& block_log::head()const { + const signed_block_header_ptr& block_log::head() const { return my->head; } - const block_id_type& block_log::head_id()const { + const block_id_type& block_log::head_id() const { return my->head_id; } @@ -985,7 +1065,7 @@ namespace eosio { namespace chain { new_block_file.close(); new_block_file.open( LOG_RW_C ); - static_assert( block_log::max_supported_version == 3, + static_assert( block_log::max_supported_version == 4, "Code was written to support version 3 format, need to update this code for latest format." ); uint32_t version = block_log::max_supported_version; new_block_file.seek(0); diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index df9e2c2408c..3df6fd2af01 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -151,7 +151,7 @@ namespace eosio { namespace chain { // Returns the maximum_pruned_padded_size. It is the caller's responsibility to // reserve enough space after the end if in-place pruning is desired. template - std::size_t pack(Stream& stream, pruned_transaction::cf_compression_type segment_compression) { + std::size_t pack(Stream& stream, pruned_transaction::cf_compression_type segment_compression) const { std::size_t padded_size = maximum_pruned_pack_size( segment_compression ); // TODO: This only handles legacy transactions. fc::raw::pack(stream, *this); diff --git a/libraries/chain/include/eosio/chain/block_header.hpp b/libraries/chain/include/eosio/chain/block_header.hpp index 286ecf528c1..7832da5b491 100644 --- a/libraries/chain/include/eosio/chain/block_header.hpp +++ b/libraries/chain/include/eosio/chain/block_header.hpp @@ -76,6 +76,8 @@ namespace eosio { namespace chain { signature_type producer_signature; }; + using signed_block_header_ptr = std::shared_ptr; + } } /// namespace eosio::chain FC_REFLECT(eosio::chain::block_header, diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index 5fbe3e771a5..66da1638643 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -12,9 +12,9 @@ namespace eosio { namespace chain { * linked list of blocks. There is a secondary index file of only block positions that enables * O(1) random access lookup by block number. * - * +---------+----------------+---------+----------------+-----+------------+-------------------+ - * | Block 1 | Pos of Block 1 | Block 2 | Pos of Block 2 | ... | Head Block | Pos of Head Block | - * +---------+----------------+---------+----------------+-----+------------+-------------------+ + * +---------------+----------------+---------------+----------------+-----+------------------+-------------------+ + * | Block 1 Entry | Pos of Block 1 | Block 2 Entry | Pos of Block 2 | ... | Head Block Entry | Pos of Head Block | + * +---------------+----------------+---------------+----------------+-----+------------------+-------------------+ * * +----------------+----------------+-----+-------------------+ * | Pos of Block 1 | Pos of Block 2 | ... | Pos of Head Block | @@ -38,27 +38,31 @@ namespace eosio { namespace chain { block_log(block_log&& other); ~block_log(); - uint64_t append(const signed_block_ptr& b); + [[deprecated]] uint64_t append(const signed_block_ptr& b); + uint64_t append(const pruned_block_ptr& block, pruned_transaction::cf_compression_type segment_compression); + void flush(); - void reset( const genesis_state& gs, const signed_block_ptr& genesis_block ); + + [[deprecated]] void reset( const genesis_state& gs, const signed_block_ptr& genesis_block ); + void reset( const genesis_state& gs, const pruned_block_ptr& genesis_block, pruned_transaction::cf_compression_type segment_compression); void reset( const chain_id_type& chain_id, uint32_t first_block_num ); - signed_block_ptr read_block(uint64_t file_pos)const; + pruned_block_ptr read_block(uint64_t file_pos) const; void read_block_header(block_header& bh, uint64_t file_pos)const; - signed_block_ptr read_block_by_num(uint32_t block_num)const; + block_id_type read_block_id_by_num(uint32_t block_num)const; - signed_block_ptr read_block_by_id(const block_id_type& id)const { - return read_block_by_num(block_header::num_from_id(id)); - } + + [[deprecated]] signed_block_ptr read_block_by_num(uint32_t block_num) const; + pruned_block_ptr read_pruned_block_by_num(uint32_t block_num) const; /** * Return offset of block in file, or block_log::npos if it does not exist. */ - uint64_t get_block_pos(uint32_t block_num) const; - signed_block_ptr read_head()const; - const signed_block_ptr& head()const; - const block_id_type& head_id()const; - uint32_t first_block_num() const; + uint64_t get_block_pos(uint32_t block_num) const; + signed_block_header_ptr read_head() const; + const signed_block_header_ptr& head() const; + const block_id_type& head_id() const; + uint32_t first_block_num() const; static const uint64_t npos = std::numeric_limits::max(); From a2ac67c5656c106158993e71f55046209b99335d Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 30 Mar 2020 19:51:07 -0500 Subject: [PATCH 002/142] Rename of pruned_transaction, pruned_block to packed_transaction, signed_block. packed_transaction, signed_block renamed to packed_transaction_v0 and signed_block_v0. Moved codebase to using new types. abi_serializer not done yet. --- libraries/chain/block.cpp | 40 ++++---- libraries/chain/block_log.cpp | 3 +- libraries/chain/controller.cpp | 8 +- .../include/eosio/chain/abi_serializer.hpp | 17 ++-- libraries/chain/include/eosio/chain/block.hpp | 88 +++++++++--------- .../chain/include/eosio/chain/transaction.hpp | 64 ++++++------- libraries/chain/transaction.cpp | 92 ++++++++----------- libraries/testing/tester.cpp | 2 +- plugins/net_plugin/net_plugin.cpp | 6 +- .../trace_api_plugin/test/test_extraction.cpp | 2 +- .../txn_test_gen_plugin.cpp | 2 +- programs/cleos/main.cpp | 12 +-- unittests/abi_tests.cpp | 2 +- unittests/block_tests.cpp | 4 +- unittests/misc_tests.cpp | 53 +++++------ .../unapplied_transaction_queue_tests.cpp | 2 +- unittests/wasm_tests.cpp | 2 +- 17 files changed, 194 insertions(+), 205 deletions(-) diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index 616c1d3975f..e7e5746ff44 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -21,14 +21,14 @@ namespace eosio { namespace chain { } } - static fc::static_variant translate_transaction_receipt(const transaction_id_type& tid, bool) { + static fc::static_variant translate_transaction_receipt(const transaction_id_type& tid, bool) { return tid; } - static fc::static_variant translate_transaction_receipt(const packed_transaction& ptrx, bool legacy) { - return pruned_transaction(ptrx, legacy); + static fc::static_variant translate_transaction_receipt(const packed_transaction_v0& ptrx, bool legacy) { + return packed_transaction(ptrx, legacy); } - pruned_transaction_receipt::pruned_transaction_receipt(const transaction_receipt& other, bool legacy) + transaction_receipt::transaction_receipt(const transaction_receipt_v0& other, bool legacy) : transaction_receipt_header(static_cast(other)), trx(other.trx.visit([&](const auto& obj) { return translate_transaction_receipt(obj, legacy); })) {} @@ -74,11 +74,11 @@ namespace eosio { namespace chain { } - flat_multimap signed_block::validate_and_extract_extensions()const { + flat_multimap signed_block_v0::validate_and_extract_extensions()const { return validate_and_extract_block_extensions( block_extensions ); } - pruned_block::pruned_block( const signed_block& other, bool legacy ) + signed_block::signed_block( const signed_block_v0& other, bool legacy ) : signed_block_header(static_cast(other)), prune_state(legacy ? prune_state_type::complete_legacy : prune_state_type::complete), block_extensions(other.block_extensions) @@ -88,46 +88,46 @@ namespace eosio { namespace chain { } } - static std::size_t pruned_trx_receipt_packed_size(const pruned_transaction& obj, pruned_transaction::cf_compression_type segment_compression) { + static std::size_t pruned_trx_receipt_packed_size(const packed_transaction& obj, packed_transaction::cf_compression_type segment_compression) { return obj.maximum_pruned_pack_size(segment_compression); } - static std::size_t pruned_trx_receipt_packed_size(const transaction_id_type& obj, pruned_transaction::cf_compression_type) { + static std::size_t pruned_trx_receipt_packed_size(const transaction_id_type& obj, packed_transaction::cf_compression_type) { return fc::raw::pack_size(obj); } - std::size_t pruned_transaction_receipt::maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const { + std::size_t transaction_receipt::maximum_pruned_pack_size( packed_transaction::cf_compression_type segment_compression ) const { return fc::raw::pack_size(*static_cast(this)) + 1 + trx.visit([&](const auto& obj){ return pruned_trx_receipt_packed_size(obj, segment_compression); }); } - std::size_t pruned_block::maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const { + std::size_t signed_block::maximum_pruned_pack_size( packed_transaction::cf_compression_type segment_compression ) const { std::size_t result = fc::raw::pack_size(fc::unsigned_int(transactions.size())); - for(const pruned_transaction_receipt& r: transactions) { + for(const transaction_receipt& r: transactions) { result += r.maximum_pruned_pack_size( segment_compression ); } return fc::raw::pack_size(*static_cast(this)) + fc::raw::pack_size(prune_state) + result + fc::raw::pack_size(block_extensions); } - flat_multimap pruned_block::validate_and_extract_extensions()const { + flat_multimap signed_block::validate_and_extract_extensions()const { return validate_and_extract_block_extensions( block_extensions ); } - signed_block_ptr pruned_block::to_signed_block() const { + signed_block_v0_ptr signed_block::to_signed_block_v0() const { if (prune_state != prune_state_type::complete_legacy) - return signed_block_ptr{}; + return signed_block_v0_ptr{}; - auto result = std::make_shared(*static_cast(this)); + auto result = std::make_shared(*static_cast(this)); result->block_extensions = this->block_extensions; auto visitor = overloaded{ - [](const transaction_id_type &id) -> transaction_receipt::trx_type { return id; }, - [](const pruned_transaction &trx) -> transaction_receipt::trx_type { + [](const transaction_id_type &id) -> transaction_receipt_v0::trx_type { return id; }, + [](const packed_transaction &trx) -> transaction_receipt_v0::trx_type { const auto& legacy = trx.get_prunable_data().prunable_data.get(); - return packed_transaction(trx.get_packed_transaction(), legacy.signatures, legacy.packed_context_free_data, trx.get_compression()); + return packed_transaction_v0(trx.get_packed_transaction(), legacy.signatures, legacy.packed_context_free_data, trx.get_compression()); }}; - for (const pruned_transaction_receipt &r : transactions){ - result->transactions.emplace_back(transaction_receipt{r, r.trx.visit(visitor)}); + for (const transaction_receipt &r : transactions){ + result->transactions.emplace_back(transaction_receipt_v0{r, r.trx.visit(visitor)}); } return result; } diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 822de1ea8b8..086351741eb 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -663,8 +663,7 @@ namespace eosio { namespace chain { } if( bad_block.valid() ) { - ilog( "Recovered only up to block number ${num}. Last block in block log was not properly committed:\n${last_block}", - ("num", block_num)("last_block", *bad_block) ); + ilog( "Recovered only up to block number ${num}. Last block in block log was not properly committed.", ("num", block_num) ); } else if( except_ptr ) { std::string error_msg; diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index a6cf5e4b198..4513d28d660 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1240,7 +1240,7 @@ struct controller_impl { signed_transaction dtrx; fc::raw::unpack(ds,static_cast(dtrx) ); - transaction_metadata_ptr trx = transaction_metadata::create_no_recover_keys( packed_transaction( dtrx ), transaction_metadata::trx_type::scheduled ); + transaction_metadata_ptr trx = transaction_metadata::create_no_recover_keys( packed_transaction( dtrx, true ), transaction_metadata::trx_type::scheduled ); trx->accepted = true; transaction_trace_ptr trace; @@ -1659,7 +1659,7 @@ struct controller_impl { try { transaction_metadata_ptr onbtrx = - transaction_metadata::create_no_recover_keys( packed_transaction( get_on_block_transaction() ), transaction_metadata::trx_type::implicit ); + transaction_metadata::create_no_recover_keys( packed_transaction( get_on_block_transaction(), true ), transaction_metadata::trx_type::implicit ); auto reset_in_trx_requiring_checks = fc::make_scoped_exit([old_value=in_trx_requiring_checks,this](){ in_trx_requiring_checks = old_value; }); @@ -1928,11 +1928,11 @@ struct controller_impl { EOS_ASSERT( trx_receipts.size() > 0, block_validate_exception, "expected a receipt", - ("block", *b)("expected_receipt", receipt) + ("block_num", b->block_num())("block_id", producer_block_id)("expected_receipt", receipt) ); EOS_ASSERT( trx_receipts.size() == num_pending_receipts + 1, block_validate_exception, "expected receipt was not added", - ("block", *b)("expected_receipt", receipt) + ("block_num", b->block_num())("block_id", producer_block_id)("expected_receipt", receipt) ); const transaction_receipt_header& r = trx_receipts.back(); EOS_ASSERT( r == static_cast(receipt), diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index c96a88bc853..569aad2015d 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -259,7 +259,7 @@ namespace impl { std::is_same::value || std::is_same::value || std::is_same::value || - std::is_same::value || + std::is_same::value || std::is_same::value; } @@ -442,7 +442,7 @@ namespace impl { } /** - * overload of to_variant_object for packed_transaction + * overload of to_variant_object for packed_transaction, providing original packed_transaction_v0 variant layout * * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of ptrx.transaction * @tparam Resolver @@ -453,18 +453,19 @@ namespace impl { template static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) { - static_assert(fc::reflector::total_member_count == 4); + static_assert(fc::reflector::total_member_count == 3); auto h = ctx.enter_scope(); mutable_variant_object mvo; auto trx = ptrx.get_transaction(); +/* TODO support old packed_transaction_v0 format? mvo("id", trx.id()); - mvo("signatures", ptrx.get_signatures()); + mvo("signatures", ptrx.get_signatures() != nullptr ? *ptrx.get_signatures() : vector() ); mvo("compression", ptrx.get_compression()); mvo("packed_context_free_data", ptrx.get_packed_context_free_data()); mvo("context_free_data", ptrx.get_context_free_data()); mvo("packed_trx", ptrx.get_packed_transaction()); add(mvo, "transaction", trx, resolver, ctx); - +*/ out(name, std::move(mvo)); } @@ -505,9 +506,9 @@ namespace impl { * block.header_extensions and block.block_extensions */ template - static void add( mutable_variant_object &out, const char* name, const signed_block& block, Resolver resolver, abi_traverse_context& ctx ) + static void add( mutable_variant_object &out, const char* name, const signed_block_v0& block, Resolver resolver, abi_traverse_context& ctx ) { - static_assert(fc::reflector::total_member_count == 12); + static_assert(fc::reflector::total_member_count == 12); auto h = ctx.enter_scope(); mutable_variant_object mvo; mvo("timestamp", block.timestamp); @@ -714,6 +715,7 @@ namespace impl { { auto h = ctx.enter_scope(); const variant_object& vo = v.get_object(); +/* TODO figure out what to do with serialization EOS_ASSERT(vo.contains("signatures"), packed_transaction_type_exception, "Missing signatures"); EOS_ASSERT(vo.contains("compression"), packed_transaction_type_exception, "Missing compression"); std::vector signatures; @@ -753,6 +755,7 @@ namespace impl { ptrx = packed_transaction( std::move( trx ), compression ); } } +*/ } }; diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index df9e2c2408c..167ee573a95 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -30,12 +30,12 @@ namespace eosio { namespace chain { fc::unsigned_int net_usage_words; ///< total billed NET usage, so we can reconstruct resource state when skipping context free data... hard failures... }; - struct transaction_receipt : public transaction_receipt_header { - using trx_type = fc::static_variant; - transaction_receipt() : transaction_receipt_header() {} - transaction_receipt(const transaction_receipt_header& header, trx_type&& t): transaction_receipt_header(header), trx(std::move(t)){} - explicit transaction_receipt( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){} - explicit transaction_receipt( const packed_transaction& ptrx ):transaction_receipt_header(executed),trx(ptrx){} + struct transaction_receipt_v0 : public transaction_receipt_header { + using trx_type = fc::static_variant; + transaction_receipt_v0() : transaction_receipt_header() {} + transaction_receipt_v0(const transaction_receipt_header& header, trx_type&& t): transaction_receipt_header(header), trx(std::move(t)){} + explicit transaction_receipt_v0( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){} + explicit transaction_receipt_v0( const packed_transaction_v0& ptrx ):transaction_receipt_header(executed),trx(ptrx){} trx_type trx; @@ -47,7 +47,7 @@ namespace eosio { namespace chain { if( trx.contains() ) fc::raw::pack( enc, trx.get() ); else - fc::raw::pack( enc, trx.get().packed_digest() ); + fc::raw::pack( enc, trx.get().packed_digest() ); return enc.result(); } }; @@ -87,33 +87,33 @@ namespace eosio { namespace chain { /** */ - struct signed_block : public signed_block_header{ + struct signed_block_v0 : public signed_block_header{ private: - signed_block( const signed_block& ) = default; + signed_block_v0( const signed_block_v0& ) = default; public: - signed_block() = default; - explicit signed_block( const signed_block_header& h ):signed_block_header(h){} - signed_block( signed_block&& ) = default; - signed_block& operator=(const signed_block&) = delete; - signed_block clone() const { return *this; } + signed_block_v0() = default; + explicit signed_block_v0( const signed_block_header& h ):signed_block_header(h){} + signed_block_v0( signed_block_v0&& ) = default; + signed_block_v0& operator=(const signed_block_v0&) = delete; + signed_block_v0 clone() const { return *this; } - deque transactions; /// new or generated transactions + deque transactions; /// new or generated transactions extensions_type block_extensions; flat_multimap validate_and_extract_extensions()const; }; - using signed_block_ptr = std::shared_ptr; + using signed_block_v0_ptr = std::shared_ptr; - struct pruned_transaction_receipt : public transaction_receipt_header { + struct transaction_receipt : public transaction_receipt_header { - pruned_transaction_receipt():transaction_receipt_header(){} - pruned_transaction_receipt(const transaction_receipt&, bool legacy); - explicit pruned_transaction_receipt( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){} - explicit pruned_transaction_receipt( const pruned_transaction& ptrx ):transaction_receipt_header(executed),trx(ptrx){} + transaction_receipt():transaction_receipt_header(){} + transaction_receipt(const transaction_receipt_v0&, bool legacy); + explicit transaction_receipt( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){} + explicit transaction_receipt( const packed_transaction& ptrx ):transaction_receipt_header(executed),trx(ptrx){} - fc::static_variant trx; + fc::static_variant trx; - std::size_t maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const; + std::size_t maximum_pruned_pack_size( packed_transaction::cf_compression_type segment_compression ) const; digest_type digest()const { digest_type::encoder enc; @@ -123,48 +123,48 @@ namespace eosio { namespace chain { if( trx.contains() ) fc::raw::pack( enc, trx.get() ); else - fc::raw::pack( enc, trx.get().packed_digest() ); + fc::raw::pack( enc, trx.get().packed_digest() ); return enc.result(); } }; - struct pruned_block : public signed_block_header{ + struct signed_block : public signed_block_header{ private: - pruned_block( const pruned_block& ) = default; + signed_block( const signed_block& ) = default; public: enum class prune_state_type : uint8_t { incomplete, complete, complete_legacy }; - pruned_block() = default; - explicit pruned_block( const signed_block_header& h ):signed_block_header(h){} - pruned_block( const signed_block&, bool legacy ); - pruned_block( pruned_block&& ) = default; - pruned_block& operator=(const pruned_block&) = delete; - pruned_block clone() const { return *this; } - signed_block_ptr to_signed_block() const; + signed_block() = default; + explicit signed_block( const signed_block_header& h ):signed_block_header(h){} + signed_block( const signed_block_v0&, bool legacy ); + signed_block( signed_block&& ) = default; + signed_block& operator=(const signed_block&) = delete; + signed_block clone() const { return *this; } + signed_block_v0_ptr to_signed_block_v0() const; fc::enum_type prune_state{prune_state_type::complete_legacy}; - deque transactions; /// new or generated transactions + deque transactions; /// new or generated transactions extensions_type block_extensions; - std::size_t maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const; + std::size_t maximum_pruned_pack_size( packed_transaction::cf_compression_type segment_compression ) const; // Returns the maximum_pruned_padded_size. It is the caller's responsibility to // reserve enough space after the end if in-place pruning is desired. template - std::size_t pack(Stream& stream, pruned_transaction::cf_compression_type segment_compression) { + std::size_t pack(Stream& stream, packed_transaction::cf_compression_type segment_compression) { std::size_t padded_size = maximum_pruned_pack_size( segment_compression ); // TODO: This only handles legacy transactions. fc::raw::pack(stream, *this); return padded_size; } template - void unpack(Stream& stream, pruned_transaction::cf_compression_type segment_compression) { + void unpack(Stream& stream, packed_transaction::cf_compression_type segment_compression) { fc::raw::unpack(stream, *this); } flat_multimap validate_and_extract_extensions()const; }; - using pruned_block_ptr = std::shared_ptr; + using signed_block_ptr = std::shared_ptr; struct producer_confirmation { block_id_type block_id; @@ -175,13 +175,15 @@ namespace eosio { namespace chain { } } /// eosio::chain -FC_REFLECT_ENUM( eosio::chain::transaction_receipt::status_enum, +FC_REFLECT_ENUM( eosio::chain::transaction_receipt_header::status_enum, (executed)(soft_fail)(hard_fail)(delayed)(expired) ) +FC_REFLECT_ENUM( eosio::chain::signed_block::prune_state_type, + (incomplete)(complete)(complete_legacy) ) FC_REFLECT(eosio::chain::transaction_receipt_header, (status)(cpu_usage_us)(net_usage_words) ) -FC_REFLECT_DERIVED(eosio::chain::transaction_receipt, (eosio::chain::transaction_receipt_header), (trx) ) +FC_REFLECT_DERIVED(eosio::chain::transaction_receipt_v0, (eosio::chain::transaction_receipt_header), (trx) ) FC_REFLECT(eosio::chain::additional_block_signatures_extension, (signatures)); -FC_REFLECT_DERIVED(eosio::chain::signed_block, (eosio::chain::signed_block_header), (transactions)(block_extensions) ) +FC_REFLECT_DERIVED(eosio::chain::signed_block_v0, (eosio::chain::signed_block_header), (transactions)(block_extensions) ) -FC_REFLECT_DERIVED(eosio::chain::pruned_transaction_receipt, (eosio::chain::transaction_receipt_header), (trx) ) -FC_REFLECT_DERIVED(eosio::chain::pruned_block, (eosio::chain::signed_block_header), (prune_state)(transactions)(block_extensions) ) +FC_REFLECT_DERIVED(eosio::chain::transaction_receipt, (eosio::chain::transaction_receipt_header), (trx) ) +FC_REFLECT_DERIVED(eosio::chain::signed_block, (eosio::chain::signed_block_header), (prune_state)(transactions)(block_extensions) ) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 14cd9d1c544..8d94ccd4af8 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -133,38 +133,38 @@ namespace eosio { namespace chain { bool allow_duplicate_keys = false )const; }; - struct packed_transaction : fc::reflect_init { + struct packed_transaction_v0 : fc::reflect_init { enum class compression_type { none = 0, zlib = 1, }; - packed_transaction() = default; - packed_transaction(packed_transaction&&) = default; - explicit packed_transaction(const packed_transaction&) = default; - packed_transaction& operator=(const packed_transaction&) = delete; - packed_transaction& operator=(packed_transaction&&) = default; + packed_transaction_v0() = default; + packed_transaction_v0(packed_transaction_v0&&) = default; + explicit packed_transaction_v0(const packed_transaction_v0&) = default; + packed_transaction_v0& operator=(const packed_transaction_v0&) = delete; + packed_transaction_v0& operator=(packed_transaction_v0&&) = default; - explicit packed_transaction(const signed_transaction& t, compression_type _compression = compression_type::none) + explicit packed_transaction_v0(const signed_transaction& t, compression_type _compression = compression_type::none) :signatures(t.signatures), compression(_compression), unpacked_trx(t), trx_id(unpacked_trx.id()) { local_pack_transaction(); local_pack_context_free_data(); } - explicit packed_transaction(signed_transaction&& t, compression_type _compression = compression_type::none) + explicit packed_transaction_v0(signed_transaction&& t, compression_type _compression = compression_type::none) :signatures(t.signatures), compression(_compression), unpacked_trx(std::move(t)), trx_id(unpacked_trx.id()) { local_pack_transaction(); local_pack_context_free_data(); } - packed_transaction(const bytes& packed_txn, const vector& sigs, const bytes& packed_cfd, compression_type _compression); + packed_transaction_v0(const bytes& packed_txn, const vector& sigs, const bytes& packed_cfd, compression_type _compression); // used by abi_serializer - packed_transaction( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); - packed_transaction( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ); - packed_transaction( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); + packed_transaction_v0( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); + packed_transaction_v0( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ); + packed_transaction_v0( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); uint32_t get_unprunable_size()const; uint32_t get_prunable_size()const; @@ -172,7 +172,6 @@ namespace eosio { namespace chain { digest_type packed_digest()const; const transaction_id_type& id()const { return trx_id; } - bytes get_raw_transaction()const; time_point_sec expiration()const { return unpacked_trx.expiration; } const vector& get_context_free_data()const { return unpacked_trx.context_free_data; } @@ -189,9 +188,9 @@ namespace eosio { namespace chain { void local_pack_transaction(); void local_pack_context_free_data(); - friend struct fc::reflector; - friend struct fc::reflector_init_visitor; - friend struct fc::has_reflector_init; + friend struct fc::reflector; + friend struct fc::reflector_init_visitor; + friend struct fc::has_reflector_init; void reflector_init(); private: vector signatures; @@ -253,19 +252,19 @@ namespace eosio { namespace chain { prunable_data_type prunable_data; }; - struct pruned_transaction : fc::reflect_init { - using compression_type = packed_transaction::compression_type; + struct packed_transaction : fc::reflect_init { + using compression_type = packed_transaction_v0::compression_type; using cf_compression_type = prunable_transaction_data::compression_type; - pruned_transaction() = default; - pruned_transaction(pruned_transaction&&) = default; - explicit pruned_transaction(const pruned_transaction&) = default; - pruned_transaction& operator=(const pruned_transaction&) = delete; - pruned_transaction& operator=(pruned_transaction&&) = default; + packed_transaction() = default; + packed_transaction(packed_transaction&&) = default; + explicit packed_transaction(const packed_transaction&) = default; + packed_transaction& operator=(const packed_transaction&) = delete; + packed_transaction& operator=(packed_transaction&&) = default; - pruned_transaction(const packed_transaction& other, bool legacy) : pruned_transaction(other.get_signed_transaction(), legacy, other.get_compression()) {} - explicit pruned_transaction(const signed_transaction& t, bool legacy, compression_type _compression = compression_type::none); - explicit pruned_transaction(signed_transaction&& t, bool legacy, compression_type _compression = compression_type::none); + packed_transaction(const packed_transaction_v0& other, bool legacy) : packed_transaction(other.get_signed_transaction(), legacy, other.get_compression()) {} + explicit packed_transaction(const signed_transaction& t, bool legacy, compression_type _compression = compression_type::none); + explicit packed_transaction(signed_transaction&& t, bool legacy, compression_type _compression = compression_type::none); #if 0 // used by abi_serializer @@ -300,9 +299,9 @@ namespace eosio { namespace chain { private: - friend struct fc::reflector; - friend struct fc::reflector_init_visitor; - friend struct fc::has_reflector_init; + friend struct fc::reflector; + friend struct fc::reflector_init_visitor; + friend struct fc::has_reflector_init; void reflector_init(); private: fc::enum_type compression; @@ -315,6 +314,7 @@ namespace eosio { namespace chain { transaction_id_type trx_id; }; + using packed_transaction_v0_ptr = std::shared_ptr; using packed_transaction_ptr = std::shared_ptr; uint128_t transaction_id_to_sender_id( const transaction_id_type& tid ); @@ -326,10 +326,10 @@ FC_REFLECT( eosio::chain::transaction_header, (expiration)(ref_block_num)(ref_bl (max_net_usage_words)(max_cpu_usage_ms)(delay_sec) ) FC_REFLECT_DERIVED( eosio::chain::transaction, (eosio::chain::transaction_header), (context_free_actions)(actions)(transaction_extensions) ) FC_REFLECT_DERIVED( eosio::chain::signed_transaction, (eosio::chain::transaction), (signatures)(context_free_data) ) -FC_REFLECT_ENUM( eosio::chain::packed_transaction::compression_type, (none)(zlib)) +FC_REFLECT_ENUM( eosio::chain::packed_transaction_v0::compression_type, (none)(zlib)) // @ignore unpacked_trx -FC_REFLECT( eosio::chain::packed_transaction, (signatures)(compression)(packed_context_free_data)(packed_trx) ) -FC_REFLECT( eosio::chain::pruned_transaction, (compression)(prunable_data)(packed_trx) ) +FC_REFLECT( eosio::chain::packed_transaction_v0, (signatures)(compression)(packed_context_free_data)(packed_trx) ) +FC_REFLECT( eosio::chain::packed_transaction, (compression)(prunable_data)(packed_trx) ) FC_REFLECT( eosio::chain::prunable_transaction_data, (prunable_data)); FC_REFLECT( eosio::chain::prunable_transaction_data::none, (prunable_digest)) FC_REFLECT( eosio::chain::prunable_transaction_data::signatures_only, (signatures)(context_free_mroot)) diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 7320f25c204..9fd9673a3e8 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -133,21 +133,21 @@ signed_transaction::get_signature_keys( const chain_id_type& chain_id, fc::time_ return transaction::get_signature_keys(signatures, chain_id, deadline, context_free_data, recovered_pub_keys, allow_duplicate_keys); } -uint32_t packed_transaction::get_unprunable_size()const { +uint32_t packed_transaction_v0::get_unprunable_size()const { uint64_t size = config::fixed_net_overhead_of_packed_trx; size += packed_trx.size(); EOS_ASSERT( size <= std::numeric_limits::max(), tx_too_big, "packed_transaction is too big" ); return static_cast(size); } -uint32_t packed_transaction::get_prunable_size()const { +uint32_t packed_transaction_v0::get_prunable_size()const { uint64_t size = fc::raw::pack_size(signatures); size += packed_context_free_data.size(); EOS_ASSERT( size <= std::numeric_limits::max(), tx_too_big, "packed_transaction is too big" ); return static_cast(size); } -digest_type packed_transaction::packed_digest()const { +digest_type packed_transaction_v0::packed_digest()const { digest_type::encoder prunable; fc::raw::pack( prunable, signatures ); fc::raw::pack( prunable, packed_context_free_data ); @@ -256,21 +256,7 @@ static bytes zlib_compress_transaction(const transaction& t) { return out; } -bytes packed_transaction::get_raw_transaction() const -{ - try { - switch(compression) { - case compression_type::none: - return packed_trx; - case compression_type::zlib: - return zlib_decompress(packed_trx); - default: - EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); - } - } FC_CAPTURE_AND_RETHROW((compression)(packed_trx)) -} - -packed_transaction::packed_transaction(const bytes& packed_txn, const vector& sigs, const bytes& packed_cfd, compression_type _compression ) +packed_transaction_v0::packed_transaction_v0(const bytes& packed_txn, const vector& sigs, const bytes& packed_cfd, compression_type _compression ) :signatures(sigs) ,compression(_compression) ,packed_context_free_data(packed_cfd) @@ -282,7 +268,7 @@ packed_transaction::packed_transaction(const bytes& packed_txn, const vector&& sigs, bytes&& packed_cfd, compression_type _compression ) +packed_transaction_v0::packed_transaction_v0( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ) :signatures(std::move(sigs)) ,compression(_compression) ,packed_context_free_data(std::move(packed_cfd)) @@ -294,7 +280,7 @@ packed_transaction::packed_transaction( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ) +packed_transaction_v0::packed_transaction_v0( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ) :signatures(std::move(sigs)) ,compression(_compression) ,packed_trx(std::move(packed_txn)) @@ -305,7 +291,7 @@ packed_transaction::packed_transaction( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ) +packed_transaction_v0::packed_transaction_v0( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ) :signatures(std::move(sigs)) ,compression(_compression) ,packed_context_free_data(std::move(packed_cfd)) @@ -318,7 +304,7 @@ packed_transaction::packed_transaction( transaction&& t, vector& } } -void packed_transaction::reflector_init() +void packed_transaction_v0::reflector_init() { // called after construction, but always on the same thread and before packed_transaction passed to any other threads static_assert(fc::raw::has_feature_reflector_init_on_unpacked_reflected_types, @@ -328,12 +314,12 @@ void packed_transaction::reflector_init() local_unpack_context_free_data(); } -static transaction unpack_transaction(const bytes& packed_trx, packed_transaction::compression_type compression) { +static transaction unpack_transaction(const bytes& packed_trx, packed_transaction_v0::compression_type compression) { try { switch( compression ) { - case packed_transaction::compression_type::none: + case packed_transaction_v0::compression_type::none: return unpack_transaction( packed_trx ); - case packed_transaction::compression_type::zlib: + case packed_transaction_v0::compression_type::zlib: return zlib_decompress_transaction( packed_trx ); default: EOS_THROW( unknown_transaction_compression, "Unknown transaction compression algorithm" ); @@ -341,19 +327,19 @@ static transaction unpack_transaction(const bytes& packed_trx, packed_transactio } FC_CAPTURE_AND_RETHROW( (compression) ) } -void packed_transaction::local_unpack_transaction(vector&& context_free_data) +void packed_transaction_v0::local_unpack_transaction(vector&& context_free_data) { unpacked_trx = signed_transaction( unpack_transaction( packed_trx, compression ), signatures, std::move(context_free_data) ); trx_id = unpacked_trx.id(); } -static vector unpack_context_free_data(const bytes& packed_context_free_data, packed_transaction::compression_type compression) +static vector unpack_context_free_data(const bytes& packed_context_free_data, packed_transaction_v0::compression_type compression) { try { switch( compression ) { - case packed_transaction::compression_type::none: + case packed_transaction_v0::compression_type::none: return unpack_context_free_data( packed_context_free_data ); - case packed_transaction::compression_type::zlib: + case packed_transaction_v0::compression_type::zlib: return zlib_decompress_context_free_data( packed_context_free_data ); default: EOS_THROW( unknown_transaction_compression, "Unknown transaction compression algorithm" ); @@ -361,18 +347,18 @@ static vector unpack_context_free_data(const bytes& packed_context_free_d } FC_CAPTURE_AND_RETHROW( (compression) ) } -void packed_transaction::local_unpack_context_free_data() +void packed_transaction_v0::local_unpack_context_free_data() { EOS_ASSERT(unpacked_trx.context_free_data.empty(), tx_decompression_error, "packed_transaction.context_free_data not empty"); unpacked_trx.context_free_data = unpack_context_free_data( packed_context_free_data, compression ); } -static bytes pack_transaction(const transaction& trx, packed_transaction::compression_type compression) { +static bytes pack_transaction(const transaction& trx, packed_transaction_v0::compression_type compression) { try { switch(compression) { - case packed_transaction::compression_type::none: + case packed_transaction_v0::compression_type::none: return pack_transaction(trx); - case packed_transaction::compression_type::zlib: + case packed_transaction_v0::compression_type::zlib: return zlib_compress_transaction(trx); default: EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); @@ -380,17 +366,17 @@ static bytes pack_transaction(const transaction& trx, packed_transaction::compre } FC_CAPTURE_AND_RETHROW((compression)) } -void packed_transaction::local_pack_transaction() +void packed_transaction_v0::local_pack_transaction() { packed_trx = pack_transaction(unpacked_trx, compression); } -static bytes pack_context_free_data( const vector& cfd, packed_transaction::compression_type compression ) { +static bytes pack_context_free_data( const vector& cfd, packed_transaction_v0::compression_type compression ) { try { switch(compression) { - case packed_transaction::compression_type::none: + case packed_transaction_v0::compression_type::none: return pack_context_free_data(cfd); - case packed_transaction::compression_type::zlib: + case packed_transaction_v0::compression_type::zlib: return zlib_compress_context_free_data(cfd); default: EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); @@ -398,7 +384,7 @@ static bytes pack_context_free_data( const vector& cfd, packed_transactio } FC_CAPTURE_AND_RETHROW((compression)) } -void packed_transaction::local_pack_context_free_data() +void packed_transaction_v0::local_pack_context_free_data() { packed_context_free_data = pack_context_free_data(unpacked_trx.context_free_data, compression); } @@ -471,7 +457,7 @@ std::size_t prunable_transaction_data::maximum_pruned_pack_size(prunable_transac return 1 + prunable_data.visit([&](const auto& t){ return padded_pack_size(t, compression); }); } -static prunable_transaction_data make_prunable_transaction_data( bool legacy, const signed_transaction& t, pruned_transaction::compression_type _compression ) { +static prunable_transaction_data make_prunable_transaction_data( bool legacy, const signed_transaction& t, packed_transaction::compression_type _compression ) { if(legacy) { return { prunable_transaction_data::full_legacy{ t.signatures, pack_context_free_data( t.context_free_data, _compression ) } }; } else { @@ -479,7 +465,7 @@ static prunable_transaction_data make_prunable_transaction_data( bool legacy, co } } -pruned_transaction::pruned_transaction(const signed_transaction& t, bool legacy, compression_type _compression) +packed_transaction::packed_transaction(const signed_transaction& t, bool legacy, compression_type _compression) : compression(_compression), prunable_data(make_prunable_transaction_data(legacy, t, _compression)), packed_trx(pack_transaction(t, compression)), @@ -487,7 +473,7 @@ pruned_transaction::pruned_transaction(const signed_transaction& t, bool legacy, trx_id(unpacked_trx.id()) {} -pruned_transaction::pruned_transaction(signed_transaction&& t, bool legacy, compression_type _compression) +packed_transaction::packed_transaction(signed_transaction&& t, bool legacy, compression_type _compression) : compression(_compression), prunable_data(make_prunable_transaction_data(legacy, t, _compression)), packed_trx(pack_transaction(t, compression)), @@ -495,7 +481,7 @@ pruned_transaction::pruned_transaction(signed_transaction&& t, bool legacy, comp trx_id(unpacked_trx.id()) {} -uint32_t pruned_transaction::get_unprunable_size()const { +uint32_t packed_transaction::get_unprunable_size()const { uint64_t size = config::fixed_net_overhead_of_packed_trx; size += packed_trx.size(); EOS_ASSERT( size <= std::numeric_limits::max(), tx_too_big, "packed_transaction is too big" ); @@ -518,11 +504,11 @@ static uint32_t get_prunable_size_impl(const T&) { EOS_THROW(tx_prune_exception, "unknown size: prunable data has been pruned."); } -uint32_t pruned_transaction::get_prunable_size()const { +uint32_t packed_transaction::get_prunable_size()const { return prunable_data.prunable_data.visit([](const auto& obj) { return get_prunable_size_impl(obj); }); } -digest_type pruned_transaction::packed_digest()const { +digest_type packed_transaction::packed_digest()const { digest_type::encoder enc; fc::raw::pack( enc, compression ); fc::raw::pack( enc, packed_trx ); @@ -533,11 +519,11 @@ digest_type pruned_transaction::packed_digest()const { template static auto maybe_get_signatures(const T& obj) -> decltype(&obj.signatures) { return &obj.signatures; } static auto maybe_get_signatures(const prunable_transaction_data::none&) -> const std::vector* { return nullptr; } -const vector* pruned_transaction::get_signatures()const { +const vector* packed_transaction::get_signatures()const { return prunable_data.prunable_data.visit([](const auto& obj) { return maybe_get_signatures(obj); }); } -const vector* pruned_transaction::get_context_free_data()const { +const vector* packed_transaction::get_context_free_data()const { if(prunable_data.prunable_data.contains() || prunable_data.prunable_data.contains()) { return &unpacked_trx.context_free_data; @@ -555,7 +541,7 @@ const bytes* maybe_get_context_free_data(const prunable_transaction_data::partia const bytes* maybe_get_context_free_data(const prunable_transaction_data::full_legacy&, std::size_t) { return nullptr; } const bytes* maybe_get_context_free_data(const prunable_transaction_data::full&, std::size_t) { return nullptr; } -const bytes* pruned_transaction::get_context_free_data(std::size_t segment_ordinal) { +const bytes* packed_transaction::get_context_free_data(std::size_t segment_ordinal) { if (segment_ordinal < unpacked_trx.context_free_data.size()) { return &unpacked_trx.context_free_data[segment_ordinal]; } else { @@ -563,28 +549,28 @@ const bytes* pruned_transaction::get_context_free_data(std::size_t segment_ordin } } -void pruned_transaction::prune_all() { +void packed_transaction::prune_all() { prunable_data = prunable_data.prune_all(); unpacked_trx.context_free_data.clear(); unpacked_trx.signatures.clear(); } -std::size_t pruned_transaction::maximum_pruned_pack_size(cf_compression_type segment_compression) const { +std::size_t packed_transaction::maximum_pruned_pack_size(cf_compression_type segment_compression) const { return fc::raw::pack_size(compression) + fc::raw::pack_size(packed_trx) + prunable_data.maximum_pruned_pack_size(segment_compression); } -static std::vector maybe_unpack_context_free_data(const prunable_transaction_data::full& obj, packed_transaction::compression_type) { +static std::vector maybe_unpack_context_free_data(const prunable_transaction_data::full& obj, packed_transaction_v0::compression_type) { return obj.context_free_segments; } -static std::vector maybe_unpack_context_free_data(const prunable_transaction_data::full_legacy& obj, packed_transaction::compression_type compression) { +static std::vector maybe_unpack_context_free_data(const prunable_transaction_data::full_legacy& obj, packed_transaction_v0::compression_type compression) { return unpack_context_free_data(obj.packed_context_free_data, compression); } template -static std::vector maybe_unpack_context_free_data(const T&, packed_transaction::compression_type) { +static std::vector maybe_unpack_context_free_data(const T&, packed_transaction_v0::compression_type) { return {}; } -void pruned_transaction::reflector_init() +void packed_transaction::reflector_init() { // called after construction, but always on the same thread and before packed_transaction passed to any other threads static_assert(fc::raw::has_feature_reflector_init_on_unpacked_reflected_types, diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index fe2d5fcbf63..0819a9d2640 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -569,7 +569,7 @@ namespace eosio { namespace testing { auto time_limit = deadline == fc::time_point::maximum() ? fc::microseconds::maximum() : fc::microseconds( deadline - fc::time_point::now() ); - auto ptrx = std::make_shared( trx, c ); + auto ptrx = std::make_shared( trx, true, c ); auto fut = transaction_metadata::start_recover_keys( ptrx, control->get_thread_pool(), control->get_chain_id(), time_limit ); auto r = control->push_transaction( fut.get(), deadline, billed_cpu_time_us, billed_cpu_time_us > 0 ); if (no_throw) return r; diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 48862ceb8fc..f8a65c77e07 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2830,9 +2830,11 @@ namespace eosio { size_t calc_trx_size( const packed_transaction_ptr& trx ) { // transaction is stored packed and unpacked, double packed_size and size of signed as an approximation of use + const auto cfd_size = trx->get_context_free_data() != nullptr ? trx->get_context_free_data()->size() : 0; + auto sig_size = trx->get_signatures() != nullptr ? trx->get_signatures()->size() : 0; + sig_size *= sizeof(signature_type); return (trx->get_packed_transaction().size() * 2 + sizeof(trx->get_signed_transaction())) * 2 + - trx->get_packed_context_free_data().size() * 4 + - trx->get_signatures().size() * sizeof(signature_type); + cfd_size + sig_size; } void connection::handle_message( packed_transaction_ptr trx ) { diff --git a/plugins/trace_api_plugin/test/test_extraction.cpp b/plugins/trace_api_plugin/test/test_extraction.cpp index b05426610dd..bff6fa75092 100644 --- a/plugins/trace_api_plugin/test/test_extraction.cpp +++ b/plugins/trace_api_plugin/test/test_extraction.cpp @@ -74,7 +74,7 @@ namespace { auto make_packed_trx( std::vector actions ) { chain::signed_transaction trx; trx.actions = std::move( actions ); - return packed_transaction( trx ); + return packed_transaction( trx, true ); } chain::action_trace make_action_trace( uint64_t global_sequence, chain::action act, chain::name receiver ) { diff --git a/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp b/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp index 8b7aaebfb7d..ca98331a44a 100644 --- a/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp +++ b/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp @@ -107,7 +107,7 @@ struct txn_test_gen_plugin_impl { chain_plugin& cp = app().get_plugin(); for (size_t i = 0; i < trxs->size(); ++i) { - cp.accept_transaction( std::make_shared(trxs->at(i)), [=](const fc::static_variant& result){ + cp.accept_transaction( std::make_shared(trxs->at(i), true), [=](const fc::static_variant& result){ if (result.contains()) { next(result.get()); } else { diff --git a/programs/cleos/main.cpp b/programs/cleos/main.cpp index 579ac62985a..10af0b464e1 100644 --- a/programs/cleos/main.cpp +++ b/programs/cleos/main.cpp @@ -409,10 +409,10 @@ fc::variant push_transaction( signed_transaction& trx, const std::vector(); - std::cout << fc::json::to_pretty_string( fc::variant( packed_transaction( trx, packed_transaction::compression_type::none ))) << std::endl; + std::cout << fc::json::to_pretty_string( fc::variant( packed_transaction_v0( trx, packed_transaction::compression_type::none ))) << std::endl; } EOS_RETHROW_EXCEPTIONS( transaction_type_exception, "Fail to convert transaction, --pack-action-data likely needed" ) } }); @@ -3478,7 +3478,7 @@ int main( int argc, char** argv ) { } if(push_trx) { - auto trx_result = call(push_txn_func, packed_transaction(trx, packed_transaction::compression_type::none)); + auto trx_result = call(push_txn_func, packed_transaction_v0(trx, packed_transaction::compression_type::none)); std::cout << fc::json::to_pretty_string(trx_result) << std::endl; } else { std::cout << fc::json::to_pretty_string(trx) << std::endl; diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index 3b3d3cc35f3..1895811af3a 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -1495,7 +1495,7 @@ BOOST_AUTO_TEST_CASE(packed_transaction) txn.max_cpu_usage_ms = 43; // pack the transaction to verify that the var unpacking logic is correct - auto packed_txn = chain::packed_transaction(txn); + auto packed_txn = chain::packed_transaction(txn, true); const char* packed_transaction_abi = R"=====( { diff --git a/unittests/block_tests.cpp b/unittests/block_tests.cpp index 8c6927bdd09..74ad40dcb41 100644 --- a/unittests/block_tests.cpp +++ b/unittests/block_tests.cpp @@ -27,7 +27,7 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_test) signed_tx.signatures.clear(); signed_tx.sign(main.get_private_key(config::system_account_name, "active"), main.control->get_chain_id()); // Replace the valid transaction with the invalid transaction - auto invalid_packed_tx = packed_transaction(signed_tx); + auto invalid_packed_tx = packed_transaction(signed_tx, true); copy_b->transactions.back().trx = invalid_packed_tx; // Re-calculate the transaction merkle @@ -67,7 +67,7 @@ std::pair corrupt_trx_in_block(validating_te signed_tx.sign(main.get_private_key(act_name, "active"), main.control->get_chain_id()); // Replace the valid transaction with the invalid transaction - auto invalid_packed_tx = packed_transaction(signed_tx, packed_trx.get_compression()); + auto invalid_packed_tx = packed_transaction(signed_tx, true, packed_trx.get_compression()); copy_b->transactions.back().trx = invalid_packed_tx; // Re-calculate the transaction merkle diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index 90460ead316..5df2cf176ba 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -739,9 +739,9 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { BOOST_CHECK_EQUAL(1u, trx.signatures.size()); trx.validate(); - packed_transaction pkt(trx, packed_transaction::compression_type::none); + packed_transaction pkt(trx, true, packed_transaction::compression_type::none); - packed_transaction pkt2(trx, packed_transaction::compression_type::zlib); + packed_transaction pkt2(trx, true, packed_transaction::compression_type::zlib); BOOST_CHECK_EQUAL(true, trx.expiration == pkt.expiration()); BOOST_CHECK_EQUAL(true, trx.expiration == pkt2.expiration()); @@ -749,11 +749,6 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { BOOST_CHECK_EQUAL(trx.id(), pkt.id()); BOOST_CHECK_EQUAL(trx.id(), pkt2.id()); - bytes raw = pkt.get_raw_transaction(); - bytes raw2 = pkt2.get_raw_transaction(); - BOOST_CHECK_EQUAL(raw.size(), raw2.size()); - BOOST_CHECK_EQUAL(true, std::equal(raw.begin(), raw.end(), raw2.begin())); - BOOST_CHECK_EQUAL(pkt.get_signed_transaction().id(), pkt2.get_signed_transaction().id()); BOOST_CHECK_EQUAL(pkt.get_signed_transaction().id(), pkt2.id()); @@ -792,8 +787,8 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { packed_transaction pkt5; fc::from_variant(pkt_v, pkt5); - bytes raw3 = pkt3.get_raw_transaction(); - bytes raw4 = pkt4.get_raw_transaction(); + bytes raw3 = pkt3.get_packed_transaction(); + bytes raw4 = pkt4.get_packed_transaction(); BOOST_CHECK_EQUAL(raw.size(), raw3.size()); BOOST_CHECK_EQUAL(raw3.size(), raw4.size()); BOOST_CHECK_EQUAL(true, std::equal(raw.begin(), raw.end(), raw3.begin())); @@ -878,11 +873,11 @@ BOOST_AUTO_TEST_CASE(transaction_metadata_test) { try { trx.sign( private_key, test.control->get_chain_id() ); BOOST_CHECK_EQUAL(1u, trx.signatures.size()); - packed_transaction pkt(trx, packed_transaction::compression_type::none); - packed_transaction pkt2(trx, packed_transaction::compression_type::zlib); + packed_transaction pkt(trx, true, packed_transaction::compression_type::none); + packed_transaction pkt2(trx, true, packed_transaction::compression_type::zlib); - packed_transaction_ptr ptrx = std::make_shared( trx, packed_transaction::compression_type::none); - packed_transaction_ptr ptrx2 = std::make_shared( trx, packed_transaction::compression_type::zlib); + packed_transaction_ptr ptrx = std::make_shared( trx, true, packed_transaction::compression_type::none); + packed_transaction_ptr ptrx2 = std::make_shared( trx, true, packed_transaction::compression_type::zlib); BOOST_CHECK_EQUAL(trx.id(), pkt.id()); BOOST_CHECK_EQUAL(trx.id(), pkt2.id()); @@ -968,8 +963,8 @@ BOOST_AUTO_TEST_CASE(pruned_transaction_test) { t.set_transaction_headers(trx); trx.sign( t.get_private_key( N(eosio), "active" ), t.control->get_chain_id() ); - packed_transaction packed(trx); - pruned_transaction pruned(trx, true); + packed_transaction_v0 packed(trx); + packed_transaction pruned(trx, true); BOOST_TEST(packed.packed_digest().str() == pruned.packed_digest().str()); BOOST_REQUIRE(pruned.get_context_free_data() != nullptr); BOOST_TEST(*pruned.get_context_free_data() == packed.get_context_free_data()); @@ -980,7 +975,7 @@ BOOST_AUTO_TEST_CASE(pruned_transaction_test) { BOOST_TEST(*pruned.get_signatures() == packed.get_signatures()); BOOST_TEST(pruned.get_prunable_size() == packed.get_prunable_size()); BOOST_TEST(pruned.get_unprunable_size() == packed.get_unprunable_size()); - std::size_t max_size = pruned.maximum_pruned_pack_size(pruned_transaction::cf_compression_type::none); + std::size_t max_size = pruned.maximum_pruned_pack_size(packed_transaction::cf_compression_type::none); BOOST_TEST(fc::raw::pack_size(pruned) <= max_size); pruned.prune_all(); @@ -995,7 +990,7 @@ BOOST_AUTO_TEST_CASE(pruned_transaction_test) { } -static checksum256_type calculate_trx_merkle( const deque& trxs ) { +static checksum256_type calculate_trx_merkle( const deque& trxs ) { deque trx_digests; for( const auto& a : trxs ) trx_digests.emplace_back( a.digest() ); @@ -1012,37 +1007,39 @@ BOOST_AUTO_TEST_CASE(pruned_block_test) { trx.sign( t.get_private_key( N(eosio), "active" ), t.control->get_chain_id() ); t.push_transaction(trx); - signed_block_ptr original = t.produce_block(); - pruned_block basic(*original, true); + signed_block_ptr produced = t.produce_block(); + signed_block_v0_ptr original = produced->to_signed_block_v0(); + BOOST_REQUIRE(original); + signed_block basic(*original, true); BOOST_TEST(basic.transaction_mroot.str() == original->transaction_mroot.str()); BOOST_TEST(basic.transaction_mroot.str() == calculate_trx_merkle(basic.transactions).str()); - signed_block_ptr recovered = basic.to_signed_block(); + signed_block_v0_ptr recovered = basic.to_signed_block_v0(); BOOST_REQUIRE(recovered); BOOST_TEST(fc::raw::pack(*original) == fc::raw::pack(*recovered)); fc::datastream size_stream; - std::size_t padded_size = basic.pack(size_stream, pruned_transaction::cf_compression_type::none); + std::size_t padded_size = basic.pack(size_stream, packed_transaction::cf_compression_type::none); BOOST_TEST(size_stream.tellp() <= padded_size); std::vector buffer(padded_size); fc::datastream stream(buffer.data(), buffer.size()); - basic.pack(stream, pruned_transaction::cf_compression_type::none); - pruned_block deserialized; + basic.pack(stream, packed_transaction::cf_compression_type::none); + signed_block deserialized; fc::datastream in(buffer.data(), buffer.size()); - deserialized.unpack(in, pruned_transaction::cf_compression_type::none); + deserialized.unpack(in, packed_transaction::cf_compression_type::none); std::size_t unpacked_size = padded_size; BOOST_TEST(in.tellp() <= buffer.size()); BOOST_TEST(deserialized.transaction_mroot.str() == original->transaction_mroot.str()); BOOST_TEST(deserialized.transaction_mroot.str() == calculate_trx_merkle(deserialized.transactions).str()); - deserialized.transactions.back().trx.get().prune_all(); - deserialized.prune_state = pruned_block::prune_state_type::incomplete; + deserialized.transactions.back().trx.get().prune_all(); + deserialized.prune_state = signed_block::prune_state_type::incomplete; BOOST_TEST(deserialized.transaction_mroot.str() == calculate_trx_merkle(deserialized.transactions).str()); fc::datastream out(buffer.data(), buffer.size()); - deserialized.pack(out, pruned_transaction::cf_compression_type::none); + deserialized.pack(out, packed_transaction::cf_compression_type::none); BOOST_TEST(out.tellp() <= buffer.size()); - BOOST_TEST(!deserialized.to_signed_block()); + BOOST_TEST(!deserialized.to_signed_block_v0()); } BOOST_AUTO_TEST_CASE(reflector_init_test) { diff --git a/unittests/unapplied_transaction_queue_tests.cpp b/unittests/unapplied_transaction_queue_tests.cpp index 6cf0aff0287..5e1a0d68731 100644 --- a/unittests/unapplied_transaction_queue_tests.cpp +++ b/unittests/unapplied_transaction_queue_tests.cpp @@ -18,7 +18,7 @@ auto unique_trx_meta_data( fc::time_point expire = fc::time_point::now() + fc::s trx.expiration = expire; trx.actions.emplace_back( vector{{creator,config::active_name}}, onerror{ nextid, "test", 4 }); - return transaction_metadata::create_no_recover_keys( packed_transaction( trx ), transaction_metadata::trx_type::input ); + return transaction_metadata::create_no_recover_keys( packed_transaction( trx, true ), transaction_metadata::trx_type::input ); } auto next( unapplied_transaction_queue& q ) { diff --git a/unittests/wasm_tests.cpp b/unittests/wasm_tests.cpp index d4d6bdc75a1..a3e05171a09 100644 --- a/unittests/wasm_tests.cpp +++ b/unittests/wasm_tests.cpp @@ -1915,7 +1915,7 @@ BOOST_AUTO_TEST_CASE( billed_cpu_test ) try { chain.set_transaction_headers( trx, ++num_secs ); // num_secs provides nonce trx.max_cpu_usage_ms = trx_max_ms; trx.sign( chain.get_private_key( acc, "active" ), chain.control->get_chain_id() ); - auto ptrx = std::make_shared(trx); + auto ptrx = std::make_shared(trx, true); auto fut = transaction_metadata::start_recover_keys( ptrx, chain.control->get_thread_pool(), chain.control->get_chain_id(), fc::microseconds::maximum() ); return fut.get(); }; From a6f3a6ed5e81f771ad9c84eb1b2e43404ef23de5 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 31 Mar 2020 07:30:09 -0500 Subject: [PATCH 003/142] Removed get_raw_transaction as not used --- unittests/misc_tests.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index 5df2cf176ba..c45e110b446 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -787,12 +787,6 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { packed_transaction pkt5; fc::from_variant(pkt_v, pkt5); - bytes raw3 = pkt3.get_packed_transaction(); - bytes raw4 = pkt4.get_packed_transaction(); - BOOST_CHECK_EQUAL(raw.size(), raw3.size()); - BOOST_CHECK_EQUAL(raw3.size(), raw4.size()); - BOOST_CHECK_EQUAL(true, std::equal(raw.begin(), raw.end(), raw3.begin())); - BOOST_CHECK_EQUAL(true, std::equal(raw.begin(), raw.end(), raw4.begin())); BOOST_CHECK_EQUAL(pkt.get_signed_transaction().id(), pkt3.get_signed_transaction().id()); BOOST_CHECK_EQUAL(pkt.get_signed_transaction().id(), pkt4.get_signed_transaction().id()); BOOST_CHECK_EQUAL(pkt.get_signed_transaction().id(), pkt5.get_signed_transaction().id()); // failure indicates reflector_init not working From 0f6fdb6a6312ff40d9bfc9c2a18d98d5dc93004e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 31 Mar 2020 11:18:12 -0500 Subject: [PATCH 004/142] Add packed_transaction & packed_transaction_v0 support to abi_serializer --- .../include/eosio/chain/abi_serializer.hpp | 60 ++++++++++++++----- .../chain/include/eosio/chain/transaction.hpp | 4 +- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 569aad2015d..02dc4ad4f9e 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -254,12 +254,14 @@ namespace impl { template constexpr bool single_type_requires_abi_v() { return std::is_base_of::value || + std::is_same::value || std::is_same::value || std::is_same::value || std::is_same::value || std::is_same::value || std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value; } @@ -441,14 +443,32 @@ namespace impl { out(name, std::move(mvo)); } + /** + * overload of to_variant_object for packed_transaction_v0 + * + * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of ptrx.transaction + */ + template + static void add( mutable_variant_object &out, const char* name, const packed_transaction_v0& ptrx, Resolver resolver, abi_traverse_context& ctx ) + { + static_assert(fc::reflector::total_member_count == 4); + auto h = ctx.enter_scope(); + mutable_variant_object mvo; + auto trx = ptrx.get_transaction(); + mvo("id", trx.id()); + mvo("signatures", ptrx.get_signatures()); + mvo("compression", ptrx.get_compression()); + mvo("packed_context_free_data", ptrx.get_packed_context_free_data()); + mvo("context_free_data", ptrx.get_context_free_data()); + mvo("packed_trx", ptrx.get_packed_transaction()); + add(mvo, "transaction", trx, resolver, ctx); + out(name, std::move(mvo)); + } + /** * overload of to_variant_object for packed_transaction, providing original packed_transaction_v0 variant layout * * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of ptrx.transaction - * @tparam Resolver - * @param act - * @param resolver - * @return */ template static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) @@ -457,15 +477,18 @@ namespace impl { auto h = ctx.enter_scope(); mutable_variant_object mvo; auto trx = ptrx.get_transaction(); -/* TODO support old packed_transaction_v0 format? mvo("id", trx.id()); - mvo("signatures", ptrx.get_signatures() != nullptr ? *ptrx.get_signatures() : vector() ); + mvo("signatures", ptrx.get_signatures() != nullptr ? *ptrx.get_signatures() : vector()); mvo("compression", ptrx.get_compression()); - mvo("packed_context_free_data", ptrx.get_packed_context_free_data()); - mvo("context_free_data", ptrx.get_context_free_data()); + if( ptrx.get_prunable_data().prunable_data.contains() ) { + const auto& legacy = ptrx.get_prunable_data().prunable_data.get(); + mvo( "packed_context_free_data", legacy.packed_context_free_data ); + } else { + mvo( "packed_context_free_data", bytes() ); + } + mvo("context_free_data", ptrx.get_context_free_data() != nullptr ? *ptrx.get_context_free_data() : vector()); mvo("packed_trx", ptrx.get_packed_transaction()); add(mvo, "transaction", trx, resolver, ctx); -*/ out(name, std::move(mvo)); } @@ -712,10 +735,17 @@ namespace impl { template static void extract( const variant& v, packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) + { + packed_transaction_v0 v0; + extract( v, v0, std::move( resolver ), ctx ); + ptrx = packed_transaction( std::move( v0 ), true ); + } + + template + static void extract( const variant& v, packed_transaction_v0& ptrx, Resolver resolver, abi_traverse_context& ctx ) { auto h = ctx.enter_scope(); const variant_object& vo = v.get_object(); -/* TODO figure out what to do with serialization EOS_ASSERT(vo.contains("signatures"), packed_transaction_type_exception, "Missing signatures"); EOS_ASSERT(vo.contains("compression"), packed_transaction_type_exception, "Missing compression"); std::vector signatures; @@ -737,26 +767,26 @@ namespace impl { bytes packed_trx; from_variant(vo["packed_trx"], packed_trx); if( use_packed_cfd ) { - ptrx = packed_transaction( std::move( packed_trx ), std::move( signatures ), std::move( packed_cfd ), compression ); + ptrx = packed_transaction_v0( std::move( packed_trx ), std::move( signatures ), std::move( packed_cfd ), compression ); } else { - ptrx = packed_transaction( std::move( packed_trx ), std::move( signatures ), std::move( cfd ), compression ); + ptrx = packed_transaction_v0( std::move( packed_trx ), std::move( signatures ), std::move( cfd ), compression ); } } else { EOS_ASSERT(vo.contains("transaction"), packed_transaction_type_exception, "Missing transaction"); if( use_packed_cfd ) { transaction trx; extract( vo["transaction"], trx, resolver, ctx ); - ptrx = packed_transaction( std::move(trx), std::move(signatures), std::move(packed_cfd), compression ); + ptrx = packed_transaction_v0( std::move(trx), std::move(signatures), std::move(packed_cfd), compression ); } else { signed_transaction trx; extract( vo["transaction"], trx, resolver, ctx ); trx.signatures = std::move( signatures ); trx.context_free_data = std::move(cfd); - ptrx = packed_transaction( std::move( trx ), compression ); + ptrx = packed_transaction_v0( std::move( trx ), compression ); } } -*/ } + }; /** diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 8d94ccd4af8..c757be24a3e 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -191,6 +191,7 @@ namespace eosio { namespace chain { friend struct fc::reflector; friend struct fc::reflector_init_visitor; friend struct fc::has_reflector_init; + friend struct packed_transaction; void reflector_init(); private: vector signatures; @@ -233,7 +234,7 @@ namespace eosio { namespace chain { struct full_legacy { std::vector signatures; - std::vector packed_context_free_data; + bytes packed_context_free_data; }; using prunable_data_type = fc::static_variant< full_legacy, @@ -263,6 +264,7 @@ namespace eosio { namespace chain { packed_transaction& operator=(packed_transaction&&) = default; packed_transaction(const packed_transaction_v0& other, bool legacy) : packed_transaction(other.get_signed_transaction(), legacy, other.get_compression()) {} + packed_transaction(packed_transaction_v0&& other, bool legacy) : packed_transaction(std::move( other.unpacked_trx ), legacy, other.get_compression()) {} explicit packed_transaction(const signed_transaction& t, bool legacy, compression_type _compression = compression_type::none); explicit packed_transaction(signed_transaction&& t, bool legacy, compression_type _compression = compression_type::none); From c0934af5b0bc86bcf52d3a418aff60ac6fbb107f Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 31 Mar 2020 11:54:35 -0500 Subject: [PATCH 005/142] Add new signed_block as signed_block_v0 support to abi_serializer --- .../chain/include/eosio/chain/abi_serializer.hpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 02dc4ad4f9e..8dd2247ac5a 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -522,8 +522,20 @@ namespace impl { out(name, std::move(mvo)); } + + /** + * overload of to_variant_object for signed_block, support old signed_block_v0 format + */ + template + static void add( mutable_variant_object &out, const char* name, const signed_block& block, Resolver resolver, abi_traverse_context& ctx ) + { + // TODO: this could be faster by going directly to variant + auto sb_v0 = block.to_signed_block_v0(); + add( out, name, sb_v0, std::move( resolver ), ctx ); + } + /** - * overload of to_variant_object for signed_block + * overload of to_variant_object for signed_block_v0 * * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of * block.header_extensions and block.block_extensions From c418e1119472a1eb6e9eaec6f2545f97da6a855f Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 31 Mar 2020 13:35:43 -0500 Subject: [PATCH 006/142] Changes for review comments and adding prune_transaction() --- libraries/chain/block_log.cpp | 213 +++++++++++------- libraries/chain/include/eosio/chain/block.hpp | 1 - .../include/eosio/chain/block_header.hpp | 4 +- .../chain/include/eosio/chain/block_log.hpp | 18 +- programs/eosio-blocklog/main.cpp | 7 +- 5 files changed, 147 insertions(+), 96 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 5cf6dbf0741..1b6299167ba 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -39,16 +39,21 @@ namespace eosio { namespace chain { namespace detail { using unique_file = std::unique_ptr; + struct log_entry { + uint32_t offset; + fc::unsigned_int compression; + pruned_block_ptr block; + }; + class block_log_impl { public: - signed_block_header_ptr head; - block_id_type head_id; - fc::cfile block_file; - fc::cfile index_file; - bool open_files = false; - bool genesis_written_to_block_log = false; - uint32_t version = 0; - uint32_t first_block_num = 0; + pruned_block_ptr head; + fc::cfile block_file; + fc::cfile index_file; + bool open_files = false; + bool genesis_written_to_block_log = false; + uint32_t version = 0; + uint32_t first_block_num = 0; inline void check_open_files() { if( !open_files ) { @@ -81,11 +86,23 @@ namespace eosio { namespace chain { static std::vector pack(const pruned_block& b, pruned_transaction::cf_compression_type segment_compression); - static void unpack(fc::cfile_datastream& ds, pruned_block& block); + static void unpack(fc::cfile_datastream& ds, log_entry& entry); uint64_t write_log_entry(const pruned_block& b, pruned_transaction::cf_compression_type segment_compression); - pruned_block_ptr read_log_entry(); + + void read_block_header(block_header& bh, uint64_t file_pos); + pruned_block_ptr read_block(uint64_t pos); + void read_head(); + log_entry read_v4_log_entry(uint64_t pos); + + /// calculate the offset from the start of serialized block entry to block start + static int offset_to_block_start(uint32_t version) { + if (version >= 4) { + return sizeof(uint32_t) + 1; // the size of offset + compression + } + return 0; + } }; void detail::block_log_impl::reopen() { @@ -260,12 +277,7 @@ namespace eosio { namespace chain { my->first_block_num = 1; } - my->head = read_head(); - if( my->head ) { - my->head_id = my->head->id(); - } else { - my->head_id = {}; - } + my->read_head(); if (index_size) { ilog("Index is nonempty"); @@ -300,18 +312,18 @@ namespace eosio { namespace chain { // In version 4 of the irreversible blocks log format, these log entries consists of the following in order: // 1. An uint32_t offset from the start of this log entry to the start of the next log entry. - // 2. An uint32_t indicating the compression status for the serialization of the pruned_block following this. + // 2. An fc::unsigned_int indicating the compression status for the serialization of the pruned_block following this. // 3. The serialization of a pruned_block representation of the block for the entry including padding. std::size_t padded_size = b.maximum_pruned_pack_size(segment_compression); - std::vector buffer(padded_size + 2* sizeof(uint32_t)); + std::vector buffer(padded_size + offset_to_block_start(4)); fc::datastream stream(buffer.data(), buffer.size()); uint32_t offset = buffer.size(); - uint32_t compression = static_cast(segment_compression); stream.write((char*)&offset, sizeof(offset)); - stream.write((char*)&compression, sizeof(compression)); - b.pack(stream, segment_compression); + fc::unsigned_int compression = static_cast(segment_compression); + fc::raw::pack(stream, compression); + fc::raw::pack(stream, b); return buffer; } @@ -322,8 +334,11 @@ namespace eosio { namespace chain { if (version >= 4) { buffer = detail::block_log_impl::pack(b, segment_compression); } else { +#warning: TODO avoid heap allocation auto block_ptr = b.to_signed_block(); EOS_ASSERT(block_ptr, block_log_append_fail, "Unable to convert block to legacy format"); + EOS_ASSERT(segment_compression == pruned_transaction::cf_compression_type::none, block_log_append_fail, + "the compression must be \"none\" for legacy format"); buffer = fc::raw::pack(*block_ptr); } block_file.write(buffer.data(), buffer.size()); @@ -332,12 +347,13 @@ namespace eosio { namespace chain { return pos; } - void detail::block_log_impl::unpack(fc::cfile_datastream& ds, pruned_block& block) { - uint32_t len; - uint32_t compression; - fc::raw::unpack(ds, len); - fc::raw::unpack(ds, compression); - block.unpack(ds, static_cast(compression)); + void detail::block_log_impl::unpack(fc::cfile_datastream& ds, detail::log_entry& entry) { + fc::raw::unpack(ds, entry.offset); + fc::raw::unpack(ds, entry.compression); + EOS_ASSERT(static_cast(entry.compression.value) == pruned_transaction::cf_compression_type::none, block_log_exception, + "Only support compression_type none"); + entry.block = std::make_shared(); + fc::raw::unpack(ds, *entry.block); } @@ -367,7 +383,6 @@ namespace eosio { namespace chain { auto pos = write_log_entry(*b, segment_compression); head = b; - head_id = b->id(); flush(); @@ -408,11 +423,13 @@ namespace eosio { namespace chain { auto totem = block_log::npos; block_file.write((char*)&totem, sizeof(totem)); + // version must be assigned before this->append() because it is used inside this->append() + version = block_log::max_supported_version; + if (first_block) { append(first_block, segment_compression); } else { head.reset(); - head_id = {}; } auto pos = block_file.tellp(); @@ -420,7 +437,6 @@ namespace eosio { namespace chain { static_assert( block_log::max_supported_version > 0, "a version number of zero is not supported" ); // going back to write correct version to indicate that all block log header data writes completed successfully - version = block_log::max_supported_version; block_file.seek( 0 ); block_file.write( (char*)&version, sizeof(version) ); block_file.seek( pos ); @@ -451,33 +467,36 @@ namespace eosio { namespace chain { block_file << chain_id; } - pruned_block_ptr detail::block_log_impl::read_log_entry() { + pruned_block_ptr detail::block_log_impl::read_block(uint64_t pos) { + check_open_files(); + block_file.seek(pos); auto ds = block_file.create_datastream(); if (version >= 4) { - pruned_block_ptr result = std::make_shared(); - detail::block_log_impl::unpack(ds, *result); - return result; + log_entry entry; + detail::block_log_impl::unpack(ds, entry); + return entry.block; } else { - signed_block_ptr result = std::make_shared(); - fc::raw::unpack(ds, *result); - return std::make_shared(*result, true); + signed_block block; + fc::raw::unpack(ds, block); +#warning: TODO: use move(block) + return std::make_shared(block, true); } } - pruned_block_ptr block_log::read_block(uint64_t pos) const { - my->check_open_files(); - my->block_file.seek(pos); - return my->read_log_entry(); + detail::log_entry detail::block_log_impl::read_v4_log_entry(uint64_t pos) { + check_open_files(); + block_file.seek(pos); + auto ds = block_file.create_datastream(); + log_entry entry; + block_log_impl::unpack(ds, entry); + return entry; } - void block_log::read_block_header(block_header& bh, uint64_t pos) const { - my->check_open_files(); - - if (my->version == 4) - pos += 2 * sizeof(uint32_t); + void detail::block_log_impl::read_block_header(block_header& bh, uint64_t pos) { + pos += offset_to_block_start(version); - my->block_file.seek(pos); - auto ds = my->block_file.create_datastream(); + block_file.seek(pos); + auto ds = block_file.create_datastream(); fc::raw::unpack(ds, bh); } @@ -491,9 +510,9 @@ namespace eosio { namespace chain { pruned_block_ptr b; uint64_t pos = get_block_pos(block_num); if (pos != npos) { - b = read_block(pos); - EOS_ASSERT(b->block_num() == block_num, reversible_blocks_exception, - "Wrong block was read from block log.", ("returned", b->block_num())("expected", block_num)); + b = my->read_block(pos); + EOS_ASSERT(b->block_num() == block_num, block_log_exception, + "Wrong block was read from block log."); } return b; } FC_LOG_AND_RETHROW() @@ -504,7 +523,7 @@ namespace eosio { namespace chain { uint64_t pos = get_block_pos(block_num); if (pos != npos) { block_header bh; - read_block_header(bh, pos); + my->read_block_header(bh, pos); EOS_ASSERT(bh.block_num() == block_num, reversible_blocks_exception, "Wrong block header was read from block log.", ("returned", bh.block_num())("expected", block_num)); return bh.id(); @@ -515,7 +534,7 @@ namespace eosio { namespace chain { uint64_t block_log::get_block_pos(uint32_t block_num) const { my->check_open_files(); - if (!(my->head && block_num <= block_header::num_from_id(my->head_id) && block_num >= my->first_block_num)) + if (!(my->head && block_num <= block_header::num_from_id(my->head->id()) && block_num >= my->first_block_num)) return npos; my->index_file.seek(sizeof(uint64_t) * (block_num - my->first_block_num)); uint64_t pos; @@ -523,31 +542,23 @@ namespace eosio { namespace chain { return pos; } - signed_block_header_ptr block_log::read_head()const { - my->check_open_files(); + void detail::block_log_impl::read_head() { uint64_t pos; - // Check that the file is not empty - my->block_file.seek_end(0); - if (my->block_file.tellp() <= sizeof(pos)) - return {}; - - my->block_file.seek_end(-sizeof(pos)); - my->block_file.read((char*)&pos, sizeof(pos)); - if (pos != npos) { - return read_block(pos); - } else { - return {}; - } + block_file.seek_end(-sizeof(pos)); + block_file.read((char*)&pos, sizeof(pos)); + if (pos != block_log::npos) { + head = read_block(pos); + } } - const signed_block_header_ptr& block_log::head() const { - return my->head; + signed_block_header* block_log::head() const { + return my->head.get(); } - const block_id_type& block_log::head_id() const { - return my->head_id; + block_id_type block_log::head_id() const { + return my->head->id(); } uint32_t block_log::first_block_num() const { @@ -834,6 +845,56 @@ namespace eosio { namespace chain { })); } + bool prune(pruned_transaction& ptx) { + ptx.prune_all(); + return true; + } + + void block_log::prune_transaction(uint32_t block_num, transaction_id_type id) { + try { + EOS_ASSERT(my->version >= 4, block_log_exception, "The block log version ${version} does not support transaction pruning.", ("version", my->version)); + uint64_t pos = get_block_pos(block_num); + EOS_ASSERT(pos != npos, block_log_exception, + "Specified block_num ${block_num} does not exist in block log.", ("block_num", block_num)); + + auto entry = my->read_v4_log_entry(pos); + + EOS_ASSERT(entry.block->block_num() == block_num, block_log_exception, + "Wrong block was read from block log."); + + auto pruner = overloaded{[](transaction_id_type&) { return false; }, + [&id](pruned_transaction& ptx) { return ptx.id() == id && prune(ptx); }}; + + for (auto trx : entry.block->transactions) { + if (trx.trx.visit(pruner)) { + my->block_file.seek(pos); + std::vector buffer = detail::block_log_impl::pack(*entry.block, static_cast(entry.compression.value)); + my->block_file.write(buffer.data(), buffer.size()); + my->block_file.flush(); + return; + } + } + + EOS_ASSERT(true, block_log_exception, + "Block ${block_num} does not contain transaction ${id}.", ("id", id)("block_num", block_num)); + + } + FC_LOG_AND_RETHROW() + } + + int trim_data::blknum_offset_from_block_entry(uint32_t block_log_version) { + + //to derive blknum_offset==14 see block_header.hpp and note on disk struct is packed + // block_timestamp_type timestamp; //bytes 0:3 + // account_name producer; //bytes 4:11 + // uint16_t confirmed; //bytes 12:13 + // block_id_type previous; //bytes 14:45, low 4 bytes is big endian block number of previous block + + int blknum_offset = 14; + blknum_offset += detail::block_log_impl::offset_to_block_start(block_log_version); + return blknum_offset; + } + detail::reverse_iterator::reverse_iterator() : _file(nullptr, &fclose) , _buffer_ptr(std::make_unique(_buf_len)) { @@ -880,10 +941,10 @@ namespace eosio { namespace chain { uint32_t bnum = 0; if (block_pos >= _start_of_buffer_position) { const uint32_t index_of_block = block_pos - _start_of_buffer_position; - bnum = *reinterpret_cast(buf + index_of_block + trim_data::blknum_offset); //block number of previous block (is big endian) + bnum = *reinterpret_cast(buf + index_of_block + trim_data::blknum_offset_from_block_entry(_version)); //block number of previous block (is big endian) } else { - const auto blknum_offset_pos = block_pos + trim_data::blknum_offset; + const auto blknum_offset_pos = block_pos + trim_data::trim_data::blknum_offset_from_block_entry(_version); auto status = fseek(_file.get(), blknum_offset_pos, SEEK_SET); EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}. Returned status: ${status}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos)("status", status) ); auto size = fread((void*)&bnum, sizeof(bnum), 1, _file.get()); @@ -1065,8 +1126,8 @@ namespace eosio { namespace chain { new_block_file.close(); new_block_file.open( LOG_RW_C ); - static_assert( block_log::max_supported_version == 4, - "Code was written to support version 3 format, need to update this code for latest format." ); + static_assert( block_log::max_supported_version >= 3, + "Code was written to support format of version 3 or greater, need to update this code for latest format." ); uint32_t version = block_log::max_supported_version; new_block_file.seek(0); new_block_file.write((char*)&version, sizeof(version)); @@ -1248,7 +1309,7 @@ namespace eosio { namespace chain { EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", index_file_name.string())("b",n) ); //read blocks.log and verify block number n is found at the determined file position - const auto calc_blknum_pos = block_n_pos + blknum_offset; + const auto calc_blknum_pos = block_n_pos + blknum_offset_from_block_entry(version); status = fseek(blk_in, calc_blknum_pos, SEEK_SET); EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", calc_blknum_pos) ); const uint64_t block_offset_pos = ftell(blk_in); diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 3df6fd2af01..640b9584f5c 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -135,7 +135,6 @@ namespace eosio { namespace chain { enum class prune_state_type : uint8_t { incomplete, complete, complete_legacy }; pruned_block() = default; - explicit pruned_block( const signed_block_header& h ):signed_block_header(h){} pruned_block( const signed_block&, bool legacy ); pruned_block( pruned_block&& ) = default; pruned_block& operator=(const pruned_block&) = delete; diff --git a/libraries/chain/include/eosio/chain/block_header.hpp b/libraries/chain/include/eosio/chain/block_header.hpp index 7832da5b491..a5c919e4bf0 100644 --- a/libraries/chain/include/eosio/chain/block_header.hpp +++ b/libraries/chain/include/eosio/chain/block_header.hpp @@ -75,9 +75,7 @@ namespace eosio { namespace chain { { signature_type producer_signature; }; - - using signed_block_header_ptr = std::shared_ptr; - + } } /// namespace eosio::chain FC_REFLECT(eosio::chain::block_header, diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index 66da1638643..fd2d78cfe9c 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -46,9 +46,6 @@ namespace eosio { namespace chain { [[deprecated]] void reset( const genesis_state& gs, const signed_block_ptr& genesis_block ); void reset( const genesis_state& gs, const pruned_block_ptr& genesis_block, pruned_transaction::cf_compression_type segment_compression); void reset( const chain_id_type& chain_id, uint32_t first_block_num ); - - pruned_block_ptr read_block(uint64_t file_pos) const; - void read_block_header(block_header& bh, uint64_t file_pos)const; block_id_type read_block_id_by_num(uint32_t block_num)const; @@ -59,11 +56,12 @@ namespace eosio { namespace chain { * Return offset of block in file, or block_log::npos if it does not exist. */ uint64_t get_block_pos(uint32_t block_num) const; - signed_block_header_ptr read_head() const; - const signed_block_header_ptr& head() const; - const block_id_type& head_id() const; + signed_block_header* head() const; + [[deprecated]] block_id_type head_id() const; // use head()->id() instead uint32_t first_block_num() const; + void prune_transaction(uint32_t block_num, transaction_id_type id); + static const uint64_t npos = std::numeric_limits::max(); static const uint32_t min_supported_version; @@ -92,12 +90,6 @@ namespace eosio { namespace chain { std::unique_ptr my; }; -//to derive blknum_offset==14 see block_header.hpp and note on disk struct is packed -// block_timestamp_type timestamp; //bytes 0:3 -// account_name producer; //bytes 4:11 -// uint16_t confirmed; //bytes 12:13 -// block_id_type previous; //bytes 14:45, low 4 bytes is big endian block number of previous block - struct trim_data { //used by trim_blocklog_front(), trim_blocklog_end(), and smoke_test() trim_data(fc::path block_dir); ~trim_data(); @@ -113,6 +105,6 @@ namespace eosio { namespace chain { uint64_t first_block_pos = 0; //file position in blocks.log for the first block in the log chain_id_type chain_id; - static constexpr int blknum_offset{14}; //offset from start of block to 4 byte block number, valid for the only allowed versions + static int blknum_offset_from_block_entry(uint32_t block_log_version); }; } } diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp index d9114e40755..33f0515291d 100644 --- a/programs/eosio-blocklog/main.cpp +++ b/programs/eosio-blocklog/main.cpp @@ -68,7 +68,7 @@ struct report_time { void blocklog::read_log() { report_time rt("reading log"); block_log block_logger(blocks_dir); - const auto end = block_logger.read_head(); + const auto end = block_logger.head(); EOS_ASSERT( end, block_log_exception, "No blocks found in block log" ); EOS_ASSERT( end->block_num() > 1, block_log_exception, "Only one block found in block log" ); @@ -249,8 +249,9 @@ void smoke_test(bfs::path block_dir) { uint64_t file_pos; auto size = fread((void*)&file_pos, sizeof(uint64_t), 1, td.blk_in); EOS_ASSERT( size == 1, block_log_exception, "${file} read fails", ("file", td.block_file_name.string()) ); - status = fseek(td.blk_in, file_pos + trim_data::blknum_offset, SEEK_SET); - EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", td.block_file_name.string())("pos", file_pos + trim_data::blknum_offset) ); + int blknum_offset = trim_data::blknum_offset_from_block_entry(td.version); + status = fseek(td.blk_in, file_pos + blknum_offset, SEEK_SET); + EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", td.block_file_name.string())("pos", file_pos + blknum_offset) ); uint32_t bnum; size = fread((void*)&bnum, sizeof(uint32_t), 1, td.blk_in); EOS_ASSERT( size == 1, block_log_exception, "${file} read fails", ("file", td.block_file_name.string()) ); From 855d83b3f27c5498a31ef1b013fe6d56d3b79750 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 31 Mar 2020 15:21:42 -0500 Subject: [PATCH 007/142] Fix serialization of transaction_receipt_v0 and signed_block to signed_block_v0 format --- .../chain/include/eosio/chain/abi_serializer.hpp | 11 ++++++++++- libraries/chain/include/eosio/chain/block.hpp | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 8dd2247ac5a..4b597ddfadc 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -257,6 +257,7 @@ namespace impl { std::is_same::value || std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || std::is_same::value || std::is_same::value || @@ -531,7 +532,7 @@ namespace impl { { // TODO: this could be faster by going directly to variant auto sb_v0 = block.to_signed_block_v0(); - add( out, name, sb_v0, std::move( resolver ), ctx ); + add( out, name, *sb_v0, std::move( resolver ), ctx ); } /** @@ -745,6 +746,14 @@ namespace impl { "Failed to deserialize data for ${account}:${name}", ("account", act.account)("name", act.name)); } + template + static void extract( const variant& v, signed_block& sb, Resolver resolver, abi_traverse_context& ctx ) + { + signed_block_v0 v0; + extract( v, v0, std::move( resolver ), ctx ); + sb = signed_block( v0, true ); + } + template static void extract( const variant& v, packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) { diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 167ee573a95..2bef05d95d2 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -139,6 +139,7 @@ namespace eosio { namespace chain { signed_block( const signed_block_v0&, bool legacy ); signed_block( signed_block&& ) = default; signed_block& operator=(const signed_block&) = delete; + signed_block& operator=(signed_block&&) = default; signed_block clone() const { return *this; } signed_block_v0_ptr to_signed_block_v0() const; From eba4d9b382571b06086a9ba1ca05b722665cba87 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 1 Apr 2020 07:26:39 -0500 Subject: [PATCH 008/142] Support receiving old/new packed_transaction & signed_block --- .../include/eosio/net_plugin/protocol.hpp | 6 ++-- plugins/net_plugin/net_plugin.cpp | 32 ++++++++++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp index 56c33909851..86e8c2bb9f9 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp @@ -142,8 +142,10 @@ namespace eosio { notice_message, request_message, sync_request_message, - signed_block, // which = 7 - packed_transaction>; // which = 8 + signed_block_v0, // which = 7 + packed_transaction_v0, // which = 8 + signed_block, // which = 9 + packed_transaction>; // which = 10 } // namespace eosio diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index f8a65c77e07..273de781141 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -390,8 +390,10 @@ namespace eosio { constexpr auto def_sync_fetch_span = 100; constexpr auto message_header_size = 4; - constexpr uint32_t signed_block_which = 7; // see protocol net_message - constexpr uint32_t packed_transaction_which = 8; // see protocol net_message + constexpr uint32_t signed_block_v0_which = 7; // see protocol net_message + constexpr uint32_t packed_transaction_v0_which = 8; // see protocol net_message + constexpr uint32_t signed_block_which = 9; // see protocol net_message + constexpr uint32_t packed_transaction_which = 10; // see protocol net_message /** * For a while, network version was a 16 bit value equal to the second set of 16 bits @@ -2365,7 +2367,7 @@ namespace eosio { auto peek_ds = pending_message_buffer.create_peek_datastream(); unsigned_int which{}; fc::raw::unpack( peek_ds, which ); - if( which == signed_block_which ) { + if( which == signed_block_v0_which || which == signed_block_which ) { block_header bh; fc::raw::unpack( peek_ds, bh ); @@ -2405,10 +2407,17 @@ namespace eosio { } } + shared_ptr ptr; auto ds = pending_message_buffer.create_datastream(); fc::raw::unpack( ds, which ); // throw away - shared_ptr ptr = std::make_shared(); - fc::raw::unpack( ds, *ptr ); + if( which == signed_block_which ) { + ptr = std::make_shared(); + fc::raw::unpack( ds, *ptr ); + } else { // signed_block_v0_which + signed_block_v0 v0; + fc::raw::unpack( ds, v0 ); + ptr = std::make_shared( std::move( v0 ), true); + } auto is_webauthn_sig = []( const fc::crypto::signature& s ) { return s.which() == fc::crypto::signature::storage_type::position(); @@ -2430,17 +2439,24 @@ namespace eosio { handle_message( blk_id, std::move( ptr ) ); - } else if( which == packed_transaction_which ) { + } else if( which == packed_transaction_v0_which || which == packed_transaction_which ) { if( !my_impl->p2p_accept_transactions ) { fc_dlog( logger, "p2p-accept-transaction=false - dropping txn" ); pending_message_buffer.advance_read_ptr( message_length ); return true; } + shared_ptr ptr; auto ds = pending_message_buffer.create_datastream(); fc::raw::unpack( ds, which ); // throw away - shared_ptr ptr = std::make_shared(); - fc::raw::unpack( ds, *ptr ); + if( which == packed_transaction_which ) { + ptr = std::make_shared(); + fc::raw::unpack( ds, *ptr ); + } else { // packed_transaction_v0_which + packed_transaction_v0 v0; + fc::raw::unpack( ds, v0 ); + ptr = std::make_shared( std::move( v0 ), true ); + } handle_message( std::move( ptr ) ); } else { From 739692f1d2278dd46b9ba661160887644b40278a Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 1 Apr 2020 12:04:57 -0500 Subject: [PATCH 009/142] Some more changes on PR comments --- libraries/chain/block_log.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 1b6299167ba..05969efad2d 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -493,10 +493,22 @@ namespace eosio { namespace chain { } void detail::block_log_impl::read_block_header(block_header& bh, uint64_t pos) { +#if defined(SUPPORT_CF_COMPRESSION) pos += offset_to_block_start(version); +#endif block_file.seek(pos); auto ds = block_file.create_datastream(); + +#if !defined(SUPPORT_CF_COMPRESSION) + if (version >= 4 ) { + uint32_t offset; + fc::unsigned_int compression; + fc::raw::unpack(ds, offset); + fc::raw::unpack(ds, compression); + EOS_ASSERT( compression.value == static_cast(pruned_transaction::cf_compression_type::none), block_log_exception , "Only \"none\" compression type is supported."); + } +#endif fc::raw::unpack(ds, bh); } @@ -534,7 +546,7 @@ namespace eosio { namespace chain { uint64_t block_log::get_block_pos(uint32_t block_num) const { my->check_open_files(); - if (!(my->head && block_num <= block_header::num_from_id(my->head->id()) && block_num >= my->first_block_num)) + if (!(my->head && block_num <= my->head->block_num() && block_num >= my->first_block_num)) return npos; my->index_file.seek(sizeof(uint64_t) * (block_num - my->first_block_num)); uint64_t pos; From 9a429fdfb07c1bc6d0265db04b8e77c8f231a925 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 1 Apr 2020 13:49:46 -0500 Subject: [PATCH 010/142] Remove unneeded calls to block id() --- plugins/state_history_plugin/state_history_plugin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 4a646ab256b..7fca1dd001c 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -455,7 +455,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->id(), + .block_id = block_state->id, .payload_size = sizeof(uint32_t) + traces_bin.size()}; trace_log->write_entry(header, block_state->block->previous, [&](auto& stream) { uint32_t s = (uint32_t)traces_bin.size(); @@ -551,7 +551,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisblock->id(), + .block_id = block_state->id, .payload_size = sizeof(uint32_t) + deltas_bin.size()}; chain_state_log->write_entry(header, block_state->block->previous, [&](auto& stream) { uint32_t s = (uint32_t)deltas_bin.size(); From f0247c26eb25e1a3014485392a68569615036dd6 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 1 Apr 2020 14:27:00 -0500 Subject: [PATCH 011/142] Add protocol version pruned_types --- plugins/chain_plugin/chain_plugin.cpp | 1 + plugins/net_plugin/net_plugin.cpp | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index 73f685c0c66..a90de6bbacd 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -2072,6 +2072,7 @@ fc::variant read_only::get_block(const read_only::get_block_params& params) cons EOS_ASSERT( block, unknown_block_exception, "Could not find block: ${block}", ("block", params.block_num_or_id)); + // serializes signed_block to variant in signed_block_v0 format fc::variant pretty_output; abi_serializer::to_variant(*block, pretty_output, make_resolver(this, abi_serializer::create_yield_function( abi_serializer_max_time )), abi_serializer::create_yield_function( abi_serializer_max_time )); diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 273de781141..be2ae512963 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -414,10 +414,11 @@ namespace eosio { * the need for compatibility hooks */ constexpr uint16_t proto_base = 0; - constexpr uint16_t proto_explicit_sync = 1; - constexpr uint16_t block_id_notify = 2; // reserved. feature was removed. next net_version should be 3 + constexpr uint16_t proto_explicit_sync = 1; // version at time of eosio 1.0 + constexpr uint16_t block_id_notify = 2; // reserved. feature was removed. next net_version should be 3 + constexpr uint16_t pruned_types = 3; // support packed_transaction & signed_block in addition to prior *_v0 versions - constexpr uint16_t net_version = proto_explicit_sync; + constexpr uint16_t net_version = pruned_types; /** * Index by start_block_num @@ -1241,6 +1242,19 @@ namespace eosio { return create_send_buffer( packed_transaction_which, trx ); } + static std::shared_ptr> create_send_buffer( const signed_block_v0_ptr& sb ) { + // this implementation is to avoid copy of signed_block to net_message + // matches which of net_message for signed_block + fc_dlog( logger, "sending block ${bn}", ("bn", sb->block_num()) ); + return create_send_buffer( signed_block_v0_which, *sb ); + } + + static std::shared_ptr> create_send_buffer( const packed_transaction_v0& trx ) { + // this implementation is to avoid copy of packed_transaction to net_message + // matches which of net_message for packed_transaction + return create_send_buffer( packed_transaction_v0_which, trx ); + } + void connection::enqueue_block( const signed_block_ptr& sb, bool to_sync_queue) { fc_dlog( logger, "enqueue block ${num}", ("num", sb->block_num()) ); verify_strand_in_this_thread( strand, __func__, __LINE__ ); @@ -1589,10 +1603,6 @@ namespace eosio { ("ep", c->peer_name())("lib", msg.last_irreversible_block_num)("head", msg.head_num) ("id", msg.head_id.str().substr(8,16)) ); c->syncing = false; - // wait for receipt of a notice message before initiating sync - if (c->protocol_version < proto_explicit_sync) { - start_sync( c, peer_lib ); - } return; } if (lib_num > msg.head_num ) { From 543399867b44de5574dc1e3bc42500c63ed1c8a1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 1 Apr 2020 16:20:13 -0500 Subject: [PATCH 012/142] Remove head_id() --- libraries/chain/block_log.cpp | 18 +++--------------- .../chain/include/eosio/chain/block_log.hpp | 1 - 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 086351741eb..018df8280de 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -40,7 +40,6 @@ namespace eosio { namespace chain { class block_log_impl { public: signed_block_ptr head; - block_id_type head_id; fc::cfile block_file; fc::cfile index_file; bool open_files = false; @@ -251,11 +250,6 @@ namespace eosio { namespace chain { } my->head = read_head(); - if( my->head ) { - my->head_id = my->head->id(); - } else { - my->head_id = {}; - } if (index_size) { ilog("Index is nonempty"); @@ -309,7 +303,6 @@ namespace eosio { namespace chain { block_file.write((char*)&pos, sizeof(pos)); index_file.write((char*)&pos, sizeof(pos)); head = b; - head_id = b->id(); flush(); @@ -354,7 +347,6 @@ namespace eosio { namespace chain { append(first_block); } else { head.reset(); - head_id = {}; } auto pos = block_file.tellp(); @@ -427,7 +419,7 @@ namespace eosio { namespace chain { read_block_header(bh, pos); EOS_ASSERT(bh.block_num() == block_num, reversible_blocks_exception, "Wrong block header was read from block log.", ("returned", bh.block_num())("expected", block_num)); - return bh.id(); + return bh.calculate_id(); } return {}; } FC_LOG_AND_RETHROW() @@ -435,7 +427,7 @@ namespace eosio { namespace chain { uint64_t block_log::get_block_pos(uint32_t block_num) const { my->check_open_files(); - if (!(my->head && block_num <= block_header::num_from_id(my->head_id) && block_num >= my->first_block_num)) + if (!(my->head && block_num <= my->head->block_num() && block_num >= my->first_block_num)) return npos; my->index_file.seek(sizeof(uint64_t) * (block_num - my->first_block_num)); uint64_t pos; @@ -466,10 +458,6 @@ namespace eosio { namespace chain { return my->head; } - const block_id_type& block_log::head_id()const { - return my->head_id; - } - uint32_t block_log::first_block_num() const { return my->first_block_num; } @@ -629,7 +617,7 @@ namespace eosio { namespace chain { break; } - auto id = tmp.id(); + auto id = tmp.calculate_id(); if( block_header::num_from_id(previous) + 1 != block_header::num_from_id(id) ) { elog( "Block ${num} (${id}) skips blocks. Previous block in block log is block ${prev_num} (${previous})", ("num", block_header::num_from_id(id))("id", id) diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index 5fbe3e771a5..9ec01a60924 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -57,7 +57,6 @@ namespace eosio { namespace chain { uint64_t get_block_pos(uint32_t block_num) const; signed_block_ptr read_head()const; const signed_block_ptr& head()const; - const block_id_type& head_id()const; uint32_t first_block_num() const; static const uint64_t npos = std::numeric_limits::max(); From b75d09c113b370a724487b97a4113e105e25ea99 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 1 Apr 2020 16:35:58 -0500 Subject: [PATCH 013/142] Rename block id() to calculate_id() to make it obvious it is not returning a cached value --- libraries/chain/block_header.cpp | 2 +- libraries/chain/block_header_state.cpp | 2 +- libraries/chain/controller.cpp | 27 ++++++++++--------- .../include/eosio/chain/block_header.hpp | 2 +- .../chain/include/eosio/chain/controller.hpp | 2 +- .../eosio/chain/reversible_block_object.hpp | 2 +- .../testing/include/eosio/testing/tester.hpp | 6 ++--- libraries/testing/tester.cpp | 8 +++--- plugins/chain_plugin/chain_plugin.cpp | 19 +++++++------ .../eosio/chain_plugin/chain_plugin.hpp | 2 -- plugins/net_plugin/net_plugin.cpp | 4 +-- plugins/producer_plugin/producer_plugin.cpp | 4 +-- .../state_history_plugin.cpp | 2 +- programs/eosio-blocklog/main.cpp | 2 +- unittests/block_tests.cpp | 2 +- unittests/database_tests.cpp | 6 ++--- unittests/forked_tests.cpp | 19 ++++++------- unittests/resource_limits_test.cpp | 8 +++--- 18 files changed, 60 insertions(+), 59 deletions(-) diff --git a/libraries/chain/block_header.cpp b/libraries/chain/block_header.cpp index 0efc7728a1f..eef0f5bee37 100644 --- a/libraries/chain/block_header.cpp +++ b/libraries/chain/block_header.cpp @@ -15,7 +15,7 @@ namespace eosio { namespace chain { return fc::endian_reverse_u32(id._hash[0]); } - block_id_type block_header::id()const + block_id_type block_header::calculate_id()const { // Do not include signed_block_header attributes in id, specifically exclude producer_signature. block_id_type result = digest(); //fc::sha256::hash(*static_cast(this)); diff --git a/libraries/chain/block_header_state.cpp b/libraries/chain/block_header_state.cpp index bdf2a4cc386..c79d7c7f3f2 100644 --- a/libraries/chain/block_header_state.cpp +++ b/libraries/chain/block_header_state.cpp @@ -292,7 +292,7 @@ namespace eosio { namespace chain { block_header_state result( std::move( *static_cast(this) ) ); - result.id = h.id(); + result.id = h.calculate_id(); result.header = h; result.header_exts = std::move(exts); diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 4513d28d660..f1bfa5d1765 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -397,7 +397,8 @@ struct controller_impl { auto root_id = fork_db.root()->id; if( log_head ) { - EOS_ASSERT( root_id == blog.head_id(), fork_database_exception, "fork database root does not match block log head" ); + // todo: move this check to startup so id does not have to be calculated + EOS_ASSERT( root_id == log_head->calculate_id(), fork_database_exception, "fork database root does not match block log head" ); } else { EOS_ASSERT( fork_db.root()->block_num == lib_num, fork_database_exception, "empty block log expects the first appended block to build off a block that is not the fork database root" ); @@ -461,7 +462,7 @@ struct controller_impl { genheader.pending_schedule.schedule_hash = fc::sha256::hash(initial_legacy_schedule); genheader.header.timestamp = genesis.initial_timestamp; genheader.header.action_mroot = genesis.compute_chain_id(); - genheader.id = genheader.header.id(); + genheader.id = genheader.header.calculate_id(); genheader.block_num = genheader.header.block_num(); head = std::make_shared(); @@ -1719,7 +1720,7 @@ struct controller_impl { block_ptr->transactions = std::move( bb._pending_trx_receipts ); - auto id = block_ptr->id(); + auto id = block_ptr->calculate_id(); // Update TaPoS table: create_block_summary( id ); @@ -1859,7 +1860,7 @@ struct controller_impl { const signed_block_ptr& b = bsp->block; const auto& new_protocol_feature_activations = bsp->get_new_protocol_feature_activations(); - auto producer_block_id = b->id(); + auto producer_block_id = bsp->id; start_block( b->timestamp, b->confirmed, new_protocol_feature_activations, s, producer_block_id); // validated in create_block_state_future() @@ -1963,11 +1964,9 @@ struct controller_impl { } } FC_CAPTURE_AND_RETHROW() } /// apply_block - std::future create_block_state_future( const signed_block_ptr& b ) { + std::future create_block_state_future( const block_id_type& id, const signed_block_ptr& b ) { EOS_ASSERT( b, block_validate_exception, "null block" ); - auto id = b->id(); - // no reason for a block_state if fork_db already knows about block auto existing = fork_db.get_block( id ); EOS_ASSERT( !existing, fork_database_exception, "we already know about this block: ${id}", ("id", id) ); @@ -1976,14 +1975,14 @@ struct controller_impl { EOS_ASSERT( prev, unlinkable_block_exception, "unlinkable block ${id}", ("id", id)("previous", b->previous) ); - return async_thread_pool( thread_pool.get_executor(), [b, prev, control=this]() { + return async_thread_pool( thread_pool.get_executor(), [b, prev, id, control=this]() { const bool skip_validate_signee = false; auto trx_mroot = calculate_trx_merkle( b->transactions ); EOS_ASSERT( b->transaction_mroot == trx_mroot, block_validate_exception, "invalid block transaction merkle root ${b} != ${c}", ("b", b->transaction_mroot)("c", trx_mroot) ); - return std::make_shared( + auto bsp = std::make_shared( *prev, move( b ), control->protocol_features.get_protocol_feature_set(), @@ -1993,6 +1992,10 @@ struct controller_impl { { control->check_protocol_features( timestamp, cur_features, new_features ); }, skip_validate_signee ); + + EOS_ASSERT( id == bsp->id, block_validate_exception, + "provided id ${id} does not match block id ${bid}", ("id", id)("bid", bsp->id) ); + return bsp; } ); } @@ -2674,8 +2677,8 @@ boost::asio::io_context& controller::get_thread_pool() { return my->thread_pool.get_executor(); } -std::future controller::create_block_state_future( const signed_block_ptr& b ) { - return my->create_block_state_future( b ); +std::future controller::create_block_state_future( const block_id_type& id, const signed_block_ptr& b ) { + return my->create_block_state_future( id, b ); } void controller::push_block( std::future& block_state_future, @@ -2858,7 +2861,7 @@ signed_block_ptr controller::fetch_block_by_id( block_id_type id )const { auto state = my->fork_db.get_block(id); if( state && state->block ) return state->block; auto bptr = fetch_block_by_number( block_header::num_from_id(id) ); - if( bptr && bptr->id() == id ) return bptr; + if( bptr && bptr->calculate_id() == id ) return bptr; return signed_block_ptr(); } diff --git a/libraries/chain/include/eosio/chain/block_header.hpp b/libraries/chain/include/eosio/chain/block_header.hpp index 286ecf528c1..f34a3ee9e52 100644 --- a/libraries/chain/include/eosio/chain/block_header.hpp +++ b/libraries/chain/include/eosio/chain/block_header.hpp @@ -63,7 +63,7 @@ namespace eosio { namespace chain { block_header() = default; digest_type digest()const; - block_id_type id() const; + block_id_type calculate_id() const; uint32_t block_num() const { return num_from_id(previous) + 1; } static uint32_t num_from_id(const block_id_type& id); diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index fe42e579dc5..72c8ef0471e 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -157,7 +157,7 @@ namespace eosio { namespace chain { void sign_block( const signer_callback_type& signer_callback ); void commit_block(); - std::future create_block_state_future( const signed_block_ptr& b ); + std::future create_block_state_future( const block_id_type& id, const signed_block_ptr& b ); /** * @param block_state_future provide from call to create_block_state_future diff --git a/libraries/chain/include/eosio/chain/reversible_block_object.hpp b/libraries/chain/include/eosio/chain/reversible_block_object.hpp index 963c2b28fa3..927fa39fc57 100644 --- a/libraries/chain/include/eosio/chain/reversible_block_object.hpp +++ b/libraries/chain/include/eosio/chain/reversible_block_object.hpp @@ -35,7 +35,7 @@ namespace eosio { namespace chain { fc::raw::unpack( ds, h ); // Only need the block id to then look up the block state in fork database, so just unpack the block_header from the stored packed data. // Avoid calling get_block() since that constructs a new signed_block in heap memory and unpacks the full signed_block from the stored packed data. - return h.id(); + return h.calculate_id(); } }; diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index ed223ecf636..e82b02a2879 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -585,7 +585,7 @@ namespace eosio { namespace testing { signed_block_ptr produce_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override { auto sb = _produce_block(skip_time, false); - auto bsf = validating_node->create_block_state_future( sb ); + auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb ); validating_node->push_block( bsf, forked_branch_callback{}, trx_meta_cache_lookup{} ); return sb; @@ -596,14 +596,14 @@ namespace eosio { namespace testing { } void validate_push_block(const signed_block_ptr& sb) { - auto bs = validating_node->create_block_state_future( sb ); + auto bs = validating_node->create_block_state_future( sb->calculate_id(), sb ); validating_node->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} ); } signed_block_ptr produce_empty_block( fc::microseconds skip_time = fc::milliseconds(config::block_interval_ms) )override { unapplied_transactions.add_aborted( control->abort_block() ); auto sb = _produce_block(skip_time, true); - auto bsf = validating_node->create_block_state_future( sb ); + auto bsf = validating_node->create_block_state_future( sb->calculate_id(), sb ); validating_node->push_block( bsf, forked_branch_callback{}, trx_meta_cache_lookup{} ); return sb; diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index 0819a9d2640..7f9f8517bd3 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -297,7 +297,7 @@ namespace eosio { namespace testing { } void base_tester::push_block(signed_block_ptr b) { - auto bsf = control->create_block_state_future(b); + auto bsf = control->create_block_state_future(b->calculate_id(), b); unapplied_transactions.add_aborted( control->abort_block() ); control->push_block( bsf, [this]( const branch_type& forked_branch ) { unapplied_transactions.add_forked( forked_branch ); @@ -306,8 +306,8 @@ namespace eosio { namespace testing { } ); auto itr = last_produced_block.find(b->producer); - if (itr == last_produced_block.end() || block_header::num_from_id(b->id()) > block_header::num_from_id(itr->second)) { - last_produced_block[b->producer] = b->id(); + if (itr == last_produced_block.end() || b->block_num() > block_header::num_from_id(itr->second)) { + last_produced_block[b->producer] = b->calculate_id(); } } @@ -1023,7 +1023,7 @@ namespace eosio { namespace testing { auto block = a.control->fetch_block_by_number(i); if( block ) { //&& !b.control->is_known_block(block->id()) ) { - auto bsf = b.control->create_block_state_future( block ); + auto bsf = b.control->create_block_state_future( block->calculate_id(), block ); b.control->abort_block(); b.control->push_block(bsf, forked_branch_callback{}, trx_meta_cache_lookup{}); //, eosio::chain::validation_steps::created_block); } diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index a90de6bbacd..6766d346734 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -1061,7 +1061,7 @@ void chain_plugin::plugin_initialize(const variables_map& options) { my->pre_accepted_block_connection = my->chain->pre_accepted_block.connect([this](const signed_block_ptr& blk) { auto itr = my->loaded_checkpoints.find( blk->block_num() ); if( itr != my->loaded_checkpoints.end() ) { - auto id = blk->id(); + auto id = blk->calculate_id(); EOS_ASSERT( itr->second == id, checkpoint_exception, "Checkpoint does not match for block number ${num}: expected: ${expected} actual: ${actual}", ("num", blk->block_num())("expected", itr->second)("actual", id) @@ -1170,11 +1170,6 @@ void chain_plugin::accept_transaction(const chain::packed_transaction_ptr& trx, my->incoming_transaction_async_method(trx, false, std::move(next)); } -bool chain_plugin::block_is_on_preferred_chain(const block_id_type& block_id) { - auto b = chain().fetch_block_by_number( block_header::num_from_id(block_id) ); - return b && b->id() == block_id; -} - bool chain_plugin::recover_reversible_blocks( const fc::path& db_dir, uint32_t cache_size, optional new_db_dir, uint32_t truncate_at_block ) { try { @@ -2077,10 +2072,11 @@ fc::variant read_only::get_block(const read_only::get_block_params& params) cons abi_serializer::to_variant(*block, pretty_output, make_resolver(this, abi_serializer::create_yield_function( abi_serializer_max_time )), abi_serializer::create_yield_function( abi_serializer_max_time )); - uint32_t ref_block_prefix = block->id()._hash[1]; + const auto id = block->calculate_id(); + const uint32_t ref_block_prefix = id._hash[1]; return fc::mutable_variant_object(pretty_output.get_object()) - ("id", block->id()) + ("id", id) ("block_num",block->block_num()) ("ref_block_prefix", ref_block_prefix); } @@ -2096,10 +2092,13 @@ fc::variant read_only::get_block_info(const read_only::get_block_info_params& pa EOS_ASSERT( block, unknown_block_exception, "Could not find block: ${block}", ("block", params.block_num)); + const auto id = block->calculate_id(); + const uint32_t ref_block_prefix = id._hash[1]; + return fc::mutable_variant_object () ("block_num", block->block_num()) ("ref_block_num", static_cast(block->block_num())) - ("id", block->id()) + ("id", id) ("timestamp", block->timestamp) ("producer", block->producer) ("confirmed", block->confirmed) @@ -2108,7 +2107,7 @@ fc::variant read_only::get_block_info(const read_only::get_block_info_params& pa ("action_mroot", block->action_mroot) ("schedule_version", block->schedule_version) ("producer_signature", block->producer_signature) - ("ref_block_prefix", static_cast(block->id()._hash[1])); + ("ref_block_prefix", ref_block_prefix); } fc::variant read_only::get_block_header_state(const get_block_header_state_params& params) const { diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp index 1fb165af3a3..81bcf715a33 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp @@ -731,8 +731,6 @@ class chain_plugin : public plugin { bool accept_block( const chain::signed_block_ptr& block, const chain::block_id_type& id ); void accept_transaction(const chain::packed_transaction_ptr& trx, chain::plugin_interface::next_function next); - bool block_is_on_preferred_chain(const chain::block_id_type& block_id); - static bool recover_reversible_blocks( const fc::path& db_dir, uint32_t cache_size, optional new_db_dir = optional(), diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index be2ae512963..68bd657c494 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2381,7 +2381,7 @@ namespace eosio { block_header bh; fc::raw::unpack( peek_ds, bh ); - const block_id_type blk_id = bh.id(); + const block_id_type blk_id = bh.calculate_id(); const uint32_t blk_num = bh.block_num(); if( my_impl->dispatcher->have_block( blk_id ) ) { fc_dlog( logger, "canceling wait on ${p}, already received block ${num}, id ${id}...", @@ -3127,7 +3127,7 @@ namespace eosio { controller& cc = chain_plug->chain(); if( cc.is_trusted_producer(block->producer) ) { dispatcher->strand.post( [this, block]() { - auto id = block->id(); + auto id = block->calculate_id(); fc_dlog( logger, "signaled pre_accepted_block, blk num = ${num}, id = ${id}", ("num", block->block_num())("id", id) ); dispatcher->bcast_block( block, id ); }); diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 2e7af343e37..afc8843dc18 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -348,7 +348,7 @@ class producer_plugin_impl : public std::enable_shared_from_thisid(); + const auto& id = block_id ? *block_id : block->calculate_id(); auto blk_num = block->block_num(); fc_dlog(_log, "received incoming block ${n} ${id}", ("n", blk_num)("id", id)); @@ -361,7 +361,7 @@ class producer_plugin_impl : public std::enable_shared_from_thischain().fetch_block_by_number(block_num); if (block) - return block->id(); + return block->calculate_id(); } catch (...) { } return {}; diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp index d9114e40755..bc9ef797ce4 100644 --- a/programs/eosio-blocklog/main.cpp +++ b/programs/eosio-blocklog/main.cpp @@ -126,7 +126,7 @@ void blocklog::read_log() { pretty_output, []( account_name n ) { return optional(); }, abi_serializer::create_yield_function( deadline )); - const auto block_id = next->id(); + const auto block_id = next->calculate_id(); const uint32_t ref_block_prefix = block_id._hash[1]; const auto enhanced_object = fc::mutable_variant_object ("block_num",next->block_num()) diff --git a/unittests/block_tests.cpp b/unittests/block_tests.cpp index 74ad40dcb41..c18131da6a4 100644 --- a/unittests/block_tests.cpp +++ b/unittests/block_tests.cpp @@ -44,7 +44,7 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_test) // Push block with invalid transaction to other chain tester validator; - auto bs = validator.control->create_block_state_future( copy_b ); + auto bs = validator.control->create_block_state_future( copy_b->calculate_id(), copy_b ); validator.control->abort_block(); BOOST_REQUIRE_EXCEPTION(validator.control->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception , [] (const fc::exception &e)->bool { diff --git a/unittests/database_tests.cpp b/unittests/database_tests.cpp index e2d047d4e88..24aeceee5a5 100644 --- a/unittests/database_tests.cpp +++ b/unittests/database_tests.cpp @@ -55,9 +55,9 @@ BOOST_AUTO_TEST_SUITE(database_tests) // Produce 200 blocks and check their IDs should match the above test.produce_blocks(num_of_blocks_to_prod); for (uint32_t i = 0; i < num_of_blocks_to_prod; ++i) { - block_ids.emplace_back(test.control->fetch_block_by_number(i + 1)->id()); + block_ids.emplace_back(test.control->fetch_block_by_number(i + 1)->calculate_id()); BOOST_TEST(block_header::num_from_id(block_ids.back()) == i + 1); - BOOST_TEST(test.control->fetch_block_by_number(i + 1)->id() == block_ids.back()); + BOOST_TEST(test.control->fetch_block_by_number(i + 1)->calculate_id() == block_ids.back()); } // Check the last irreversible block number is set correctly, with one producer, irreversibility should only just 1 block before @@ -76,7 +76,7 @@ BOOST_AUTO_TEST_SUITE(database_tests) // Previous nonexisting future block should exist by now BOOST_CHECK_NO_THROW(test.control->fetch_block_by_number(nonexisting_future_block_num)); // Check the latest head block match - BOOST_TEST(test.control->fetch_block_by_number(test.control->head_block_num())->id() == + BOOST_TEST(test.control->fetch_block_by_number(test.control->head_block_num())->calculate_id() == test.control->head_block_id()); } FC_LOG_AND_RETHROW() } diff --git a/unittests/forked_tests.cpp b/unittests/forked_tests.cpp index d24511babea..92215809216 100644 --- a/unittests/forked_tests.cpp +++ b/unittests/forked_tests.cpp @@ -78,7 +78,7 @@ BOOST_AUTO_TEST_CASE( fork_with_bad_block ) try { copy_b->action_mroot._hash[0] ^= 0x1ULL; } else if (j < i) { // link to a corrupted chain - copy_b->previous = fork.blocks.back()->id(); + copy_b->previous = fork.blocks.back()->calculate_id(); } // re-sign the block @@ -87,7 +87,7 @@ BOOST_AUTO_TEST_CASE( fork_with_bad_block ) try { copy_b->producer_signature = remote.get_private_key(N(b), "active").sign(sig_digest); // add this new block to our corrupted block merkle - fork.block_merkle.append(copy_b->id()); + fork.block_merkle.append(copy_b->calculate_id()); fork.blocks.emplace_back(copy_b); } else { fork.blocks.emplace_back(b); @@ -105,7 +105,7 @@ BOOST_AUTO_TEST_CASE( fork_with_bad_block ) try { for (size_t fidx = 0; fidx < fork.blocks.size() - 1; fidx++) { const auto& b = fork.blocks.at(fidx); // push the block only if its not known already - if (!bios.control->fetch_block_by_id(b->id())) { + if (!bios.control->fetch_block_by_id(b->calculate_id())) { bios.push_block(b); } } @@ -266,7 +266,8 @@ BOOST_AUTO_TEST_CASE( forking ) try { wlog( "now push dan's block to c1 but first corrupt it so it is a bad block" ); signed_block bad_block = std::move(*b); bad_block.action_mroot = bad_block.previous; - auto bad_block_bs = c.control->create_block_state_future( std::make_shared(std::move(bad_block)) ); + auto bad_id = bad_block.calculate_id(); + auto bad_block_bs = c.control->create_block_state_future( bad_id, std::make_shared(std::move(bad_block)) ); c.control->abort_block(); BOOST_REQUIRE_EXCEPTION(c.control->push_block( bad_block_bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), fc::exception, [] (const fc::exception &ex)->bool { @@ -362,7 +363,7 @@ BOOST_AUTO_TEST_CASE( validator_accepts_valid_blocks ) try { BOOST_REQUIRE( first_block ); first_block->verify_signee(); - BOOST_CHECK_EQUAL( first_block->header.id(), first_block->block->id() ); + BOOST_CHECK_EQUAL( first_block->header.calculate_id(), first_block->block->calculate_id() ); BOOST_CHECK( first_block->header.producer_signature == first_block->block->producer_signature ); c.disconnect(); @@ -643,7 +644,7 @@ BOOST_AUTO_TEST_CASE( push_block_returns_forked_transactions ) try { .active = active_auth, }); trx.expiration = c.control->head_block_time() + fc::seconds( 60 ); - trx.set_reference_block( cb->id() ); + trx.set_reference_block( cb->calculate_id() ); trx.sign( get_private_key( config::system_account_name, "active" ), c.control->get_chain_id() ); trace1 = c.push_transaction( trx ); } @@ -660,7 +661,7 @@ BOOST_AUTO_TEST_CASE( push_block_returns_forked_transactions ) try { .active = active_auth, }); trx.expiration = c.control->head_block_time() + fc::seconds( 60 ); - trx.set_reference_block( cb->id() ); + trx.set_reference_block( cb->calculate_id() ); trx.sign( get_private_key( config::system_account_name, "active" ), c.control->get_chain_id() ); trace2 = c.push_transaction( trx ); } @@ -676,7 +677,7 @@ BOOST_AUTO_TEST_CASE( push_block_returns_forked_transactions ) try { .active = active_auth, }); trx.expiration = c.control->head_block_time() + fc::seconds( 60 ); - trx.set_reference_block( cb->id() ); + trx.set_reference_block( cb->calculate_id() ); trx.sign( get_private_key( config::system_account_name, "active" ), c.control->get_chain_id() ); trace3 = c.push_transaction( trx ); } @@ -692,7 +693,7 @@ BOOST_AUTO_TEST_CASE( push_block_returns_forked_transactions ) try { .active = active_auth, }); trx.expiration = c.control->head_block_time() + fc::seconds( 60 ); - trx.set_reference_block( b->id() ); // tapos to dan's block should be rejected on fork switch + trx.set_reference_block( b->calculate_id() ); // tapos to dan's block should be rejected on fork switch trx.sign( get_private_key( config::system_account_name, "active" ), c.control->get_chain_id() ); trace4 = c.push_transaction( trx ); BOOST_CHECK( trace4->receipt->status == transaction_receipt_header::executed ); diff --git a/unittests/resource_limits_test.cpp b/unittests/resource_limits_test.cpp index 0914f785e26..60c1ba28bbe 100644 --- a/unittests/resource_limits_test.cpp +++ b/unittests/resource_limits_test.cpp @@ -497,7 +497,7 @@ BOOST_AUTO_TEST_SUITE(resource_limits_test) // Push trigger block to validator node. // This should fail because the NET bill calculated by the fully-validating node will differ from the one in the block. { - auto bs = validator.control->create_block_state_future( trigger_block ); + auto bs = validator.control->create_block_state_future( trigger_block->calculate_id(), trigger_block ); validator.control->abort_block(); BOOST_REQUIRE_EXCEPTION( validator.control->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), @@ -510,7 +510,7 @@ BOOST_AUTO_TEST_SUITE(resource_limits_test) // Because validator2 is in light validation mode, it does not compute the NET bill itself and instead only relies on the value in the block. // The failure will be due to failing check_net_usage within transaction_context::finalize because the NET bill in the block is too high. { - auto bs = validator2.control->create_block_state_future( trigger_block ); + auto bs = validator2.control->create_block_state_future( trigger_block->calculate_id(), trigger_block ); validator2.control->abort_block(); BOOST_REQUIRE_EXCEPTION( validator2.control->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), @@ -539,7 +539,7 @@ BOOST_AUTO_TEST_SUITE(resource_limits_test) // Push new trigger block to validator node. // This should still fail because the NET bill is incorrect. { - auto bs = validator.control->create_block_state_future( trigger_block ); + auto bs = validator.control->create_block_state_future( trigger_block->calculate_id(), trigger_block ); validator.control->abort_block(); BOOST_REQUIRE_EXCEPTION( validator.control->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} ), @@ -550,7 +550,7 @@ BOOST_AUTO_TEST_SUITE(resource_limits_test) // Push new trigger block to validator2 node. // Because validator2 is in light validation mode, this will not fail despite the fact that the NET bill is incorrect. { - auto bs = validator2.control->create_block_state_future( trigger_block ); + auto bs = validator2.control->create_block_state_future( trigger_block->calculate_id(), trigger_block ); validator2.control->abort_block(); validator2.control->push_block( bs, forked_branch_callback{}, trx_meta_cache_lookup{} ); } From 01e3b2299a88802e9af89276bba2ba45434ffa28 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 2 Apr 2020 14:39:33 -0500 Subject: [PATCH 014/142] refactor repair_log with mmap() --- libraries/chain/block_log.cpp | 449 ++++++++++-------- .../chain/include/eosio/chain/block_log.hpp | 3 +- .../chain/include/eosio/chain/transaction.hpp | 2 +- 3 files changed, 248 insertions(+), 206 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 05969efad2d..5245d34e26c 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -4,7 +4,7 @@ #include #include #include - +#include #define LOG_READ (std::ios::in | std::ios::binary) #define LOG_WRITE (std::ios::out | std::ios::binary | std::ios::app) @@ -40,11 +40,20 @@ namespace eosio { namespace chain { using unique_file = std::unique_ptr; struct log_entry { - uint32_t offset; - fc::unsigned_int compression; - pruned_block_ptr block; + uint32_t offset; + pruned_transaction::cf_compression_type compression; + pruned_block* block; }; + template + void unpack(Stream& ds, log_entry& entry){ + fc::raw::unpack(ds, entry.offset); + fc::raw::unpack(ds, entry.compression); + EOS_ASSERT(entry.compression == pruned_transaction::cf_compression_type::none, block_log_exception, + "Only support compression_type none"); + fc::raw::unpack(ds, *entry.block); + } + class block_log_impl { public: pruned_block_ptr head; @@ -84,9 +93,8 @@ namespace eosio { namespace chain { template static fc::optional extract_chain_context( const fc::path& data_dir, Lambda&& lambda ); - static std::vector pack(const pruned_block& b, - pruned_transaction::cf_compression_type segment_compression); - static void unpack(fc::cfile_datastream& ds, log_entry& entry); + static std::vector pack_v4_log_entry(const pruned_block& b, + pruned_transaction::cf_compression_type segment_compression); uint64_t write_log_entry(const pruned_block& b, pruned_transaction::cf_compression_type segment_compression); @@ -94,14 +102,12 @@ namespace eosio { namespace chain { void read_block_header(block_header& bh, uint64_t file_pos); pruned_block_ptr read_block(uint64_t pos); void read_head(); - log_entry read_v4_log_entry(uint64_t pos); + void read_v4_log_entry(uint64_t pos, log_entry&); /// calculate the offset from the start of serialized block entry to block start static int offset_to_block_start(uint32_t version) { - if (version >= 4) { - return sizeof(uint32_t) + 1; // the size of offset + compression - } - return 0; + if (version < 4) return 0; + return sizeof(uint32_t) + sizeof(pruned_transaction::cf_compression_type); } }; @@ -206,6 +212,44 @@ namespace eosio { namespace chain { FILE* const _file; const std::string _filename; }; + + /* + * @brief A utility class to wrap mmap() system call for RAII usage + * + */ + class mmap_readonly_data { + int fd; + uint64_t _size; + void* _addr; + + public: + mmap_readonly_data(const char* filename) { + int fd = ::open(filename, O_RDONLY); + EOS_ASSERT(fd != -1, block_log_exception, "Unable to open file: ${filename}", ("filename", filename)); + + struct stat st; + stat(filename, &st); + _size = st.st_size; + + _addr = mmap(0, st.st_size, PROT_READ, MAP_SHARED, fd, 0); + if (_addr == MAP_FAILED) { + int err_num = errno; + close(fd); + EOS_ASSERT(true, block_log_exception, "mmap() failure: ${msg}", ("msg", (strerror(err_num)))); + } + } + ~mmap_readonly_data() { + munmap(_addr, _size); + close(fd); + } + + mmap_readonly_data(const mmap_readonly_data&) = delete; + mmap_readonly_data& operator=(const mmap_readonly_data&) = delete; + + const char* addr() const { return static_cast(_addr); } + uint64_t size() const { return _size; } + fc::datastream create_datastream() const { return fc::datastream(addr(), size()); } + }; } block_log::block_log(const fc::path& data_dir) @@ -308,11 +352,11 @@ namespace eosio { namespace chain { } } - std::vector detail::block_log_impl::pack(const pruned_block& b, pruned_transaction::cf_compression_type segment_compression) { + std::vector detail::block_log_impl::pack_v4_log_entry(const pruned_block& b, pruned_transaction::cf_compression_type segment_compression) { // In version 4 of the irreversible blocks log format, these log entries consists of the following in order: // 1. An uint32_t offset from the start of this log entry to the start of the next log entry. - // 2. An fc::unsigned_int indicating the compression status for the serialization of the pruned_block following this. + // 2. An uint8_t indicating the compression status for the serialization of the pruned_block following this. // 3. The serialization of a pruned_block representation of the block for the entry including padding. std::size_t padded_size = b.maximum_pruned_pack_size(segment_compression); @@ -321,8 +365,7 @@ namespace eosio { namespace chain { uint32_t offset = buffer.size(); stream.write((char*)&offset, sizeof(offset)); - fc::unsigned_int compression = static_cast(segment_compression); - fc::raw::pack(stream, compression); + fc::raw::pack(stream, segment_compression); fc::raw::pack(stream, b); return buffer; } @@ -332,7 +375,7 @@ namespace eosio { namespace chain { std::vector buffer; if (version >= 4) { - buffer = detail::block_log_impl::pack(b, segment_compression); + buffer = detail::block_log_impl::pack_v4_log_entry(b, segment_compression); } else { #warning: TODO avoid heap allocation auto block_ptr = b.to_signed_block(); @@ -347,17 +390,6 @@ namespace eosio { namespace chain { return pos; } - void detail::block_log_impl::unpack(fc::cfile_datastream& ds, detail::log_entry& entry) { - fc::raw::unpack(ds, entry.offset); - fc::raw::unpack(ds, entry.compression); - EOS_ASSERT(static_cast(entry.compression.value) == pruned_transaction::cf_compression_type::none, block_log_exception, - "Only support compression_type none"); - entry.block = std::make_shared(); - fc::raw::unpack(ds, *entry.block); - } - - - uint64_t block_log::append(const signed_block_ptr& b) { return this->append(std::make_shared(*b, true), pruned_transaction::cf_compression_type::none); } @@ -468,13 +500,14 @@ namespace eosio { namespace chain { } pruned_block_ptr detail::block_log_impl::read_block(uint64_t pos) { - check_open_files(); block_file.seek(pos); auto ds = block_file.create_datastream(); if (version >= 4) { log_entry entry; - detail::block_log_impl::unpack(ds, entry); - return entry.block; + pruned_block_ptr block = std::make_shared(); + entry.block = block.get(); + detail::unpack(ds, entry); + return block; } else { signed_block block; fc::raw::unpack(ds, block); @@ -483,24 +516,16 @@ namespace eosio { namespace chain { } } - detail::log_entry detail::block_log_impl::read_v4_log_entry(uint64_t pos) { - check_open_files(); + void detail::block_log_impl::read_v4_log_entry(uint64_t pos, detail::log_entry& entry) { block_file.seek(pos); auto ds = block_file.create_datastream(); - log_entry entry; - block_log_impl::unpack(ds, entry); - return entry; + unpack(ds, entry); } void detail::block_log_impl::read_block_header(block_header& bh, uint64_t pos) { -#if defined(SUPPORT_CF_COMPRESSION) - pos += offset_to_block_start(version); -#endif - block_file.seek(pos); auto ds = block_file.create_datastream(); -#if !defined(SUPPORT_CF_COMPRESSION) if (version >= 4 ) { uint32_t offset; fc::unsigned_int compression; @@ -508,7 +533,6 @@ namespace eosio { namespace chain { fc::raw::unpack(ds, compression); EOS_ASSERT( compression.value == static_cast(pruned_transaction::cf_compression_type::none), block_log_exception , "Only \"none\" compression type is supported."); } -#endif fc::raw::unpack(ds, bh); } @@ -618,189 +642,200 @@ namespace eosio { namespace chain { index.complete(); } - fc::path block_log::repair_log( const fc::path& data_dir, uint32_t truncate_at_block ) { - ilog("Recovering Block Log..."); - EOS_ASSERT( fc::is_directory(data_dir) && fc::is_regular_file(data_dir / "blocks.log"), block_log_not_found, - "Block log not found in '${blocks_dir}'", ("blocks_dir", data_dir) ); - - auto now = fc::time_point::now(); - - auto blocks_dir = fc::canonical( data_dir ); - if( blocks_dir.filename().generic_string() == "." ) { - blocks_dir = blocks_dir.parent_path(); - } - auto backup_dir = blocks_dir.parent_path(); - auto blocks_dir_name = blocks_dir.filename(); - EOS_ASSERT( blocks_dir_name.generic_string() != ".", block_log_exception, "Invalid path to blocks directory" ); - backup_dir = backup_dir / blocks_dir_name.generic_string().append("-").append( now ); - - EOS_ASSERT( !fc::exists(backup_dir), block_log_backup_dir_exist, - "Cannot move existing blocks directory to already existing directory '${new_blocks_dir}'", - ("new_blocks_dir", backup_dir) ); - - fc::rename( blocks_dir, backup_dir ); - ilog( "Moved existing blocks directory to backup location: '${new_blocks_dir}'", ("new_blocks_dir", backup_dir) ); - - fc::create_directories(blocks_dir); - auto block_log_path = blocks_dir / "blocks.log"; - - ilog( "Reconstructing '${new_block_log}' from backed up block log", ("new_block_log", block_log_path) ); - - std::fstream old_block_stream; - std::fstream new_block_stream; - - old_block_stream.open( (backup_dir / "blocks.log").generic_string().c_str(), LOG_READ ); - new_block_stream.open( block_log_path.generic_string().c_str(), LOG_WRITE ); - - old_block_stream.seekg( 0, std::ios::end ); - uint64_t end_pos = old_block_stream.tellg(); - old_block_stream.seekg( 0 ); - + static uint32_t verify_log_preamble_and_return_version(fc::datastream& ds, const char* block_file_name) { uint32_t version = 0; - old_block_stream.read( (char*)&version, sizeof(version) ); - EOS_ASSERT( version > 0, block_log_exception, "Block log was not setup properly" ); - EOS_ASSERT( is_supported_version(version), block_log_unsupported_version, - "Unsupported version of block log. Block log version is ${version} while code supports version(s) [${min},${max}]", - ("version", version)("min", block_log::min_supported_version)("max", block_log::max_supported_version) ); - - new_block_stream.write( (char*)&version, sizeof(version) ); + ds.read((char*)&version, sizeof(version)); + EOS_ASSERT(version > 0, block_log_exception, "Block log was not setup properly"); + EOS_ASSERT( + block_log::is_supported_version(version), block_log_unsupported_version, + "Unsupported version of block log. Block log version is ${version} while code supports version(s) " + "[${min},${max}]", + ("version", version)("min", block_log::min_supported_version)("max", block_log::max_supported_version)); uint32_t first_block_num = 1; if (version != 1) { - old_block_stream.read ( (char*)&first_block_num, sizeof(first_block_num) ); + ds.read((char*)&first_block_num, sizeof(first_block_num)); // this assert is only here since repair_log is only used for --hard-replay-blockchain, which removes any // existing state, if another API needs to use it, this can be removed and the check for the first block's // previous block id will need to accommodate this. - EOS_ASSERT( first_block_num == 1, block_log_exception, - "Block log ${file} must contain a genesis state and start at block number 1. This block log " - "starts at block number ${first_block_num}.", - ("file", (backup_dir / "blocks.log").generic_string())("first_block_num", first_block_num)); - - new_block_stream.write( (char*)&first_block_num, sizeof(first_block_num) ); + EOS_ASSERT(first_block_num == 1, block_log_exception, + "Block log ${file} must contain a genesis state and start at block number 1. This block log " + "starts at block number ${first_block_num}.", + ("file", block_file_name)("first_block_num", first_block_num)); } - if (contains_genesis_state(version, first_block_num)) { + if (block_log::contains_genesis_state(version, first_block_num)) { genesis_state gs; - fc::raw::unpack(old_block_stream, gs); + fc::raw::unpack(ds, gs); - auto data = fc::raw::pack( gs ); - new_block_stream.write( data.data(), data.size() ); + auto data = fc::raw::pack(gs); + } else if (block_log::contains_chain_id(version, first_block_num)) { + fc::sha256 chain_id; // I couldn't use chain_id_type because its constructor is private + ds >> chain_id; + } else { + EOS_THROW(block_log_exception, + "Block log ${file} is not supported. version: ${ver} and first_block_num: ${fbn} does not contain " + "a genesis_state nor a chain_id.", + ("file", block_file_name)("ver", version)("fbn", first_block_num)); } - else if (contains_chain_id(version, first_block_num)) { - chain_id_type chain_id; - old_block_stream >> chain_id; - new_block_stream << chain_id; + if (version != 1) { + auto expected_totem = block_log::npos; + std::decay_t actual_totem; + ds.read((char*)&actual_totem, sizeof(actual_totem)); + + EOS_ASSERT( + actual_totem == expected_totem, block_log_exception, + "Expected separator between block log header and blocks was not found( expected: ${e}, actual: ${a} )", + ("e", fc::to_hex((char*)&expected_totem, sizeof(expected_totem)))( + "a", fc::to_hex((char*)&actual_totem, sizeof(actual_totem)))); } - else { - EOS_THROW( block_log_exception, - "Block log ${file} is not supported. version: ${ver} and first_block_num: ${fbn} does not contain " - "a genesis_state nor a chain_id.", - ("file", (backup_dir / "blocks.log").generic_string())("ver", version)("fbn", first_block_num)); + return version; + } + + static bool verify_block(fc::datastream& ds, block_id_type previous, uint64_t pos, block_header& header) { + auto id = header.id(); + if (block_header::num_from_id(previous) + 1 != block_header::num_from_id(id)) { + elog("Block ${num} (${id}) skips blocks. Previous block in block log is block ${prev_num} (${previous})", + ("num", block_header::num_from_id(id))("id", id)("prev_num", block_header::num_from_id(previous))( + "previous", previous)); + } + if (previous != header.previous) { + elog("Block ${num} (${id}) does not link back to previous block. " + "Expected previous: ${expected}. Actual previous: ${actual}.", + ("num", block_header::num_from_id(id))("id", id)("expected", previous)("actual", header.previous)); } - if (version != 1) { - auto expected_totem = npos; - std::decay_t actual_totem; - old_block_stream.read ( (char*)&actual_totem, sizeof(actual_totem) ); + uint64_t tmp_pos = std::numeric_limits::max(); + if (ds.remaining() >= sizeof(tmp_pos)) { + ds.read(reinterpret_cast(&tmp_pos), sizeof(tmp_pos)); + } + if (pos != tmp_pos) { + return false; + } + return true; + } - EOS_ASSERT(actual_totem == expected_totem, block_log_exception, - "Expected separator between block log header and blocks was not found( expected: ${e}, actual: ${a} )", - ("e", fc::to_hex((char*)&expected_totem, sizeof(expected_totem) ))("a", fc::to_hex((char*)&actual_totem, sizeof(actual_totem) ))); + static void write_incomplete_block_data(const fc::path& blocks_dir, fc::time_point now, uint32_t block_num, const char* start, int size) { + auto tail_path = blocks_dir / std::string("blocks-bad-tail-").append(now).append(".log"); + if (!fc::exists(tail_path)) { + std::fstream tail_stream; + tail_stream.open(tail_path.generic_string().c_str(), LOG_WRITE); + tail_stream.write(start, size); - new_block_stream.write( (char*)&actual_totem, sizeof(actual_totem) ); + ilog("Data at tail end of block log which should contain the (incomplete) serialization of block ${num} " + "has been written out to '${tail_path}'.", + ("num", block_num + 1)("tail_path", tail_path)); } + } - std::exception_ptr except_ptr; - vector incomplete_block_data; - optional bad_block; - uint32_t block_num = 0; + fc::path block_log::repair_log(const fc::path& data_dir, uint32_t truncate_at_block) { + ilog("Recovering Block Log..."); + EOS_ASSERT(fc::is_directory(data_dir) && fc::is_regular_file(data_dir / "blocks.log"), block_log_not_found, + "Block log not found in '${blocks_dir}'", ("blocks_dir", data_dir)); - block_id_type previous; + auto now = fc::time_point::now(); - uint64_t pos = old_block_stream.tellg(); - while( pos < end_pos ) { - signed_block tmp; - - try { - fc::raw::unpack(old_block_stream, tmp); - } catch( ... ) { - except_ptr = std::current_exception(); - incomplete_block_data.resize( end_pos - pos ); - old_block_stream.read( incomplete_block_data.data(), incomplete_block_data.size() ); - break; - } + auto blocks_dir = fc::canonical(data_dir); + if (blocks_dir.filename().generic_string() == ".") { + blocks_dir = blocks_dir.parent_path(); + } + auto backup_dir = blocks_dir.parent_path(); + auto blocks_dir_name = blocks_dir.filename(); + EOS_ASSERT(blocks_dir_name.generic_string() != ".", block_log_exception, "Invalid path to blocks directory"); + backup_dir = backup_dir / blocks_dir_name.generic_string().append("-").append(now); - auto id = tmp.id(); - if( block_header::num_from_id(previous) + 1 != block_header::num_from_id(id) ) { - elog( "Block ${num} (${id}) skips blocks. Previous block in block log is block ${prev_num} (${previous})", - ("num", block_header::num_from_id(id))("id", id) - ("prev_num", block_header::num_from_id(previous))("previous", previous) ); - } - if( previous != tmp.previous ) { - elog( "Block ${num} (${id}) does not link back to previous block. " - "Expected previous: ${expected}. Actual previous: ${actual}.", - ("num", block_header::num_from_id(id))("id", id)("expected", previous)("actual", tmp.previous) ); - } - previous = id; + EOS_ASSERT(!fc::exists(backup_dir), block_log_backup_dir_exist, + "Cannot move existing blocks directory to already existing directory '${new_blocks_dir}'", + ("new_blocks_dir", backup_dir)); - uint64_t tmp_pos = std::numeric_limits::max(); - if( (static_cast(old_block_stream.tellg()) + sizeof(pos)) <= end_pos ) { - old_block_stream.read( reinterpret_cast(&tmp_pos), sizeof(tmp_pos) ); - } - if( pos != tmp_pos ) { - bad_block.emplace(std::move(tmp)); - break; - } + fc::rename(blocks_dir, backup_dir); + ilog("Moved existing blocks directory to backup location: '${new_blocks_dir}'", ("new_blocks_dir", backup_dir)); - auto data = fc::raw::pack(tmp); - new_block_stream.write( data.data(), data.size() ); - new_block_stream.write( reinterpret_cast(&pos), sizeof(pos) ); - block_num = tmp.block_num(); - if(block_num % 1000 == 0) - ilog( "Recovered block ${num}", ("num", block_num) ); - pos = new_block_stream.tellp(); - if( block_num == truncate_at_block ) - break; - } + fc::create_directories(blocks_dir); + auto block_log_path = blocks_dir / "blocks.log"; - if( bad_block.valid() ) { - ilog( "Recovered only up to block number ${num}. Last block in block log was not properly committed:\n${last_block}", - ("num", block_num)("last_block", *bad_block) ); - } else if( except_ptr ) { - std::string error_msg; - - try { - std::rethrow_exception(except_ptr); - } catch( const fc::exception& e ) { - error_msg = e.what(); - } catch( const std::exception& e ) { - error_msg = e.what(); - } catch( ... ) { - error_msg = "unrecognized exception"; - } + ilog("Reconstructing '${new_block_log}' from backed up block log", ("new_block_log", block_log_path)); + auto block_log_path_string = block_log_path.generic_string(); + const char* block_file_name = block_log_path_string.c_str(); - ilog( "Recovered only up to block number ${num}. " - "The block ${next_num} could not be deserialized from the block log due to error:\n${error_msg}", - ("num", block_num)("next_num", block_num+1)("error_msg", error_msg) ); + auto pos = 0; + uint32_t block_num = 0; + block_id_type previous; - auto tail_path = blocks_dir / std::string("blocks-bad-tail-").append( now ).append(".log"); - if( !fc::exists(tail_path) && incomplete_block_data.size() > 0 ) { - std::fstream tail_stream; - tail_stream.open( tail_path.generic_string().c_str(), LOG_WRITE ); - tail_stream.write( incomplete_block_data.data(), incomplete_block_data.size() ); + detail::mmap_readonly_data log_data((backup_dir / "blocks.log").generic_string().c_str()); + fc::datastream ds = log_data.create_datastream(); + uint32_t version = verify_log_preamble_and_return_version(ds, block_file_name); + pos = ds.tellp(); + std::string error_msg; - ilog( "Data at tail end of block log which should contain the (incomplete) serialization of block ${num} " - "has been written out to '${tail_path}'.", - ("num", block_num+1)("tail_path", tail_path) ); + try { + signed_block signed_tmp; + pruned_block pruned_tmp; + signed_block_header* hdr = version >=4 ? static_cast(&pruned_tmp) : &signed_tmp; + while (ds.remaining() > 0) { + try { + if (version >= 4) { + detail::log_entry entry; + entry.block = &pruned_tmp; + unpack(ds, entry); + } else { + fc::raw::unpack(ds, &signed_tmp); + } + } catch (...) { + write_incomplete_block_data(blocks_dir, now, block_num, log_data.addr() + pos, log_data.size() - pos); + throw; + } + + if (!verify_block(ds, previous, pos, *hdr)) { + #warning TODO: require fc::reflector to work, change after merged with PR 8901 + // variant block; + // if (version >= 4) + // to_variant(pruned_tmp, block); + // else + // to_variant(signed_tmp, block); + // ilog("Recovered only up to block number ${num}. Last block in block log was not properly " + // "committed:\n${last_block}", + // ("num", block_num)("last_block", block)); + break; + } + + previous = hdr->id(); + block_num = hdr->block_num(); + + if (block_num % 1000 == 0) + ilog("Verified block ${num}", ("num", block_num)); + pos = ds.tellp(); + if (block_num == truncate_at_block) + break; } - } else if( block_num == truncate_at_block && pos < end_pos ) { - ilog( "Stopped recovery of block log early at specified block number: ${stop}.", ("stop", truncate_at_block) ); - } else { - ilog( "Existing block log was undamaged. Recovered all irreversible blocks up to block number ${num}.", ("num", block_num) ); + } catch (const fc::exception& e) { + error_msg = e.what(); + } catch (const std::exception& e) { + error_msg = e.what(); + } catch (...) { + error_msg = "unrecognized exception"; } + int fd_to = creat(block_file_name, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + EOS_ASSERT(fd_to > 0, block_log_exception, "Could not create block log file: ${name}", ("name", block_file_name)); + int nwritten = write(fd_to, log_data.addr(), pos); + EOS_ASSERT( + nwritten == pos, block_log_exception, + "Unable to write the entire block log file: ${name}, expected size=${expected}, written size=${written}", + ("name", block_file_name)("expected", pos)("written", nwritten)); + + if (error_msg.size()) { + ilog("Recovered only up to block number ${num}. " + "The block ${next_num} could not be deserialized from the block log due to error:\n${error_msg}", + ("num", block_num)("next_num", block_num + 1)("error_msg", error_msg)); + } else if (block_num == truncate_at_block && pos < log_data.size()) { + ilog("Stopped recovery of block log early at specified block number: ${stop}.", ("stop", truncate_at_block)); + } else { + ilog("Existing block log was undamaged. Recovered all irreversible blocks up to block number ${num}.", + ("num", block_num)); + } return backup_dir; } @@ -862,36 +897,40 @@ namespace eosio { namespace chain { return true; } - void block_log::prune_transaction(uint32_t block_num, transaction_id_type id) { + bool block_log::prune_transaction(uint32_t block_num, transaction_id_type id) { try { EOS_ASSERT(my->version >= 4, block_log_exception, "The block log version ${version} does not support transaction pruning.", ("version", my->version)); uint64_t pos = get_block_pos(block_num); EOS_ASSERT(pos != npos, block_log_exception, "Specified block_num ${block_num} does not exist in block log.", ("block_num", block_num)); - auto entry = my->read_v4_log_entry(pos); + detail::log_entry entry; + pruned_block block; + entry.block = █ + my->read_v4_log_entry(pos, entry); - EOS_ASSERT(entry.block->block_num() == block_num, block_log_exception, + EOS_ASSERT(block.block_num() == block_num, block_log_exception, "Wrong block was read from block log."); auto pruner = overloaded{[](transaction_id_type&) { return false; }, [&id](pruned_transaction& ptx) { return ptx.id() == id && prune(ptx); }}; - for (auto trx : entry.block->transactions) { + for (auto trx : block.transactions) { if (trx.trx.visit(pruner)) { my->block_file.seek(pos); - std::vector buffer = detail::block_log_impl::pack(*entry.block, static_cast(entry.compression.value)); + std::vector buffer = detail::block_log_impl::pack_v4_log_entry(*entry.block, static_cast(entry.compression)); + EOS_ASSERT(buffer.size() <= entry.offset, block_log_exception, ""); my->block_file.write(buffer.data(), buffer.size()); my->block_file.flush(); - return; + return true; } } - EOS_ASSERT(true, block_log_exception, - "Block ${block_num} does not contain transaction ${id}.", ("id", id)("block_num", block_num)); - + elog("Block ${block_num} does not contain transaction ${id}.", ("id", id)("block_num", block_num)); + return false; } - FC_LOG_AND_RETHROW() + FC_CAPTURE_AND_LOG(()) + return false; } int trim_data::blknum_offset_from_block_entry(uint32_t block_log_version) { @@ -918,6 +957,8 @@ namespace eosio { namespace chain { EOS_ASSERT( _file, block_log_exception, "Could not open Block log file at '${blocks_log}'", ("blocks_log", _block_file_name) ); _end_of_buffer_position = _unset_position; + // TODO: reimplemented with mmap_readonly_data() and verify_log_preamble_and_return_version() + //read block log to see if version 1 or 2 and get first blocknum (implicit 1 if version 1) _version = 0; auto size = fread((char*)&_version, sizeof(_version), 1, _file.get()); @@ -1138,8 +1179,8 @@ namespace eosio { namespace chain { new_block_file.close(); new_block_file.open( LOG_RW_C ); - static_assert( block_log::max_supported_version >= 3, - "Code was written to support format of version 3 or greater, need to update this code for latest format." ); + static_assert( block_log::max_supported_version >= 4, + "Code was written to support format of version 4 or greater, need to update this code for latest format." ); uint32_t version = block_log::max_supported_version; new_block_file.seek(0); new_block_file.write((char*)&version, sizeof(version)); diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index fd2d78cfe9c..fa5620b7cae 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -56,11 +56,12 @@ namespace eosio { namespace chain { * Return offset of block in file, or block_log::npos if it does not exist. */ uint64_t get_block_pos(uint32_t block_num) const; +#warning TODO: change to pruned_block after merged with PR 8901 signed_block_header* head() const; [[deprecated]] block_id_type head_id() const; // use head()->id() instead uint32_t first_block_num() const; - void prune_transaction(uint32_t block_num, transaction_id_type id); + bool prune_transaction(uint32_t block_num, transaction_id_type id); static const uint64_t npos = std::numeric_limits::max(); diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 14cd9d1c544..93dbf5e396b 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -206,7 +206,7 @@ namespace eosio { namespace chain { }; struct prunable_transaction_data { - enum class compression_type { + enum class compression_type : uint8_t { none = 0, zlib = 1, }; From 2a125a53caf30843786a4390fb1624d8075ed6d5 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 2 Apr 2020 15:41:08 -0500 Subject: [PATCH 015/142] Use boost mapped_file_source --- libraries/chain/block_log.cpp | 56 ++++++------------------------- unittests/restart_chain_tests.cpp | 2 ++ 2 files changed, 12 insertions(+), 46 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 5245d34e26c..00d3b58dc35 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #define LOG_READ (std::ios::in | std::ios::binary) #define LOG_WRITE (std::ios::out | std::ios::binary | std::ios::app) @@ -212,44 +212,6 @@ namespace eosio { namespace chain { FILE* const _file; const std::string _filename; }; - - /* - * @brief A utility class to wrap mmap() system call for RAII usage - * - */ - class mmap_readonly_data { - int fd; - uint64_t _size; - void* _addr; - - public: - mmap_readonly_data(const char* filename) { - int fd = ::open(filename, O_RDONLY); - EOS_ASSERT(fd != -1, block_log_exception, "Unable to open file: ${filename}", ("filename", filename)); - - struct stat st; - stat(filename, &st); - _size = st.st_size; - - _addr = mmap(0, st.st_size, PROT_READ, MAP_SHARED, fd, 0); - if (_addr == MAP_FAILED) { - int err_num = errno; - close(fd); - EOS_ASSERT(true, block_log_exception, "mmap() failure: ${msg}", ("msg", (strerror(err_num)))); - } - } - ~mmap_readonly_data() { - munmap(_addr, _size); - close(fd); - } - - mmap_readonly_data(const mmap_readonly_data&) = delete; - mmap_readonly_data& operator=(const mmap_readonly_data&) = delete; - - const char* addr() const { return static_cast(_addr); } - uint64_t size() const { return _size; } - fc::datastream create_datastream() const { return fc::datastream(addr(), size()); } - }; } block_log::block_log(const fc::path& data_dir) @@ -764,8 +726,8 @@ namespace eosio { namespace chain { uint32_t block_num = 0; block_id_type previous; - detail::mmap_readonly_data log_data((backup_dir / "blocks.log").generic_string().c_str()); - fc::datastream ds = log_data.create_datastream(); + boost::iostreams::mapped_file_source log_data((backup_dir / "blocks.log").generic_string()); + fc::datastream ds(log_data.data(), log_data.size()); uint32_t version = verify_log_preamble_and_return_version(ds, block_file_name); pos = ds.tellp(); std::string error_msg; @@ -784,7 +746,7 @@ namespace eosio { namespace chain { fc::raw::unpack(ds, &signed_tmp); } } catch (...) { - write_incomplete_block_data(blocks_dir, now, block_num, log_data.addr() + pos, log_data.size() - pos); + write_incomplete_block_data(blocks_dir, now, block_num, log_data.data() + pos, log_data.size() - pos); throw; } @@ -818,9 +780,11 @@ namespace eosio { namespace chain { error_msg = "unrecognized exception"; } - int fd_to = creat(block_file_name, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); - EOS_ASSERT(fd_to > 0, block_log_exception, "Could not create block log file: ${name}", ("name", block_file_name)); - int nwritten = write(fd_to, log_data.addr(), pos); + FILE* fp = fopen(block_file_name, "w"); + EOS_ASSERT(fp != nullptr, block_log_exception, "Could not create block log file: ${name}", ("name", block_file_name)); + int nwritten = fwrite(log_data.data(), 1, pos, fp); + fclose(fp); + EOS_ASSERT( nwritten == pos, block_log_exception, "Unable to write the entire block log file: ${name}, expected size=${expected}, written size=${written}", @@ -957,7 +921,7 @@ namespace eosio { namespace chain { EOS_ASSERT( _file, block_log_exception, "Could not open Block log file at '${blocks_log}'", ("blocks_log", _block_file_name) ); _end_of_buffer_position = _unset_position; - // TODO: reimplemented with mmap_readonly_data() and verify_log_preamble_and_return_version() + // TODO: reimplemented with mapped_file_source and verify_log_preamble_and_return_version() //read block log to see if version 1 or 2 and get first blocknum (implicit 1 if version 1) _version = 0; diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index c5df85eadc2..653e5b16fa3 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -162,6 +162,8 @@ BOOST_AUTO_TEST_CASE(test_restart_from_block_log) { BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay1))); BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay2))); BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay3))); + + BOOST_REQUIRE_NO_THROW(block_log::repair_log(copied_config.blocks_dir, 0)); } BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log) { From eeb89fa5650e0c17ba9ea27f1fb6ccc32fd1df9a Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 2 Apr 2020 15:45:54 -0500 Subject: [PATCH 016/142] Update comments --- .../chain/include/eosio/chain/abi_serializer.hpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 4b597ddfadc..7690f317827 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -447,10 +447,11 @@ namespace impl { /** * overload of to_variant_object for packed_transaction_v0 * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of ptrx.transaction + * This matches the FC_REFLECT for packed_transaction_v0 type with the addition of "transaction" which is provided + * to allow extracting the contents of ptrx.get_transaction() */ template - static void add( mutable_variant_object &out, const char* name, const packed_transaction_v0& ptrx, Resolver resolver, abi_traverse_context& ctx ) + static void add( mutable_variant_object& out, const char* name, const packed_transaction_v0& ptrx, Resolver resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 4); auto h = ctx.enter_scope(); @@ -469,10 +470,11 @@ namespace impl { /** * overload of to_variant_object for packed_transaction, providing original packed_transaction_v0 variant layout * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of ptrx.transaction + * This matches the FC_REFLECT for packed_transaction_v0 type with the addition of "transaction" which is provided + * to allow extracting the contents of ptrx.get_transaction(). The generated variant should match above method. */ template - static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) + static void add( mutable_variant_object& out, const char* name, const packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) { static_assert(fc::reflector::total_member_count == 3); auto h = ctx.enter_scope(); @@ -531,6 +533,7 @@ namespace impl { static void add( mutable_variant_object &out, const char* name, const signed_block& block, Resolver resolver, abi_traverse_context& ctx ) { // TODO: this could be faster by going directly to variant + // TODO: Conversion to packed_transaction_v0 could be done via above if we go directly to variant auto sb_v0 = block.to_signed_block_v0(); add( out, name, *sb_v0, std::move( resolver ), ctx ); } From 43eba6b3b22acc36feb6674b5f90cb5121bb66b3 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 2 Apr 2020 15:52:47 -0500 Subject: [PATCH 017/142] Add estimated size to packed_transaction to more closely track size of transaction --- .../chain/include/eosio/chain/transaction.hpp | 6 ++- .../chain/unapplied_transaction_queue.hpp | 2 +- libraries/chain/transaction.cpp | 48 ++++++++++++------- libraries/chain/transaction_context.cpp | 2 +- plugins/chain_plugin/chain_plugin.cpp | 2 +- .../eosio/chain_plugin/chain_plugin.hpp | 2 +- plugins/net_plugin/net_plugin.cpp | 7 +-- unittests/misc_tests.cpp | 1 + 8 files changed, 41 insertions(+), 29 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index c757be24a3e..8180462c041 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -277,6 +277,7 @@ namespace eosio { namespace chain { uint32_t get_unprunable_size()const; uint32_t get_prunable_size()const; + uint32_t get_estimated_size()const { return estimated_size; } digest_type packed_digest()const; @@ -290,7 +291,7 @@ namespace eosio { namespace chain { // Returns nullptr if any context_free_data segment was pruned const vector* get_context_free_data()const; // Returns nullptr if the context_free_data segment was pruned or segment_ordinal is out of range. - const bytes* get_context_free_data(std::size_t segment_ordinal); + const bytes* get_context_free_data(std::size_t segment_ordinal)const; const fc::enum_type& get_compression()const { return compression; } const bytes& get_packed_transaction()const { return packed_trx; } const prunable_transaction_data& get_prunable_data() const { return prunable_data; } @@ -306,12 +307,13 @@ namespace eosio { namespace chain { friend struct fc::has_reflector_init; void reflector_init(); private: + uint32_t estimated_size = 0; fc::enum_type compression; prunable_transaction_data prunable_data; bytes packed_trx; private: - // cache unpacked trx, for thread safety do not modify after construction + // cache unpacked trx, for thread safety do not modify any attributes after construction signed_transaction unpacked_trx; transaction_id_type trx_id; }; diff --git a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp index 9d46159a72c..c086ff4fc26 100644 --- a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp +++ b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp @@ -255,7 +255,7 @@ class unapplied_transaction_queue { static uint64_t calc_size( const transaction_metadata_ptr& trx ) { // packed_trx caches unpacked transaction so double - return (trx->packed_trx()->get_unprunable_size() + trx->packed_trx()->get_prunable_size()) * 2 + sizeof( *trx ); + return trx->packed_trx()->get_estimated_size() + sizeof( *trx ); } }; diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 9fd9673a3e8..b10c00081b5 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -215,8 +215,9 @@ static vector zlib_decompress_context_free_data(const bytes& data) { return unpack_context_free_data(out); } -static transaction zlib_decompress_transaction(const bytes& data) { +static transaction zlib_decompress_transaction(const bytes& data, uint32_t& packed_size) { bytes out = zlib_decompress(data); + packed_size = out.size(); return unpack_transaction(out); } @@ -245,8 +246,9 @@ static bytes zlib_compress_context_free_data(const vector& cfd ) { return out; } -static bytes zlib_compress_transaction(const transaction& t) { +static bytes zlib_compress_transaction(const transaction& t, uint32_t& packed_size) { bytes in = pack_transaction(t); + packed_size = in.size(); bytes out; bio::filtering_ostream comp; comp.push(bio::zlib_compressor(bio::zlib::best_compression)); @@ -309,18 +311,19 @@ void packed_transaction_v0::reflector_init() // called after construction, but always on the same thread and before packed_transaction passed to any other threads static_assert(fc::raw::has_feature_reflector_init_on_unpacked_reflected_types, "FC unpack needs to call reflector_init otherwise unpacked_trx will not be initialized"); - EOS_ASSERT( unpacked_trx.expiration == time_point_sec(), tx_decompression_error, "packed_transaction already unpacked" ); + EOS_ASSERT( unpacked_trx.expiration == time_point_sec(), tx_decompression_error, "packed_transaction_v0 already unpacked" ); local_unpack_transaction({}); local_unpack_context_free_data(); } -static transaction unpack_transaction(const bytes& packed_trx, packed_transaction_v0::compression_type compression) { +static transaction unpack_transaction(const bytes& packed_trx, uint32_t& packed_size, packed_transaction_v0::compression_type compression) { try { switch( compression ) { case packed_transaction_v0::compression_type::none: + packed_size = packed_trx.size(); return unpack_transaction( packed_trx ); case packed_transaction_v0::compression_type::zlib: - return zlib_decompress_transaction( packed_trx ); + return zlib_decompress_transaction( packed_trx, packed_size ); default: EOS_THROW( unknown_transaction_compression, "Unknown transaction compression algorithm" ); } @@ -329,7 +332,8 @@ static transaction unpack_transaction(const bytes& packed_trx, packed_transactio void packed_transaction_v0::local_unpack_transaction(vector&& context_free_data) { - unpacked_trx = signed_transaction( unpack_transaction( packed_trx, compression ), signatures, std::move(context_free_data) ); + uint32_t packed_size = 0; + unpacked_trx = signed_transaction( unpack_transaction( packed_trx, packed_size, compression ), signatures, std::move(context_free_data) ); trx_id = unpacked_trx.id(); } @@ -353,13 +357,16 @@ void packed_transaction_v0::local_unpack_context_free_data() unpacked_trx.context_free_data = unpack_context_free_data( packed_context_free_data, compression ); } -static bytes pack_transaction(const transaction& trx, packed_transaction_v0::compression_type compression) { +static bytes pack_transaction(const transaction& trx, uint32_t& packed_size, packed_transaction_v0::compression_type compression) { try { switch(compression) { - case packed_transaction_v0::compression_type::none: - return pack_transaction(trx); + case packed_transaction_v0::compression_type::none: { + bytes result = pack_transaction( trx ); + packed_size = result.size(); + return result; + } case packed_transaction_v0::compression_type::zlib: - return zlib_compress_transaction(trx); + return zlib_compress_transaction(trx, packed_size); default: EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); } @@ -368,7 +375,8 @@ static bytes pack_transaction(const transaction& trx, packed_transaction_v0::com void packed_transaction_v0::local_pack_transaction() { - packed_trx = pack_transaction(unpacked_trx, compression); + uint32_t packed_size = 0; + packed_trx = pack_transaction(unpacked_trx, packed_size, compression); } static bytes pack_context_free_data( const vector& cfd, packed_transaction_v0::compression_type compression ) { @@ -468,18 +476,22 @@ static prunable_transaction_data make_prunable_transaction_data( bool legacy, co packed_transaction::packed_transaction(const signed_transaction& t, bool legacy, compression_type _compression) : compression(_compression), prunable_data(make_prunable_transaction_data(legacy, t, _compression)), - packed_trx(pack_transaction(t, compression)), + packed_trx(pack_transaction(t, estimated_size, compression)), unpacked_trx(t), trx_id(unpacked_trx.id()) -{} +{ + estimated_size = estimated_size * 2 + get_prunable_size(); +} packed_transaction::packed_transaction(signed_transaction&& t, bool legacy, compression_type _compression) : compression(_compression), prunable_data(make_prunable_transaction_data(legacy, t, _compression)), - packed_trx(pack_transaction(t, compression)), + packed_trx(pack_transaction(t, estimated_size, compression)), unpacked_trx(std::move(t)), trx_id(unpacked_trx.id()) -{} +{ + estimated_size = estimated_size * 2 + get_prunable_size(); +} uint32_t packed_transaction::get_unprunable_size()const { uint64_t size = config::fixed_net_overhead_of_packed_trx; @@ -541,7 +553,7 @@ const bytes* maybe_get_context_free_data(const prunable_transaction_data::partia const bytes* maybe_get_context_free_data(const prunable_transaction_data::full_legacy&, std::size_t) { return nullptr; } const bytes* maybe_get_context_free_data(const prunable_transaction_data::full&, std::size_t) { return nullptr; } -const bytes* packed_transaction::get_context_free_data(std::size_t segment_ordinal) { +const bytes* packed_transaction::get_context_free_data(std::size_t segment_ordinal) const { if (segment_ordinal < unpacked_trx.context_free_data.size()) { return &unpacked_trx.context_free_data[segment_ordinal]; } else { @@ -577,10 +589,12 @@ void packed_transaction::reflector_init() "FC unpack needs to call reflector_init otherwise unpacked_trx will not be initialized"); EOS_ASSERT( unpacked_trx.expiration == time_point_sec(), tx_decompression_error, "packed_transaction already unpacked" ); const auto* signatures = get_signatures(); - unpacked_trx = signed_transaction(unpack_transaction(packed_trx, compression), + uint32_t packed_size = 0; + unpacked_trx = signed_transaction(unpack_transaction(packed_trx, packed_size, compression), signatures ? *signatures : std::vector{}, prunable_data.prunable_data.visit([&](const auto& obj) { return maybe_unpack_context_free_data(obj, compression); })); trx_id = unpacked_trx.id(); + estimated_size = packed_size * 2 + get_prunable_size(); // x2 since stored packed and unpacked } } } // eosio::chain diff --git a/libraries/chain/transaction_context.cpp b/libraries/chain/transaction_context.cpp index 25346fd32e3..5ca3a304d0b 100644 --- a/libraries/chain/transaction_context.cpp +++ b/libraries/chain/transaction_context.cpp @@ -52,7 +52,7 @@ namespace eosio { namespace chain { ,trx(t) ,id(trx_id) ,undo_session() - ,trace(std::make_shared()) + ,trace(std::make_unique()) ,start(s) ,transaction_timer(std::move(tmr)) ,net_usage(trace->net_usage) diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index 6766d346734..ea660a507f6 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -2135,7 +2135,7 @@ fc::variant read_only::get_block_header_state(const get_block_header_state_param void read_write::push_block(read_write::push_block_params&& params, next_function next) { try { - app().get_method()(std::make_shared(std::move(params)), {}); + app().get_method()(std::make_shared(params, true), {}); next(read_write::push_block_results{}); } catch ( boost::interprocess::bad_alloc& ) { chain_plugin::handle_db_exhaustion(); diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp index 81bcf715a33..3bab68aa970 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp @@ -618,7 +618,7 @@ class read_write { read_write(controller& db, const fc::microseconds& abi_serializer_max_time, bool api_accept_transactions); void validate() const; - using push_block_params = chain::signed_block; + using push_block_params = chain::signed_block_v0; using push_block_results = empty; void push_block(push_block_params&& params, chain::plugin_interface::next_function next); diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 68bd657c494..b524fdccbe7 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2855,12 +2855,7 @@ namespace eosio { } size_t calc_trx_size( const packed_transaction_ptr& trx ) { - // transaction is stored packed and unpacked, double packed_size and size of signed as an approximation of use - const auto cfd_size = trx->get_context_free_data() != nullptr ? trx->get_context_free_data()->size() : 0; - auto sig_size = trx->get_signatures() != nullptr ? trx->get_signatures()->size() : 0; - sig_size *= sizeof(signature_type); - return (trx->get_packed_transaction().size() * 2 + sizeof(trx->get_signed_transaction())) * 2 + - cfd_size + sig_size; + return trx->get_estimated_size(); } void connection::handle_message( packed_transaction_ptr trx ) { diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index c45e110b446..f7b6f86a05f 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -971,6 +971,7 @@ BOOST_AUTO_TEST_CASE(pruned_transaction_test) { BOOST_TEST(pruned.get_unprunable_size() == packed.get_unprunable_size()); std::size_t max_size = pruned.maximum_pruned_pack_size(packed_transaction::cf_compression_type::none); BOOST_TEST(fc::raw::pack_size(pruned) <= max_size); + BOOST_TEST(fc::raw::pack_size(pruned) < pruned.get_estimated_size()); pruned.prune_all(); BOOST_TEST(packed.packed_digest().str() == pruned.packed_digest().str()); From f83d51a5ad7ae6d662f84cf04fc899252310383a Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 2 Apr 2020 16:52:16 -0500 Subject: [PATCH 018/142] Use packed_transaction_v0::compression_type where appropriate --- .../include/eosio/chain/abi_serializer.hpp | 10 +-------- programs/cleos/main.cpp | 22 +++++++++---------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 7690f317827..612d1432793 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -749,14 +749,6 @@ namespace impl { "Failed to deserialize data for ${account}:${name}", ("account", act.account)("name", act.name)); } - template - static void extract( const variant& v, signed_block& sb, Resolver resolver, abi_traverse_context& ctx ) - { - signed_block_v0 v0; - extract( v, v0, std::move( resolver ), ctx ); - sb = signed_block( v0, true ); - } - template static void extract( const variant& v, packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) { @@ -773,7 +765,7 @@ namespace impl { EOS_ASSERT(vo.contains("signatures"), packed_transaction_type_exception, "Missing signatures"); EOS_ASSERT(vo.contains("compression"), packed_transaction_type_exception, "Missing compression"); std::vector signatures; - packed_transaction::compression_type compression; + packed_transaction_v0::compression_type compression; from_variant(vo["signatures"], signatures); from_variant(vo["compression"], compression); diff --git a/programs/cleos/main.cpp b/programs/cleos/main.cpp index 10af0b464e1..5d62d94d55b 100644 --- a/programs/cleos/main.cpp +++ b/programs/cleos/main.cpp @@ -370,7 +370,7 @@ void sign_transaction(signed_transaction& trx, fc::variant& required_keys, const } fc::variant push_transaction( signed_transaction& trx, const std::vector& signing_keys = std::vector(), - packed_transaction::compression_type compression = packed_transaction::compression_type::none ) { + packed_transaction_v0::compression_type compression = packed_transaction_v0::compression_type::none ) { auto info = get_info(); if (trx.signatures.size() == 0) { // #5445 can't change txn content if already signed @@ -434,7 +434,7 @@ fc::variant push_transaction( signed_transaction& trx, const std::vector&& actions, packed_transaction::compression_type compression = packed_transaction::compression_type::none, const std::vector& signing_keys = std::vector() ) { +fc::variant push_actions(std::vector&& actions, packed_transaction_v0::compression_type compression = packed_transaction_v0::compression_type::none, const std::vector& signing_keys = std::vector() ) { signed_transaction trx; trx.actions = std::forward(actions); @@ -567,7 +567,7 @@ void print_result( const fc::variant& result ) { try { } FC_CAPTURE_AND_RETHROW( (result) ) } using std::cout; -void send_actions(std::vector&& actions, const std::vector& signing_keys = std::vector(), packed_transaction::compression_type compression = packed_transaction::compression_type::none ) { +void send_actions(std::vector&& actions, const std::vector& signing_keys = std::vector(), packed_transaction_v0::compression_type compression = packed_transaction_v0::compression_type::none ) { std::ofstream out; if (tx_json_save_file.length()) { out.open(tx_json_save_file); @@ -2521,11 +2521,11 @@ int main( int argc, char** argv ) { abi_serializer::from_variant( trx_var, trx, abi_serializer_resolver, abi_serializer::create_yield_function( abi_serializer_max_time ) ); } EOS_RETHROW_EXCEPTIONS( transaction_type_exception, "Invalid transaction format: '${data}'", ("data", fc::json::to_string(trx_var, fc::time_point::maximum()))) - std::cout << fc::json::to_pretty_string( packed_transaction_v0( trx, packed_transaction::compression_type::none )) << std::endl; + std::cout << fc::json::to_pretty_string( packed_transaction_v0( trx, packed_transaction_v0::compression_type::none )) << std::endl; } else { try { signed_transaction trx = trx_var.as(); - std::cout << fc::json::to_pretty_string( fc::variant( packed_transaction_v0( trx, packed_transaction::compression_type::none ))) << std::endl; + std::cout << fc::json::to_pretty_string( fc::variant( packed_transaction_v0( trx, packed_transaction_v0::compression_type::none ))) << std::endl; } EOS_RETHROW_EXCEPTIONS( transaction_type_exception, "Fail to convert transaction, --pack-action-data likely needed" ) } }); @@ -2538,9 +2538,9 @@ int main( int argc, char** argv ) { unpack_transaction->add_flag("--unpack-action-data", unpack_action_data_flag, localized("Unpack all action data within transaction, needs interaction with ${n}", ("n", node_executable_name))); unpack_transaction->set_callback([&] { fc::variant packed_trx_var = json_from_file_or_string( packed_transaction_json ); - packed_transaction packed_trx; + packed_transaction_v0 packed_trx; try { - fc::from_variant( packed_trx_var, packed_trx ); + fc::from_variant( packed_trx_var, packed_trx ); } EOS_RETHROW_EXCEPTIONS( transaction_type_exception, "Invalid packed transaction format: '${data}'", ("data", fc::json::to_string(packed_trx_var, fc::time_point::maximum()))) signed_transaction strx = packed_trx.get_signed_transaction(); @@ -3110,7 +3110,7 @@ int main( int argc, char** argv ) { actions.emplace_back( create_setcode(name(account), code_bytes ) ); if ( shouldSend ) { std::cerr << localized("Setting Code...") << std::endl; - send_actions(std::move(actions), signing_keys_opt.get_keys(), packed_transaction::compression_type::zlib); + send_actions(std::move(actions), signing_keys_opt.get_keys(), packed_transaction_v0::compression_type::zlib); } } else { std::cerr << localized("Skipping set code because the new code is the same as the existing code") << std::endl; @@ -3158,7 +3158,7 @@ int main( int argc, char** argv ) { } EOS_RETHROW_EXCEPTIONS(abi_type_exception, "Fail to parse ABI JSON") if ( shouldSend ) { std::cerr << localized("Setting ABI...") << std::endl; - send_actions(std::move(actions), signing_keys_opt.get_keys(), packed_transaction::compression_type::zlib); + send_actions(std::move(actions), signing_keys_opt.get_keys(), packed_transaction_v0::compression_type::zlib); } } else { std::cerr << localized("Skipping set abi because the new abi is the same as the existing abi") << std::endl; @@ -3175,7 +3175,7 @@ int main( int argc, char** argv ) { set_abi_callback(); if (actions.size()) { std::cerr << localized("Publishing contract...") << std::endl; - send_actions(std::move(actions), signing_keys_opt.get_keys(), packed_transaction::compression_type::zlib); + send_actions(std::move(actions), signing_keys_opt.get_keys(), packed_transaction_v0::compression_type::zlib); } else { std::cout << "no transaction is sent" << std::endl; } @@ -3478,7 +3478,7 @@ int main( int argc, char** argv ) { } if(push_trx) { - auto trx_result = call(push_txn_func, packed_transaction_v0(trx, packed_transaction::compression_type::none)); + auto trx_result = call(push_txn_func, packed_transaction_v0(trx, packed_transaction_v0::compression_type::none)); std::cout << fc::json::to_pretty_string(trx_result) << std::endl; } else { std::cout << fc::json::to_pretty_string(trx) << std::endl; From c1a61c4e7fa82a4148037c82016857f6610631d2 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 2 Apr 2020 16:56:09 -0500 Subject: [PATCH 019/142] Updated comments. Going to work new net_version in diff PR. --- plugins/net_plugin/net_plugin.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index b524fdccbe7..7efe2be5462 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -418,7 +418,7 @@ namespace eosio { constexpr uint16_t block_id_notify = 2; // reserved. feature was removed. next net_version should be 3 constexpr uint16_t pruned_types = 3; // support packed_transaction & signed_block in addition to prior *_v0 versions - constexpr uint16_t net_version = pruned_types; + constexpr uint16_t net_version = proto_explicit_sync; /** * Index by start_block_num @@ -1243,15 +1243,15 @@ namespace eosio { } static std::shared_ptr> create_send_buffer( const signed_block_v0_ptr& sb ) { - // this implementation is to avoid copy of signed_block to net_message - // matches which of net_message for signed_block + // this implementation is to avoid copy of signed_block_v0 to net_message + // matches which of net_message for signed_block_v0 fc_dlog( logger, "sending block ${bn}", ("bn", sb->block_num()) ); return create_send_buffer( signed_block_v0_which, *sb ); } static std::shared_ptr> create_send_buffer( const packed_transaction_v0& trx ) { - // this implementation is to avoid copy of packed_transaction to net_message - // matches which of net_message for packed_transaction + // this implementation is to avoid copy of packed_transaction_v0 to net_message + // matches which of net_message for packed_transaction_v0 return create_send_buffer( packed_transaction_v0_which, trx ); } From eef41b48a584d7d6fd7cd4aec616746557fb3459 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 08:50:33 -0500 Subject: [PATCH 020/142] Added support for add of signed_block. Removed support for extract of signed_block & packed_transaction. --- .../include/eosio/chain/abi_serializer.hpp | 63 ++++++++++--------- plugins/chain_plugin/chain_plugin.cpp | 16 +++-- unittests/abi_tests.cpp | 2 +- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 612d1432793..eb3a235d8c7 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -525,29 +525,10 @@ namespace impl { out(name, std::move(mvo)); } - - /** - * overload of to_variant_object for signed_block, support old signed_block_v0 format - */ - template - static void add( mutable_variant_object &out, const char* name, const signed_block& block, Resolver resolver, abi_traverse_context& ctx ) - { - // TODO: this could be faster by going directly to variant - // TODO: Conversion to packed_transaction_v0 could be done via above if we go directly to variant - auto sb_v0 = block.to_signed_block_v0(); - add( out, name, *sb_v0, std::move( resolver ), ctx ); - } - - /** - * overload of to_variant_object for signed_block_v0 - * - * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of - * block.header_extensions and block.block_extensions - */ - template - static void add( mutable_variant_object &out, const char* name, const signed_block_v0& block, Resolver resolver, abi_traverse_context& ctx ) + template + static void add_signed_block( mutable_variant_object& out, const char* name, const SignedBlock& block, Resolver resolver, abi_traverse_context& ctx ) { - static_assert(fc::reflector::total_member_count == 12); + static_assert( std::is_same_v || std::is_same_v ); auto h = ctx.enter_scope(); mutable_variant_object mvo; mvo("timestamp", block.timestamp); @@ -562,7 +543,8 @@ namespace impl { // process contents of block.header_extensions flat_multimap header_exts = block.validate_and_extract_header_extensions(); if ( header_exts.count(protocol_feature_activation::extension_id() > 0) ) { - const auto& new_protocol_features = header_exts.lower_bound(protocol_feature_activation::extension_id())->second.get().protocol_features; + const auto& new_protocol_features = + header_exts.lower_bound(protocol_feature_activation::extension_id())->second.template get().protocol_features; vector pf_array; pf_array.reserve(new_protocol_features.size()); for (auto feature : new_protocol_features) { @@ -573,7 +555,8 @@ namespace impl { mvo("new_protocol_features", pf_array); } if ( header_exts.count(producer_schedule_change_extension::extension_id())) { - const auto& new_producer_schedule = header_exts.lower_bound(producer_schedule_change_extension::extension_id())->second.get(); + const auto& new_producer_schedule = + header_exts.lower_bound(producer_schedule_change_extension::extension_id())->second.template get(); mvo("new_producer_schedule", new_producer_schedule); } @@ -583,12 +566,36 @@ namespace impl { // process contents of block.block_extensions auto block_exts = block.validate_and_extract_extensions(); if ( block_exts.count(additional_block_signatures_extension::extension_id()) > 0) { - const auto& additional_signatures = block_exts.lower_bound(additional_block_signatures_extension::extension_id())->second.get(); + const auto& additional_signatures = + block_exts.lower_bound(additional_block_signatures_extension::extension_id())->second.template get(); mvo("additional_signatures", additional_signatures); } out(name, std::move(mvo)); } + + /** + * overload of to_variant_object for signed_block, support old signed_block_v0 format + */ + template + static void add( mutable_variant_object& out, const char* name, const signed_block& block, Resolver resolver, abi_traverse_context& ctx ) + { + static_assert(fc::reflector::total_member_count == 13); + add_signed_block( out, name, block, std::move(resolver), ctx ); + } + + /** + * overload of to_variant_object for signed_block_v0 + * + * This matches the FC_REFLECT for this type, but this is provided to allow extracting the contents of + * block.header_extensions and block.block_extensions + */ + template + static void add( mutable_variant_object &out, const char* name, const signed_block_v0& block, Resolver resolver, abi_traverse_context& ctx ) + { + static_assert(fc::reflector::total_member_count == 12); + add_signed_block( out, name, block, std::move(resolver), ctx ); + } }; /** @@ -752,9 +759,7 @@ namespace impl { template static void extract( const variant& v, packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) { - packed_transaction_v0 v0; - extract( v, v0, std::move( resolver ), ctx ); - ptrx = packed_transaction( std::move( v0 ), true ); + EOS_THROW( packed_transaction_type_exception, "not implemented, use packed_transaction_v0" ) } template @@ -872,6 +877,8 @@ void abi_serializer::to_variant( const T& o, variant& vo, Resolver resolver, con template void abi_serializer::from_variant( const variant& v, T& o, Resolver resolver, const yield_function_t& yield ) try { + static_assert( !std::is_same_v, "use packed_transaction_v0" ); + static_assert( !std::is_same_v, "use signed_block_v0" ); impl::abi_traverse_context ctx( yield ); impl::abi_from_variant::extract(v, o, resolver, ctx); } FC_RETHROW_EXCEPTIONS(error, "Failed to deserialize variant", ("variant",v)) diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index ea660a507f6..d79b826f90a 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -2146,13 +2146,15 @@ void read_write::push_block(read_write::push_block_params&& params, next_functio void read_write::push_transaction(const read_write::push_transaction_params& params, next_function next) { try { - auto pretty_input = std::make_shared(); + packed_transaction_v0 input_trx_v0; auto resolver = make_resolver(this, abi_serializer::create_yield_function( abi_serializer_max_time )); + packed_transaction_ptr input_trx; try { - abi_serializer::from_variant(params, *pretty_input, std::move( resolver ), abi_serializer::create_yield_function( abi_serializer_max_time )); + abi_serializer::from_variant(params, input_trx_v0, std::move( resolver ), abi_serializer::create_yield_function( abi_serializer_max_time )); + input_trx = std::make_shared( input_trx_v0, true ); } EOS_RETHROW_EXCEPTIONS(chain::packed_transaction_type_exception, "Invalid packed transaction") - app().get_method()(pretty_input, true, + app().get_method()(input_trx, true, [this, next](const fc::static_variant& result) -> void { if (result.contains()) { next(result.get()); @@ -2265,13 +2267,15 @@ void read_write::push_transactions(const read_write::push_transactions_params& p void read_write::send_transaction(const read_write::send_transaction_params& params, next_function next) { try { - auto pretty_input = std::make_shared(); + packed_transaction_v0 input_trx_v0; auto resolver = make_resolver(this, abi_serializer::create_yield_function( abi_serializer_max_time )); + packed_transaction_ptr input_trx; try { - abi_serializer::from_variant(params, *pretty_input, resolver, abi_serializer::create_yield_function( abi_serializer_max_time )); + abi_serializer::from_variant(params, input_trx_v0, std::move( resolver ), abi_serializer::create_yield_function( abi_serializer_max_time )); + input_trx = std::make_shared( input_trx_v0, true ); } EOS_RETHROW_EXCEPTIONS(chain::packed_transaction_type_exception, "Invalid packed transaction") - app().get_method()(pretty_input, true, + app().get_method()(input_trx, true, [this, next](const fc::static_variant& result) -> void { if (result.contains()) { next(result.get()); diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index 1895811af3a..ebede3aa21a 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -1559,7 +1559,7 @@ BOOST_AUTO_TEST_CASE(packed_transaction) fc::variant var; abi_serializer::to_variant(packed_txn, var, get_resolver(fc::json::from_string(packed_transaction_abi).as()), abi_serializer::create_yield_function( max_serialization_time )); - chain::packed_transaction packed_txn2; + chain::packed_transaction_v0 packed_txn2; abi_serializer::from_variant(var, packed_txn2, get_resolver(fc::json::from_string(packed_transaction_abi).as()), abi_serializer::create_yield_function( max_serialization_time )); const auto txn2 = packed_txn2.get_transaction(); From 1e7705d7b61d0ca0f4e541738ccdb57254d011e3 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 3 Apr 2020 08:59:13 -0500 Subject: [PATCH 021/142] Add test for trim_blocklog_end --- libraries/chain/block_log.cpp | 23 ++++++++ .../chain/include/eosio/chain/block_log.hpp | 1 + programs/eosio-blocklog/main.cpp | 22 +------- unittests/restart_chain_tests.cpp | 55 ++++++++++++++++++- 4 files changed, 81 insertions(+), 20 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 00d3b58dc35..e44e3c084d4 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #define LOG_READ (std::ios::in | std::ios::binary) #define LOG_WRITE (std::ios::out | std::ios::binary | std::ios::app) @@ -1342,4 +1343,26 @@ namespace eosio { namespace chain { return block_n_pos; } + int block_log::trim_blocklog_end(fc::path block_dir, uint32_t n) { //n is last block to keep (remove later blocks) + trim_data td(block_dir); + + ilog("In directory ${block_dir} will trim all blocks after block ${n} from ${block_file} and ${index_file}", + ("block_dir", block_dir.generic_string())("n", n)("block_file",td.block_file_name.generic_string())("index_file", td.index_file_name.generic_string())); + + if (n < td.first_block) { + elog("All blocks are after block ${n} so do nothing (trim_end would delete entire blocks.log)",("n", n)); + return 1; + } + if (n >= td.last_block) { + elog("There are no blocks after block ${n} so do nothing",("n", n)); + return 2; + } + const uint64_t end_of_new_file = td.block_pos(n + 1); + boost::filesystem::resize_file(td.block_file_name, end_of_new_file); + const uint64_t index_end= td.block_index(n) + sizeof(uint64_t); //advance past record for block n + boost::filesystem::resize_file(td.index_file_name, index_end); + ilog("blocks.index has been trimmed to ${index_end} bytes", ("index_end", index_end)); + return 0; + } + } } /// eosio::chain diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index fa5620b7cae..d0e1eab4e39 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -83,6 +83,7 @@ namespace eosio { namespace chain { static bool is_supported_version(uint32_t version); static bool trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block); + static int trim_blocklog_end(fc::path block_dir, uint32_t n); private: void open(const fc::path& data_dir); diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp index 33f0515291d..d488f8109dd 100644 --- a/programs/eosio-blocklog/main.cpp +++ b/programs/eosio-blocklog/main.cpp @@ -210,26 +210,10 @@ void blocklog::initialize(const variables_map& options) { } int trim_blocklog_end(bfs::path block_dir, uint32_t n) { //n is last block to keep (remove later blocks) - report_time rt("trimming blocklog end"); - using namespace std; - trim_data td(block_dir); - cout << "\nIn directory " << block_dir << " will trim all blocks after block " << n << " from " - << td.block_file_name.generic_string() << " and " << td.index_file_name.generic_string() << ".\n"; - if (n < td.first_block) { - cerr << "All blocks are after block " << n << " so do nothing (trim_end would delete entire blocks.log)\n"; - return 1; - } - if (n >= td.last_block) { - cerr << "There are no blocks after block " << n << " so do nothing\n"; - return 2; - } - const uint64_t end_of_new_file = td.block_pos(n + 1); - bfs::resize_file(td.block_file_name, end_of_new_file); - const uint64_t index_end= td.block_index(n) + sizeof(uint64_t); //advance past record for block n - bfs::resize_file(td.index_file_name, index_end); - cout << "blocks.index has been trimmed to " << index_end << " bytes\n"; + report_time rt("trimming blocklog end"); + int ret = block_log::trim_blocklog_end(block_dir, n); rt.report(); - return 0; + return ret; } bool trim_blocklog_front(bfs::path block_dir, uint32_t n) { //n is first block to keep (remove prior blocks) diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index 653e5b16fa3..e5706bf53c4 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -10,6 +10,7 @@ #include #include +#include using namespace eosio; using namespace testing; @@ -28,6 +29,11 @@ void remove_existing_states(controller::config& config) { fc::create_directories(state_path); } +void remove_reversible_blocks(controller::config& config) { + auto blocks_path = config.blocks_dir; + remove_all(blocks_path/"reversible"); +} + struct dummy_action { static eosio::chain::name get_name() { return N(dummyaction); } static eosio::chain::name get_account() { return N(testapi); } @@ -163,7 +169,16 @@ BOOST_AUTO_TEST_CASE(test_restart_from_block_log) { BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay2))); BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay3))); - BOOST_REQUIRE_NO_THROW(block_log::repair_log(copied_config.blocks_dir, 0)); + fc::path backup_dir; + + BOOST_REQUIRE_NO_THROW(backup_dir = block_log::repair_log(copied_config.blocks_dir, 0)); + + using boost::iostreams::mapped_file_source; + mapped_file_source recovered_log((copied_config.blocks_dir / "blocks.log").generic_string()); + mapped_file_source backup_log((backup_dir / "blocks.log").generic_string()); + + BOOST_REQUIRE(recovered_log.size() == backup_log.size()); + BOOST_CHECK(memcmp(recovered_log.data(), backup_log.data(), backup_log.size()) == 0); } BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log) { @@ -209,6 +224,7 @@ BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log) { // remove the state files to make sure we are starting from block log remove_existing_states(copied_config); + transaction_trace_ptr other_trace; replay_tester from_block_log_chain(copied_config, *genesis, @@ -236,4 +252,41 @@ BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log) { BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->digest(), other_trace->action_traces.at(1).receipt->digest()); } +BOOST_AUTO_TEST_CASE(test_trim_blocklog_end) { + tester chain; + + chain.create_account(N(replay1)); + chain.produce_blocks(1); + chain.create_account(N(replay2)); + chain.produce_blocks(1); + chain.create_account(N(replay3)); + chain.produce_blocks(1); + auto cutoff_block = chain.produce_block(); + auto cutoff_block_num = cutoff_block->block_num(); + chain.produce_blocks(10); + + BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay1))); + BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay2))); + BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay3))); + + chain.close(); + + controller::config copied_config = chain.get_config(); + auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); + BOOST_REQUIRE(genesis); + + // remove the state files to make sure we are starting from block log + remove_existing_states(copied_config); + remove_reversible_blocks(copied_config); + // trim the block file + block_log::trim_blocklog_end(copied_config.blocks_dir, cutoff_block_num); + + tester from_block_log_chain(copied_config, *genesis); + BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay1))); + BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay2))); + BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay3))); +} + + + BOOST_AUTO_TEST_SUITE_END() From 50dc7e03564aeece1e7e4455eb8aac9d3ec2e200 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 09:25:27 -0500 Subject: [PATCH 022/142] Prevent implicit copy of transaction/signed_transaction. fix some unneeded copies. --- .../chain/include/eosio/chain/abi_serializer.hpp | 4 ++-- libraries/chain/include/eosio/chain/transaction.hpp | 12 ++++++++++-- programs/cleos/main.cpp | 2 +- unittests/abi_tests.cpp | 2 +- unittests/block_tests.cpp | 4 ++-- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index eb3a235d8c7..2a2bc35bc36 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -456,7 +456,7 @@ namespace impl { static_assert(fc::reflector::total_member_count == 4); auto h = ctx.enter_scope(); mutable_variant_object mvo; - auto trx = ptrx.get_transaction(); + const auto& trx = ptrx.get_transaction(); mvo("id", trx.id()); mvo("signatures", ptrx.get_signatures()); mvo("compression", ptrx.get_compression()); @@ -479,7 +479,7 @@ namespace impl { static_assert(fc::reflector::total_member_count == 3); auto h = ctx.enter_scope(); mutable_variant_object mvo; - auto trx = ptrx.get_transaction(); + const auto& trx = ptrx.get_transaction(); mvo("id", trx.id()); mvo("signatures", ptrx.get_signatures() != nullptr ? *ptrx.get_signatures() : vector()); mvo("compression", ptrx.get_compression()); diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 8180462c041..9231768adde 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -81,6 +81,12 @@ namespace eosio { namespace chain { * read and write scopes. */ struct transaction : public transaction_header { + transaction() = default; + explicit transaction( const transaction& ) = default; + transaction( transaction&& ) = default; + transaction& operator=(const transaction&) = delete; + transaction& operator=(transaction&&) = default; + vector context_free_actions; vector actions; extensions_type transaction_extensions; @@ -110,8 +116,10 @@ namespace eosio { namespace chain { struct signed_transaction : public transaction { signed_transaction() = default; -// signed_transaction( const signed_transaction& ) = default; -// signed_transaction( signed_transaction&& ) = default; + explicit signed_transaction( const signed_transaction& ) = default; + signed_transaction( signed_transaction&& ) = default; + signed_transaction& operator=(const signed_transaction&) = delete; + signed_transaction& operator=(signed_transaction&&) = default; signed_transaction( transaction&& trx, const vector& signatures, const vector& context_free_data) : transaction(std::move(trx)) , signatures(signatures) diff --git a/programs/cleos/main.cpp b/programs/cleos/main.cpp index 5d62d94d55b..cee7fa4103b 100644 --- a/programs/cleos/main.cpp +++ b/programs/cleos/main.cpp @@ -2543,7 +2543,7 @@ int main( int argc, char** argv ) { fc::from_variant( packed_trx_var, packed_trx ); } EOS_RETHROW_EXCEPTIONS( transaction_type_exception, "Invalid packed transaction format: '${data}'", ("data", fc::json::to_string(packed_trx_var, fc::time_point::maximum()))) - signed_transaction strx = packed_trx.get_signed_transaction(); + const signed_transaction& strx = packed_trx.get_signed_transaction(); fc::variant trx_var; if( unpack_action_data_flag ) { abi_serializer::to_variant( strx, trx_var, abi_serializer_resolver, abi_serializer::create_yield_function( abi_serializer_max_time ) ); diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index ebede3aa21a..d4dcd5c5df5 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -1562,7 +1562,7 @@ BOOST_AUTO_TEST_CASE(packed_transaction) chain::packed_transaction_v0 packed_txn2; abi_serializer::from_variant(var, packed_txn2, get_resolver(fc::json::from_string(packed_transaction_abi).as()), abi_serializer::create_yield_function( max_serialization_time )); - const auto txn2 = packed_txn2.get_transaction(); + const auto& txn2 = packed_txn2.get_transaction(); BOOST_REQUIRE_EQUAL(txn.ref_block_num, txn2.ref_block_num); BOOST_REQUIRE_EQUAL(txn.ref_block_prefix, txn2.ref_block_prefix); diff --git a/unittests/block_tests.cpp b/unittests/block_tests.cpp index c18131da6a4..7b2aaa9afb3 100644 --- a/unittests/block_tests.cpp +++ b/unittests/block_tests.cpp @@ -17,7 +17,7 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_test) // Make a copy of the valid block and corrupt the transaction auto copy_b = std::make_shared(std::move(*b)); - auto signed_tx = copy_b->transactions.back().trx.get().get_signed_transaction(); + auto signed_tx = signed_transaction( copy_b->transactions.back().trx.get().get_signed_transaction() ); auto& act = signed_tx.actions.back(); auto act_data = act.data_as(); // Make the transaction invalid by having the new account name the same as the creator name @@ -61,7 +61,7 @@ std::pair corrupt_trx_in_block(validating_te // Make a copy of the valid block and corrupt the transaction auto copy_b = std::make_shared(b->clone()); const auto& packed_trx = copy_b->transactions.back().trx.get(); - auto signed_tx = packed_trx.get_signed_transaction(); + auto signed_tx = signed_transaction( packed_trx.get_signed_transaction() ); // Corrupt one signature signed_tx.signatures.clear(); signed_tx.sign(main.get_private_key(act_name, "active"), main.control->get_chain_id()); From ebf6e3ffd5ee4429a0d0a6031a360a22e97dd036 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 09:35:29 -0500 Subject: [PATCH 023/142] Made copy constructor explicit --- libraries/chain/include/eosio/chain/transaction.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 9231768adde..f6298f0ea97 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -57,6 +57,12 @@ namespace eosio { namespace chain { * region. */ struct transaction_header { + transaction_header() = default; + explicit transaction_header( const transaction_header& ) = default; + transaction_header( transaction_header&& ) = default; + transaction_header& operator=(const transaction_header&) = delete; + transaction_header& operator=(transaction_header&&) = default; + time_point_sec expiration; ///< the time at which a transaction expires uint16_t ref_block_num = 0U; ///< specifies a block num in the last 2^16 blocks. uint32_t ref_block_prefix = 0UL; ///< specifies the lower 32 bits of the blockid at get_ref_blocknum From d5d4dba7c52293c591249dc6dc638d052fb305d7 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 10:27:00 -0500 Subject: [PATCH 024/142] Get block id appropriately --- plugins/mongo_db_plugin/mongo_db_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mongo_db_plugin/mongo_db_plugin.cpp b/plugins/mongo_db_plugin/mongo_db_plugin.cpp index 541e522f593..5d0bec9b918 100644 --- a/plugins/mongo_db_plugin/mongo_db_plugin.cpp +++ b/plugins/mongo_db_plugin/mongo_db_plugin.cpp @@ -1019,7 +1019,7 @@ void mongo_db_plugin_impl::_process_irreversible_block(const chain::block_state_ using bsoncxx::builder::basic::kvp; - const auto block_id = bs->block->id(); + const auto& block_id = bs->id; const auto block_id_str = block_id.str(); auto now = std::chrono::duration_cast( From 718e222ca96455dc6643343a5fa760c0d61fa500 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 3 Apr 2020 11:13:51 -0500 Subject: [PATCH 025/142] refactor test cases and add prune_transaction test --- libraries/chain/block_log.cpp | 2 +- unittests/restart_chain_tests.cpp | 270 ++++++++++++++---------------- 2 files changed, 126 insertions(+), 146 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index e44e3c084d4..cfee7047494 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -880,7 +880,7 @@ namespace eosio { namespace chain { auto pruner = overloaded{[](transaction_id_type&) { return false; }, [&id](pruned_transaction& ptx) { return ptx.id() == id && prune(ptx); }}; - for (auto trx : block.transactions) { + for (auto& trx : block.transactions) { if (trx.trx.visit(pruner)) { my->block_file.seek(pos); std::vector buffer = detail::block_log_impl::pack_v4_log_entry(*entry.block, static_cast(entry.compression)); diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index e5706bf53c4..bf80eeccdf4 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -29,11 +29,6 @@ void remove_existing_states(controller::config& config) { fc::create_directories(state_path); } -void remove_reversible_blocks(controller::config& config) { - auto blocks_path = config.blocks_dir; - remove_all(blocks_path/"reversible"); -} - struct dummy_action { static eosio::chain::name get_name() { return N(dummyaction); } static eosio::chain::name get_account() { return N(testapi); } @@ -85,6 +80,120 @@ class replay_tester : public base_tester { bool validate() { return true; } }; +struct restart_from_block_log_test_fixture { + tester chain; + uint32_t cutoff_block_num; + restart_from_block_log_test_fixture() { + chain.create_account(N(replay1)); + chain.produce_blocks(1); + chain.create_account(N(replay2)); + chain.produce_blocks(1); + chain.create_account(N(replay3)); + chain.produce_blocks(1); + auto cutoff_block = chain.produce_block(); + cutoff_block_num = cutoff_block->block_num(); + chain.produce_blocks(10); + + BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay1))); + BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay2))); + BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay3))); + + chain.close(); + } + ~restart_from_block_log_test_fixture() { + controller::config copied_config = chain.get_config(); + auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); + BOOST_REQUIRE(genesis); + + // remove the state files to make sure we are starting from block log + remove_existing_states(copied_config); + tester from_block_log_chain(copied_config, *genesis); + BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay1))); + BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay2))); + BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay3))); + } +}; + +struct light_validation_restart_from_block_log_test_fixture { + tester chain; + eosio::chain::transaction_trace_ptr trace; + light_validation_restart_from_block_log_test_fixture() + : chain(setup_policy::full) { + chain.create_account(N(testapi)); + chain.create_account(N(dummy)); + chain.produce_block(); + chain.set_code(N(testapi), contracts::test_api_wasm()); + chain.produce_block(); + + cf_action cfa; + signed_transaction trx; + action act({}, cfa); + trx.context_free_actions.push_back(act); + trx.context_free_data.emplace_back(fc::raw::pack(100)); // verify payload matches context free data + trx.context_free_data.emplace_back(fc::raw::pack(200)); + // add a normal action along with cfa + dummy_action da = {DUMMY_ACTION_DEFAULT_A, DUMMY_ACTION_DEFAULT_B, DUMMY_ACTION_DEFAULT_C}; + action act1(vector{{N(testapi), config::active_name}}, da); + trx.actions.push_back(act1); + chain.set_transaction_headers(trx); + // run normal passing case + auto sigs = trx.sign(chain.get_private_key(N(testapi), "active"), chain.control->get_chain_id()); + trace = chain.push_transaction(trx); + chain.produce_block(); + + BOOST_REQUIRE(trace->receipt); + BOOST_CHECK_EQUAL(trace->receipt->status, transaction_receipt::executed); + BOOST_CHECK_EQUAL(2, trace->action_traces.size()); + + BOOST_CHECK(trace->action_traces.at(0).context_free); // cfa + BOOST_CHECK_EQUAL("test\n", trace->action_traces.at(0).console); // cfa executed + + BOOST_CHECK(!trace->action_traces.at(1).context_free); // non-cfa + BOOST_CHECK_EQUAL("", trace->action_traces.at(1).console); + + chain.close(); + } + + ~light_validation_restart_from_block_log_test_fixture() { + controller::config copied_config = chain.get_config(); + auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); + BOOST_REQUIRE(genesis); + + // remove the state files to make sure we are starting from block log + remove_existing_states(copied_config); + + transaction_trace_ptr other_trace; + + replay_tester from_block_log_chain(copied_config, *genesis, + [&](std::tuple x) { + auto& t = std::get<0>(x); + if (t && t->id == trace->id) { + other_trace = t; + } + }); + + BOOST_REQUIRE(other_trace); + BOOST_REQUIRE(other_trace->receipt); + BOOST_CHECK_EQUAL(other_trace->receipt->status, transaction_receipt::executed); + BOOST_CHECK(*trace->receipt == *other_trace->receipt); + BOOST_CHECK_EQUAL(2, other_trace->action_traces.size()); + + BOOST_CHECK(other_trace->action_traces.at(0).context_free); // cfa + BOOST_CHECK_EQUAL("", other_trace->action_traces.at(0).console); // cfa not executed for replay + BOOST_CHECK_EQUAL(trace->action_traces.at(0).receipt->global_sequence, + other_trace->action_traces.at(0).receipt->global_sequence); + BOOST_CHECK_EQUAL(trace->action_traces.at(0).receipt->digest(), + other_trace->action_traces.at(0).receipt->digest()); + + BOOST_CHECK(!other_trace->action_traces.at(1).context_free); // non-cfa + BOOST_CHECK_EQUAL("", other_trace->action_traces.at(1).console); + BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->global_sequence, + other_trace->action_traces.at(1).receipt->global_sequence); + BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->digest(), + other_trace->action_traces.at(1).receipt->digest()); + } +}; + BOOST_AUTO_TEST_SUITE(restart_chain_tests) BOOST_AUTO_TEST_CASE(test_existing_state_without_block_log) { @@ -140,153 +249,24 @@ BOOST_AUTO_TEST_CASE(test_restart_with_different_chain_id) { fc_exception_message_starts_with("chain ID in state ")); } -BOOST_AUTO_TEST_CASE(test_restart_from_block_log) { - tester chain; - - chain.create_account(N(replay1)); - chain.produce_blocks(1); - chain.create_account(N(replay2)); - chain.produce_blocks(1); - chain.create_account(N(replay3)); - chain.produce_blocks(1); - - BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay1))); - BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay2))); - BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay3))); - - chain.close(); - - controller::config copied_config = chain.get_config(); - auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); - BOOST_REQUIRE(genesis); - - // remove the state files to make sure we are starting from block log - remove_existing_states(copied_config); - - tester from_block_log_chain(copied_config, *genesis); - - BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay1))); - BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay2))); - BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay3))); - - fc::path backup_dir; - - BOOST_REQUIRE_NO_THROW(backup_dir = block_log::repair_log(copied_config.blocks_dir, 0)); +BOOST_FIXTURE_TEST_CASE(test_restart_from_block_log, restart_from_block_log_test_fixture) { +} - using boost::iostreams::mapped_file_source; - mapped_file_source recovered_log((copied_config.blocks_dir / "blocks.log").generic_string()); - mapped_file_source backup_log((backup_dir / "blocks.log").generic_string()); - BOOST_REQUIRE(recovered_log.size() == backup_log.size()); - BOOST_CHECK(memcmp(recovered_log.data(), backup_log.data(), backup_log.size()) == 0); +BOOST_FIXTURE_TEST_CASE(test_restart_from_trimed_block_log, restart_from_block_log_test_fixture) { + auto& config = chain.get_config(); + auto blocks_path = config.blocks_dir; + remove_all(blocks_path/"reversible"); + block_log::trim_blocklog_end(config.blocks_dir, cutoff_block_num); } - -BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log) { - tester chain(setup_policy::full); - - chain.create_account(N(testapi)); - chain.create_account(N(dummy)); - chain.produce_block(); - chain.set_code(N(testapi), contracts::test_api_wasm()); - chain.produce_block(); - - cf_action cfa; - signed_transaction trx; - action act({}, cfa); - trx.context_free_actions.push_back(act); - trx.context_free_data.emplace_back(fc::raw::pack(100)); // verify payload matches context free data - trx.context_free_data.emplace_back(fc::raw::pack(200)); - // add a normal action along with cfa - dummy_action da = {DUMMY_ACTION_DEFAULT_A, DUMMY_ACTION_DEFAULT_B, DUMMY_ACTION_DEFAULT_C}; - action act1(vector{{N(testapi), config::active_name}}, da); - trx.actions.push_back(act1); - chain.set_transaction_headers(trx); - // run normal passing case - auto sigs = trx.sign(chain.get_private_key(N(testapi), "active"), chain.control->get_chain_id()); - auto trace = chain.push_transaction(trx); - chain.produce_block(); - - BOOST_REQUIRE(trace->receipt); - BOOST_CHECK_EQUAL(trace->receipt->status, transaction_receipt::executed); - BOOST_CHECK_EQUAL(2, trace->action_traces.size()); - - BOOST_CHECK(trace->action_traces.at(0).context_free); // cfa - BOOST_CHECK_EQUAL("test\n", trace->action_traces.at(0).console); // cfa executed - - BOOST_CHECK(!trace->action_traces.at(1).context_free); // non-cfa - BOOST_CHECK_EQUAL("", trace->action_traces.at(1).console); - - chain.close(); - - controller::config copied_config = chain.get_config(); - auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); - BOOST_REQUIRE(genesis); - - // remove the state files to make sure we are starting from block log - remove_existing_states(copied_config); - transaction_trace_ptr other_trace; - - replay_tester from_block_log_chain(copied_config, *genesis, - [&](std::tuple x) { - auto& t = std::get<0>(x); - if (t && t->id == trace->id) { - other_trace = t; - } - }); - - BOOST_REQUIRE(other_trace); - BOOST_REQUIRE(other_trace->receipt); - BOOST_CHECK_EQUAL(other_trace->receipt->status, transaction_receipt::executed); - BOOST_CHECK(*trace->receipt == *other_trace->receipt); - BOOST_CHECK_EQUAL(2, other_trace->action_traces.size()); - - BOOST_CHECK(other_trace->action_traces.at(0).context_free); // cfa - BOOST_CHECK_EQUAL("", other_trace->action_traces.at(0).console); // cfa not executed for replay - BOOST_CHECK_EQUAL(trace->action_traces.at(0).receipt->global_sequence, other_trace->action_traces.at(0).receipt->global_sequence); - BOOST_CHECK_EQUAL(trace->action_traces.at(0).receipt->digest(), other_trace->action_traces.at(0).receipt->digest()); - - BOOST_CHECK(!other_trace->action_traces.at(1).context_free); // non-cfa - BOOST_CHECK_EQUAL("", other_trace->action_traces.at(1).console); - BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->global_sequence, other_trace->action_traces.at(1).receipt->global_sequence); - BOOST_CHECK_EQUAL(trace->action_traces.at(1).receipt->digest(), other_trace->action_traces.at(1).receipt->digest()); +BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log, light_validation_restart_from_block_log_test_fixture) { } -BOOST_AUTO_TEST_CASE(test_trim_blocklog_end) { - tester chain; - - chain.create_account(N(replay1)); - chain.produce_blocks(1); - chain.create_account(N(replay2)); - chain.produce_blocks(1); - chain.create_account(N(replay3)); - chain.produce_blocks(1); - auto cutoff_block = chain.produce_block(); - auto cutoff_block_num = cutoff_block->block_num(); - chain.produce_blocks(10); - - BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay1))); - BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay2))); - BOOST_REQUIRE_NO_THROW(chain.control->get_account(N(replay3))); - - chain.close(); - - controller::config copied_config = chain.get_config(); - auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir); - BOOST_REQUIRE(genesis); - - // remove the state files to make sure we are starting from block log - remove_existing_states(copied_config); - remove_reversible_blocks(copied_config); - // trim the block file - block_log::trim_blocklog_end(copied_config.blocks_dir, cutoff_block_num); - - tester from_block_log_chain(copied_config, *genesis); - BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay1))); - BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay2))); - BOOST_REQUIRE_NO_THROW(from_block_log_chain.control->get_account(N(replay3))); +BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log_with_pruned_trx, light_validation_restart_from_block_log_test_fixture) { + block_log blog(chain.get_config().blocks_dir); + blog.prune_transaction(trace->block_num, trace->id); } - BOOST_AUTO_TEST_SUITE_END() From 62d61d3dfca72d0e5343d89225058a545b9044e8 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 12:35:47 -0500 Subject: [PATCH 026/142] Revert "Support receiving old/new packed_transaction & signed_block" This reverts commit eba4d9b382571b06086a9ba1ca05b722665cba87. --- .../include/eosio/net_plugin/protocol.hpp | 6 ++-- plugins/net_plugin/net_plugin.cpp | 32 +++++-------------- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp index 86e8c2bb9f9..56c33909851 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp @@ -142,10 +142,8 @@ namespace eosio { notice_message, request_message, sync_request_message, - signed_block_v0, // which = 7 - packed_transaction_v0, // which = 8 - signed_block, // which = 9 - packed_transaction>; // which = 10 + signed_block, // which = 7 + packed_transaction>; // which = 8 } // namespace eosio diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 7efe2be5462..ba44d950da5 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -390,10 +390,8 @@ namespace eosio { constexpr auto def_sync_fetch_span = 100; constexpr auto message_header_size = 4; - constexpr uint32_t signed_block_v0_which = 7; // see protocol net_message - constexpr uint32_t packed_transaction_v0_which = 8; // see protocol net_message - constexpr uint32_t signed_block_which = 9; // see protocol net_message - constexpr uint32_t packed_transaction_which = 10; // see protocol net_message + constexpr uint32_t signed_block_which = 7; // see protocol net_message + constexpr uint32_t packed_transaction_which = 8; // see protocol net_message /** * For a while, network version was a 16 bit value equal to the second set of 16 bits @@ -2377,7 +2375,7 @@ namespace eosio { auto peek_ds = pending_message_buffer.create_peek_datastream(); unsigned_int which{}; fc::raw::unpack( peek_ds, which ); - if( which == signed_block_v0_which || which == signed_block_which ) { + if( which == signed_block_which ) { block_header bh; fc::raw::unpack( peek_ds, bh ); @@ -2417,17 +2415,10 @@ namespace eosio { } } - shared_ptr ptr; auto ds = pending_message_buffer.create_datastream(); fc::raw::unpack( ds, which ); // throw away - if( which == signed_block_which ) { - ptr = std::make_shared(); - fc::raw::unpack( ds, *ptr ); - } else { // signed_block_v0_which - signed_block_v0 v0; - fc::raw::unpack( ds, v0 ); - ptr = std::make_shared( std::move( v0 ), true); - } + shared_ptr ptr = std::make_shared(); + fc::raw::unpack( ds, *ptr ); auto is_webauthn_sig = []( const fc::crypto::signature& s ) { return s.which() == fc::crypto::signature::storage_type::position(); @@ -2449,24 +2440,17 @@ namespace eosio { handle_message( blk_id, std::move( ptr ) ); - } else if( which == packed_transaction_v0_which || which == packed_transaction_which ) { + } else if( which == packed_transaction_which ) { if( !my_impl->p2p_accept_transactions ) { fc_dlog( logger, "p2p-accept-transaction=false - dropping txn" ); pending_message_buffer.advance_read_ptr( message_length ); return true; } - shared_ptr ptr; auto ds = pending_message_buffer.create_datastream(); fc::raw::unpack( ds, which ); // throw away - if( which == packed_transaction_which ) { - ptr = std::make_shared(); - fc::raw::unpack( ds, *ptr ); - } else { // packed_transaction_v0_which - packed_transaction_v0 v0; - fc::raw::unpack( ds, v0 ); - ptr = std::make_shared( std::move( v0 ), true ); - } + shared_ptr ptr = std::make_shared(); + fc::raw::unpack( ds, *ptr ); handle_message( std::move( ptr ) ); } else { From 8e35ae544f9c9bc2beb1d1366cf299f56fae608f Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 12:36:54 -0500 Subject: [PATCH 027/142] Revert make_unique, going to work on that some other time. --- libraries/chain/transaction_context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/transaction_context.cpp b/libraries/chain/transaction_context.cpp index 5ca3a304d0b..25346fd32e3 100644 --- a/libraries/chain/transaction_context.cpp +++ b/libraries/chain/transaction_context.cpp @@ -52,7 +52,7 @@ namespace eosio { namespace chain { ,trx(t) ,id(trx_id) ,undo_session() - ,trace(std::make_unique()) + ,trace(std::make_shared()) ,start(s) ,transaction_timer(std::move(tmr)) ,net_usage(trace->net_usage) From 175bae2dcee9aecc7734c84813231b52c52098d7 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 12:37:26 -0500 Subject: [PATCH 028/142] Make signed_block_v0_ptr shared_prt to const --- libraries/chain/include/eosio/chain/block.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 2bef05d95d2..5ce7e2a2883 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -102,7 +102,7 @@ namespace eosio { namespace chain { flat_multimap validate_and_extract_extensions()const; }; - using signed_block_v0_ptr = std::shared_ptr; + using signed_block_v0_ptr = std::shared_ptr; struct transaction_receipt : public transaction_receipt_header { From 2acbf7c8d13f457c60b510399737d9063adf8701 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 12:37:47 -0500 Subject: [PATCH 029/142] Remove extract for packed_transaction --- libraries/chain/include/eosio/chain/abi_serializer.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 2a2bc35bc36..face7f6f409 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -756,12 +756,6 @@ namespace impl { "Failed to deserialize data for ${account}:${name}", ("account", act.account)("name", act.name)); } - template - static void extract( const variant& v, packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) - { - EOS_THROW( packed_transaction_type_exception, "not implemented, use packed_transaction_v0" ) - } - template static void extract( const variant& v, packed_transaction_v0& ptrx, Resolver resolver, abi_traverse_context& ctx ) { From 6bf76062971e9b339206aca61a9884e9a0b9c278 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 12:49:32 -0500 Subject: [PATCH 030/142] Revert net_plugin changes for now. --- plugins/net_plugin/net_plugin.cpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index ba44d950da5..0009eef082a 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -414,7 +414,6 @@ namespace eosio { constexpr uint16_t proto_base = 0; constexpr uint16_t proto_explicit_sync = 1; // version at time of eosio 1.0 constexpr uint16_t block_id_notify = 2; // reserved. feature was removed. next net_version should be 3 - constexpr uint16_t pruned_types = 3; // support packed_transaction & signed_block in addition to prior *_v0 versions constexpr uint16_t net_version = proto_explicit_sync; @@ -1240,19 +1239,6 @@ namespace eosio { return create_send_buffer( packed_transaction_which, trx ); } - static std::shared_ptr> create_send_buffer( const signed_block_v0_ptr& sb ) { - // this implementation is to avoid copy of signed_block_v0 to net_message - // matches which of net_message for signed_block_v0 - fc_dlog( logger, "sending block ${bn}", ("bn", sb->block_num()) ); - return create_send_buffer( signed_block_v0_which, *sb ); - } - - static std::shared_ptr> create_send_buffer( const packed_transaction_v0& trx ) { - // this implementation is to avoid copy of packed_transaction_v0 to net_message - // matches which of net_message for packed_transaction_v0 - return create_send_buffer( packed_transaction_v0_which, trx ); - } - void connection::enqueue_block( const signed_block_ptr& sb, bool to_sync_queue) { fc_dlog( logger, "enqueue block ${num}", ("num", sb->block_num()) ); verify_strand_in_this_thread( strand, __func__, __LINE__ ); From a933a221fbbad9cf54aeacf41416e85e54385b17 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 14:11:56 -0500 Subject: [PATCH 031/142] For now just convert to/from packed_transaction/packed_transaction_v0 and signed_block/signed_block_v0 in net_plugin to support current net protocol --- .../chain/include/eosio/chain/transaction.hpp | 12 ++++------ libraries/chain/transaction.cpp | 4 ++++ .../include/eosio/net_plugin/protocol.hpp | 4 ++-- plugins/net_plugin/net_plugin.cpp | 24 +++++++++++-------- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index f6298f0ea97..01956293514 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -267,6 +267,8 @@ namespace eosio { namespace chain { prunable_data_type prunable_data; }; + using packed_transaction_v0_ptr = std::shared_ptr; + struct packed_transaction : fc::reflect_init { using compression_type = packed_transaction_v0::compression_type; using cf_compression_type = prunable_transaction_data::compression_type; @@ -282,12 +284,7 @@ namespace eosio { namespace chain { explicit packed_transaction(const signed_transaction& t, bool legacy, compression_type _compression = compression_type::none); explicit packed_transaction(signed_transaction&& t, bool legacy, compression_type _compression = compression_type::none); -#if 0 - // used by abi_serializer - packed_transaction( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); - packed_transaction( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ); - packed_transaction( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); -#endif + packed_transaction_v0_ptr to_packed_transaction_v0() const; uint32_t get_unprunable_size()const; uint32_t get_prunable_size()const; @@ -332,8 +329,7 @@ namespace eosio { namespace chain { transaction_id_type trx_id; }; - using packed_transaction_v0_ptr = std::shared_ptr; - using packed_transaction_ptr = std::shared_ptr; + using packed_transaction_ptr = std::shared_ptr; uint128_t transaction_id_to_sender_id( const transaction_id_type& tid ); diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index b10c00081b5..9c69de7b20e 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -493,6 +493,10 @@ packed_transaction::packed_transaction(signed_transaction&& t, bool legacy, comp estimated_size = estimated_size * 2 + get_prunable_size(); } +packed_transaction_v0_ptr packed_transaction::to_packed_transaction_v0() const { + return std::make_shared( get_signed_transaction(), get_compression() ); +} + uint32_t packed_transaction::get_unprunable_size()const { uint64_t size = config::fixed_net_overhead_of_packed_trx; size += packed_trx.size(); diff --git a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp index 56c33909851..3658c933695 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp @@ -142,8 +142,8 @@ namespace eosio { notice_message, request_message, sync_request_message, - signed_block, // which = 7 - packed_transaction>; // which = 8 + signed_block_v0, // which = 7 + packed_transaction_v0>; // which = 8 } // namespace eosio diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 0009eef082a..1f2c0a08fa4 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -390,8 +390,8 @@ namespace eosio { constexpr auto def_sync_fetch_span = 100; constexpr auto message_header_size = 4; - constexpr uint32_t signed_block_which = 7; // see protocol net_message - constexpr uint32_t packed_transaction_which = 8; // see protocol net_message + constexpr uint32_t signed_block_v0_which = 7; // see protocol net_message + constexpr uint32_t packed_transaction_v0_which = 8; // see protocol net_message /** * For a while, network version was a 16 bit value equal to the second set of 16 bits @@ -1230,13 +1230,15 @@ namespace eosio { // this implementation is to avoid copy of signed_block to net_message // matches which of net_message for signed_block fc_dlog( logger, "sending block ${bn}", ("bn", sb->block_num()) ); - return create_send_buffer( signed_block_which, *sb ); + auto sb_v0 = sb->to_signed_block_v0(); + return create_send_buffer( signed_block_v0_which, *sb_v0 ); } static std::shared_ptr> create_send_buffer( const packed_transaction& trx ) { // this implementation is to avoid copy of packed_transaction to net_message // matches which of net_message for packed_transaction - return create_send_buffer( packed_transaction_which, trx ); + auto pt_v0 = trx.to_packed_transaction_v0(); + return create_send_buffer( packed_transaction_v0_which, *pt_v0 ); } void connection::enqueue_block( const signed_block_ptr& sb, bool to_sync_queue) { @@ -2361,7 +2363,7 @@ namespace eosio { auto peek_ds = pending_message_buffer.create_peek_datastream(); unsigned_int which{}; fc::raw::unpack( peek_ds, which ); - if( which == signed_block_which ) { + if( which == signed_block_v0_which ) { block_header bh; fc::raw::unpack( peek_ds, bh ); @@ -2403,8 +2405,9 @@ namespace eosio { auto ds = pending_message_buffer.create_datastream(); fc::raw::unpack( ds, which ); // throw away - shared_ptr ptr = std::make_shared(); - fc::raw::unpack( ds, *ptr ); + signed_block_v0 sb_v0; + fc::raw::unpack( ds, sb_v0 ); + shared_ptr ptr = std::make_shared( std::move( sb_v0 ), true ); auto is_webauthn_sig = []( const fc::crypto::signature& s ) { return s.which() == fc::crypto::signature::storage_type::position(); @@ -2426,7 +2429,7 @@ namespace eosio { handle_message( blk_id, std::move( ptr ) ); - } else if( which == packed_transaction_which ) { + } else if( which == packed_transaction_v0_which ) { if( !my_impl->p2p_accept_transactions ) { fc_dlog( logger, "p2p-accept-transaction=false - dropping txn" ); pending_message_buffer.advance_read_ptr( message_length ); @@ -2435,8 +2438,9 @@ namespace eosio { auto ds = pending_message_buffer.create_datastream(); fc::raw::unpack( ds, which ); // throw away - shared_ptr ptr = std::make_shared(); - fc::raw::unpack( ds, *ptr ); + packed_transaction_v0 pt_v0; + fc::raw::unpack( ds, pt_v0 ); + shared_ptr ptr = std::make_shared( pt_v0, true ); handle_message( std::move( ptr ) ); } else { From 7653f7ee454cc1e34fe0b7a671c1cca0d6003efe Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 3 Apr 2020 14:17:08 -0500 Subject: [PATCH 032/142] refactor smoke_test --- .../chain/include/eosio/chain/block_log.hpp | 20 ++----------- programs/eosio-blocklog/main.cpp | 28 ++----------------- unittests/restart_chain_tests.cpp | 2 +- 3 files changed, 5 insertions(+), 45 deletions(-) diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index d0e1eab4e39..f47b880e9dd 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -84,6 +84,8 @@ namespace eosio { namespace chain { static bool trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block); static int trim_blocklog_end(fc::path block_dir, uint32_t n); + static void smoke_test(fc::path block_dir); + private: void open(const fc::path& data_dir); @@ -91,22 +93,4 @@ namespace eosio { namespace chain { std::unique_ptr my; }; - - struct trim_data { //used by trim_blocklog_front(), trim_blocklog_end(), and smoke_test() - trim_data(fc::path block_dir); - ~trim_data(); - uint64_t block_index(uint32_t n) const; - uint64_t block_pos(uint32_t n); - fc::path block_file_name, index_file_name; //full pathname for blocks.log and blocks.index - uint32_t version = 0; //blocklog version - uint32_t first_block = 0; //first block in blocks.log - uint32_t last_block = 0; //last block in blocks.log - FILE* blk_in = nullptr; //C style files for reading blocks.log and blocks.index - FILE* ind_in = nullptr; //C style files for reading blocks.log and blocks.index - //we use low level file IO because it is distinctly faster than C++ filebuf or iostream - uint64_t first_block_pos = 0; //file position in blocks.log for the first block in the log - chain_id_type chain_id; - - static int blknum_offset_from_block_entry(uint32_t block_log_version); - }; } } diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp index d488f8109dd..779ab3994e8 100644 --- a/programs/eosio-blocklog/main.cpp +++ b/programs/eosio-blocklog/main.cpp @@ -227,32 +227,8 @@ bool trim_blocklog_front(bfs::path block_dir, uint32_t n) { //n is first void smoke_test(bfs::path block_dir) { using namespace std; cout << "\nSmoke test of blocks.log and blocks.index in directory " << block_dir << '\n'; - trim_data td(block_dir); - auto status = fseek(td.blk_in, -sizeof(uint64_t), SEEK_END); //get last_block from blocks.log, compare to from blocks.index - EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", td.block_file_name.string())("pos", sizeof(uint64_t)) ); - uint64_t file_pos; - auto size = fread((void*)&file_pos, sizeof(uint64_t), 1, td.blk_in); - EOS_ASSERT( size == 1, block_log_exception, "${file} read fails", ("file", td.block_file_name.string()) ); - int blknum_offset = trim_data::blknum_offset_from_block_entry(td.version); - status = fseek(td.blk_in, file_pos + blknum_offset, SEEK_SET); - EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", td.block_file_name.string())("pos", file_pos + blknum_offset) ); - uint32_t bnum; - size = fread((void*)&bnum, sizeof(uint32_t), 1, td.blk_in); - EOS_ASSERT( size == 1, block_log_exception, "${file} read fails", ("file", td.block_file_name.string()) ); - bnum = endian_reverse_u32(bnum) + 1; //convert from big endian to little endian and add 1 - EOS_ASSERT( td.last_block == bnum, block_log_exception, "blocks.log says last block is ${lb} which disagrees with blocks.index", ("lb", bnum) ); - cout << "blocks.log and blocks.index agree on number of blocks\n"; - uint32_t delta = (td.last_block + 8 - td.first_block) >> 3; - if (delta < 1) - delta = 1; - for (uint32_t n = td.first_block; ; n += delta) { - if (n > td.last_block) - n = td.last_block; - td.block_pos(n); //check block 'n' is where blocks.index says - if (n == td.last_block) - break; - } - cout << "\nno problems found\n"; //if get here there were no exceptions + block_log::smoke_test(block_dir); + cout << "\nno problems found\n"; // if get here there were no exceptions } int main(int argc, char** argv) { diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index bf80eeccdf4..8b467c4dd13 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -250,9 +250,9 @@ BOOST_AUTO_TEST_CASE(test_restart_with_different_chain_id) { } BOOST_FIXTURE_TEST_CASE(test_restart_from_block_log, restart_from_block_log_test_fixture) { + BOOST_REQUIRE_NO_THROW(block_log::smoke_test(chain.get_config().blocks_dir)); } - BOOST_FIXTURE_TEST_CASE(test_restart_from_trimed_block_log, restart_from_block_log_test_fixture) { auto& config = chain.get_config(); auto blocks_path = config.blocks_dir; From ddfd7a5f6a4c2aaa8e79b0b20bc7433df6ccb4f0 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 3 Apr 2020 14:22:04 -0500 Subject: [PATCH 033/142] Add move conversion constructor to pruned_block --- libraries/chain/block.cpp | 35 +++++-- libraries/chain/block_log.cpp | 98 ++++++++++++++----- libraries/chain/include/eosio/chain/block.hpp | 2 + .../chain/include/eosio/chain/transaction.hpp | 21 ++-- libraries/chain/transaction.cpp | 39 +++++++- libraries/fc | 2 +- 6 files changed, 157 insertions(+), 40 deletions(-) diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index 616c1d3975f..8647ea71c47 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -21,16 +21,27 @@ namespace eosio { namespace chain { } } - static fc::static_variant translate_transaction_receipt(const transaction_id_type& tid, bool) { - return tid; - } - static fc::static_variant translate_transaction_receipt(const packed_transaction& ptrx, bool legacy) { - return pruned_transaction(ptrx, legacy); - } + struct transaction_receipt_translator { + bool legacy; + fc::static_variant operator()(const transaction_id_type& tid) const { + return tid; + } + fc::static_variant operator()(const packed_transaction& ptrx) const { + return pruned_transaction(ptrx, legacy); + } + fc::static_variant operator()(packed_transaction&& ptrx) const { + return pruned_transaction(std::move(ptrx), legacy); + } + }; pruned_transaction_receipt::pruned_transaction_receipt(const transaction_receipt& other, bool legacy) : transaction_receipt_header(static_cast(other)), - trx(other.trx.visit([&](const auto& obj) { return translate_transaction_receipt(obj, legacy); })) + trx(other.trx.visit(transaction_receipt_translator{legacy})) + {} + + pruned_transaction_receipt::pruned_transaction_receipt(transaction_receipt&& other, bool legacy) + : transaction_receipt_header(static_cast(other)), + trx( std::move(other.trx).visit(transaction_receipt_translator{legacy})) {} static flat_multimap validate_and_extract_block_extensions(const extensions_type& block_extensions) { @@ -88,6 +99,16 @@ namespace eosio { namespace chain { } } + pruned_block::pruned_block( signed_block&& other, bool legacy ) + : signed_block_header(std::move(static_cast(other))), + prune_state(legacy ? prune_state_type::complete_legacy : prune_state_type::complete), + block_extensions(std::move(other.block_extensions)) + { + for(auto& trx : other.transactions) { + transactions.emplace_back(std::move(trx), legacy); + } + } + static std::size_t pruned_trx_receipt_packed_size(const pruned_transaction& obj, pruned_transaction::cf_compression_type segment_compression) { return obj.maximum_pruned_pack_size(segment_compression); } diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index cfee7047494..31c0daa17cb 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -55,6 +55,9 @@ namespace eosio { namespace chain { fc::raw::unpack(ds, *entry.block); } + + + class block_log_impl { public: pruned_block_ptr head; @@ -110,6 +113,19 @@ namespace eosio { namespace chain { if (version < 4) return 0; return sizeof(uint32_t) + sizeof(pruned_transaction::cf_compression_type); } + + static int blknum_offset_from_block_entry(uint32_t block_log_version) { + + //to derive blknum_offset==14 see block_header.hpp and note on disk struct is packed + // block_timestamp_type timestamp; //bytes 0:3 + // account_name producer; //bytes 4:11 + // uint16_t confirmed; //bytes 12:13 + // block_id_type previous; //bytes 14:45, low 4 bytes is big endian block number of previous block + + int blknum_offset = 14; + blknum_offset += detail::block_log_impl::offset_to_block_start(block_log_version); + return blknum_offset; + } }; void detail::block_log_impl::reopen() { @@ -605,7 +621,7 @@ namespace eosio { namespace chain { index.complete(); } - static uint32_t verify_log_preamble_and_return_version(fc::datastream& ds, const char* block_file_name) { + static std::tuple get_blocklog_version_and_first_block_num(fc::datastream& ds, const char* block_file_name) { uint32_t version = 0; ds.read((char*)&version, sizeof(version)); EOS_ASSERT(version > 0, block_log_exception, "Block log was not setup properly"); @@ -654,7 +670,7 @@ namespace eosio { namespace chain { ("e", fc::to_hex((char*)&expected_totem, sizeof(expected_totem)))( "a", fc::to_hex((char*)&actual_totem, sizeof(actual_totem)))); } - return version; + return std::make_tuple(version, first_block_num); } static bool verify_block(fc::datastream& ds, block_id_type previous, uint64_t pos, block_header& header) { @@ -729,7 +745,7 @@ namespace eosio { namespace chain { boost::iostreams::mapped_file_source log_data((backup_dir / "blocks.log").generic_string()); fc::datastream ds(log_data.data(), log_data.size()); - uint32_t version = verify_log_preamble_and_return_version(ds, block_file_name); + auto [version, first_block_num] = get_blocklog_version_and_first_block_num(ds, block_file_name); pos = ds.tellp(); std::string error_msg; @@ -898,19 +914,7 @@ namespace eosio { namespace chain { return false; } - int trim_data::blknum_offset_from_block_entry(uint32_t block_log_version) { - - //to derive blknum_offset==14 see block_header.hpp and note on disk struct is packed - // block_timestamp_type timestamp; //bytes 0:3 - // account_name producer; //bytes 4:11 - // uint16_t confirmed; //bytes 12:13 - // block_id_type previous; //bytes 14:45, low 4 bytes is big endian block number of previous block - - int blknum_offset = 14; - blknum_offset += detail::block_log_impl::offset_to_block_start(block_log_version); - return blknum_offset; - } - + detail::reverse_iterator::reverse_iterator() : _file(nullptr, &fclose) , _buffer_ptr(std::make_unique(_buf_len)) { @@ -959,10 +963,10 @@ namespace eosio { namespace chain { uint32_t bnum = 0; if (block_pos >= _start_of_buffer_position) { const uint32_t index_of_block = block_pos - _start_of_buffer_position; - bnum = *reinterpret_cast(buf + index_of_block + trim_data::blknum_offset_from_block_entry(_version)); //block number of previous block (is big endian) + bnum = *reinterpret_cast(buf + index_of_block + block_log_impl::blknum_offset_from_block_entry(_version)); //block number of previous block (is big endian) } else { - const auto blknum_offset_pos = block_pos + trim_data::trim_data::blknum_offset_from_block_entry(_version); + const auto blknum_offset_pos = block_pos + block_log_impl::blknum_offset_from_block_entry(_version); auto status = fseek(_file.get(), blknum_offset_pos, SEEK_SET); EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}. Returned status: ${status}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos)("status", status) ); auto size = fread((void*)&bnum, sizeof(bnum), 1, _file.get()); @@ -1115,6 +1119,23 @@ namespace eosio { namespace chain { return std::clamp(version, min_supported_version, max_supported_version) == version; } + struct trim_data { //used by trim_blocklog_front(), trim_blocklog_end(), and smoke_test() + trim_data(fc::path block_dir); + ~trim_data(); + uint64_t block_index(uint32_t n) const; + uint64_t block_pos(uint32_t n); + fc::path block_file_name, index_file_name; //full pathname for blocks.log and blocks.index + uint32_t version = 0; //blocklog version + uint32_t first_block = 0; //first block in blocks.log + uint32_t last_block = 0; //last block in blocks.log + FILE* blk_in = nullptr; //C style files for reading blocks.log and blocks.index + FILE* ind_in = nullptr; //C style files for reading blocks.log and blocks.index + //we use low level file IO because it is distinctly faster than C++ filebuf or iostream + uint64_t first_block_pos = 0; //file position in blocks.log for the first block in the log + chain_id_type chain_id; + }; + + bool block_log::trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block) { using namespace std; EOS_ASSERT( block_dir != temp_dir, block_log_exception, "block_dir and temp_dir need to be different directories" ); @@ -1224,17 +1245,17 @@ namespace eosio { namespace chain { return true; } + + trim_data::trim_data(fc::path block_dir) { // code should follow logic in block_log::repair_log using namespace std; block_file_name = block_dir / "blocks.log"; - index_file_name = block_dir / "blocks.index"; blk_in = FC_FOPEN(block_file_name.generic_string().c_str(), "rb"); EOS_ASSERT( blk_in != nullptr, block_log_not_found, "cannot read file ${file}", ("file",block_file_name.string()) ); - ind_in = FC_FOPEN(index_file_name.generic_string().c_str(), "rb"); - EOS_ASSERT( ind_in != nullptr, block_log_not_found, "cannot read file ${file}", ("file",index_file_name.string()) ); + auto size = fread((void*)&version,sizeof(version), 1, blk_in); EOS_ASSERT( size == 1, block_log_unsupported_version, "invalid format for file ${file}", ("file",block_file_name.string())); ilog("block log version= ${version}",("version",version)); @@ -1279,6 +1300,10 @@ namespace eosio { namespace chain { const uint64_t start_of_blocks = ftell(blk_in); + index_file_name = block_dir / "blocks.index"; + ind_in = FC_FOPEN(index_file_name.generic_string().c_str(), "rb"); + EOS_ASSERT( ind_in != nullptr, block_log_not_found, "cannot read file ${file}", ("file",index_file_name.string()) ); + const auto status = fseek(ind_in, 0, SEEK_END); //get length of blocks.index (gives number of blocks) EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} end", ("file", index_file_name.string()) ); const uint64_t file_end = ftell(ind_in); //get length of blocks.index (gives number of blocks) @@ -1327,7 +1352,7 @@ namespace eosio { namespace chain { EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", index_file_name.string())("b",n) ); //read blocks.log and verify block number n is found at the determined file position - const auto calc_blknum_pos = block_n_pos + blknum_offset_from_block_entry(version); + const auto calc_blknum_pos = block_n_pos + detail::block_log_impl::blknum_offset_from_block_entry(version); status = fseek(blk_in, calc_blknum_pos, SEEK_SET); EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", calc_blknum_pos) ); const uint64_t block_offset_pos = ftell(blk_in); @@ -1365,4 +1390,31 @@ namespace eosio { namespace chain { return 0; } - } } /// eosio::chain + void block_log::smoke_test(fc::path block_dir) { + trim_data td(block_dir); + auto status = fseek(td.blk_in, -sizeof(uint64_t), SEEK_END); //get last_block from blocks.log, compare to from blocks.index + EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", td.block_file_name.string())("pos", sizeof(uint64_t)) ); + uint64_t file_pos; + auto size = fread((void*)&file_pos, sizeof(uint64_t), 1, td.blk_in); + EOS_ASSERT( size == 1, block_log_exception, "${file} read fails", ("file", td.block_file_name.string()) ); + int blknum_offset = detail::block_log_impl::blknum_offset_from_block_entry(td.version); + status = fseek(td.blk_in, file_pos + blknum_offset, SEEK_SET); + EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", td.block_file_name.string())("pos", file_pos + blknum_offset) ); + uint32_t bnum; + size = fread((void*)&bnum, sizeof(uint32_t), 1, td.blk_in); + EOS_ASSERT( size == 1, block_log_exception, "${file} read fails", ("file", td.block_file_name.string()) ); + bnum = fc::endian_reverse_u32(bnum) + 1; //convert from big endian to little endian and add 1 + EOS_ASSERT( td.last_block == bnum, block_log_exception, "blocks.log says last block is ${lb} which disagrees with blocks.index", ("lb", bnum) ); + ilog("blocks.log and blocks.index agree on number of blocks"); + uint32_t delta = (td.last_block + 8 - td.first_block) >> 3; + if (delta < 1) + delta = 1; + for (uint32_t n = td.first_block; ; n += delta) { + if (n > td.last_block) + n = td.last_block; + td.block_pos(n); //check block 'n' is where blocks.index says + if (n == td.last_block) + break; + } + } +}} /// eosio::chain diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 640b9584f5c..7c4d5c8d2ea 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -108,6 +108,7 @@ namespace eosio { namespace chain { pruned_transaction_receipt():transaction_receipt_header(){} pruned_transaction_receipt(const transaction_receipt&, bool legacy); + pruned_transaction_receipt(transaction_receipt&&, bool legacy); explicit pruned_transaction_receipt( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){} explicit pruned_transaction_receipt( const pruned_transaction& ptrx ):transaction_receipt_header(executed),trx(ptrx){} @@ -136,6 +137,7 @@ namespace eosio { namespace chain { pruned_block() = default; pruned_block( const signed_block&, bool legacy ); + pruned_block( signed_block&&, bool legacy ); pruned_block( pruned_block&& ) = default; pruned_block& operator=(const pruned_block&) = delete; pruned_block clone() const { return *this; } diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 93dbf5e396b..5b31c7aa153 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -194,10 +194,12 @@ namespace eosio { namespace chain { friend struct fc::has_reflector_init; void reflector_init(); private: - vector signatures; - fc::enum_type compression; - bytes packed_context_free_data; - bytes packed_trx; + friend struct pruned_transaction; + friend struct prunable_transaction_data; + vector signatures; + fc::enum_type compression; + bytes packed_context_free_data; + bytes packed_trx; private: // cache unpacked trx, for thread safety do not modify after construction @@ -251,6 +253,11 @@ namespace eosio { namespace chain { std::size_t maximum_pruned_pack_size( compression_type segment_compression ) const; prunable_data_type prunable_data; + + static prunable_transaction_data make(bool legacy, const signed_transaction& t, + packed_transaction::compression_type _compression); + static prunable_transaction_data make(bool legacy, const packed_transaction& t); + static prunable_transaction_data make(bool legacy, packed_transaction&& t); }; struct pruned_transaction : fc::reflect_init { @@ -263,8 +270,10 @@ namespace eosio { namespace chain { pruned_transaction& operator=(const pruned_transaction&) = delete; pruned_transaction& operator=(pruned_transaction&&) = default; - pruned_transaction(const packed_transaction& other, bool legacy) : pruned_transaction(other.get_signed_transaction(), legacy, other.get_compression()) {} - explicit pruned_transaction(const signed_transaction& t, bool legacy, compression_type _compression = compression_type::none); + pruned_transaction(const packed_transaction& other, bool legacy); + pruned_transaction(packed_transaction&& other, bool legacy); + explicit pruned_transaction(const signed_transaction& t, bool legacy, + compression_type _compression = compression_type::none); explicit pruned_transaction(signed_transaction&& t, bool legacy, compression_type _compression = compression_type::none); #if 0 diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 7320f25c204..472e9dba152 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -471,7 +471,7 @@ std::size_t prunable_transaction_data::maximum_pruned_pack_size(prunable_transac return 1 + prunable_data.visit([&](const auto& t){ return padded_pack_size(t, compression); }); } -static prunable_transaction_data make_prunable_transaction_data( bool legacy, const signed_transaction& t, pruned_transaction::compression_type _compression ) { +prunable_transaction_data prunable_transaction_data::make( bool legacy, const signed_transaction& t, packed_transaction::compression_type _compression ) { if(legacy) { return { prunable_transaction_data::full_legacy{ t.signatures, pack_context_free_data( t.context_free_data, _compression ) } }; } else { @@ -479,9 +479,42 @@ static prunable_transaction_data make_prunable_transaction_data( bool legacy, co } } +prunable_transaction_data prunable_transaction_data::make( bool legacy, const packed_transaction& t) { + if(legacy) { + return { prunable_transaction_data::full_legacy{ t.signatures, t.packed_context_free_data } }; + } else { + return { prunable_transaction_data::full{ t.signatures, t.get_context_free_data() } }; + } +} + +prunable_transaction_data prunable_transaction_data::make( bool legacy, packed_transaction&& t) { + if(legacy) { + return { prunable_transaction_data::full_legacy{ std::move(t.signatures), std::move(t.packed_context_free_data) } }; + } else { + return { prunable_transaction_data::full{ std::move(t.signatures), t.get_context_free_data() } }; + } +} + +pruned_transaction::pruned_transaction(const packed_transaction& other, bool legacy) + : compression(other.compression), + prunable_data(prunable_transaction_data::make(legacy, other)), + packed_trx(other.packed_trx), + unpacked_trx(other.unpacked_trx), + trx_id(other.id()) +{} + + +pruned_transaction::pruned_transaction(packed_transaction&& other, bool legacy) + : compression(other.compression), + prunable_data(prunable_transaction_data::make(legacy, std::move(other))), + packed_trx(std::move(other.packed_trx)), + unpacked_trx(std::move(other.unpacked_trx)), + trx_id(other.id()) +{} + pruned_transaction::pruned_transaction(const signed_transaction& t, bool legacy, compression_type _compression) : compression(_compression), - prunable_data(make_prunable_transaction_data(legacy, t, _compression)), + prunable_data(prunable_transaction_data::make(legacy, t, _compression)), packed_trx(pack_transaction(t, compression)), unpacked_trx(t), trx_id(unpacked_trx.id()) @@ -489,7 +522,7 @@ pruned_transaction::pruned_transaction(const signed_transaction& t, bool legacy, pruned_transaction::pruned_transaction(signed_transaction&& t, bool legacy, compression_type _compression) : compression(_compression), - prunable_data(make_prunable_transaction_data(legacy, t, _compression)), + prunable_data(prunable_transaction_data::make(legacy, t, _compression)), packed_trx(pack_transaction(t, compression)), unpacked_trx(std::move(t)), trx_id(unpacked_trx.id()) diff --git a/libraries/fc b/libraries/fc index 1b92cb14666..dae9acc60f5 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit 1b92cb14666d18e663b6626ab812badf338509c3 +Subproject commit dae9acc60f594cf7c3a04f8de0315941c4bec3e9 From 3272c0c6b5d3aa0adae0ce1f7dc16506a8137a5e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 14:23:14 -0500 Subject: [PATCH 034/142] Multiply prunable_size by 2 since it is also stored in unpacked transaction --- libraries/chain/transaction.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 9c69de7b20e..082e85da589 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -480,7 +480,7 @@ packed_transaction::packed_transaction(const signed_transaction& t, bool legacy, unpacked_trx(t), trx_id(unpacked_trx.id()) { - estimated_size = estimated_size * 2 + get_prunable_size(); + estimated_size = estimated_size * 2 + get_prunable_size() * 2; } packed_transaction::packed_transaction(signed_transaction&& t, bool legacy, compression_type _compression) @@ -490,7 +490,7 @@ packed_transaction::packed_transaction(signed_transaction&& t, bool legacy, comp unpacked_trx(std::move(t)), trx_id(unpacked_trx.id()) { - estimated_size = estimated_size * 2 + get_prunable_size(); + estimated_size = estimated_size * 2 + get_prunable_size() * 2; } packed_transaction_v0_ptr packed_transaction::to_packed_transaction_v0() const { @@ -598,7 +598,7 @@ void packed_transaction::reflector_init() signatures ? *signatures : std::vector{}, prunable_data.prunable_data.visit([&](const auto& obj) { return maybe_unpack_context_free_data(obj, compression); })); trx_id = unpacked_trx.id(); - estimated_size = packed_size * 2 + get_prunable_size(); // x2 since stored packed and unpacked + estimated_size = packed_size * 2 + get_prunable_size() * 2; // x2 since stored packed and unpacked } } } // eosio::chain From 5dff59aca0ec233813aa29b0d20cf866988511fe Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 14:23:28 -0500 Subject: [PATCH 035/142] Add missing moves --- plugins/chain_plugin/chain_plugin.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index d79b826f90a..c904923eb34 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -2135,7 +2135,7 @@ fc::variant read_only::get_block_header_state(const get_block_header_state_param void read_write::push_block(read_write::push_block_params&& params, next_function next) { try { - app().get_method()(std::make_shared(params, true), {}); + app().get_method()(std::make_shared( std::move( params ), true), {}); next(read_write::push_block_results{}); } catch ( boost::interprocess::bad_alloc& ) { chain_plugin::handle_db_exhaustion(); @@ -2151,7 +2151,7 @@ void read_write::push_transaction(const read_write::push_transaction_params& par packed_transaction_ptr input_trx; try { abi_serializer::from_variant(params, input_trx_v0, std::move( resolver ), abi_serializer::create_yield_function( abi_serializer_max_time )); - input_trx = std::make_shared( input_trx_v0, true ); + input_trx = std::make_shared( std::move( input_trx_v0 ), true ); } EOS_RETHROW_EXCEPTIONS(chain::packed_transaction_type_exception, "Invalid packed transaction") app().get_method()(input_trx, true, @@ -2272,7 +2272,7 @@ void read_write::send_transaction(const read_write::send_transaction_params& par packed_transaction_ptr input_trx; try { abi_serializer::from_variant(params, input_trx_v0, std::move( resolver ), abi_serializer::create_yield_function( abi_serializer_max_time )); - input_trx = std::make_shared( input_trx_v0, true ); + input_trx = std::make_shared( std::move( input_trx_v0 ), true ); } EOS_RETHROW_EXCEPTIONS(chain::packed_transaction_type_exception, "Invalid packed transaction") app().get_method()(input_trx, true, From 11f510e745f90ba67c7626b047d9ab0c62dfc9cb Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 3 Apr 2020 14:24:46 -0500 Subject: [PATCH 036/142] use move constructor --- libraries/chain/block_log.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 31c0daa17cb..1f4651d6f9b 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -490,8 +490,7 @@ namespace eosio { namespace chain { } else { signed_block block; fc::raw::unpack(ds, block); -#warning: TODO: use move(block) - return std::make_shared(block, true); + return std::make_shared(std::move(block), true); } } From 1ec37a522a5716dc0636d015b5e2980ca85ec6e2 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 3 Apr 2020 14:39:47 -0500 Subject: [PATCH 037/142] Add get_estimated_size to transaction_metadata --- libraries/chain/include/eosio/chain/transaction_metadata.hpp | 1 + .../chain/include/eosio/chain/unapplied_transaction_queue.hpp | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction_metadata.hpp b/libraries/chain/include/eosio/chain/transaction_metadata.hpp index bb48c71e9b4..f39dda27cdc 100644 --- a/libraries/chain/include/eosio/chain/transaction_metadata.hpp +++ b/libraries/chain/include/eosio/chain/transaction_metadata.hpp @@ -69,6 +69,7 @@ class transaction_metadata { const transaction_id_type& id()const { return _packed_trx->id(); } fc::microseconds signature_cpu_usage()const { return _sig_cpu_usage; } const flat_set& recovered_keys()const { return _recovered_pub_keys; } + uint32_t get_estimated_size() const { return sizeof(transaction_metadata) + _recovered_pub_keys.size() * sizeof(public_key_type); } /// Thread safe. /// @returns transaction_metadata_ptr or exception via future diff --git a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp index c086ff4fc26..a86c66f2577 100644 --- a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp +++ b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp @@ -254,8 +254,7 @@ class unapplied_transaction_queue { } static uint64_t calc_size( const transaction_metadata_ptr& trx ) { - // packed_trx caches unpacked transaction so double - return trx->packed_trx()->get_estimated_size() + sizeof( *trx ); + return trx->packed_trx()->get_estimated_size() + sizeof(unapplied_transaction) + trx->get_estimated_size(); } }; From 43dba9231033c0580f05b92155104f8ed6a811d1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Sat, 4 Apr 2020 20:39:23 -0500 Subject: [PATCH 038/142] Updated add of packed_transaction to always work, just put in empty signatures and empty context free data if not available --- .../chain/include/eosio/chain/abi_serializer.hpp | 11 ++++------- libraries/fc | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index face7f6f409..50b664e3821 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -481,14 +481,11 @@ namespace impl { mutable_variant_object mvo; const auto& trx = ptrx.get_transaction(); mvo("id", trx.id()); - mvo("signatures", ptrx.get_signatures() != nullptr ? *ptrx.get_signatures() : vector()); + const auto* sigs = ptrx.get_signatures(); + mvo("signatures", sigs != nullptr ? *sigs : vector()); mvo("compression", ptrx.get_compression()); - if( ptrx.get_prunable_data().prunable_data.contains() ) { - const auto& legacy = ptrx.get_prunable_data().prunable_data.get(); - mvo( "packed_context_free_data", legacy.packed_context_free_data ); - } else { - mvo( "packed_context_free_data", bytes() ); - } + const auto* context_free_data = ptrx.get_context_free_data(); + mvo("packed_context_free_data", context_free_data != nullptr ? *context_free_data : vector() ); mvo("context_free_data", ptrx.get_context_free_data() != nullptr ? *ptrx.get_context_free_data() : vector()); mvo("packed_trx", ptrx.get_packed_transaction()); add(mvo, "transaction", trx, resolver, ctx); diff --git a/libraries/fc b/libraries/fc index 1b92cb14666..dae9acc60f5 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit 1b92cb14666d18e663b6626ab812badf338509c3 +Subproject commit dae9acc60f594cf7c3a04f8de0315941c4bec3e9 From 2d8860e87bcd74fd3f4cce17ac6c68b68b87a68a Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Sat, 4 Apr 2020 20:41:51 -0500 Subject: [PATCH 039/142] Add move constructor for v0 types --- libraries/chain/block.cpp | 35 +++++++++++++++---- libraries/chain/include/eosio/chain/block.hpp | 4 ++- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index e7e5746ff44..b4bccd66113 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -21,16 +21,27 @@ namespace eosio { namespace chain { } } - static fc::static_variant translate_transaction_receipt(const transaction_id_type& tid, bool) { - return tid; - } - static fc::static_variant translate_transaction_receipt(const packed_transaction_v0& ptrx, bool legacy) { - return packed_transaction(ptrx, legacy); - } + struct transaction_receipt_translator { + bool legacy = true; + fc::static_variant operator()(const transaction_id_type& tid) const { + return tid; + } + fc::static_variant operator()(const packed_transaction_v0& ptrx) const { + return packed_transaction(ptrx, legacy); + } + fc::static_variant operator()(packed_transaction_v0&& ptrx) const { + return packed_transaction(std::move(ptrx), legacy); + } + }; transaction_receipt::transaction_receipt(const transaction_receipt_v0& other, bool legacy) : transaction_receipt_header(static_cast(other)), - trx(other.trx.visit([&](const auto& obj) { return translate_transaction_receipt(obj, legacy); })) + trx( std::move(other.trx).visit(transaction_receipt_translator{legacy})) + {} + + transaction_receipt::transaction_receipt(transaction_receipt_v0&& other, bool legacy) + : transaction_receipt_header(std::move(static_cast(other))), + trx( std::move(other.trx).visit(transaction_receipt_translator{legacy})) {} static flat_multimap validate_and_extract_block_extensions(const extensions_type& block_extensions) { @@ -88,6 +99,16 @@ namespace eosio { namespace chain { } } + signed_block::signed_block( signed_block_v0&& other, bool legacy ) + : signed_block_header(std::move(static_cast(other))), + prune_state(legacy ? prune_state_type::complete_legacy : prune_state_type::complete), + block_extensions(std::move(other.block_extensions)) + { + for(auto& trx : other.transactions) { + transactions.emplace_back(std::move(trx), legacy); + } + } + static std::size_t pruned_trx_receipt_packed_size(const packed_transaction& obj, packed_transaction::cf_compression_type segment_compression) { return obj.maximum_pruned_pack_size(segment_compression); } diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 5ce7e2a2883..35af9531050 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -108,6 +108,7 @@ namespace eosio { namespace chain { transaction_receipt():transaction_receipt_header(){} transaction_receipt(const transaction_receipt_v0&, bool legacy); + transaction_receipt(transaction_receipt_v0&&, bool legacy); explicit transaction_receipt( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){} explicit transaction_receipt( const packed_transaction& ptrx ):transaction_receipt_header(executed),trx(ptrx){} @@ -137,6 +138,7 @@ namespace eosio { namespace chain { signed_block() = default; explicit signed_block( const signed_block_header& h ):signed_block_header(h){} signed_block( const signed_block_v0&, bool legacy ); + signed_block( signed_block_v0&&, bool legacy ); signed_block( signed_block&& ) = default; signed_block& operator=(const signed_block&) = delete; signed_block& operator=(signed_block&&) = default; @@ -152,7 +154,7 @@ namespace eosio { namespace chain { // Returns the maximum_pruned_padded_size. It is the caller's responsibility to // reserve enough space after the end if in-place pruning is desired. template - std::size_t pack(Stream& stream, packed_transaction::cf_compression_type segment_compression) { + std::size_t pack(Stream& stream, packed_transaction::cf_compression_type segment_compression) const { std::size_t padded_size = maximum_pruned_pack_size( segment_compression ); // TODO: This only handles legacy transactions. fc::raw::pack(stream, *this); From a345c648540477d70ede3de6a22321514b6772b1 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Sat, 4 Apr 2020 19:03:22 -0500 Subject: [PATCH 040/142] add trim_blocklog_front test --- unittests/restart_chain_tests.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index 8b467c4dd13..008fe28dac2 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -257,7 +257,7 @@ BOOST_FIXTURE_TEST_CASE(test_restart_from_trimed_block_log, restart_from_block_l auto& config = chain.get_config(); auto blocks_path = config.blocks_dir; remove_all(blocks_path/"reversible"); - block_log::trim_blocklog_end(config.blocks_dir, cutoff_block_num); + BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_end(config.blocks_dir, cutoff_block_num)); } BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log, light_validation_restart_from_block_log_test_fixture) { @@ -268,5 +268,16 @@ BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log_with_pruned blog.prune_transaction(trace->block_num, trace->id); } +BOOST_AUTO_TEST_CASE(test_trim_blocklog_front) { + tester chain; + chain.produce_blocks(10); + chain.produce_blocks(20); + chain.close(); + + auto blocks_dir = chain.get_config().blocks_dir; + auto temp = boost::filesystem::unique_path(); + BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_front(blocks_dir, temp, 10)); + BOOST_REQUIRE_NO_THROW(block_log::smoke_test(blocks_dir)); +} BOOST_AUTO_TEST_SUITE_END() From 6361dd909d145f94aa6e5e9ce9b5d240dd03f099 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 6 Apr 2020 07:45:26 -0500 Subject: [PATCH 041/142] - Pulled in constructor fixes from #8891. Updated with peer review comments on #8891. - Changed `packed_transaction` to store a cached `transaction` instead of a cached `signed_transaction` since signatures and context free data already stored on `packed_transaction` allowing for move constructor from `packed_transaction_v0`. - Added `apply_context::get_packed_transaction_size` to optimize intrinsic `transaction_size`. - Optimized `apply_context::get_packed_transaction` used by intrinsic `read_transaction` to avoid `fc::raw::pack` when packed form already available. - Store `packed_transaction_ptr` on `transaction_context` providing access to context free data. - Change `controller` signal `applied_transaction` to emit a `packed_transaction_ptr` instead of a `signed_transaction` - Added two new exceptions `tx_no_signature` & `tx_no_context_free_data` --- libraries/chain/apply_context.cpp | 41 ++++++++--- libraries/chain/controller.cpp | 32 +++++---- .../include/eosio/chain/apply_context.hpp | 1 + .../chain/include/eosio/chain/controller.hpp | 2 +- .../chain/include/eosio/chain/exceptions.hpp | 4 ++ .../chain/include/eosio/chain/transaction.hpp | 29 ++++---- .../eosio/chain/transaction_context.hpp | 6 +- .../eosio/chain/transaction_metadata.hpp | 11 +-- libraries/chain/transaction.cpp | 70 ++++++++++++++----- libraries/chain/transaction_context.cpp | 21 ++++-- libraries/chain/transaction_metadata.cpp | 9 ++- libraries/chain/wasm_interface.cpp | 8 +-- libraries/testing/tester.cpp | 2 +- plugins/chain_plugin/chain_plugin.cpp | 2 +- plugins/history_plugin/history_plugin.cpp | 8 ++- plugins/mongo_db_plugin/mongo_db_plugin.cpp | 5 +- .../state_history_plugin.hpp | 22 +++--- .../state_history_plugin.cpp | 4 +- .../eosio/trace_api/chain_extraction.hpp | 6 +- .../trace_api_plugin/test/test_extraction.cpp | 14 ++-- plugins/trace_api_plugin/trace_api_plugin.cpp | 2 +- unittests/api_tests.cpp | 14 ++-- unittests/block_tests.cpp | 4 +- unittests/misc_tests.cpp | 21 +++--- unittests/protocol_feature_tests.cpp | 4 +- unittests/restart_chain_tests.cpp | 2 +- unittests/whitelist_blacklist_tests.cpp | 2 +- 27 files changed, 211 insertions(+), 135 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 6617b6cdf5c..46867d857e1 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -390,15 +390,15 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a "deferred transaction generaction context contains mismatching sender_id", ("expected", sender_id)("actual", context.sender_id) ); - EOS_ASSERT( context.sender_trx_id == trx_context.id, ill_formed_deferred_transaction_generation_context, + EOS_ASSERT( context.sender_trx_id == trx_context.packed_trx->id(), ill_formed_deferred_transaction_generation_context, "deferred transaction generaction context contains mismatching sender_trx_id", - ("expected", trx_context.id)("actual", context.sender_trx_id) + ("expected", trx_context.packed_trx->id())("actual", context.sender_trx_id) ); } else { emplace_extension( trx.transaction_extensions, deferred_transaction_generation_context::extension_id(), - fc::raw::pack( deferred_transaction_generation_context( trx_context.id, sender_id, receiver ) ) + fc::raw::pack( deferred_transaction_generation_context( trx_context.packed_trx->id(), sender_id, receiver ) ) ); } trx.expiration = time_point_sec(); @@ -612,8 +612,19 @@ vector apply_context::get_active_producers() const { } bytes apply_context::get_packed_transaction() { - auto r = fc::raw::pack( static_cast(trx_context.trx) ); - return r; + if( trx_context.packed_trx->get_compression() == packed_transaction::compression_type::none) { + return trx_context.packed_trx->get_packed_transaction(); + } else { + return fc::raw::pack( static_cast( trx_context.packed_trx->get_transaction() ) ); + } +} + +size_t apply_context::get_packed_transaction_size() { + if( trx_context.packed_trx->get_compression() == packed_transaction::compression_type::none) { + return trx_context.packed_trx->get_packed_transaction().size(); + } else { + return fc::raw::pack_size( static_cast( trx_context.packed_trx->get_transaction() ) ); + } } void apply_context::update_db_usage( const account_name& payer, int64_t delta ) { @@ -632,7 +643,7 @@ void apply_context::update_db_usage( const account_name& payer, int64_t delta ) int apply_context::get_action( uint32_t type, uint32_t index, char* buffer, size_t buffer_size )const { - const auto& trx = trx_context.trx; + const auto& trx = trx_context.packed_trx->get_transaction(); const action* act_ptr = nullptr; if( type == 0 ) { @@ -658,15 +669,25 @@ int apply_context::get_action( uint32_t type, uint32_t index, char* buffer, size int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t buffer_size )const { - const auto& trx = trx_context.trx; + const prunable_transaction_data::prunable_data_type& data = trx_context.packed_trx->get_prunable_data().prunable_data; + if( data.contains() || data.contains() || + data.contains() ) // TODO: for partial determine if index is pruned and throw only in that case + { + // TODO: throw an exception like disallowed_transaction_extensions_bad_block_exception && subjective_block_production_exception + EOS_ASSERT( false, subjective_block_production_exception, "todo" ); + } + + std::vector context_free_data = data.contains() ? + data.get().context_free_segments : + data.get().context_free_segments; - if( index >= trx.context_free_data.size() ) return -1; + if( index >= context_free_data.size() ) return -1; - auto s = trx.context_free_data[index].size(); + auto s = context_free_data[index].size(); if( buffer_size == 0 ) return s; auto copy_size = std::min( buffer_size, s ); - memcpy( buffer, trx.context_free_data[index].data(), copy_size ); + memcpy( buffer, context_free_data[index].data(), copy_size ); return copy_size; } diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index f1bfa5d1765..86e18fc33b7 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1140,7 +1140,8 @@ struct controller_impl { } transaction_checktime_timer trx_timer(timer); - transaction_context trx_context( self, etrx, etrx.id(), std::move(trx_timer), start ); + packed_transaction_ptr trx = std::make_shared( std::move( etrx ), true ); + transaction_context trx_context( self, trx, std::move(trx_timer), start ); trx_context.deadline = deadline; trx_context.explicit_billed_cpu_time = explicit_billed_cpu_time; trx_context.billed_cpu_time_us = billed_cpu_time_us; @@ -1149,7 +1150,7 @@ struct controller_impl { try { trx_context.init_for_implicit_trx(); trx_context.published = gtrx.published; - trx_context.execute_action( trx_context.schedule_action( etrx.actions.back(), gtrx.sender, false, 0, 0 ), 0 ); + trx_context.execute_action( trx_context.schedule_action( trx->get_transaction().actions.back(), gtrx.sender, false, 0, 0 ), 0 ); trx_context.finalize(); // Automatically rounds up network and CPU usage in trace and bills payers if successful auto restore = make_block_restore_point(); @@ -1241,7 +1242,8 @@ struct controller_impl { signed_transaction dtrx; fc::raw::unpack(ds,static_cast(dtrx) ); - transaction_metadata_ptr trx = transaction_metadata::create_no_recover_keys( packed_transaction( dtrx, true ), transaction_metadata::trx_type::scheduled ); + transaction_metadata_ptr trx = + transaction_metadata::create_no_recover_keys( packed_transaction( std::move(dtrx), true ), transaction_metadata::trx_type::scheduled ); trx->accepted = true; transaction_trace_ptr trace; @@ -1255,7 +1257,7 @@ struct controller_impl { trace->receipt = push_receipt( gtrx.trx_id, transaction_receipt::expired, billed_cpu_time_us, 0 ); // expire the transaction trace->account_ram_delta = account_delta( gtrx.payer, trx_removal_ram_delta ); emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, dtrx) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); undo_session.squash(); return trace; } @@ -1268,7 +1270,7 @@ struct controller_impl { uint32_t cpu_time_to_bill_us = billed_cpu_time_us; transaction_checktime_timer trx_timer(timer); - transaction_context trx_context( self, dtrx, gtrx.trx_id, std::move(trx_timer) ); + transaction_context trx_context( self, trx->packed_trx(), std::move(trx_timer) ); trx_context.leeway = fc::microseconds(0); // avoid stealing cpu resource trx_context.deadline = deadline; trx_context.explicit_billed_cpu_time = explicit_billed_cpu_time; @@ -1280,7 +1282,7 @@ struct controller_impl { if( trx_context.enforce_whiteblacklist && pending->_block_status == controller::block_status::incomplete ) { flat_set actors; - for( const auto& act : trx_context.trx.actions ) { + for( const auto& act : trx->packed_trx()->get_transaction().actions ) { for( const auto& auth : act.authorization ) { actors.insert( auth.actor ); } @@ -1304,7 +1306,7 @@ struct controller_impl { trace->account_ram_delta = account_delta( gtrx.payer, trx_removal_ram_delta ); emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, dtrx) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); trx_context.squash(); undo_session.squash(); @@ -1338,7 +1340,7 @@ struct controller_impl { if( !trace->except_ptr ) { trace->account_ram_delta = account_delta( gtrx.payer, trx_removal_ram_delta ); emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, dtrx) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); undo_session.squash(); return trace; } @@ -1379,12 +1381,12 @@ struct controller_impl { trace->account_ram_delta = account_delta( gtrx.payer, trx_removal_ram_delta ); emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, dtrx) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); undo_session.squash(); } else { emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, dtrx) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); } return trace; @@ -1440,9 +1442,8 @@ struct controller_impl { } } - const signed_transaction& trn = trx->packed_trx()->get_signed_transaction(); transaction_checktime_timer trx_timer(timer); - transaction_context trx_context(self, trn, trx->id(), std::move(trx_timer), start); + transaction_context trx_context(self, trx->packed_trx(), std::move(trx_timer), start); if ((bool)subjective_cpu_leeway && pending->_block_status == controller::block_status::incomplete) { trx_context.leeway = *subjective_cpu_leeway; } @@ -1451,6 +1452,7 @@ struct controller_impl { trx_context.billed_cpu_time_us = billed_cpu_time_us; trace = trx_context.trace; try { + const transaction& trn = trx->packed_trx()->get_transaction(); if( trx->implicit ) { EOS_ASSERT( !explicit_net_usage_words.valid(), transaction_exception, "NET usage cannot be explicitly set for implicit transactions" ); trx_context.init_for_implicit_trx(); @@ -1507,7 +1509,7 @@ struct controller_impl { emit( self.accepted_transaction, trx); } - emit(self.applied_transaction, std::tie(trace, trn)); + emit(self.applied_transaction, std::tie(trace, trx->packed_trx())); if ( read_mode != db_read_mode::SPECULATIVE && pending->_block_status == controller::block_status::incomplete ) { @@ -1530,7 +1532,7 @@ struct controller_impl { } emit( self.accepted_transaction, trx ); - emit( self.applied_transaction, std::tie(trace, trn) ); + emit( self.applied_transaction, std::tie(trace, trx->packed_trx()) ); return trace; } FC_CAPTURE_AND_RETHROW((trace)) @@ -1883,7 +1885,7 @@ struct controller_impl { trx_metas.emplace_back( std::move( trx_meta_ptr ), recover_keys_future{} ); } else if( skip_auth_checks ) { trx_metas.emplace_back( - transaction_metadata::create_no_recover_keys( pt, transaction_metadata::trx_type::input ), + transaction_metadata::create_no_recover_keys( packed_transaction( pt ), transaction_metadata::trx_type::input ), recover_keys_future{} ); } else { auto ptrx = std::make_shared( pt ); diff --git a/libraries/chain/include/eosio/chain/apply_context.hpp b/libraries/chain/include/eosio/chain/apply_context.hpp index c50c10daca5..573218326eb 100644 --- a/libraries/chain/include/eosio/chain/apply_context.hpp +++ b/libraries/chain/include/eosio/chain/apply_context.hpp @@ -538,6 +538,7 @@ class apply_context { int get_context_free_data( uint32_t index, char* buffer, size_t buffer_size )const; vector get_active_producers() const; bytes get_packed_transaction(); + size_t get_packed_transaction_size(); uint64_t next_global_sequence(); uint64_t next_recv_sequence( const account_metadata_object& receiver_account ); diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 72c8ef0471e..c0997da8028 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -307,7 +307,7 @@ namespace eosio { namespace chain { signal accepted_block; signal irreversible_block; signal accepted_transaction; - signal)> applied_transaction; + signal)> applied_transaction; signal bad_alloc; /* diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index b534844ea53..a80c4bb0ef8 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -279,6 +279,10 @@ namespace eosio { namespace chain { 3040018, "Transaction exceeded transient resource limit" ) FC_DECLARE_DERIVED_EXCEPTION( tx_prune_exception, transaction_exception, 3040019, "Prunable data not found" ) + FC_DECLARE_DERIVED_EXCEPTION( tx_no_signature, transaction_exception, + 3040020, "Transaction has no signatures" ) + FC_DECLARE_DERIVED_EXCEPTION( tx_no_context_free_data, transaction_exception, + 3040021, "Transaction context free data pruned" ) FC_DECLARE_DERIVED_EXCEPTION( action_validate_exception, chain_exception, diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 01956293514..35b456a5358 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -126,14 +126,9 @@ namespace eosio { namespace chain { signed_transaction( signed_transaction&& ) = default; signed_transaction& operator=(const signed_transaction&) = delete; signed_transaction& operator=(signed_transaction&&) = default; - signed_transaction( transaction&& trx, const vector& signatures, const vector& context_free_data) + signed_transaction( transaction&& trx, vector signatures, vector context_free_data) : transaction(std::move(trx)) - , signatures(signatures) - , context_free_data(context_free_data) - {} - signed_transaction( transaction&& trx, const vector& signatures, vector&& context_free_data) - : transaction(std::move(trx)) - , signatures(signatures) + , signatures(std::move(signatures)) , context_free_data(std::move(context_free_data)) {} @@ -208,6 +203,8 @@ namespace eosio { namespace chain { friend struct packed_transaction; void reflector_init(); private: + friend struct packed_transaction; + vector signatures; fc::enum_type compression; bytes packed_context_free_data; @@ -220,10 +217,13 @@ namespace eosio { namespace chain { }; struct prunable_transaction_data { - enum class compression_type { + enum class compression_type : uint8_t { none = 0, zlib = 1, + COMPRESSION_TYPE_COUNT }; + // Do not exceed 127 as that will break compatibility in serialization format + static_assert( static_cast(compression_type::COMPRESSION_TYPE_COUNT) <= 127 ); struct none { digest_type prunable_digest; @@ -237,18 +237,20 @@ namespace eosio { namespace chain { using segment_type = fc::static_variant>; struct partial { + // TODO: will need indication of what was pruned so correct exception can be thrown in apply_context.get_context_free_data std::vector signatures; std::vector context_free_segments; }; struct full { std::vector signatures; - std::vector> context_free_segments; + std::vector context_free_segments; }; struct full_legacy { std::vector signatures; bytes packed_context_free_data; + vector context_free_segments; }; using prunable_data_type = fc::static_variant< full_legacy, @@ -279,8 +281,8 @@ namespace eosio { namespace chain { packed_transaction& operator=(const packed_transaction&) = delete; packed_transaction& operator=(packed_transaction&&) = default; - packed_transaction(const packed_transaction_v0& other, bool legacy) : packed_transaction(other.get_signed_transaction(), legacy, other.get_compression()) {} - packed_transaction(packed_transaction_v0&& other, bool legacy) : packed_transaction(std::move( other.unpacked_trx ), legacy, other.get_compression()) {} + packed_transaction(const packed_transaction_v0& other, bool legacy); + packed_transaction(packed_transaction_v0&& other, bool legacy); explicit packed_transaction(const signed_transaction& t, bool legacy, compression_type _compression = compression_type::none); explicit packed_transaction(signed_transaction&& t, bool legacy, compression_type _compression = compression_type::none); @@ -296,7 +298,6 @@ namespace eosio { namespace chain { time_point_sec expiration()const { return unpacked_trx.expiration; } const transaction& get_transaction()const { return unpacked_trx; } - const signed_transaction& get_signed_transaction()const { return unpacked_trx; } // Returns nullptr if the signatures were pruned const vector* get_signatures()const; // Returns nullptr if any context_free_data segment was pruned @@ -321,11 +322,11 @@ namespace eosio { namespace chain { uint32_t estimated_size = 0; fc::enum_type compression; prunable_transaction_data prunable_data; - bytes packed_trx; + bytes packed_trx; // packed and compressed (according to compression) transaction private: // cache unpacked trx, for thread safety do not modify any attributes after construction - signed_transaction unpacked_trx; + transaction unpacked_trx; transaction_id_type trx_id; }; diff --git a/libraries/chain/include/eosio/chain/transaction_context.hpp b/libraries/chain/include/eosio/chain/transaction_context.hpp index 5517eb83f52..4de1a136253 100644 --- a/libraries/chain/include/eosio/chain/transaction_context.hpp +++ b/libraries/chain/include/eosio/chain/transaction_context.hpp @@ -38,8 +38,7 @@ namespace eosio { namespace chain { public: transaction_context( controller& c, - const signed_transaction& t, - const transaction_id_type& trx_id, + packed_transaction_ptr t, transaction_checktime_timer&& timer, fc::time_point start = fc::time_point::now() ); @@ -134,8 +133,7 @@ namespace eosio { namespace chain { public: controller& control; - const signed_transaction& trx; - transaction_id_type id; + const packed_transaction_ptr packed_trx; optional undo_session; transaction_trace_ptr trace; fc::time_point start; diff --git a/libraries/chain/include/eosio/chain/transaction_metadata.hpp b/libraries/chain/include/eosio/chain/transaction_metadata.hpp index f39dda27cdc..970fd1129ac 100644 --- a/libraries/chain/include/eosio/chain/transaction_metadata.hpp +++ b/libraries/chain/include/eosio/chain/transaction_metadata.hpp @@ -40,10 +40,13 @@ class transaction_metadata { private: struct private_type{}; - static void check_variable_sig_size(const packed_transaction_ptr& trx, uint32_t max) { - for(const signature_type& sig : trx->get_signed_transaction().signatures) + static const vector& check_variable_sig_size(const packed_transaction_ptr& trx, uint32_t max) { + const vector* sigs = trx->get_signatures(); + EOS_ASSERT( sigs, tx_no_signature, "No signaures on packed_transaction" ); + for(const signature_type& sig : *sigs) EOS_ASSERT(sig.variable_size() <= max, sig_variable_size_limit_exception, "signature variable length component size (${s}) greater than subjective maximum (${m})", ("s", sig.variable_size())("m", max)); + return *sigs; } public: @@ -80,9 +83,9 @@ class transaction_metadata { /// @returns constructed transaction_metadata with no key recovery (sig_cpu_usage=0, recovered_pub_keys=empty) static transaction_metadata_ptr - create_no_recover_keys( const packed_transaction& trx, trx_type t ) { + create_no_recover_keys( packed_transaction trx, trx_type t ) { return std::make_shared( private_type(), - std::make_shared( trx ), fc::microseconds(), flat_set(), + std::make_shared( std::move(trx) ), fc::microseconds(), flat_set(), t == trx_type::implicit, t == trx_type::scheduled ); } diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 082e85da589..510ed20b6f6 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -465,17 +465,20 @@ std::size_t prunable_transaction_data::maximum_pruned_pack_size(prunable_transac return 1 + prunable_data.visit([&](const auto& t){ return padded_pack_size(t, compression); }); } -static prunable_transaction_data make_prunable_transaction_data( bool legacy, const signed_transaction& t, packed_transaction::compression_type _compression ) { +static prunable_transaction_data make_prunable_transaction_data( bool legacy, vector signatures, + vector context_free_data, + packed_transaction::compression_type _compression ) { if(legacy) { - return { prunable_transaction_data::full_legacy{ t.signatures, pack_context_free_data( t.context_free_data, _compression ) } }; + bytes packed_cfd = pack_context_free_data( context_free_data, _compression ); + return { prunable_transaction_data::full_legacy{ std::move(signatures), std::move(packed_cfd), std::move(context_free_data) } }; } else { - return { prunable_transaction_data::full{ t.signatures, t.context_free_data } }; + return { prunable_transaction_data::full{ std::move(signatures), std::move(context_free_data) } }; } } packed_transaction::packed_transaction(const signed_transaction& t, bool legacy, compression_type _compression) : compression(_compression), - prunable_data(make_prunable_transaction_data(legacy, t, _compression)), + prunable_data(make_prunable_transaction_data(legacy, t.signatures, t.context_free_data, _compression)), packed_trx(pack_transaction(t, estimated_size, compression)), unpacked_trx(t), trx_id(unpacked_trx.id()) @@ -485,7 +488,7 @@ packed_transaction::packed_transaction(const signed_transaction& t, bool legacy, packed_transaction::packed_transaction(signed_transaction&& t, bool legacy, compression_type _compression) : compression(_compression), - prunable_data(make_prunable_transaction_data(legacy, t, _compression)), + prunable_data(make_prunable_transaction_data(legacy, std::move(t.signatures), std::move(t.context_free_data), _compression)), packed_trx(pack_transaction(t, estimated_size, compression)), unpacked_trx(std::move(t)), trx_id(unpacked_trx.id()) @@ -493,8 +496,37 @@ packed_transaction::packed_transaction(signed_transaction&& t, bool legacy, comp estimated_size = estimated_size * 2 + get_prunable_size() * 2; } +packed_transaction::packed_transaction(const packed_transaction_v0& other, bool legacy) + : compression(other.compression), + prunable_data(legacy ? prunable_transaction_data{ prunable_transaction_data::full_legacy{ other.signatures, + other.packed_context_free_data, + other.unpacked_trx.context_free_data } } + : prunable_transaction_data{ prunable_transaction_data::full{ other.signatures, + other.unpacked_trx.context_free_data } }), + packed_trx(other.packed_trx), + unpacked_trx(other.unpacked_trx), + trx_id(other.id()) +{} + +packed_transaction::packed_transaction(packed_transaction_v0&& other, bool legacy) + : compression(other.compression), + prunable_data(legacy ? prunable_transaction_data{ prunable_transaction_data::full_legacy{ std::move(other.signatures), + std::move(other.packed_context_free_data), + std::move(other.unpacked_trx.context_free_data) } } + : prunable_transaction_data{ prunable_transaction_data::full{ std::move(other.signatures), + std::move(other.unpacked_trx.context_free_data) } }), + packed_trx(std::move(other.packed_trx)), + unpacked_trx(std::move(other.unpacked_trx)), + trx_id(other.id()) +{} + packed_transaction_v0_ptr packed_transaction::to_packed_transaction_v0() const { - return std::make_shared( get_signed_transaction(), get_compression() ); + const auto* sigs = get_signatures(); + const auto* context_free_data = get_context_free_data(); + signed_transaction strx( transaction( get_transaction() ), + sigs != nullptr ? *sigs : vector(), + context_free_data != nullptr ? *context_free_data : vector() ); + return std::make_shared( std::move( strx ) ); } uint32_t packed_transaction::get_unprunable_size()const { @@ -540,9 +572,10 @@ const vector* packed_transaction::get_signatures()const { } const vector* packed_transaction::get_context_free_data()const { - if(prunable_data.prunable_data.contains() || - prunable_data.prunable_data.contains()) { - return &unpacked_trx.context_free_data; + if( prunable_data.prunable_data.contains() ) { + return &prunable_data.prunable_data.get().context_free_segments; + } else if( prunable_data.prunable_data.contains() ) { + return &prunable_data.prunable_data.get().context_free_segments; } else { return nullptr; } @@ -553,22 +586,21 @@ const bytes* maybe_get_context_free_data(const prunable_transaction_data::signat const bytes* maybe_get_context_free_data(const prunable_transaction_data::partial&, std::size_t) { EOS_THROW(tx_prune_exception, "unimplemented"); } -// full uses the copy in unpacked_trx -const bytes* maybe_get_context_free_data(const prunable_transaction_data::full_legacy&, std::size_t) { return nullptr; } -const bytes* maybe_get_context_free_data(const prunable_transaction_data::full&, std::size_t) { return nullptr; } +const bytes* maybe_get_context_free_data(const prunable_transaction_data::full_legacy& full_leg, std::size_t i) { + if( full_leg.context_free_segments.size() <= i ) return nullptr; + return &full_leg.context_free_segments[i]; +} +const bytes* maybe_get_context_free_data(const prunable_transaction_data::full& f, std::size_t i) { + if( f.context_free_segments.size() <= i ) return nullptr; + return &f.context_free_segments[i]; +} const bytes* packed_transaction::get_context_free_data(std::size_t segment_ordinal) const { - if (segment_ordinal < unpacked_trx.context_free_data.size()) { - return &unpacked_trx.context_free_data[segment_ordinal]; - } else { - return prunable_data.prunable_data.visit([&](const auto& obj) { return maybe_get_context_free_data(obj, segment_ordinal); }); - } + return prunable_data.prunable_data.visit([&](const auto& obj) { return maybe_get_context_free_data(obj, segment_ordinal); }); } void packed_transaction::prune_all() { prunable_data = prunable_data.prune_all(); - unpacked_trx.context_free_data.clear(); - unpacked_trx.signatures.clear(); } std::size_t packed_transaction::maximum_pruned_pack_size(cf_compression_type segment_compression) const { diff --git a/libraries/chain/transaction_context.cpp b/libraries/chain/transaction_context.cpp index 25346fd32e3..18d3a5a32c0 100644 --- a/libraries/chain/transaction_context.cpp +++ b/libraries/chain/transaction_context.cpp @@ -44,13 +44,11 @@ namespace eosio { namespace chain { } transaction_context::transaction_context( controller& c, - const signed_transaction& t, - const transaction_id_type& trx_id, + packed_transaction_ptr t, transaction_checktime_timer&& tmr, fc::time_point s ) :control(c) - ,trx(t) - ,id(trx_id) + ,packed_trx(std::move(t)) ,undo_session() ,trace(std::make_shared()) ,start(s) @@ -61,7 +59,7 @@ namespace eosio { namespace chain { if (!c.skip_db_sessions()) { undo_session = c.mutable_db().start_undo_session(true); } - trace->id = id; + trace->id = packed_trx->id(); trace->block_num = c.head_block_num() + 1; trace->block_time = c.pending_block_time(); trace->producer_block_id = c.pending_producer_block_id(); @@ -100,6 +98,7 @@ namespace eosio { namespace chain { _deadline = start + objective_duration_limit; } + const transaction& trx = packed_trx->get_transaction(); // Possibly lower net_limit to optional limit set in the transaction header uint64_t trx_specified_net_usage_limit = static_cast(trx.max_net_usage_words.value) * 8; if( trx_specified_net_usage_limit > 0 && trx_specified_net_usage_limit <= net_limit ) { @@ -192,6 +191,7 @@ namespace eosio { namespace chain { void transaction_context::init_for_implicit_trx( uint64_t initial_net_usage ) { + const transaction& trx = packed_trx->get_transaction(); if( trx.transaction_extensions.size() > 0 ) { disallow_transaction_extensions( "no transaction extensions supported yet for implicit transactions" ); } @@ -204,6 +204,7 @@ namespace eosio { namespace chain { uint64_t packed_trx_prunable_size, bool skip_recording ) { + const transaction& trx = packed_trx->get_transaction(); if( trx.transaction_extensions.size() > 0 ) { disallow_transaction_extensions( "no transaction extensions supported yet for input transactions" ); } @@ -236,6 +237,7 @@ namespace eosio { namespace chain { void transaction_context::init_for_input_trx_with_explicit_net( uint32_t explicit_net_usage_words, bool skip_recording ) { + const transaction& trx = packed_trx->get_transaction(); if( trx.transaction_extensions.size() > 0 ) { disallow_transaction_extensions( "no transaction extensions supported yet for input transactions" ); } @@ -250,6 +252,7 @@ namespace eosio { namespace chain { { published = control.pending_block_time(); is_input = true; + const transaction& trx = packed_trx->get_transaction(); if (!control.skip_trx_checks()) { control.validate_expiration(trx); control.validate_tapos(trx); @@ -257,11 +260,12 @@ namespace eosio { namespace chain { } init( initial_net_usage ); if (!skip_recording) - record_transaction( id, trx.expiration ); /// checks for dupes + record_transaction( packed_trx->id(), trx.expiration ); /// checks for dupes } void transaction_context::init_for_deferred_trx( fc::time_point p ) { + const transaction& trx = packed_trx->get_transaction(); if( (trx.expiration.sec_since_epoch() != 0) && (trx.transaction_extensions.size() > 0) ) { disallow_transaction_extensions( "no transaction extensions supported yet for deferred transactions" ); } @@ -279,6 +283,7 @@ namespace eosio { namespace chain { void transaction_context::exec() { EOS_ASSERT( is_initialized, transaction_exception, "must first initialize" ); + const transaction& trx = packed_trx->get_transaction(); if( apply_context_free ) { for( const auto& act : trx.context_free_actions ) { schedule_action( act, act.account, true, 0, 0 ); @@ -306,6 +311,7 @@ namespace eosio { namespace chain { EOS_ASSERT( is_initialized, transaction_exception, "must first initialize" ); if( is_input ) { + const transaction& trx = packed_trx->get_transaction(); auto& am = control.get_mutable_authorization_manager(); for( const auto& act : trx.actions ) { for( const auto& auth : act.authorization ) { @@ -611,6 +617,7 @@ namespace eosio { namespace chain { void transaction_context::schedule_transaction() { // Charge ahead of time for the additional net usage needed to retire the delayed transaction // whether that be by successfully executing, soft failure, hard failure, or expiration. + const transaction& trx = packed_trx->get_transaction(); if( trx.delay_sec.value == 0 ) { // Do not double bill. Only charge if we have not already charged for the delay. const auto& cfg = control.get_global_properties().configuration; add_net_usage( static_cast(cfg.base_per_transaction_net_usage) @@ -621,7 +628,7 @@ namespace eosio { namespace chain { uint32_t trx_size = 0; const auto& cgto = control.mutable_db().create( [&]( auto& gto ) { - gto.trx_id = id; + gto.trx_id = packed_trx->id(); gto.payer = first_auth; gto.sender = account_name(); /// delayed transactions have no sender gto.sender_id = transaction_id_to_sender_id( gto.trx_id ); diff --git a/libraries/chain/transaction_metadata.cpp b/libraries/chain/transaction_metadata.cpp index 00a9e2c329a..abd9f3f77bb 100644 --- a/libraries/chain/transaction_metadata.cpp +++ b/libraries/chain/transaction_metadata.cpp @@ -13,10 +13,13 @@ recover_keys_future transaction_metadata::start_recover_keys( packed_transaction return async_thread_pool( thread_pool, [trx{std::move(trx)}, chain_id, time_limit, max_variable_sig_size]() mutable { fc::time_point deadline = time_limit == fc::microseconds::maximum() ? fc::time_point::maximum() : fc::time_point::now() + time_limit; - check_variable_sig_size( trx, max_variable_sig_size ); - const signed_transaction& trn = trx->get_signed_transaction(); + const vector& sigs = check_variable_sig_size( trx, max_variable_sig_size ); + const vector* context_free_data = trx->get_context_free_data(); + EOS_ASSERT( context_free_data, tx_no_context_free_data, "context free data pruned from packed_transaction" ); flat_set recovered_pub_keys; - fc::microseconds cpu_usage = trn.get_signature_keys( chain_id, deadline, recovered_pub_keys ); + const bool allow_duplicate_keys = false; + fc::microseconds cpu_usage = + trx->get_transaction().get_signature_keys(sigs, chain_id, deadline, *context_free_data, recovered_pub_keys, allow_duplicate_keys); return std::make_shared( private_type(), std::move( trx ), cpu_usage, std::move( recovered_pub_keys ) ); } ); diff --git a/libraries/chain/wasm_interface.cpp b/libraries/chain/wasm_interface.cpp index 480f0186148..199baded266 100644 --- a/libraries/chain/wasm_interface.cpp +++ b/libraries/chain/wasm_interface.cpp @@ -1521,18 +1521,18 @@ class context_free_transaction_api : public context_aware_api { } int transaction_size() { - return context.get_packed_transaction().size(); + return context.get_packed_transaction_size(); } int expiration() { - return context.trx_context.trx.expiration.sec_since_epoch(); + return context.trx_context.packed_trx->get_transaction().expiration.sec_since_epoch(); } int tapos_block_num() { - return context.trx_context.trx.ref_block_num; + return context.trx_context.packed_trx->get_transaction().ref_block_num; } int tapos_block_prefix() { - return context.trx_context.trx.ref_block_prefix; + return context.trx_context.packed_trx->get_transaction().ref_block_prefix; } int get_action( uint32_t type, uint32_t index, array_ptr buffer, uint32_t buffer_size )const { diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index 7f9f8517bd3..391dd4c8180 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -570,7 +570,7 @@ namespace eosio { namespace testing { fc::microseconds::maximum() : fc::microseconds( deadline - fc::time_point::now() ); auto ptrx = std::make_shared( trx, true, c ); - auto fut = transaction_metadata::start_recover_keys( ptrx, control->get_thread_pool(), control->get_chain_id(), time_limit ); + auto fut = transaction_metadata::start_recover_keys( std::move( ptrx ), control->get_thread_pool(), control->get_chain_id(), time_limit ); auto r = control->push_transaction( fut.get(), deadline, billed_cpu_time_us, billed_cpu_time_us > 0 ); if (no_throw) return r; if( r->except_ptr ) std::rethrow_exception( r->except_ptr ); diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index c904923eb34..dd12542b24e 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -1090,7 +1090,7 @@ void chain_plugin::plugin_initialize(const variables_map& options) { } ); my->applied_transaction_connection = my->chain->applied_transaction.connect( - [this]( std::tuple t ) { + [this]( std::tuple t ) { my->applied_transaction_channel.publish( priority::low, std::get<0>(t) ); } ); diff --git a/plugins/history_plugin/history_plugin.cpp b/plugins/history_plugin/history_plugin.cpp index 89ac1728986..4463fa4c7d6 100644 --- a/plugins/history_plugin/history_plugin.cpp +++ b/plugins/history_plugin/history_plugin.cpp @@ -352,7 +352,7 @@ namespace eosio { db.add_index(); my->applied_transaction_connection.emplace( - chain.applied_transaction.connect( [&]( std::tuple t ) { + chain.applied_transaction.connect( [&]( std::tuple t ) { my->on_applied_transaction( std::get<0>(t) ); } )); } FC_LOG_AND_RETHROW() @@ -498,7 +498,8 @@ namespace eosio { auto &pt = receipt.trx.get(); if (pt.id() == result.id) { fc::mutable_variant_object r("receipt", receipt); - r("trx", chain.to_variant_with_abi(pt.get_signed_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time ))); + auto ptv0 = pt.to_packed_transaction_v0(); + r("trx", chain.to_variant_with_abi(ptv0->get_signed_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time ))); result.trx = move(r); break; } @@ -526,7 +527,8 @@ namespace eosio { result.block_num = *p.block_num_hint; result.block_time = blk->timestamp; fc::mutable_variant_object r("receipt", receipt); - r("trx", chain.to_variant_with_abi(pt.get_signed_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time ))); + auto ptv0 = pt.to_packed_transaction_v0(); + r("trx", chain.to_variant_with_abi(ptv0->get_signed_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time ))); result.trx = move(r); found = true; break; diff --git a/plugins/mongo_db_plugin/mongo_db_plugin.cpp b/plugins/mongo_db_plugin/mongo_db_plugin.cpp index 5d0bec9b918..44bd120179a 100644 --- a/plugins/mongo_db_plugin/mongo_db_plugin.cpp +++ b/plugins/mongo_db_plugin/mongo_db_plugin.cpp @@ -742,7 +742,8 @@ void mongo_db_plugin_impl::_process_accepted_transaction( const chain::transacti using bsoncxx::builder::basic::make_array; namespace bbb = bsoncxx::builder::basic; - const signed_transaction& trx = t->packed_trx()->get_signed_transaction(); + const packed_transaction_v0_ptr ptv0 = t->packed_trx()->to_packed_transaction_v0(); + const signed_transaction& trx = ptv0->get_signed_transaction(); if( !filter_include( trx ) ) return; @@ -1658,7 +1659,7 @@ void mongo_db_plugin::plugin_initialize(const variables_map& options) my->accepted_transaction( t ); } )); my->applied_transaction_connection.emplace( - chain.applied_transaction.connect( [&]( std::tuple t ) { + chain.applied_transaction.connect( [&]( std::tuple t ) { my->applied_transaction( std::get<0>(t) ); } )); diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index 6a00d746f93..e82018532f8 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -29,16 +29,16 @@ struct partial_transaction { vector signatures = {}; vector context_free_data = {}; - partial_transaction(const chain::signed_transaction& t) - : expiration(t.expiration) - , ref_block_num(t.ref_block_num) - , ref_block_prefix(t.ref_block_prefix) - , max_net_usage_words(t.max_net_usage_words) - , max_cpu_usage_ms(t.max_cpu_usage_ms) - , delay_sec(t.delay_sec) - , transaction_extensions(t.transaction_extensions) - , signatures(t.signatures) - , context_free_data(t.context_free_data) {} + partial_transaction(const chain::packed_transaction_ptr& t) + : expiration(t->get_transaction().expiration) + , ref_block_num(t->get_transaction().ref_block_num) + , ref_block_prefix(t->get_transaction().ref_block_prefix) + , max_net_usage_words(t->get_transaction().max_net_usage_words) + , max_cpu_usage_ms(t->get_transaction().max_cpu_usage_ms) + , delay_sec(t->get_transaction().delay_sec) + , transaction_extensions(t->get_transaction().transaction_extensions) + , signatures(t->get_signatures() != nullptr ? *t->get_signatures() : vector{}) + , context_free_data(t->get_context_free_data() != nullptr ? *t->get_context_free_data() : vector{}) {} }; struct augmented_transaction_trace { @@ -57,7 +57,7 @@ struct augmented_transaction_trace { : trace{trace} , partial{partial} {} - augmented_transaction_trace(const chain::transaction_trace_ptr& trace, const chain::signed_transaction& t) + augmented_transaction_trace(const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& t) : trace{trace} , partial{std::make_shared(t)} {} diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 0b554ffba4b..2c8a8b8934c 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -406,7 +406,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisreceipt && trace_log) { if (is_onblock(p)) onblock_trace.emplace(p, t); @@ -590,7 +590,7 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { EOS_ASSERT(my->chain_plug, chain::missing_chain_plugin_exception, ""); auto& chain = my->chain_plug->chain(); my->applied_transaction_connection.emplace( - chain.applied_transaction.connect([&](std::tuple t) { + chain.applied_transaction.connect([&](std::tuple t) { my->on_applied_transaction(std::get<0>(t), std::get<1>(t)); })); my->accepted_block_connection.emplace( diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp index bf5e2506775..38a1b3a6a52 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp @@ -26,8 +26,8 @@ class chain_extraction_impl_type { {} /// connect to chain controller applied_transaction signal - void signal_applied_transaction( const chain::transaction_trace_ptr& trace, const chain::signed_transaction& strx ) { - on_applied_transaction( trace, strx ); + void signal_applied_transaction( const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& ptrx ) { + on_applied_transaction( trace, ptrx ); } /// connect to chain controller accepted_block signal @@ -53,7 +53,7 @@ class chain_extraction_impl_type { auth.permission == eosio::chain::config::active_name; } - void on_applied_transaction(const chain::transaction_trace_ptr& trace, const chain::signed_transaction& t) { + void on_applied_transaction(const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& t) { if( !trace->receipt ) return; // include only executed transactions; soft_fail included so that onerror (and any inlines via onerror) are included if((trace->receipt->status != chain::transaction_receipt_header::executed && diff --git a/plugins/trace_api_plugin/test/test_extraction.cpp b/plugins/trace_api_plugin/test/test_extraction.cpp index bff6fa75092..e00ead6d0c0 100644 --- a/plugins/trace_api_plugin/test/test_extraction.cpp +++ b/plugins/trace_api_plugin/test/test_extraction.cpp @@ -180,8 +180,8 @@ struct extraction_test_fixture { { } - void signal_applied_transaction( const chain::transaction_trace_ptr& trace, const chain::signed_transaction& strx ) { - extraction_impl.signal_applied_transaction(trace, strx); + void signal_applied_transaction( const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& ptrx ) { + extraction_impl.signal_applied_transaction(trace, ptrx); } void signal_accepted_block( const chain::block_state_ptr& bsp ) { @@ -212,7 +212,7 @@ BOOST_AUTO_TEST_SUITE(block_extraction) signal_applied_transaction( make_transaction_trace( ptrx1.id(), 1, 1, chain::transaction_receipt_header::executed, { actt1, actt2, actt3 } ), - ptrx1.get_signed_transaction() ); + std::make_shared(ptrx1) ); // accept the block with one transaction auto bsp1 = make_block_state( chain::block_id_type(), 1, 1, "bp.one"_n, @@ -268,15 +268,15 @@ BOOST_AUTO_TEST_SUITE(block_extraction) signal_applied_transaction( make_transaction_trace( ptrx1.id(), 1, 1, chain::transaction_receipt_header::executed, { actt1 } ), - ptrx1.get_signed_transaction() ); + std::make_shared( ptrx1 ) ); signal_applied_transaction( make_transaction_trace( ptrx2.id(), 1, 1, chain::transaction_receipt_header::executed, { actt2 } ), - ptrx2.get_signed_transaction() ); + std::make_shared( ptrx2 ) ); signal_applied_transaction( make_transaction_trace( ptrx3.id(), 1, 1, chain::transaction_receipt_header::executed, { actt3 } ), - ptrx3.get_signed_transaction() ); + std::make_shared( ptrx3 ) ); // accept the block with three transaction auto bsp1 = make_block_state( chain::block_id_type(), 1, 1, "bp.one"_n, @@ -344,7 +344,7 @@ BOOST_AUTO_TEST_SUITE(block_extraction) { actt2 } ); onerror_trace->failed_dtrx_trace = transfer_trace; - signal_applied_transaction( onerror_trace, transfer_trx.get_signed_transaction() ); + signal_applied_transaction( onerror_trace, std::make_shared( transfer_trx ) ); auto bsp1 = make_block_state( chain::block_id_type(), 1, 1, "bp.one"_n, { chain::packed_transaction(transfer_trx) } ); diff --git a/plugins/trace_api_plugin/trace_api_plugin.cpp b/plugins/trace_api_plugin/trace_api_plugin.cpp index 0e2748e3897..68906ef4180 100644 --- a/plugins/trace_api_plugin/trace_api_plugin.cpp +++ b/plugins/trace_api_plugin/trace_api_plugin.cpp @@ -310,7 +310,7 @@ struct trace_api_plugin_impl { auto& chain = app().find_plugin()->chain(); applied_transaction_connection.emplace( - chain.applied_transaction.connect([this](std::tuple t) { + chain.applied_transaction.connect([this](std::tuple t) { emit_killer([&](){ extraction->signal_applied_transaction(std::get<0>(t), std::get<1>(t)); }); diff --git a/unittests/api_tests.cpp b/unittests/api_tests.cpp index 3b31d26c34d..3498f880c10 100644 --- a/unittests/api_tests.cpp +++ b/unittests/api_tests.cpp @@ -895,7 +895,7 @@ BOOST_AUTO_TEST_CASE(light_validation_skip_cfa) try { transaction_trace_ptr other_trace; - auto cc = other.control->applied_transaction.connect( [&](std::tuple x) { + auto cc = other.control->applied_transaction.connect( [&](std::tuple x) { auto& t = std::get<0>(x); if( t && t->id == trace->id ) { other_trace = t; @@ -1167,7 +1167,7 @@ BOOST_FIXTURE_TEST_CASE(transaction_tests, TESTER) { try { { produce_blocks(10); transaction_trace_ptr trace; - auto c = control->applied_transaction.connect([&](std::tuple x) { + auto c = control->applied_transaction.connect([&](std::tuple x) { auto& t = std::get<0>(x); if (t && t->receipt && t->receipt->status != transaction_receipt::executed) { trace = t; } } ); @@ -1239,7 +1239,7 @@ BOOST_FIXTURE_TEST_CASE(deferred_transaction_tests, TESTER) { try { //schedule { transaction_trace_ptr trace; - auto c = control->applied_transaction.connect([&](std::tuple x) { + auto c = control->applied_transaction.connect([&](std::tuple x) { auto& t = std::get<0>(x); if (t->scheduled) { trace = t; } } ); @@ -1262,7 +1262,7 @@ BOOST_FIXTURE_TEST_CASE(deferred_transaction_tests, TESTER) { try { { transaction_trace_ptr trace; uint32_t count = 0; - auto c = control->applied_transaction.connect([&](std::tuple x) { + auto c = control->applied_transaction.connect([&](std::tuple x) { auto& t = std::get<0>(x); if (t && t->scheduled) { trace = t; ++count; } } ); @@ -1288,7 +1288,7 @@ BOOST_FIXTURE_TEST_CASE(deferred_transaction_tests, TESTER) { try { { transaction_trace_ptr trace; uint32_t count = 0; - auto c = control->applied_transaction.connect([&](std::tuple x) { + auto c = control->applied_transaction.connect([&](std::tuple x) { auto& t = std::get<0>(x); if (t && t->scheduled) { trace = t; ++count; } } ); @@ -1314,7 +1314,7 @@ BOOST_FIXTURE_TEST_CASE(deferred_transaction_tests, TESTER) { try { //schedule and cancel { transaction_trace_ptr trace; - auto c = control->applied_transaction.connect([&](std::tuple x) { + auto c = control->applied_transaction.connect([&](std::tuple x) { auto& t = std::get<0>(x); if (t && t->scheduled) { trace = t; } } ); @@ -1337,7 +1337,7 @@ BOOST_FIXTURE_TEST_CASE(deferred_transaction_tests, TESTER) { try { //repeated deferred transactions { vector traces; - auto c = control->applied_transaction.connect([&](std::tuple x) { + auto c = control->applied_transaction.connect([&](std::tuple x) { auto& t = std::get<0>(x); if (t && t->scheduled) { traces.push_back( t ); diff --git a/unittests/block_tests.cpp b/unittests/block_tests.cpp index 7b2aaa9afb3..ae3696598ed 100644 --- a/unittests/block_tests.cpp +++ b/unittests/block_tests.cpp @@ -17,7 +17,7 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_test) // Make a copy of the valid block and corrupt the transaction auto copy_b = std::make_shared(std::move(*b)); - auto signed_tx = signed_transaction( copy_b->transactions.back().trx.get().get_signed_transaction() ); + auto signed_tx = signed_transaction( copy_b->transactions.back().trx.get().to_packed_transaction_v0()->get_signed_transaction() ); auto& act = signed_tx.actions.back(); auto act_data = act.data_as(); // Make the transaction invalid by having the new account name the same as the creator name @@ -61,7 +61,7 @@ std::pair corrupt_trx_in_block(validating_te // Make a copy of the valid block and corrupt the transaction auto copy_b = std::make_shared(b->clone()); const auto& packed_trx = copy_b->transactions.back().trx.get(); - auto signed_tx = signed_transaction( packed_trx.get_signed_transaction() ); + auto signed_tx = signed_transaction( packed_trx.to_packed_transaction_v0()->get_signed_transaction() ); // Corrupt one signature signed_tx.signatures.clear(); signed_tx.sign(main.get_private_key(act_name, "active"), main.control->get_chain_id()); diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index f7b6f86a05f..9a365e5d559 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -749,15 +749,16 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { BOOST_CHECK_EQUAL(trx.id(), pkt.id()); BOOST_CHECK_EQUAL(trx.id(), pkt2.id()); - BOOST_CHECK_EQUAL(pkt.get_signed_transaction().id(), pkt2.get_signed_transaction().id()); - BOOST_CHECK_EQUAL(pkt.get_signed_transaction().id(), pkt2.id()); + BOOST_CHECK_EQUAL(pkt.to_packed_transaction_v0()->get_signed_transaction().id(), pkt2.to_packed_transaction_v0()->get_signed_transaction().id()); + BOOST_CHECK_EQUAL(pkt.to_packed_transaction_v0()->get_transaction().id(), pkt2.id()); + BOOST_CHECK_EQUAL(pkt.get_transaction().id(), pkt2.id()); flat_set keys; - auto cpu_time1 = pkt.get_signed_transaction().get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); + auto cpu_time1 = pkt.to_packed_transaction_v0()->get_signed_transaction().get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); BOOST_CHECK_EQUAL(1u, keys.size()); BOOST_CHECK_EQUAL(public_key, *keys.begin()); keys.clear(); - auto cpu_time2 = pkt.get_signed_transaction().get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); + auto cpu_time2 = pkt.to_packed_transaction_v0()->get_signed_transaction().get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); BOOST_CHECK_EQUAL(1u, keys.size()); BOOST_CHECK_EQUAL(public_key, *keys.begin()); @@ -787,14 +788,14 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { packed_transaction pkt5; fc::from_variant(pkt_v, pkt5); - BOOST_CHECK_EQUAL(pkt.get_signed_transaction().id(), pkt3.get_signed_transaction().id()); - BOOST_CHECK_EQUAL(pkt.get_signed_transaction().id(), pkt4.get_signed_transaction().id()); - BOOST_CHECK_EQUAL(pkt.get_signed_transaction().id(), pkt5.get_signed_transaction().id()); // failure indicates reflector_init not working - BOOST_CHECK_EQUAL(pkt.id(), pkt4.get_signed_transaction().id()); + BOOST_CHECK_EQUAL(pkt.get_transaction().id(), pkt3.get_transaction().id()); + BOOST_CHECK_EQUAL(pkt.get_transaction().id(), pkt4.get_transaction().id()); + BOOST_CHECK_EQUAL(pkt.get_transaction().id(), pkt5.get_transaction().id()); // failure indicates reflector_init not working + BOOST_CHECK_EQUAL(pkt.id(), pkt4.get_transaction().id()); BOOST_CHECK_EQUAL(true, trx.expiration == pkt4.expiration()); - BOOST_CHECK_EQUAL(true, trx.expiration == pkt4.get_signed_transaction().expiration); + BOOST_CHECK_EQUAL(true, trx.expiration == pkt4.get_transaction().expiration); keys.clear(); - pkt4.get_signed_transaction().get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); + pkt4.to_packed_transaction_v0()->get_signed_transaction().get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); BOOST_CHECK_EQUAL(1u, keys.size()); BOOST_CHECK_EQUAL(public_key, *keys.begin()); diff --git a/unittests/protocol_feature_tests.cpp b/unittests/protocol_feature_tests.cpp index 8afede4ddad..0137b8d9a6b 100644 --- a/unittests/protocol_feature_tests.cpp +++ b/unittests/protocol_feature_tests.cpp @@ -536,7 +536,7 @@ BOOST_AUTO_TEST_CASE( no_duplicate_deferred_id_test ) try { c2.produce_empty_block( fc::minutes(10) ); transaction_trace_ptr trace0; - auto h2 = c2.control->applied_transaction.connect( [&](std::tuple x) { + auto h2 = c2.control->applied_transaction.connect( [&](std::tuple x) { auto& t = std::get<0>(x); if( t && t->receipt && t->receipt->status == transaction_receipt::expired) { trace0 = t; @@ -554,7 +554,7 @@ BOOST_AUTO_TEST_CASE( no_duplicate_deferred_id_test ) try { const auto& index = c.control->db().get_index(); transaction_trace_ptr trace1; - auto h = c.control->applied_transaction.connect( [&](std::tuple x) { + auto h = c.control->applied_transaction.connect( [&](std::tuple x) { auto& t = std::get<0>(x); if( t && t->receipt && t->receipt->status == transaction_receipt::executed) { trace1 = t; diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index c5df85eadc2..c1e8684daec 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -210,7 +210,7 @@ BOOST_AUTO_TEST_CASE(test_light_validation_restart_from_block_log) { transaction_trace_ptr other_trace; replay_tester from_block_log_chain(copied_config, *genesis, - [&](std::tuple x) { + [&](std::tuple x) { auto& t = std::get<0>(x); if (t && t->id == trace->id) { other_trace = t; diff --git a/unittests/whitelist_blacklist_tests.cpp b/unittests/whitelist_blacklist_tests.cpp index faabea39220..d43089cf958 100644 --- a/unittests/whitelist_blacklist_tests.cpp +++ b/unittests/whitelist_blacklist_tests.cpp @@ -475,7 +475,7 @@ BOOST_AUTO_TEST_CASE( actor_blacklist_inline_deferred ) { try { tester2.chain->push_block( b ); } - auto log_trxs = [&](std::tuple x) { + auto log_trxs = [&](std::tuple x) { auto& t = std::get<0>(x); if( !t || t->action_traces.size() == 0 ) return; From 52960177928177aa3f2d9a40d7ad9baa92af5ed3 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 6 Apr 2020 08:10:50 -0500 Subject: [PATCH 042/142] Remove duplicate friend. Fix compile issue. --- libraries/chain/include/eosio/chain/transaction.hpp | 2 -- plugins/mongo_db_plugin/mongo_db_plugin.cpp | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 35b456a5358..60c33c759b1 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -203,8 +203,6 @@ namespace eosio { namespace chain { friend struct packed_transaction; void reflector_init(); private: - friend struct packed_transaction; - vector signatures; fc::enum_type compression; bytes packed_context_free_data; diff --git a/plugins/mongo_db_plugin/mongo_db_plugin.cpp b/plugins/mongo_db_plugin/mongo_db_plugin.cpp index 44bd120179a..c63bf6c1d85 100644 --- a/plugins/mongo_db_plugin/mongo_db_plugin.cpp +++ b/plugins/mongo_db_plugin/mongo_db_plugin.cpp @@ -40,6 +40,8 @@ using chain::signed_transaction; using chain::signed_block; using chain::transaction_id_type; using chain::packed_transaction; +using chain::packed_transaction_v0; +using chain::packed_transaction_v0_ptr; static appbase::abstract_plugin& _mongo_db_plugin = app().register_plugin(); @@ -1065,7 +1067,7 @@ void mongo_db_plugin_impl::_process_irreversible_block(const chain::block_state_ string trx_id_str; if( receipt.trx.contains() ) { const auto& pt = receipt.trx.get(); - if( !filter_include( pt.get_signed_transaction() ) ) continue; + if( !filter_include( pt.get_transaction() ) ) continue; const auto& id = pt.id(); trx_id_str = id.str(); } else { From 24a579d65ce6779d63441e2d5c2864a2997e7962 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 6 Apr 2020 09:13:03 -0500 Subject: [PATCH 043/142] Fix some more PR comments --- libraries/chain/block_log.cpp | 40 ++++++++----------- .../chain/include/eosio/chain/block_log.hpp | 4 +- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 1f4651d6f9b..8a46306621e 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -49,15 +49,14 @@ namespace eosio { namespace chain { template void unpack(Stream& ds, log_entry& entry){ fc::raw::unpack(ds, entry.offset); - fc::raw::unpack(ds, entry.compression); - EOS_ASSERT(entry.compression == pruned_transaction::cf_compression_type::none, block_log_exception, + uint8_t compression; + fc::raw::unpack(ds, compression); + EOS_ASSERT(compression == static_cast(pruned_transaction::cf_compression_type::none), block_log_exception, "Only support compression_type none"); + entry.compression = static_cast(compression); fc::raw::unpack(ds, *entry.block); } - - - class block_log_impl { public: pruned_block_ptr head; @@ -111,7 +110,7 @@ namespace eosio { namespace chain { /// calculate the offset from the start of serialized block entry to block start static int offset_to_block_start(uint32_t version) { if (version < 4) return 0; - return sizeof(uint32_t) + sizeof(pruned_transaction::cf_compression_type); + return sizeof(uint32_t) + 1; } static int blknum_offset_from_block_entry(uint32_t block_log_version) { @@ -345,7 +344,7 @@ namespace eosio { namespace chain { uint32_t offset = buffer.size(); stream.write((char*)&offset, sizeof(offset)); fc::raw::pack(stream, segment_compression); - fc::raw::pack(stream, b); + b.pack(stream, segment_compression); return buffer; } @@ -506,10 +505,10 @@ namespace eosio { namespace chain { if (version >= 4 ) { uint32_t offset; - fc::unsigned_int compression; + uint8_t compression; fc::raw::unpack(ds, offset); fc::raw::unpack(ds, compression); - EOS_ASSERT( compression.value == static_cast(pruned_transaction::cf_compression_type::none), block_log_exception , "Only \"none\" compression type is supported."); + EOS_ASSERT( compression == static_cast(pruned_transaction::cf_compression_type::none), block_log_exception , "Only \"none\" compression type is supported."); } fc::raw::unpack(ds, bh); } @@ -767,7 +766,7 @@ namespace eosio { namespace chain { } if (!verify_block(ds, previous, pos, *hdr)) { - #warning TODO: require fc::reflector to work, change after merged with PR 8901 + #warning TODO: require fc::reflector to work, change after merged with PR 8900 // variant block; // if (version >= 4) // to_variant(pruned_tmp, block); @@ -877,7 +876,7 @@ namespace eosio { namespace chain { return true; } - bool block_log::prune_transaction(uint32_t block_num, transaction_id_type id) { + void block_log::prune_transaction(uint32_t block_num, transaction_id_type id) { try { EOS_ASSERT(my->version >= 4, block_log_exception, "The block log version ${version} does not support transaction pruning.", ("version", my->version)); uint64_t pos = get_block_pos(block_num); @@ -898,19 +897,14 @@ namespace eosio { namespace chain { for (auto& trx : block.transactions) { if (trx.trx.visit(pruner)) { my->block_file.seek(pos); - std::vector buffer = detail::block_log_impl::pack_v4_log_entry(*entry.block, static_cast(entry.compression)); - EOS_ASSERT(buffer.size() <= entry.offset, block_log_exception, ""); + std::vector buffer = detail::block_log_impl::pack_v4_log_entry(*entry.block, entry.compression); + EOS_ASSERT(buffer.size() <= entry.offset, block_log_exception, "Not enough space reserved in block log entry to serialize pruned block."); my->block_file.write(buffer.data(), buffer.size()); my->block_file.flush(); - return true; } } - - elog("Block ${block_num} does not contain transaction ${id}.", ("id", id)("block_num", block_num)); - return false; } - FC_CAPTURE_AND_LOG(()) - return false; + FC_LOG_AND_RETHROW() } @@ -1164,8 +1158,8 @@ namespace eosio { namespace chain { new_block_file.close(); new_block_file.open( LOG_RW_C ); - static_assert( block_log::max_supported_version >= 4, - "Code was written to support format of version 4 or greater, need to update this code for latest format." ); + static_assert( block_log::max_supported_version == 4, + "Code was written to support format of version 4, need to update this code for latest format." ); uint32_t version = block_log::max_supported_version; new_block_file.seek(0); new_block_file.write((char*)&version, sizeof(version)); @@ -1374,11 +1368,11 @@ namespace eosio { namespace chain { ("block_dir", block_dir.generic_string())("n", n)("block_file",td.block_file_name.generic_string())("index_file", td.index_file_name.generic_string())); if (n < td.first_block) { - elog("All blocks are after block ${n} so do nothing (trim_end would delete entire blocks.log)",("n", n)); + dlog("All blocks are after block ${n} so do nothing (trim_end would delete entire blocks.log)",("n", n)); return 1; } if (n >= td.last_block) { - elog("There are no blocks after block ${n} so do nothing",("n", n)); + dlog("There are no blocks after block ${n} so do nothing",("n", n)); return 2; } const uint64_t end_of_new_file = td.block_pos(n + 1); diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index f47b880e9dd..f15dae7b5d4 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -56,12 +56,12 @@ namespace eosio { namespace chain { * Return offset of block in file, or block_log::npos if it does not exist. */ uint64_t get_block_pos(uint32_t block_num) const; -#warning TODO: change to pruned_block after merged with PR 8901 +#warning TODO: change to pruned_block after merged with PR 8900 signed_block_header* head() const; [[deprecated]] block_id_type head_id() const; // use head()->id() instead uint32_t first_block_num() const; - bool prune_transaction(uint32_t block_num, transaction_id_type id); + void prune_transaction(uint32_t block_num, transaction_id_type id); static const uint64_t npos = std::numeric_limits::max(); From ab70ea3e74ce265824357bafb476761eb0f22f2e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 6 Apr 2020 11:10:27 -0500 Subject: [PATCH 044/142] Added two new categories of execptions: subjective_block_production_exception & objective_block_production_exception. objective_block_production_exception derived types can now be used for execeptions that need to propagate out of transaciton processing. --- libraries/chain/apply_context.cpp | 7 +++++-- libraries/chain/controller.cpp | 12 +++-------- .../chain/include/eosio/chain/exceptions.hpp | 21 +++++++++++++------ 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 46867d857e1..77b977be27b 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -673,8 +673,11 @@ int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t b if( data.contains() || data.contains() || data.contains() ) // TODO: for partial determine if index is pruned and throw only in that case { - // TODO: throw an exception like disallowed_transaction_extensions_bad_block_exception && subjective_block_production_exception - EOS_ASSERT( false, subjective_block_production_exception, "todo" ); + if( control.is_producing_block() ) { + EOS_THROW( subjective_block_production_exception, "pruned context free data not available" ); + } else { + EOS_THROW( pruned_context_free_data_bad_block_exception, "pruned context free data not available" ); + } } std::vector context_free_data = data.contains() ? diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 86e18fc33b7..abb02238e35 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1162,9 +1162,7 @@ struct controller_impl { trx_context.squash(); restore.cancel(); return trace; - } catch( const disallowed_transaction_extensions_bad_block_exception& ) { - throw; - } catch( const protocol_feature_bad_block_exception& ) { + } catch( const objective_block_production_exception& ) { throw; } catch( const fc::exception& e ) { cpu_time_to_bill_us = trx_context.update_billed_cpu_time( fc::time_point::now() ); @@ -1314,9 +1312,7 @@ struct controller_impl { restore.cancel(); return trace; - } catch( const disallowed_transaction_extensions_bad_block_exception& ) { - throw; - } catch( const protocol_feature_bad_block_exception& ) { + } catch( const objective_block_production_exception& ) { throw; } catch( const fc::exception& e ) { cpu_time_to_bill_us = trx_context.update_billed_cpu_time( fc::time_point::now() ); @@ -1521,9 +1517,7 @@ struct controller_impl { } return trace; - } catch( const disallowed_transaction_extensions_bad_block_exception& ) { - throw; - } catch( const protocol_feature_bad_block_exception& ) { + } catch( const objective_block_production_exception& ) { throw; } catch (const fc::exception& e) { trace->error_code = controller::convert_exception_to_error_code( e ); diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index a80c4bb0ef8..7d66e1683f3 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -273,8 +273,6 @@ namespace eosio { namespace chain { 3040015, "Invalid transaction extension" ) FC_DECLARE_DERIVED_EXCEPTION( ill_formed_deferred_transaction_generation_context, transaction_exception, 3040016, "Transaction includes an ill-formed deferred transaction generation context extension" ) - FC_DECLARE_DERIVED_EXCEPTION( disallowed_transaction_extensions_bad_block_exception, transaction_exception, - 3040017, "Transaction includes disallowed extensions (invalid block)" ) FC_DECLARE_DERIVED_EXCEPTION( tx_resource_exhaustion, transaction_exception, 3040018, "Transaction exceeded transient resource limit" ) FC_DECLARE_DERIVED_EXCEPTION( tx_prune_exception, transaction_exception, @@ -404,8 +402,6 @@ namespace eosio { namespace chain { 3100004, "Corrupted reversible block database was fixed" ) FC_DECLARE_DERIVED_EXCEPTION( extract_genesis_state_exception, misc_exception, 3100005, "Extracted genesis state from blocks.log" ) - FC_DECLARE_DERIVED_EXCEPTION( subjective_block_production_exception, misc_exception, - 3100006, "Subjective exception thrown during block production" ) FC_DECLARE_DERIVED_EXCEPTION( multiple_voter_info, misc_exception, 3100007, "Multiple voter info detected" ) FC_DECLARE_DERIVED_EXCEPTION( unsupported_feature, misc_exception, @@ -634,8 +630,21 @@ namespace eosio { namespace chain { 3250000, "Protocol feature exception" ) FC_DECLARE_DERIVED_EXCEPTION( protocol_feature_validation_exception, protocol_feature_exception, 3250001, "Protocol feature validation exception" ) - FC_DECLARE_DERIVED_EXCEPTION( protocol_feature_bad_block_exception, protocol_feature_exception, - 3250002, "Protocol feature exception (invalid block)" ) FC_DECLARE_DERIVED_EXCEPTION( protocol_feature_iterator_exception, protocol_feature_exception, 3250003, "Protocol feature iterator exception" ) + + // Any derived types of subjective_block_production_exception need to update controller::failure_is_subjective + FC_DECLARE_DERIVED_EXCEPTION( subjective_block_production_exception, chain_exception, + 3260000, "Subjective exception thrown during block production" ) + + // Objective block production exceptions are throw out of transaction processing to stop block production + FC_DECLARE_DERIVED_EXCEPTION( objective_block_production_exception, chain_exception, + 3270000, "Objective exception thrown during block production" ) + FC_DECLARE_DERIVED_EXCEPTION( disallowed_transaction_extensions_bad_block_exception, objective_block_production_exception, + 3270001, "Transaction includes disallowed extensions (invalid block)" ) + FC_DECLARE_DERIVED_EXCEPTION( protocol_feature_bad_block_exception, objective_block_production_exception, + 3270002, "Protocol feature exception (invalid block)" ) + FC_DECLARE_DERIVED_EXCEPTION( pruned_context_free_data_bad_block_exception, objective_block_production_exception, + 3270002, "Protocol feature exception (invalid block)" ) + } } // eosio::chain From 6e71c85fa0cd81a4ec18e048cefb751d34c86cec Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 6 Apr 2020 09:41:31 -0500 Subject: [PATCH 045/142] Change prune_transaction API --- libraries/chain/block_log.cpp | 4 ++-- libraries/chain/include/eosio/chain/block_log.hpp | 2 +- unittests/restart_chain_tests.cpp | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 8a46306621e..13f204db516 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -876,7 +876,7 @@ namespace eosio { namespace chain { return true; } - void block_log::prune_transaction(uint32_t block_num, transaction_id_type id) { + void block_log::prune_transactions(uint32_t block_num, const std::vector& ids) { try { EOS_ASSERT(my->version >= 4, block_log_exception, "The block log version ${version} does not support transaction pruning.", ("version", my->version)); uint64_t pos = get_block_pos(block_num); @@ -892,7 +892,7 @@ namespace eosio { namespace chain { "Wrong block was read from block log."); auto pruner = overloaded{[](transaction_id_type&) { return false; }, - [&id](pruned_transaction& ptx) { return ptx.id() == id && prune(ptx); }}; + [&ids](pruned_transaction& ptx) { return std::find(ids.begin(), ids.end(), ptx.id()) != ids.end() && prune(ptx); }}; for (auto& trx : block.transactions) { if (trx.trx.visit(pruner)) { diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index f15dae7b5d4..9cbfa0d6eef 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -61,7 +61,7 @@ namespace eosio { namespace chain { [[deprecated]] block_id_type head_id() const; // use head()->id() instead uint32_t first_block_num() const; - void prune_transaction(uint32_t block_num, transaction_id_type id); + void prune_transactions(uint32_t block_num, const vector& ids); static const uint64_t npos = std::numeric_limits::max(); diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index 008fe28dac2..42109c00ac3 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -139,7 +139,7 @@ struct light_validation_restart_from_block_log_test_fixture { // run normal passing case auto sigs = trx.sign(chain.get_private_key(N(testapi), "active"), chain.control->get_chain_id()); trace = chain.push_transaction(trx); - chain.produce_block(); + chain.produce_blocks(10); BOOST_REQUIRE(trace->receipt); BOOST_CHECK_EQUAL(trace->receipt->status, transaction_receipt::executed); @@ -265,7 +265,8 @@ BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log, light_vali BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log_with_pruned_trx, light_validation_restart_from_block_log_test_fixture) { block_log blog(chain.get_config().blocks_dir); - blog.prune_transaction(trace->block_num, trace->id); +#warning: TODO re-enable after merge with PR 8900, it doesn't work now because pruned_block cannot be converted back to signed_block + // BOOST_REQUIRE_NO_THROW(blog.prune_transactions(trace->block_num, std::vector{trace->id})); } BOOST_AUTO_TEST_CASE(test_trim_blocklog_front) { From e2dd59d51014596f2d1aba39836c0751b40203d1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 6 Apr 2020 12:20:29 -0500 Subject: [PATCH 046/142] Combine packed_transaction constructors that take signed_transaction --- .../chain/include/eosio/chain/transaction.hpp | 5 ++-- libraries/chain/transaction.cpp | 27 ++----------------- .../testing/include/eosio/testing/tester.hpp | 4 +-- libraries/testing/tester.cpp | 6 ++--- .../trace_api_plugin/test/test_extraction.cpp | 2 +- .../txn_test_gen_plugin.cpp | 3 ++- unittests/abi_tests.cpp | 2 +- unittests/block_tests.cpp | 4 +-- unittests/misc_tests.cpp | 22 ++++++++++----- .../unapplied_transaction_queue_tests.cpp | 2 +- unittests/wasm_tests.cpp | 2 +- 11 files changed, 32 insertions(+), 47 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 60c33c759b1..f5ad5f3883d 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -126,7 +126,7 @@ namespace eosio { namespace chain { signed_transaction( signed_transaction&& ) = default; signed_transaction& operator=(const signed_transaction&) = delete; signed_transaction& operator=(signed_transaction&&) = default; - signed_transaction( transaction&& trx, vector signatures, vector context_free_data) + signed_transaction( transaction trx, vector signatures, vector context_free_data) : transaction(std::move(trx)) , signatures(std::move(signatures)) , context_free_data(std::move(context_free_data)) @@ -281,8 +281,7 @@ namespace eosio { namespace chain { packed_transaction(const packed_transaction_v0& other, bool legacy); packed_transaction(packed_transaction_v0&& other, bool legacy); - explicit packed_transaction(const signed_transaction& t, bool legacy, compression_type _compression = compression_type::none); - explicit packed_transaction(signed_transaction&& t, bool legacy, compression_type _compression = compression_type::none); + explicit packed_transaction(signed_transaction t, bool legacy, compression_type _compression = compression_type::none); packed_transaction_v0_ptr to_packed_transaction_v0() const; diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 510ed20b6f6..d4414bf0fcc 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -476,17 +476,7 @@ static prunable_transaction_data make_prunable_transaction_data( bool legacy, ve } } -packed_transaction::packed_transaction(const signed_transaction& t, bool legacy, compression_type _compression) - : compression(_compression), - prunable_data(make_prunable_transaction_data(legacy, t.signatures, t.context_free_data, _compression)), - packed_trx(pack_transaction(t, estimated_size, compression)), - unpacked_trx(t), - trx_id(unpacked_trx.id()) -{ - estimated_size = estimated_size * 2 + get_prunable_size() * 2; -} - -packed_transaction::packed_transaction(signed_transaction&& t, bool legacy, compression_type _compression) +packed_transaction::packed_transaction(signed_transaction t, bool legacy, compression_type _compression) : compression(_compression), prunable_data(make_prunable_transaction_data(legacy, std::move(t.signatures), std::move(t.context_free_data), _compression)), packed_trx(pack_transaction(t, estimated_size, compression)), @@ -607,17 +597,6 @@ std::size_t packed_transaction::maximum_pruned_pack_size(cf_compression_type seg return fc::raw::pack_size(compression) + fc::raw::pack_size(packed_trx) + prunable_data.maximum_pruned_pack_size(segment_compression); } -static std::vector maybe_unpack_context_free_data(const prunable_transaction_data::full& obj, packed_transaction_v0::compression_type) { - return obj.context_free_segments; -} -static std::vector maybe_unpack_context_free_data(const prunable_transaction_data::full_legacy& obj, packed_transaction_v0::compression_type compression) { - return unpack_context_free_data(obj.packed_context_free_data, compression); -} -template -static std::vector maybe_unpack_context_free_data(const T&, packed_transaction_v0::compression_type) { - return {}; -} - void packed_transaction::reflector_init() { // called after construction, but always on the same thread and before packed_transaction passed to any other threads @@ -626,9 +605,7 @@ void packed_transaction::reflector_init() EOS_ASSERT( unpacked_trx.expiration == time_point_sec(), tx_decompression_error, "packed_transaction already unpacked" ); const auto* signatures = get_signatures(); uint32_t packed_size = 0; - unpacked_trx = signed_transaction(unpack_transaction(packed_trx, packed_size, compression), - signatures ? *signatures : std::vector{}, - prunable_data.prunable_data.visit([&](const auto& obj) { return maybe_unpack_context_free_data(obj, compression); })); + unpacked_trx = unpack_transaction(packed_trx, packed_size, compression); trx_id = unpacked_trx.id(); estimated_size = packed_size * 2 + get_prunable_size() * 2; // x2 since stored packed and unpacked } diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index e82b02a2879..20fc39e1bf1 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -195,8 +195,8 @@ namespace eosio { namespace testing { vector get_scheduled_transactions() const; unapplied_transaction_queue& get_unapplied_transaction_queue() { return unapplied_transactions; } - transaction_trace_ptr push_transaction( packed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US ); - transaction_trace_ptr push_transaction( signed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US, bool no_throw = false ); + transaction_trace_ptr push_transaction( const packed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US ); + transaction_trace_ptr push_transaction( const signed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US, bool no_throw = false ); [[nodiscard]] action_result push_action(action&& cert_act, uint64_t authorizer); // TODO/QUESTION: Is this needed? diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index 391dd4c8180..46ddb249ec1 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -533,7 +533,7 @@ namespace eosio { namespace testing { return push_transaction( trx ); } - transaction_trace_ptr base_tester::push_transaction( packed_transaction& trx, + transaction_trace_ptr base_tester::push_transaction( const packed_transaction& trx, fc::time_point deadline, uint32_t billed_cpu_time_us ) @@ -552,7 +552,7 @@ namespace eosio { namespace testing { return r; } FC_RETHROW_EXCEPTIONS( warn, "transaction_header: ${header}", ("header", transaction_header(trx.get_transaction()) )) } - transaction_trace_ptr base_tester::push_transaction( signed_transaction& trx, + transaction_trace_ptr base_tester::push_transaction( const signed_transaction& trx, fc::time_point deadline, uint32_t billed_cpu_time_us, bool no_throw @@ -569,7 +569,7 @@ namespace eosio { namespace testing { auto time_limit = deadline == fc::time_point::maximum() ? fc::microseconds::maximum() : fc::microseconds( deadline - fc::time_point::now() ); - auto ptrx = std::make_shared( trx, true, c ); + auto ptrx = std::make_shared( signed_transaction(trx), true, c ); auto fut = transaction_metadata::start_recover_keys( std::move( ptrx ), control->get_thread_pool(), control->get_chain_id(), time_limit ); auto r = control->push_transaction( fut.get(), deadline, billed_cpu_time_us, billed_cpu_time_us > 0 ); if (no_throw) return r; diff --git a/plugins/trace_api_plugin/test/test_extraction.cpp b/plugins/trace_api_plugin/test/test_extraction.cpp index e00ead6d0c0..53ce576ed3c 100644 --- a/plugins/trace_api_plugin/test/test_extraction.cpp +++ b/plugins/trace_api_plugin/test/test_extraction.cpp @@ -74,7 +74,7 @@ namespace { auto make_packed_trx( std::vector actions ) { chain::signed_transaction trx; trx.actions = std::move( actions ); - return packed_transaction( trx, true ); + return packed_transaction( std::move(trx), true ); } chain::action_trace make_action_trace( uint64_t global_sequence, chain::action act, chain::name receiver ) { diff --git a/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp b/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp index ca98331a44a..d559d8b0184 100644 --- a/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp +++ b/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp @@ -107,7 +107,8 @@ struct txn_test_gen_plugin_impl { chain_plugin& cp = app().get_plugin(); for (size_t i = 0; i < trxs->size(); ++i) { - cp.accept_transaction( std::make_shared(trxs->at(i), true), [=](const fc::static_variant& result){ + cp.accept_transaction( std::make_shared(signed_transaction(trxs->at(i)), true), + [=](const fc::static_variant& result){ if (result.contains()) { next(result.get()); } else { diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index d4dcd5c5df5..2534b903578 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -1495,7 +1495,7 @@ BOOST_AUTO_TEST_CASE(packed_transaction) txn.max_cpu_usage_ms = 43; // pack the transaction to verify that the var unpacking logic is correct - auto packed_txn = chain::packed_transaction(txn, true); + auto packed_txn = chain::packed_transaction(signed_transaction(txn), true); const char* packed_transaction_abi = R"=====( { diff --git a/unittests/block_tests.cpp b/unittests/block_tests.cpp index ae3696598ed..cb67f8bb53a 100644 --- a/unittests/block_tests.cpp +++ b/unittests/block_tests.cpp @@ -27,7 +27,7 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_test) signed_tx.signatures.clear(); signed_tx.sign(main.get_private_key(config::system_account_name, "active"), main.control->get_chain_id()); // Replace the valid transaction with the invalid transaction - auto invalid_packed_tx = packed_transaction(signed_tx, true); + auto invalid_packed_tx = packed_transaction(std::move(signed_tx), true); copy_b->transactions.back().trx = invalid_packed_tx; // Re-calculate the transaction merkle @@ -67,7 +67,7 @@ std::pair corrupt_trx_in_block(validating_te signed_tx.sign(main.get_private_key(act_name, "active"), main.control->get_chain_id()); // Replace the valid transaction with the invalid transaction - auto invalid_packed_tx = packed_transaction(signed_tx, true, packed_trx.get_compression()); + auto invalid_packed_tx = packed_transaction(std::move(signed_tx), true, packed_trx.get_compression()); copy_b->transactions.back().trx = invalid_packed_tx; // Re-calculate the transaction merkle diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index 9a365e5d559..52e5d9d417e 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -739,9 +739,9 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { BOOST_CHECK_EQUAL(1u, trx.signatures.size()); trx.validate(); - packed_transaction pkt(trx, true, packed_transaction::compression_type::none); + packed_transaction pkt(signed_transaction(trx), true, packed_transaction::compression_type::none); - packed_transaction pkt2(trx, true, packed_transaction::compression_type::zlib); + packed_transaction pkt2(signed_transaction(trx), true, packed_transaction::compression_type::zlib); BOOST_CHECK_EQUAL(true, trx.expiration == pkt.expiration()); BOOST_CHECK_EQUAL(true, trx.expiration == pkt2.expiration()); @@ -799,6 +799,14 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { BOOST_CHECK_EQUAL(1u, keys.size()); BOOST_CHECK_EQUAL(public_key, *keys.begin()); + packed_transaction pkt6(*pkt.to_packed_transaction_v0(), true); + BOOST_CHECK_EQUAL(pkt.id(), pkt6.id()); + BOOST_CHECK(pkt6.get_estimated_size() > 0); + keys.clear(); + pkt6.to_packed_transaction_v0()->get_signed_transaction().get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); + BOOST_CHECK_EQUAL(1u, keys.size()); + BOOST_CHECK_EQUAL(public_key, *keys.begin()); + } FC_LOG_AND_RETHROW() } @@ -868,11 +876,11 @@ BOOST_AUTO_TEST_CASE(transaction_metadata_test) { try { trx.sign( private_key, test.control->get_chain_id() ); BOOST_CHECK_EQUAL(1u, trx.signatures.size()); - packed_transaction pkt(trx, true, packed_transaction::compression_type::none); - packed_transaction pkt2(trx, true, packed_transaction::compression_type::zlib); + packed_transaction pkt(signed_transaction(trx), true, packed_transaction::compression_type::none); + packed_transaction pkt2(signed_transaction(trx), true, packed_transaction::compression_type::zlib); - packed_transaction_ptr ptrx = std::make_shared( trx, true, packed_transaction::compression_type::none); - packed_transaction_ptr ptrx2 = std::make_shared( trx, true, packed_transaction::compression_type::zlib); + packed_transaction_ptr ptrx = std::make_shared( signed_transaction(trx), true, packed_transaction::compression_type::none); + packed_transaction_ptr ptrx2 = std::make_shared( signed_transaction(trx), true, packed_transaction::compression_type::zlib); BOOST_CHECK_EQUAL(trx.id(), pkt.id()); BOOST_CHECK_EQUAL(trx.id(), pkt2.id()); @@ -959,7 +967,7 @@ BOOST_AUTO_TEST_CASE(pruned_transaction_test) { trx.sign( t.get_private_key( N(eosio), "active" ), t.control->get_chain_id() ); packed_transaction_v0 packed(trx); - packed_transaction pruned(trx, true); + packed_transaction pruned(std::move(trx), true); BOOST_TEST(packed.packed_digest().str() == pruned.packed_digest().str()); BOOST_REQUIRE(pruned.get_context_free_data() != nullptr); BOOST_TEST(*pruned.get_context_free_data() == packed.get_context_free_data()); diff --git a/unittests/unapplied_transaction_queue_tests.cpp b/unittests/unapplied_transaction_queue_tests.cpp index 5e1a0d68731..19b5b4fddad 100644 --- a/unittests/unapplied_transaction_queue_tests.cpp +++ b/unittests/unapplied_transaction_queue_tests.cpp @@ -18,7 +18,7 @@ auto unique_trx_meta_data( fc::time_point expire = fc::time_point::now() + fc::s trx.expiration = expire; trx.actions.emplace_back( vector{{creator,config::active_name}}, onerror{ nextid, "test", 4 }); - return transaction_metadata::create_no_recover_keys( packed_transaction( trx, true ), transaction_metadata::trx_type::input ); + return transaction_metadata::create_no_recover_keys( packed_transaction( std::move(trx), true ), transaction_metadata::trx_type::input ); } auto next( unapplied_transaction_queue& q ) { diff --git a/unittests/wasm_tests.cpp b/unittests/wasm_tests.cpp index a3e05171a09..aca6a3ef4b9 100644 --- a/unittests/wasm_tests.cpp +++ b/unittests/wasm_tests.cpp @@ -1915,7 +1915,7 @@ BOOST_AUTO_TEST_CASE( billed_cpu_test ) try { chain.set_transaction_headers( trx, ++num_secs ); // num_secs provides nonce trx.max_cpu_usage_ms = trx_max_ms; trx.sign( chain.get_private_key( acc, "active" ), chain.control->get_chain_id() ); - auto ptrx = std::make_shared(trx, true); + auto ptrx = std::make_shared(std::move(trx), true); auto fut = transaction_metadata::start_recover_keys( ptrx, chain.control->get_thread_pool(), chain.control->get_chain_id(), fc::microseconds::maximum() ); return fut.get(); }; From b80a5e435c2f281a9dff759dfc7c4be08a9d2db3 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 6 Apr 2020 12:28:15 -0500 Subject: [PATCH 047/142] revert repair_log changes --- libraries/chain/block_log.cpp | 240 ++++++++++++------ libraries/chain/include/eosio/chain/block.hpp | 2 + 2 files changed, 162 insertions(+), 80 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 13f204db516..3e068600ea6 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -709,113 +709,193 @@ namespace eosio { namespace chain { fc::path block_log::repair_log(const fc::path& data_dir, uint32_t truncate_at_block) { ilog("Recovering Block Log..."); - EOS_ASSERT(fc::is_directory(data_dir) && fc::is_regular_file(data_dir / "blocks.log"), block_log_not_found, - "Block log not found in '${blocks_dir}'", ("blocks_dir", data_dir)); + EOS_ASSERT( fc::is_directory(data_dir) && fc::is_regular_file(data_dir / "blocks.log"), block_log_not_found, + "Block log not found in '${blocks_dir}'", ("blocks_dir", data_dir) ); auto now = fc::time_point::now(); - auto blocks_dir = fc::canonical(data_dir); - if (blocks_dir.filename().generic_string() == ".") { + auto blocks_dir = fc::canonical( data_dir ); + if( blocks_dir.filename().generic_string() == "." ) { blocks_dir = blocks_dir.parent_path(); } - auto backup_dir = blocks_dir.parent_path(); + auto backup_dir = blocks_dir.parent_path(); auto blocks_dir_name = blocks_dir.filename(); - EOS_ASSERT(blocks_dir_name.generic_string() != ".", block_log_exception, "Invalid path to blocks directory"); - backup_dir = backup_dir / blocks_dir_name.generic_string().append("-").append(now); + EOS_ASSERT( blocks_dir_name.generic_string() != ".", block_log_exception, "Invalid path to blocks directory" ); + backup_dir = backup_dir / blocks_dir_name.generic_string().append("-").append( now ); - EOS_ASSERT(!fc::exists(backup_dir), block_log_backup_dir_exist, + EOS_ASSERT( !fc::exists(backup_dir), block_log_backup_dir_exist, "Cannot move existing blocks directory to already existing directory '${new_blocks_dir}'", - ("new_blocks_dir", backup_dir)); + ("new_blocks_dir", backup_dir) ); - fc::rename(blocks_dir, backup_dir); - ilog("Moved existing blocks directory to backup location: '${new_blocks_dir}'", ("new_blocks_dir", backup_dir)); + fc::rename( blocks_dir, backup_dir ); + ilog( "Moved existing blocks directory to backup location: '${new_blocks_dir}'", ("new_blocks_dir", backup_dir) ); fc::create_directories(blocks_dir); auto block_log_path = blocks_dir / "blocks.log"; - ilog("Reconstructing '${new_block_log}' from backed up block log", ("new_block_log", block_log_path)); - auto block_log_path_string = block_log_path.generic_string(); - const char* block_file_name = block_log_path_string.c_str(); + ilog( "Reconstructing '${new_block_log}' from backed up block log", ("new_block_log", block_log_path) ); + + std::fstream old_block_stream; + std::fstream new_block_stream; + + old_block_stream.open( (backup_dir / "blocks.log").generic_string().c_str(), LOG_READ ); + new_block_stream.open( block_log_path.generic_string().c_str(), LOG_WRITE ); + + old_block_stream.seekg( 0, std::ios::end ); + uint64_t end_pos = old_block_stream.tellg(); + old_block_stream.seekg( 0 ); + + uint32_t version = 0; + old_block_stream.read( (char*)&version, sizeof(version) ); + EOS_ASSERT( version > 0, block_log_exception, "Block log was not setup properly" ); + EOS_ASSERT( is_supported_version(version), block_log_unsupported_version, + "Unsupported version of block log. Block log version is ${version} while code supports version(s) [${min},${max}]", + ("version", version)("min", block_log::min_supported_version)("max", block_log::max_supported_version) ); + + new_block_stream.write( (char*)&version, sizeof(version) ); + + uint32_t first_block_num = 1; + if (version != 1) { + old_block_stream.read ( (char*)&first_block_num, sizeof(first_block_num) ); + + // this assert is only here since repair_log is only used for --hard-replay-blockchain, which removes any + // existing state, if another API needs to use it, this can be removed and the check for the first block's + // previous block id will need to accommodate this. + EOS_ASSERT( first_block_num == 1, block_log_exception, + "Block log ${file} must contain a genesis state and start at block number 1. This block log " + "starts at block number ${first_block_num}.", + ("file", (backup_dir / "blocks.log").generic_string())("first_block_num", first_block_num)); + + new_block_stream.write( (char*)&first_block_num, sizeof(first_block_num) ); + } + + if (contains_genesis_state(version, first_block_num)) { + genesis_state gs; + fc::raw::unpack(old_block_stream, gs); + + auto data = fc::raw::pack( gs ); + new_block_stream.write( data.data(), data.size() ); + } + else if (contains_chain_id(version, first_block_num)) { + chain_id_type chain_id; + old_block_stream >> chain_id; + + new_block_stream << chain_id; + } + else { + EOS_THROW( block_log_exception, + "Block log ${file} is not supported. version: ${ver} and first_block_num: ${fbn} does not contain " + "a genesis_state nor a chain_id.", + ("file", (backup_dir / "blocks.log").generic_string())("ver", version)("fbn", first_block_num)); + } + + if (version != 1) { + auto expected_totem = npos; + std::decay_t actual_totem; + old_block_stream.read ( (char*)&actual_totem, sizeof(actual_totem) ); + + EOS_ASSERT(actual_totem == expected_totem, block_log_exception, + "Expected separator between block log header and blocks was not found( expected: ${e}, actual: ${a} )", + ("e", fc::to_hex((char*)&expected_totem, sizeof(expected_totem) ))("a", fc::to_hex((char*)&actual_totem, sizeof(actual_totem) ))); + + new_block_stream.write( (char*)&actual_totem, sizeof(actual_totem) ); + } + + std::exception_ptr except_ptr; + vector incomplete_block_data; + optional bad_block; + uint32_t block_num = 0; - auto pos = 0; - uint32_t block_num = 0; block_id_type previous; - boost::iostreams::mapped_file_source log_data((backup_dir / "blocks.log").generic_string()); - fc::datastream ds(log_data.data(), log_data.size()); - auto [version, first_block_num] = get_blocklog_version_and_first_block_num(ds, block_file_name); - pos = ds.tellp(); - std::string error_msg; + uint64_t pos = old_block_stream.tellg(); - try { - signed_block signed_tmp; - pruned_block pruned_tmp; - signed_block_header* hdr = version >=4 ? static_cast(&pruned_tmp) : &signed_tmp; - while (ds.remaining() > 0) { - try { - if (version >= 4) { - detail::log_entry entry; - entry.block = &pruned_tmp; - unpack(ds, entry); - } else { - fc::raw::unpack(ds, &signed_tmp); - } - } catch (...) { - write_incomplete_block_data(blocks_dir, now, block_num, log_data.data() + pos, log_data.size() - pos); - throw; - } + EOS_ASSERT(version == 4, block_log_exception, "Unsupported block log version"); + pruned_block tmp; + detail::log_entry entry; + entry.block = &tmp; - if (!verify_block(ds, previous, pos, *hdr)) { - #warning TODO: require fc::reflector to work, change after merged with PR 8900 - // variant block; - // if (version >= 4) - // to_variant(pruned_tmp, block); - // else - // to_variant(signed_tmp, block); - // ilog("Recovered only up to block number ${num}. Last block in block log was not properly " - // "committed:\n${last_block}", - // ("num", block_num)("last_block", block)); - break; - } + while( pos < end_pos ) { + try { + unpack(old_block_stream, entry); + } catch (...) { + except_ptr = std::current_exception(); + incomplete_block_data.resize( end_pos - pos ); + old_block_stream.read( incomplete_block_data.data(), incomplete_block_data.size() ); + break; + } - previous = hdr->id(); - block_num = hdr->block_num(); + auto id = tmp.id(); + if( block_header::num_from_id(previous) + 1 != block_header::num_from_id(id) ) { + elog( "Block ${num} (${id}) skips blocks. Previous block in block log is block ${prev_num} (${previous})", + ("num", block_header::num_from_id(id))("id", id) + ("prev_num", block_header::num_from_id(previous))("previous", previous) ); + } + if( previous != tmp.previous ) { + elog( "Block ${num} (${id}) does not link back to previous block. " + "Expected previous: ${expected}. Actual previous: ${actual}.", + ("num", block_header::num_from_id(id))("id", id)("expected", previous)("actual", tmp.previous) ); + } + previous = id; - if (block_num % 1000 == 0) - ilog("Verified block ${num}", ("num", block_num)); - pos = ds.tellp(); - if (block_num == truncate_at_block) - break; + uint64_t tmp_pos = std::numeric_limits::max(); + if( (static_cast(old_block_stream.tellg()) + sizeof(pos)) <= end_pos ) { + old_block_stream.read( reinterpret_cast(&tmp_pos), sizeof(tmp_pos) ); + } + if( pos != tmp_pos ) { + bad_block.emplace(std::move(tmp)); + break; } - } catch (const fc::exception& e) { - error_msg = e.what(); - } catch (const std::exception& e) { - error_msg = e.what(); - } catch (...) { - error_msg = "unrecognized exception"; + + auto data = detail::block_log_impl::pack_v4_log_entry(tmp, entry.compression); + new_block_stream.write( data.data(), data.size() ); + new_block_stream.write( reinterpret_cast(&pos), sizeof(pos) ); + block_num = tmp.block_num(); + if(block_num % 1000 == 0) + ilog( "Recovered block ${num}", ("num", block_num) ); + pos = new_block_stream.tellp(); + if( block_num == truncate_at_block ) + break; } - FILE* fp = fopen(block_file_name, "w"); - EOS_ASSERT(fp != nullptr, block_log_exception, "Could not create block log file: ${name}", ("name", block_file_name)); - int nwritten = fwrite(log_data.data(), 1, pos, fp); - fclose(fp); + if( bad_block.valid() ) { + ilog( "Recovered only up to block number ${num}. Last block in block log was not properly committed:\n${last_block}", + ("num", block_num)("last_block", *bad_block) ); + } else if( except_ptr ) { + std::string error_msg; + + try { + std::rethrow_exception(except_ptr); + } catch( const fc::exception& e ) { + error_msg = e.what(); + } catch( const std::exception& e ) { + error_msg = e.what(); + } catch( ... ) { + error_msg = "unrecognized exception"; + } - EOS_ASSERT( - nwritten == pos, block_log_exception, - "Unable to write the entire block log file: ${name}, expected size=${expected}, written size=${written}", - ("name", block_file_name)("expected", pos)("written", nwritten)); - - if (error_msg.size()) { - ilog("Recovered only up to block number ${num}. " - "The block ${next_num} could not be deserialized from the block log due to error:\n${error_msg}", - ("num", block_num)("next_num", block_num + 1)("error_msg", error_msg)); - } else if (block_num == truncate_at_block && pos < log_data.size()) { - ilog("Stopped recovery of block log early at specified block number: ${stop}.", ("stop", truncate_at_block)); + ilog( "Recovered only up to block number ${num}. " + "The block ${next_num} could not be deserialized from the block log due to error:\n${error_msg}", + ("num", block_num)("next_num", block_num+1)("error_msg", error_msg) ); + + auto tail_path = blocks_dir / std::string("blocks-bad-tail-").append( now ).append(".log"); + if( !fc::exists(tail_path) && incomplete_block_data.size() > 0 ) { + std::fstream tail_stream; + tail_stream.open( tail_path.generic_string().c_str(), LOG_WRITE ); + tail_stream.write( incomplete_block_data.data(), incomplete_block_data.size() ); + + ilog( "Data at tail end of block log which should contain the (incomplete) serialization of block ${num} " + "has been written out to '${tail_path}'.", + ("num", block_num+1)("tail_path", tail_path) ); + } + } else if( block_num == truncate_at_block && pos < end_pos ) { + ilog( "Stopped recovery of block log early at specified block number: ${stop}.", ("stop", truncate_at_block) ); } else { - ilog("Existing block log was undamaged. Recovered all irreversible blocks up to block number ${num}.", - ("num", block_num)); + ilog( "Existing block log was undamaged. Recovered all irreversible blocks up to block number ${num}.", ("num", block_num) ); } + return backup_dir; + } template diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 7c4d5c8d2ea..3b27910e6c9 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -178,6 +178,8 @@ namespace eosio { namespace chain { FC_REFLECT_ENUM( eosio::chain::transaction_receipt::status_enum, (executed)(soft_fail)(hard_fail)(delayed)(expired) ) +FC_REFLECT_ENUM(eosio::chain::pruned_block::prune_state_type, + (incomplete)(complete)(complete_legacy)); FC_REFLECT(eosio::chain::transaction_receipt_header, (status)(cpu_usage_us)(net_usage_words) ) FC_REFLECT_DERIVED(eosio::chain::transaction_receipt, (eosio::chain::transaction_receipt_header), (trx) ) From c86853a52b3aaa4f0b2a9d574c3bc8d6baf7819e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 6 Apr 2020 17:59:29 -0500 Subject: [PATCH 048/142] Reworked how get_estimated_size is calculated --- .../chain/include/eosio/chain/transaction.hpp | 9 +-- libraries/chain/transaction.cpp | 80 +++++++++++++------ 2 files changed, 60 insertions(+), 29 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index f5ad5f3883d..10d421a95f6 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -214,6 +214,8 @@ namespace eosio { namespace chain { transaction_id_type trx_id; }; + using packed_transaction_v0_ptr = std::shared_ptr; + struct prunable_transaction_data { enum class compression_type : uint8_t { none = 0, @@ -232,10 +234,9 @@ namespace eosio { namespace chain { digest_type context_free_mroot; }; - using segment_type = fc::static_variant>; + using segment_type = fc::static_variant; struct partial { - // TODO: will need indication of what was pruned so correct exception can be thrown in apply_context.get_context_free_data std::vector signatures; std::vector context_free_segments; }; @@ -267,8 +268,6 @@ namespace eosio { namespace chain { prunable_data_type prunable_data; }; - using packed_transaction_v0_ptr = std::shared_ptr; - struct packed_transaction : fc::reflect_init { using compression_type = packed_transaction_v0::compression_type; using cf_compression_type = prunable_transaction_data::compression_type; @@ -310,11 +309,11 @@ namespace eosio { namespace chain { std::size_t maximum_pruned_pack_size( cf_compression_type segment_compression ) const; private: - friend struct fc::reflector; friend struct fc::reflector_init_visitor; friend struct fc::has_reflector_init; void reflector_init(); + uint32_t calculate_estimated_size() const; private: uint32_t estimated_size = 0; fc::enum_type compression; diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index d4414bf0fcc..2c0d1086588 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -215,9 +215,8 @@ static vector zlib_decompress_context_free_data(const bytes& data) { return unpack_context_free_data(out); } -static transaction zlib_decompress_transaction(const bytes& data, uint32_t& packed_size) { +static transaction zlib_decompress_transaction(const bytes& data) { bytes out = zlib_decompress(data); - packed_size = out.size(); return unpack_transaction(out); } @@ -246,9 +245,8 @@ static bytes zlib_compress_context_free_data(const vector& cfd ) { return out; } -static bytes zlib_compress_transaction(const transaction& t, uint32_t& packed_size) { +static bytes zlib_compress_transaction(const transaction& t) { bytes in = pack_transaction(t); - packed_size = in.size(); bytes out; bio::filtering_ostream comp; comp.push(bio::zlib_compressor(bio::zlib::best_compression)); @@ -316,14 +314,13 @@ void packed_transaction_v0::reflector_init() local_unpack_context_free_data(); } -static transaction unpack_transaction(const bytes& packed_trx, uint32_t& packed_size, packed_transaction_v0::compression_type compression) { +static transaction unpack_transaction(const bytes& packed_trx, packed_transaction_v0::compression_type compression) { try { switch( compression ) { case packed_transaction_v0::compression_type::none: - packed_size = packed_trx.size(); return unpack_transaction( packed_trx ); case packed_transaction_v0::compression_type::zlib: - return zlib_decompress_transaction( packed_trx, packed_size ); + return zlib_decompress_transaction( packed_trx ); default: EOS_THROW( unknown_transaction_compression, "Unknown transaction compression algorithm" ); } @@ -332,8 +329,7 @@ static transaction unpack_transaction(const bytes& packed_trx, uint32_t& packed_ void packed_transaction_v0::local_unpack_transaction(vector&& context_free_data) { - uint32_t packed_size = 0; - unpacked_trx = signed_transaction( unpack_transaction( packed_trx, packed_size, compression ), signatures, std::move(context_free_data) ); + unpacked_trx = signed_transaction( unpack_transaction( packed_trx, compression ), signatures, std::move(context_free_data) ); trx_id = unpacked_trx.id(); } @@ -357,16 +353,14 @@ void packed_transaction_v0::local_unpack_context_free_data() unpacked_trx.context_free_data = unpack_context_free_data( packed_context_free_data, compression ); } -static bytes pack_transaction(const transaction& trx, uint32_t& packed_size, packed_transaction_v0::compression_type compression) { +static bytes pack_transaction(const transaction& trx, packed_transaction_v0::compression_type compression) { try { switch(compression) { case packed_transaction_v0::compression_type::none: { - bytes result = pack_transaction( trx ); - packed_size = result.size(); - return result; + return pack_transaction( trx ); } case packed_transaction_v0::compression_type::zlib: - return zlib_compress_transaction(trx, packed_size); + return zlib_compress_transaction(trx); default: EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); } @@ -375,8 +369,7 @@ static bytes pack_transaction(const transaction& trx, uint32_t& packed_size, pac void packed_transaction_v0::local_pack_transaction() { - uint32_t packed_size = 0; - packed_trx = pack_transaction(unpacked_trx, packed_size, compression); + packed_trx = pack_transaction(unpacked_trx, compression); } static bytes pack_context_free_data( const vector& cfd, packed_transaction_v0::compression_type compression ) { @@ -479,11 +472,11 @@ static prunable_transaction_data make_prunable_transaction_data( bool legacy, ve packed_transaction::packed_transaction(signed_transaction t, bool legacy, compression_type _compression) : compression(_compression), prunable_data(make_prunable_transaction_data(legacy, std::move(t.signatures), std::move(t.context_free_data), _compression)), - packed_trx(pack_transaction(t, estimated_size, compression)), + packed_trx(pack_transaction(t, compression)), unpacked_trx(std::move(t)), trx_id(unpacked_trx.id()) { - estimated_size = estimated_size * 2 + get_prunable_size() * 2; + estimated_size = calculate_estimated_size(); } packed_transaction::packed_transaction(const packed_transaction_v0& other, bool legacy) @@ -496,7 +489,9 @@ packed_transaction::packed_transaction(const packed_transaction_v0& other, bool packed_trx(other.packed_trx), unpacked_trx(other.unpacked_trx), trx_id(other.id()) -{} +{ + estimated_size = calculate_estimated_size(); +} packed_transaction::packed_transaction(packed_transaction_v0&& other, bool legacy) : compression(other.compression), @@ -508,7 +503,9 @@ packed_transaction::packed_transaction(packed_transaction_v0&& other, bool legac packed_trx(std::move(other.packed_trx)), unpacked_trx(std::move(other.unpacked_trx)), trx_id(other.id()) -{} +{ + estimated_size = calculate_estimated_size(); +} packed_transaction_v0_ptr packed_transaction::to_packed_transaction_v0() const { const auto* sigs = get_signatures(); @@ -546,6 +543,43 @@ uint32_t packed_transaction::get_prunable_size()const { return prunable_data.prunable_data.visit([](const auto& obj) { return get_prunable_size_impl(obj); }); } +uint32_t packed_transaction::calculate_estimated_size() const { + auto est_size = overloaded{ + [](const std::vector& vec) { + uint32_t s = 0; + for( const auto& v : vec ) s += v.size(); + return s; + }, + [](const std::vector& vec) { + uint32_t s = 0; + for( const auto& v : vec ) { + s += v.visit( overloaded{ + [](const digest_type& t) { return sizeof(t); }, + [](const bytes& vec) { return vec.size(); } + } ); + } + return s; + } + }; + auto visitor = overloaded{ + [](const prunable_transaction_data::none& v) { return 0ul; }, + [](const prunable_transaction_data::signatures_only& v) { + return v.signatures.size() * sizeof(signature_type); + }, + [&](const prunable_transaction_data::partial& v) { + return v.signatures.size() * sizeof(signature_type) + est_size(v.context_free_segments); + }, + [&](const prunable_transaction_data::full& v) { + return v.signatures.size() * sizeof(signature_type) + est_size(v.context_free_segments); + }, + [&](const prunable_transaction_data::full_legacy& v) { + return v.signatures.size() * sizeof(signature_type) + v.packed_context_free_data.size() + est_size(v.context_free_segments); + } + }; + + return sizeof(*this) + packed_trx.size() * 2 + sizeof(prunable_data) + prunable_data.prunable_data.visit(visitor); +} + digest_type packed_transaction::packed_digest()const { digest_type::encoder enc; fc::raw::pack( enc, compression ); @@ -603,11 +637,9 @@ void packed_transaction::reflector_init() static_assert(fc::raw::has_feature_reflector_init_on_unpacked_reflected_types, "FC unpack needs to call reflector_init otherwise unpacked_trx will not be initialized"); EOS_ASSERT( unpacked_trx.expiration == time_point_sec(), tx_decompression_error, "packed_transaction already unpacked" ); - const auto* signatures = get_signatures(); - uint32_t packed_size = 0; - unpacked_trx = unpack_transaction(packed_trx, packed_size, compression); + unpacked_trx = unpack_transaction(packed_trx, compression); trx_id = unpacked_trx.id(); - estimated_size = packed_size * 2 + get_prunable_size() * 2; // x2 since stored packed and unpacked + estimated_size = calculate_estimated_size(); } } } // eosio::chain From 15e111dc97ab328ee6158b5f71015cd4c4940481 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 6 Apr 2020 18:51:44 -0500 Subject: [PATCH 049/142] Revert change of adding packed_context_free_data --- libraries/chain/include/eosio/chain/abi_serializer.hpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 50b664e3821..e436fde0f42 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -484,9 +484,14 @@ namespace impl { const auto* sigs = ptrx.get_signatures(); mvo("signatures", sigs != nullptr ? *sigs : vector()); mvo("compression", ptrx.get_compression()); + if( ptrx.get_prunable_data().prunable_data.contains() ) { + const auto& legacy = ptrx.get_prunable_data().prunable_data.get(); + mvo( "packed_context_free_data", legacy.packed_context_free_data ); + } else { + mvo( "packed_context_free_data", bytes() ); + } const auto* context_free_data = ptrx.get_context_free_data(); - mvo("packed_context_free_data", context_free_data != nullptr ? *context_free_data : vector() ); - mvo("context_free_data", ptrx.get_context_free_data() != nullptr ? *ptrx.get_context_free_data() : vector()); + mvo("context_free_data", context_free_data != nullptr ? *context_free_data : vector()); mvo("packed_trx", ptrx.get_packed_transaction()); add(mvo, "transaction", trx, resolver, ctx); out(name, std::move(mvo)); From 03d36a4ea78d5f464799f667c83fda41c492657c Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 6 Apr 2020 18:52:39 -0500 Subject: [PATCH 050/142] Store const packed_transaction& instead of shared_ptr in transaction_context to avoid having to create packed_transaction on heap in onerror --- libraries/chain/apply_context.cpp | 22 +++++++-------- libraries/chain/controller.cpp | 8 +++--- .../eosio/chain/transaction_context.hpp | 4 +-- libraries/chain/transaction_context.cpp | 28 +++++++++---------- libraries/chain/wasm_interface.cpp | 6 ++-- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 77b977be27b..fe470a8e2ba 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -390,15 +390,15 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a "deferred transaction generaction context contains mismatching sender_id", ("expected", sender_id)("actual", context.sender_id) ); - EOS_ASSERT( context.sender_trx_id == trx_context.packed_trx->id(), ill_formed_deferred_transaction_generation_context, + EOS_ASSERT( context.sender_trx_id == trx_context.packed_trx.id(), ill_formed_deferred_transaction_generation_context, "deferred transaction generaction context contains mismatching sender_trx_id", - ("expected", trx_context.packed_trx->id())("actual", context.sender_trx_id) + ("expected", trx_context.packed_trx.id())("actual", context.sender_trx_id) ); } else { emplace_extension( trx.transaction_extensions, deferred_transaction_generation_context::extension_id(), - fc::raw::pack( deferred_transaction_generation_context( trx_context.packed_trx->id(), sender_id, receiver ) ) + fc::raw::pack( deferred_transaction_generation_context( trx_context.packed_trx.id(), sender_id, receiver ) ) ); } trx.expiration = time_point_sec(); @@ -612,18 +612,18 @@ vector apply_context::get_active_producers() const { } bytes apply_context::get_packed_transaction() { - if( trx_context.packed_trx->get_compression() == packed_transaction::compression_type::none) { - return trx_context.packed_trx->get_packed_transaction(); + if( trx_context.packed_trx.get_compression() == packed_transaction::compression_type::none) { + return trx_context.packed_trx.get_packed_transaction(); } else { - return fc::raw::pack( static_cast( trx_context.packed_trx->get_transaction() ) ); + return fc::raw::pack( static_cast( trx_context.packed_trx.get_transaction() ) ); } } size_t apply_context::get_packed_transaction_size() { - if( trx_context.packed_trx->get_compression() == packed_transaction::compression_type::none) { - return trx_context.packed_trx->get_packed_transaction().size(); + if( trx_context.packed_trx.get_compression() == packed_transaction::compression_type::none) { + return trx_context.packed_trx.get_packed_transaction().size(); } else { - return fc::raw::pack_size( static_cast( trx_context.packed_trx->get_transaction() ) ); + return fc::raw::pack_size( static_cast( trx_context.packed_trx.get_transaction() ) ); } } @@ -643,7 +643,7 @@ void apply_context::update_db_usage( const account_name& payer, int64_t delta ) int apply_context::get_action( uint32_t type, uint32_t index, char* buffer, size_t buffer_size )const { - const auto& trx = trx_context.packed_trx->get_transaction(); + const auto& trx = trx_context.packed_trx.get_transaction(); const action* act_ptr = nullptr; if( type == 0 ) { @@ -669,7 +669,7 @@ int apply_context::get_action( uint32_t type, uint32_t index, char* buffer, size int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t buffer_size )const { - const prunable_transaction_data::prunable_data_type& data = trx_context.packed_trx->get_prunable_data().prunable_data; + const prunable_transaction_data::prunable_data_type& data = trx_context.packed_trx.get_prunable_data().prunable_data; if( data.contains() || data.contains() || data.contains() ) // TODO: for partial determine if index is pruned and throw only in that case { diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index abb02238e35..2ea1b8eec85 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1140,7 +1140,7 @@ struct controller_impl { } transaction_checktime_timer trx_timer(timer); - packed_transaction_ptr trx = std::make_shared( std::move( etrx ), true ); + const packed_transaction trx( std::move( etrx ), true ); transaction_context trx_context( self, trx, std::move(trx_timer), start ); trx_context.deadline = deadline; trx_context.explicit_billed_cpu_time = explicit_billed_cpu_time; @@ -1150,7 +1150,7 @@ struct controller_impl { try { trx_context.init_for_implicit_trx(); trx_context.published = gtrx.published; - trx_context.execute_action( trx_context.schedule_action( trx->get_transaction().actions.back(), gtrx.sender, false, 0, 0 ), 0 ); + trx_context.execute_action( trx_context.schedule_action( trx.get_transaction().actions.back(), gtrx.sender, false, 0, 0 ), 0 ); trx_context.finalize(); // Automatically rounds up network and CPU usage in trace and bills payers if successful auto restore = make_block_restore_point(); @@ -1268,7 +1268,7 @@ struct controller_impl { uint32_t cpu_time_to_bill_us = billed_cpu_time_us; transaction_checktime_timer trx_timer(timer); - transaction_context trx_context( self, trx->packed_trx(), std::move(trx_timer) ); + transaction_context trx_context( self, *trx->packed_trx(), std::move(trx_timer) ); trx_context.leeway = fc::microseconds(0); // avoid stealing cpu resource trx_context.deadline = deadline; trx_context.explicit_billed_cpu_time = explicit_billed_cpu_time; @@ -1439,7 +1439,7 @@ struct controller_impl { } transaction_checktime_timer trx_timer(timer); - transaction_context trx_context(self, trx->packed_trx(), std::move(trx_timer), start); + transaction_context trx_context(self, *trx->packed_trx(), std::move(trx_timer), start); if ((bool)subjective_cpu_leeway && pending->_block_status == controller::block_status::incomplete) { trx_context.leeway = *subjective_cpu_leeway; } diff --git a/libraries/chain/include/eosio/chain/transaction_context.hpp b/libraries/chain/include/eosio/chain/transaction_context.hpp index 4de1a136253..88034aa8f0c 100644 --- a/libraries/chain/include/eosio/chain/transaction_context.hpp +++ b/libraries/chain/include/eosio/chain/transaction_context.hpp @@ -38,7 +38,7 @@ namespace eosio { namespace chain { public: transaction_context( controller& c, - packed_transaction_ptr t, + const packed_transaction& t, transaction_checktime_timer&& timer, fc::time_point start = fc::time_point::now() ); @@ -133,7 +133,7 @@ namespace eosio { namespace chain { public: controller& control; - const packed_transaction_ptr packed_trx; + const packed_transaction& packed_trx; optional undo_session; transaction_trace_ptr trace; fc::time_point start; diff --git a/libraries/chain/transaction_context.cpp b/libraries/chain/transaction_context.cpp index 18d3a5a32c0..3d137d89583 100644 --- a/libraries/chain/transaction_context.cpp +++ b/libraries/chain/transaction_context.cpp @@ -44,11 +44,11 @@ namespace eosio { namespace chain { } transaction_context::transaction_context( controller& c, - packed_transaction_ptr t, + const packed_transaction& t, transaction_checktime_timer&& tmr, fc::time_point s ) :control(c) - ,packed_trx(std::move(t)) + ,packed_trx(t) ,undo_session() ,trace(std::make_shared()) ,start(s) @@ -59,7 +59,7 @@ namespace eosio { namespace chain { if (!c.skip_db_sessions()) { undo_session = c.mutable_db().start_undo_session(true); } - trace->id = packed_trx->id(); + trace->id = packed_trx.id(); trace->block_num = c.head_block_num() + 1; trace->block_time = c.pending_block_time(); trace->producer_block_id = c.pending_producer_block_id(); @@ -98,7 +98,7 @@ namespace eosio { namespace chain { _deadline = start + objective_duration_limit; } - const transaction& trx = packed_trx->get_transaction(); + const transaction& trx = packed_trx.get_transaction(); // Possibly lower net_limit to optional limit set in the transaction header uint64_t trx_specified_net_usage_limit = static_cast(trx.max_net_usage_words.value) * 8; if( trx_specified_net_usage_limit > 0 && trx_specified_net_usage_limit <= net_limit ) { @@ -191,7 +191,7 @@ namespace eosio { namespace chain { void transaction_context::init_for_implicit_trx( uint64_t initial_net_usage ) { - const transaction& trx = packed_trx->get_transaction(); + const transaction& trx = packed_trx.get_transaction(); if( trx.transaction_extensions.size() > 0 ) { disallow_transaction_extensions( "no transaction extensions supported yet for implicit transactions" ); } @@ -204,7 +204,7 @@ namespace eosio { namespace chain { uint64_t packed_trx_prunable_size, bool skip_recording ) { - const transaction& trx = packed_trx->get_transaction(); + const transaction& trx = packed_trx.get_transaction(); if( trx.transaction_extensions.size() > 0 ) { disallow_transaction_extensions( "no transaction extensions supported yet for input transactions" ); } @@ -237,7 +237,7 @@ namespace eosio { namespace chain { void transaction_context::init_for_input_trx_with_explicit_net( uint32_t explicit_net_usage_words, bool skip_recording ) { - const transaction& trx = packed_trx->get_transaction(); + const transaction& trx = packed_trx.get_transaction(); if( trx.transaction_extensions.size() > 0 ) { disallow_transaction_extensions( "no transaction extensions supported yet for input transactions" ); } @@ -252,7 +252,7 @@ namespace eosio { namespace chain { { published = control.pending_block_time(); is_input = true; - const transaction& trx = packed_trx->get_transaction(); + const transaction& trx = packed_trx.get_transaction(); if (!control.skip_trx_checks()) { control.validate_expiration(trx); control.validate_tapos(trx); @@ -260,12 +260,12 @@ namespace eosio { namespace chain { } init( initial_net_usage ); if (!skip_recording) - record_transaction( packed_trx->id(), trx.expiration ); /// checks for dupes + record_transaction( packed_trx.id(), trx.expiration ); /// checks for dupes } void transaction_context::init_for_deferred_trx( fc::time_point p ) { - const transaction& trx = packed_trx->get_transaction(); + const transaction& trx = packed_trx.get_transaction(); if( (trx.expiration.sec_since_epoch() != 0) && (trx.transaction_extensions.size() > 0) ) { disallow_transaction_extensions( "no transaction extensions supported yet for deferred transactions" ); } @@ -283,7 +283,7 @@ namespace eosio { namespace chain { void transaction_context::exec() { EOS_ASSERT( is_initialized, transaction_exception, "must first initialize" ); - const transaction& trx = packed_trx->get_transaction(); + const transaction& trx = packed_trx.get_transaction(); if( apply_context_free ) { for( const auto& act : trx.context_free_actions ) { schedule_action( act, act.account, true, 0, 0 ); @@ -311,7 +311,7 @@ namespace eosio { namespace chain { EOS_ASSERT( is_initialized, transaction_exception, "must first initialize" ); if( is_input ) { - const transaction& trx = packed_trx->get_transaction(); + const transaction& trx = packed_trx.get_transaction(); auto& am = control.get_mutable_authorization_manager(); for( const auto& act : trx.actions ) { for( const auto& auth : act.authorization ) { @@ -617,7 +617,7 @@ namespace eosio { namespace chain { void transaction_context::schedule_transaction() { // Charge ahead of time for the additional net usage needed to retire the delayed transaction // whether that be by successfully executing, soft failure, hard failure, or expiration. - const transaction& trx = packed_trx->get_transaction(); + const transaction& trx = packed_trx.get_transaction(); if( trx.delay_sec.value == 0 ) { // Do not double bill. Only charge if we have not already charged for the delay. const auto& cfg = control.get_global_properties().configuration; add_net_usage( static_cast(cfg.base_per_transaction_net_usage) @@ -628,7 +628,7 @@ namespace eosio { namespace chain { uint32_t trx_size = 0; const auto& cgto = control.mutable_db().create( [&]( auto& gto ) { - gto.trx_id = packed_trx->id(); + gto.trx_id = packed_trx.id(); gto.payer = first_auth; gto.sender = account_name(); /// delayed transactions have no sender gto.sender_id = transaction_id_to_sender_id( gto.trx_id ); diff --git a/libraries/chain/wasm_interface.cpp b/libraries/chain/wasm_interface.cpp index 199baded266..a7f69982afa 100644 --- a/libraries/chain/wasm_interface.cpp +++ b/libraries/chain/wasm_interface.cpp @@ -1525,14 +1525,14 @@ class context_free_transaction_api : public context_aware_api { } int expiration() { - return context.trx_context.packed_trx->get_transaction().expiration.sec_since_epoch(); + return context.trx_context.packed_trx.get_transaction().expiration.sec_since_epoch(); } int tapos_block_num() { - return context.trx_context.packed_trx->get_transaction().ref_block_num; + return context.trx_context.packed_trx.get_transaction().ref_block_num; } int tapos_block_prefix() { - return context.trx_context.packed_trx->get_transaction().ref_block_prefix; + return context.trx_context.packed_trx.get_transaction().ref_block_prefix; } int get_action( uint32_t type, uint32_t index, array_ptr buffer, uint32_t buffer_size )const { From adeb5e13b81039721428bd966ab5d256c7a0ec51 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 7 Apr 2020 08:32:43 -0500 Subject: [PATCH 051/142] Make transaction_metadata get_estimated_size more accurate --- libraries/chain/include/eosio/chain/transaction_metadata.hpp | 2 +- .../chain/include/eosio/chain/unapplied_transaction_queue.hpp | 2 +- libraries/chain/transaction_metadata.cpp | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction_metadata.hpp b/libraries/chain/include/eosio/chain/transaction_metadata.hpp index 970fd1129ac..e9f1c3693f3 100644 --- a/libraries/chain/include/eosio/chain/transaction_metadata.hpp +++ b/libraries/chain/include/eosio/chain/transaction_metadata.hpp @@ -72,7 +72,7 @@ class transaction_metadata { const transaction_id_type& id()const { return _packed_trx->id(); } fc::microseconds signature_cpu_usage()const { return _sig_cpu_usage; } const flat_set& recovered_keys()const { return _recovered_pub_keys; } - uint32_t get_estimated_size() const { return sizeof(transaction_metadata) + _recovered_pub_keys.size() * sizeof(public_key_type); } + uint32_t get_estimated_size() const; /// Thread safe. /// @returns transaction_metadata_ptr or exception via future diff --git a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp index a86c66f2577..c854874bf9c 100644 --- a/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp +++ b/libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp @@ -254,7 +254,7 @@ class unapplied_transaction_queue { } static uint64_t calc_size( const transaction_metadata_ptr& trx ) { - return trx->packed_trx()->get_estimated_size() + sizeof(unapplied_transaction) + trx->get_estimated_size(); + return sizeof(unapplied_transaction) + trx->get_estimated_size(); } }; diff --git a/libraries/chain/transaction_metadata.cpp b/libraries/chain/transaction_metadata.cpp index abd9f3f77bb..d6d1861e50d 100644 --- a/libraries/chain/transaction_metadata.cpp +++ b/libraries/chain/transaction_metadata.cpp @@ -25,4 +25,8 @@ recover_keys_future transaction_metadata::start_recover_keys( packed_transaction ); } +uint32_t transaction_metadata::get_estimated_size() const { + return sizeof(*this) + _recovered_pub_keys.size() * sizeof(public_key_type) + packed_trx()->get_estimated_size(); +} + } } // eosio::chain From 6ab191ee2817505a1a2269fdda665d0558eec501 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 7 Apr 2020 12:07:01 -0500 Subject: [PATCH 052/142] remove unused code --- libraries/chain/block_log.cpp | 88 ----------------------------------- 1 file changed, 88 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 3e068600ea6..52f49c58536 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -619,94 +619,6 @@ namespace eosio { namespace chain { index.complete(); } - static std::tuple get_blocklog_version_and_first_block_num(fc::datastream& ds, const char* block_file_name) { - uint32_t version = 0; - ds.read((char*)&version, sizeof(version)); - EOS_ASSERT(version > 0, block_log_exception, "Block log was not setup properly"); - EOS_ASSERT( - block_log::is_supported_version(version), block_log_unsupported_version, - "Unsupported version of block log. Block log version is ${version} while code supports version(s) " - "[${min},${max}]", - ("version", version)("min", block_log::min_supported_version)("max", block_log::max_supported_version)); - - uint32_t first_block_num = 1; - if (version != 1) { - ds.read((char*)&first_block_num, sizeof(first_block_num)); - - // this assert is only here since repair_log is only used for --hard-replay-blockchain, which removes any - // existing state, if another API needs to use it, this can be removed and the check for the first block's - // previous block id will need to accommodate this. - EOS_ASSERT(first_block_num == 1, block_log_exception, - "Block log ${file} must contain a genesis state and start at block number 1. This block log " - "starts at block number ${first_block_num}.", - ("file", block_file_name)("first_block_num", first_block_num)); - } - - if (block_log::contains_genesis_state(version, first_block_num)) { - genesis_state gs; - fc::raw::unpack(ds, gs); - - auto data = fc::raw::pack(gs); - } else if (block_log::contains_chain_id(version, first_block_num)) { - fc::sha256 chain_id; // I couldn't use chain_id_type because its constructor is private - ds >> chain_id; - } else { - EOS_THROW(block_log_exception, - "Block log ${file} is not supported. version: ${ver} and first_block_num: ${fbn} does not contain " - "a genesis_state nor a chain_id.", - ("file", block_file_name)("ver", version)("fbn", first_block_num)); - } - - if (version != 1) { - auto expected_totem = block_log::npos; - std::decay_t actual_totem; - ds.read((char*)&actual_totem, sizeof(actual_totem)); - - EOS_ASSERT( - actual_totem == expected_totem, block_log_exception, - "Expected separator between block log header and blocks was not found( expected: ${e}, actual: ${a} )", - ("e", fc::to_hex((char*)&expected_totem, sizeof(expected_totem)))( - "a", fc::to_hex((char*)&actual_totem, sizeof(actual_totem)))); - } - return std::make_tuple(version, first_block_num); - } - - static bool verify_block(fc::datastream& ds, block_id_type previous, uint64_t pos, block_header& header) { - auto id = header.id(); - if (block_header::num_from_id(previous) + 1 != block_header::num_from_id(id)) { - elog("Block ${num} (${id}) skips blocks. Previous block in block log is block ${prev_num} (${previous})", - ("num", block_header::num_from_id(id))("id", id)("prev_num", block_header::num_from_id(previous))( - "previous", previous)); - } - if (previous != header.previous) { - elog("Block ${num} (${id}) does not link back to previous block. " - "Expected previous: ${expected}. Actual previous: ${actual}.", - ("num", block_header::num_from_id(id))("id", id)("expected", previous)("actual", header.previous)); - } - - uint64_t tmp_pos = std::numeric_limits::max(); - if (ds.remaining() >= sizeof(tmp_pos)) { - ds.read(reinterpret_cast(&tmp_pos), sizeof(tmp_pos)); - } - if (pos != tmp_pos) { - return false; - } - return true; - } - - static void write_incomplete_block_data(const fc::path& blocks_dir, fc::time_point now, uint32_t block_num, const char* start, int size) { - auto tail_path = blocks_dir / std::string("blocks-bad-tail-").append(now).append(".log"); - if (!fc::exists(tail_path)) { - std::fstream tail_stream; - tail_stream.open(tail_path.generic_string().c_str(), LOG_WRITE); - tail_stream.write(start, size); - - ilog("Data at tail end of block log which should contain the (incomplete) serialization of block ${num} " - "has been written out to '${tail_path}'.", - ("num", block_num + 1)("tail_path", tail_path)); - } - } - fc::path block_log::repair_log(const fc::path& data_dir, uint32_t truncate_at_block) { ilog("Recovering Block Log..."); EOS_ASSERT( fc::is_directory(data_dir) && fc::is_regular_file(data_dir / "blocks.log"), block_log_not_found, From d0ce4392e93ef2b8cde8ae0a87d865c57e46fdee Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 7 Apr 2020 16:21:58 -0500 Subject: [PATCH 053/142] Construct variant manually to avoid to_packed_transaction_v0() call --- plugins/history_plugin/history_plugin.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/plugins/history_plugin/history_plugin.cpp b/plugins/history_plugin/history_plugin.cpp index 4463fa4c7d6..540aba79917 100644 --- a/plugins/history_plugin/history_plugin.cpp +++ b/plugins/history_plugin/history_plugin.cpp @@ -498,8 +498,13 @@ namespace eosio { auto &pt = receipt.trx.get(); if (pt.id() == result.id) { fc::mutable_variant_object r("receipt", receipt); - auto ptv0 = pt.to_packed_transaction_v0(); - r("trx", chain.to_variant_with_abi(ptv0->get_signed_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time ))); + fc::variant v = chain.to_variant_with_abi(pt.get_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time )); + fc::mutable_variant_object tmp(v.get_object()); + const auto* sigs = pt.get_signatures(); + tmp("signatures", sigs != nullptr ? *sigs : vector()); + const auto* context_free_data = pt.get_context_free_data(); + tmp("context_free_data", context_free_data != nullptr ? *context_free_data : vector()); + r("trx", std::move(tmp) ); result.trx = move(r); break; } @@ -527,8 +532,13 @@ namespace eosio { result.block_num = *p.block_num_hint; result.block_time = blk->timestamp; fc::mutable_variant_object r("receipt", receipt); - auto ptv0 = pt.to_packed_transaction_v0(); - r("trx", chain.to_variant_with_abi(ptv0->get_signed_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time ))); + fc::variant v = chain.to_variant_with_abi(pt.get_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time )); + fc::mutable_variant_object tmp(v.get_object()); + const auto* sigs = pt.get_signatures(); + tmp("signatures", sigs != nullptr ? *sigs : vector()); + const auto* context_free_data = pt.get_context_free_data(); + tmp("context_free_data", context_free_data != nullptr ? *context_free_data : vector()); + r("trx", std::move(tmp) ); result.trx = move(r); found = true; break; From 30d3b375a070b112a5b31e7dd990904b37fe96b0 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 7 Apr 2020 16:43:01 -0500 Subject: [PATCH 054/142] Renames and better descriptions --- libraries/chain/controller.cpp | 6 +++--- .../chain/include/eosio/chain/exceptions.hpp | 18 +++++++++--------- .../eosio/chain/transaction_metadata.hpp | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 2ea1b8eec85..16c582ba6a6 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1162,7 +1162,7 @@ struct controller_impl { trx_context.squash(); restore.cancel(); return trace; - } catch( const objective_block_production_exception& ) { + } catch( const objective_block_validation_exception& ) { throw; } catch( const fc::exception& e ) { cpu_time_to_bill_us = trx_context.update_billed_cpu_time( fc::time_point::now() ); @@ -1312,7 +1312,7 @@ struct controller_impl { restore.cancel(); return trace; - } catch( const objective_block_production_exception& ) { + } catch( const objective_block_validation_exception& ) { throw; } catch( const fc::exception& e ) { cpu_time_to_bill_us = trx_context.update_billed_cpu_time( fc::time_point::now() ); @@ -1517,7 +1517,7 @@ struct controller_impl { } return trace; - } catch( const objective_block_production_exception& ) { + } catch( const objective_block_validation_exception& ) { throw; } catch (const fc::exception& e) { trace->error_code = controller::convert_exception_to_error_code( e ); diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index 7d66e1683f3..aec56070ea4 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -278,7 +278,7 @@ namespace eosio { namespace chain { FC_DECLARE_DERIVED_EXCEPTION( tx_prune_exception, transaction_exception, 3040019, "Prunable data not found" ) FC_DECLARE_DERIVED_EXCEPTION( tx_no_signature, transaction_exception, - 3040020, "Transaction has no signatures" ) + 3040020, "Transaction signatures pruned" ) FC_DECLARE_DERIVED_EXCEPTION( tx_no_context_free_data, transaction_exception, 3040021, "Transaction context free data pruned" ) @@ -634,17 +634,17 @@ namespace eosio { namespace chain { 3250003, "Protocol feature iterator exception" ) // Any derived types of subjective_block_production_exception need to update controller::failure_is_subjective - FC_DECLARE_DERIVED_EXCEPTION( subjective_block_production_exception, chain_exception, + FC_DECLARE_DERIVED_EXCEPTION( subjective_block_production_exception, chain_exception, 3260000, "Subjective exception thrown during block production" ) - // Objective block production exceptions are throw out of transaction processing to stop block production - FC_DECLARE_DERIVED_EXCEPTION( objective_block_production_exception, chain_exception, - 3270000, "Objective exception thrown during block production" ) - FC_DECLARE_DERIVED_EXCEPTION( disallowed_transaction_extensions_bad_block_exception, objective_block_production_exception, + // Objective block validation exceptions are throw out of transaction processing to stop block production + FC_DECLARE_DERIVED_EXCEPTION( objective_block_validation_exception, chain_exception, + 3270000, "Objective exception thrown during block validation" ) + FC_DECLARE_DERIVED_EXCEPTION( disallowed_transaction_extensions_bad_block_exception, objective_block_validation_exception, 3270001, "Transaction includes disallowed extensions (invalid block)" ) - FC_DECLARE_DERIVED_EXCEPTION( protocol_feature_bad_block_exception, objective_block_production_exception, - 3270002, "Protocol feature exception (invalid block)" ) - FC_DECLARE_DERIVED_EXCEPTION( pruned_context_free_data_bad_block_exception, objective_block_production_exception, + FC_DECLARE_DERIVED_EXCEPTION( protocol_feature_bad_block_exception, objective_block_validation_exception, 3270002, "Protocol feature exception (invalid block)" ) + FC_DECLARE_DERIVED_EXCEPTION( pruned_context_free_data_bad_block_exception, objective_block_validation_exception, + 3270003, "Context free data pruned (invalid block)" ) } } // eosio::chain diff --git a/libraries/chain/include/eosio/chain/transaction_metadata.hpp b/libraries/chain/include/eosio/chain/transaction_metadata.hpp index e9f1c3693f3..48ecfe1cb60 100644 --- a/libraries/chain/include/eosio/chain/transaction_metadata.hpp +++ b/libraries/chain/include/eosio/chain/transaction_metadata.hpp @@ -42,7 +42,7 @@ class transaction_metadata { static const vector& check_variable_sig_size(const packed_transaction_ptr& trx, uint32_t max) { const vector* sigs = trx->get_signatures(); - EOS_ASSERT( sigs, tx_no_signature, "No signaures on packed_transaction" ); + EOS_ASSERT( sigs, tx_no_signature, "signatures pruned from packed_transaction" ); for(const signature_type& sig : *sigs) EOS_ASSERT(sig.variable_size() <= max, sig_variable_size_limit_exception, "signature variable length component size (${s}) greater than subjective maximum (${m})", ("s", sig.variable_size())("m", max)); From 4bd8fd941bdddf744a431ba8c69f511d57a99503 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 7 Apr 2020 17:02:21 -0500 Subject: [PATCH 055/142] Move get_packed_transaction and get_packed_transaction_size ipmlementations to wasm_interface so they can be optimized. --- libraries/chain/apply_context.cpp | 16 ---------------- .../include/eosio/chain/apply_context.hpp | 2 -- libraries/chain/wasm_interface.cpp | 18 +++++++++++++----- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index fe470a8e2ba..21b1ba1a4b3 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -611,22 +611,6 @@ vector apply_context::get_active_producers() const { return accounts; } -bytes apply_context::get_packed_transaction() { - if( trx_context.packed_trx.get_compression() == packed_transaction::compression_type::none) { - return trx_context.packed_trx.get_packed_transaction(); - } else { - return fc::raw::pack( static_cast( trx_context.packed_trx.get_transaction() ) ); - } -} - -size_t apply_context::get_packed_transaction_size() { - if( trx_context.packed_trx.get_compression() == packed_transaction::compression_type::none) { - return trx_context.packed_trx.get_packed_transaction().size(); - } else { - return fc::raw::pack_size( static_cast( trx_context.packed_trx.get_transaction() ) ); - } -} - void apply_context::update_db_usage( const account_name& payer, int64_t delta ) { if( delta > 0 ) { if( !(privileged || payer == account_name(receiver) diff --git a/libraries/chain/include/eosio/chain/apply_context.hpp b/libraries/chain/include/eosio/chain/apply_context.hpp index 573218326eb..ba4877055f0 100644 --- a/libraries/chain/include/eosio/chain/apply_context.hpp +++ b/libraries/chain/include/eosio/chain/apply_context.hpp @@ -537,8 +537,6 @@ class apply_context { int get_action( uint32_t type, uint32_t index, char* buffer, size_t buffer_size )const; int get_context_free_data( uint32_t index, char* buffer, size_t buffer_size )const; vector get_active_producers() const; - bytes get_packed_transaction(); - size_t get_packed_transaction_size(); uint64_t next_global_sequence(); uint64_t next_recv_sequence( const account_metadata_object& receiver_account ); diff --git a/libraries/chain/wasm_interface.cpp b/libraries/chain/wasm_interface.cpp index a7f69982afa..ea2bae27504 100644 --- a/libraries/chain/wasm_interface.cpp +++ b/libraries/chain/wasm_interface.cpp @@ -1509,19 +1509,27 @@ class context_free_transaction_api : public context_aware_api { :context_aware_api(ctx,true){} int read_transaction( array_ptr data, uint32_t buffer_size ) { - bytes trx = context.get_packed_transaction(); + if( buffer_size == 0 ) return transaction_size(); - auto s = trx.size(); - if( buffer_size == 0) return s; + const packed_transaction& packed_trx = context.trx_context.packed_trx; + const bytes& trx = + packed_trx.get_compression() == packed_transaction::compression_type::none ? + packed_trx.get_packed_transaction() : + fc::raw::pack( static_cast( packed_trx.get_transaction() ) ); - auto copy_size = std::min( static_cast(buffer_size), s ); + size_t copy_size = std::min( static_cast(buffer_size), trx.size() ); memcpy( data, trx.data(), copy_size ); return copy_size; } int transaction_size() { - return context.get_packed_transaction_size(); + const packed_transaction& packed_trx = context.trx_context.packed_trx; + if( packed_trx.get_compression() == packed_transaction::compression_type::none) { + return packed_trx.get_packed_transaction().size(); + } else { + return fc::raw::pack_size( static_cast( packed_trx.get_transaction() ) ); + } } int expiration() { From 7ccfaf9271505b3dbefdf8ebecae5d9e79296417 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 8 Apr 2020 08:59:00 -0500 Subject: [PATCH 056/142] Add partial support to apply_context::get_context_free_data --- libraries/chain/apply_context.cpp | 31 ++++++++++++++++++++----------- libraries/chain/transaction.cpp | 9 +++++++-- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 21b1ba1a4b3..3aedb5109a5 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -654,9 +654,24 @@ int apply_context::get_action( uint32_t type, uint32_t index, char* buffer, size int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t buffer_size )const { const prunable_transaction_data::prunable_data_type& data = trx_context.packed_trx.get_prunable_data().prunable_data; - if( data.contains() || data.contains() || - data.contains() ) // TODO: for partial determine if index is pruned and throw only in that case - { + bool throw_exception = false; + const bytes* cfd = nullptr; + if( data.contains() || data.contains() ) { + } else if( data.contains() ) { + if( index >= data.get().context_free_segments.size() ) return -1; + + cfd = trx_context.packed_trx.get_context_free_data(index); + } else { + const std::vector& context_free_data = + data.contains() ? + data.get().context_free_segments : + data.get().context_free_segments; + if( index >= context_free_data.size() ) return -1; + + cfd = &context_free_data[index]; + } + + if( !cfd ) { if( control.is_producing_block() ) { EOS_THROW( subjective_block_production_exception, "pruned context free data not available" ); } else { @@ -664,17 +679,11 @@ int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t b } } - std::vector context_free_data = data.contains() ? - data.get().context_free_segments : - data.get().context_free_segments; - - if( index >= context_free_data.size() ) return -1; - - auto s = context_free_data[index].size(); + auto s = cfd->size(); if( buffer_size == 0 ) return s; auto copy_size = std::min( buffer_size, s ); - memcpy( buffer, context_free_data[index].data(), copy_size ); + memcpy( buffer, cfd->data(), copy_size ); return copy_size; } diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 2c0d1086588..0a47c10a3b5 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -607,8 +607,13 @@ const vector* packed_transaction::get_context_free_data()const { const bytes* maybe_get_context_free_data(const prunable_transaction_data::none&, std::size_t) { return nullptr; } const bytes* maybe_get_context_free_data(const prunable_transaction_data::signatures_only&, std::size_t) { return nullptr; } -const bytes* maybe_get_context_free_data(const prunable_transaction_data::partial&, std::size_t) { - EOS_THROW(tx_prune_exception, "unimplemented"); +const bytes* maybe_get_context_free_data(const prunable_transaction_data::partial& p, std::size_t i) { + if( p.context_free_segments.size() <= i ) return nullptr; + return p.context_free_segments[i].visit( + overloaded{ + []( const digest_type& t ) -> const bytes* { return nullptr; }, + []( const bytes& vec ) { return &vec; } + } ); } const bytes* maybe_get_context_free_data(const prunable_transaction_data::full_legacy& full_leg, std::size_t i) { if( full_leg.context_free_segments.size() <= i ) return nullptr; From c9522cd2b5e45ee63c2e8564ddc708d38b89242c Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 8 Apr 2020 13:13:32 -0500 Subject: [PATCH 057/142] avoid restart_chain unit tests running concurrently --- unittests/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 1849492e5d2..e2159d30d89 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -64,6 +64,9 @@ foreach(TEST_SUITE ${UNIT_TESTS}) # create an independent target for each test s string(APPEND ctest_tests "|") endif() string(APPEND ctest_tests ${TRIMMED_SUITE_NAME}_unit_test_$RUNTIME) + if(TRIMMED_SUITE_NAME STREQUAL "restart_chain") + set_tests_properties(${TRIMMED_SUITE_NAME}_unit_test_${RUNTIME} PROPERTIES RESOURCE_LOCK "restart_chain") + endif() endforeach() endif() endforeach(TEST_SUITE) From c1be4bfffb659b3196d55b9fc0d2f2896dd7f85c Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 8 Apr 2020 13:18:30 -0500 Subject: [PATCH 058/142] Avoid uneeded copy of transaction data --- .../state_history_plugin.hpp | 35 +++---------------- .../state_history_serialization.hpp | 31 ++++++++-------- 2 files changed, 21 insertions(+), 45 deletions(-) diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index e82018532f8..e13054a4133 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -18,32 +18,9 @@ using std::shared_ptr; typedef shared_ptr state_history_ptr; -struct partial_transaction { - chain::time_point_sec expiration = {}; - uint16_t ref_block_num = {}; - uint32_t ref_block_prefix = {}; - fc::unsigned_int max_net_usage_words = {}; - uint8_t max_cpu_usage_ms = {}; - fc::unsigned_int delay_sec = {}; - chain::extensions_type transaction_extensions = {}; - vector signatures = {}; - vector context_free_data = {}; - - partial_transaction(const chain::packed_transaction_ptr& t) - : expiration(t->get_transaction().expiration) - , ref_block_num(t->get_transaction().ref_block_num) - , ref_block_prefix(t->get_transaction().ref_block_prefix) - , max_net_usage_words(t->get_transaction().max_net_usage_words) - , max_cpu_usage_ms(t->get_transaction().max_cpu_usage_ms) - , delay_sec(t->get_transaction().delay_sec) - , transaction_extensions(t->get_transaction().transaction_extensions) - , signatures(t->get_signatures() != nullptr ? *t->get_signatures() : vector{}) - , context_free_data(t->get_context_free_data() != nullptr ? *t->get_context_free_data() : vector{}) {} -}; - struct augmented_transaction_trace { chain::transaction_trace_ptr trace; - std::shared_ptr partial; + chain::packed_transaction_ptr packed_trx; augmented_transaction_trace() = default; augmented_transaction_trace(const augmented_transaction_trace&) = default; @@ -52,14 +29,10 @@ struct augmented_transaction_trace { augmented_transaction_trace(const chain::transaction_trace_ptr& trace) : trace{trace} {} - augmented_transaction_trace(const chain::transaction_trace_ptr& trace, - const std::shared_ptr& partial) - : trace{trace} - , partial{partial} {} - - augmented_transaction_trace(const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& t) + augmented_transaction_trace(const chain::transaction_trace_ptr& trace, + const chain::packed_transaction_ptr& packed_trx) : trace{trace} - , partial{std::make_shared(t)} {} + , packed_trx{packed_trx} {} augmented_transaction_trace& operator=(const augmented_transaction_trace&) = default; augmented_transaction_trace& operator=(augmented_transaction_trace&&) = default; diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index 8691f4fc69a..45ec7101128 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -650,23 +650,26 @@ datastream& operator<<(datastream& std::pair context = std::make_pair(stat, debug_mode); fc::raw::pack( // ds, make_history_context_wrapper( - obj.db, context, eosio::augmented_transaction_trace{trace.failed_dtrx_trace, obj.obj.partial})); + obj.db, context, eosio::augmented_transaction_trace{trace.failed_dtrx_trace, obj.obj.packed_trx})); } - bool include_partial = obj.obj.partial && !trace.failed_dtrx_trace; - fc::raw::pack(ds, include_partial); - if (include_partial) { - auto& partial = *obj.obj.partial; + bool include_packed_trx = obj.obj.packed_trx && !trace.failed_dtrx_trace; + fc::raw::pack(ds, include_packed_trx); + if (include_packed_trx) { + const auto& pt = *obj.obj.packed_trx; + const auto& trx = pt.get_transaction(); fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(partial.expiration)); - fc::raw::pack(ds, as_type(partial.ref_block_num)); - fc::raw::pack(ds, as_type(partial.ref_block_prefix)); - fc::raw::pack(ds, as_type(partial.max_net_usage_words)); - fc::raw::pack(ds, as_type(partial.max_cpu_usage_ms)); - fc::raw::pack(ds, as_type(partial.delay_sec)); - fc::raw::pack(ds, as_type(partial.transaction_extensions)); - fc::raw::pack(ds, as_type>(partial.signatures)); - fc::raw::pack(ds, as_type>(partial.context_free_data)); + fc::raw::pack(ds, as_type(trx.expiration)); + fc::raw::pack(ds, as_type(trx.ref_block_num)); + fc::raw::pack(ds, as_type(trx.ref_block_prefix)); + fc::raw::pack(ds, as_type(trx.max_net_usage_words)); + fc::raw::pack(ds, as_type(trx.max_cpu_usage_ms)); + fc::raw::pack(ds, as_type(trx.delay_sec)); + fc::raw::pack(ds, as_type(trx.transaction_extensions)); + const auto* sigs = pt.get_signatures(); + const auto* cfd = pt.get_context_free_data(); + fc::raw::pack(ds, as_type>(sigs ? *sigs : std::vector{})); + fc::raw::pack(ds, as_type>(cfd ? *cfd : std::vector{})); } return ds; From a38602b14aec4dc457a7e72520ffa16e47cdf992 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 8 Apr 2020 13:49:05 -0500 Subject: [PATCH 059/142] Use shared_ptr aliasing constructor to avoid copy of packed_transaction --- libraries/chain/controller.cpp | 11 +++++++---- .../include/eosio/chain/transaction_metadata.hpp | 7 +++---- unittests/unapplied_transaction_queue_tests.cpp | 3 ++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 16c582ba6a6..22365baa0b9 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1241,7 +1241,8 @@ struct controller_impl { signed_transaction dtrx; fc::raw::unpack(ds,static_cast(dtrx) ); transaction_metadata_ptr trx = - transaction_metadata::create_no_recover_keys( packed_transaction( std::move(dtrx), true ), transaction_metadata::trx_type::scheduled ); + transaction_metadata::create_no_recover_keys( std::make_shared( std::move(dtrx), true ), + transaction_metadata::trx_type::scheduled ); trx->accepted = true; transaction_trace_ptr trace; @@ -1656,7 +1657,8 @@ struct controller_impl { try { transaction_metadata_ptr onbtrx = - transaction_metadata::create_no_recover_keys( packed_transaction( get_on_block_transaction(), true ), transaction_metadata::trx_type::implicit ); + transaction_metadata::create_no_recover_keys( std::make_shared( get_on_block_transaction(), true ), + transaction_metadata::trx_type::implicit ); auto reset_in_trx_requiring_checks = fc::make_scoped_exit([old_value=in_trx_requiring_checks,this](){ in_trx_requiring_checks = old_value; }); @@ -1878,11 +1880,12 @@ struct controller_impl { if( trx_meta_ptr && ( skip_auth_checks || !trx_meta_ptr->recovered_keys().empty() ) ) { trx_metas.emplace_back( std::move( trx_meta_ptr ), recover_keys_future{} ); } else if( skip_auth_checks ) { + packed_transaction_ptr ptrx( b, &pt ); // alias signed_block_ptr trx_metas.emplace_back( - transaction_metadata::create_no_recover_keys( packed_transaction( pt ), transaction_metadata::trx_type::input ), + transaction_metadata::create_no_recover_keys( ptrx, transaction_metadata::trx_type::input ), recover_keys_future{} ); } else { - auto ptrx = std::make_shared( pt ); + packed_transaction_ptr ptrx( b, &pt ); // alias signed_block_ptr auto fut = transaction_metadata::start_recover_keys( std::move( ptrx ), thread_pool.get_executor(), chain_id, microseconds::maximum() ); trx_metas.emplace_back( transaction_metadata_ptr{}, std::move( fut ) ); diff --git a/libraries/chain/include/eosio/chain/transaction_metadata.hpp b/libraries/chain/include/eosio/chain/transaction_metadata.hpp index 48ecfe1cb60..2508c602de3 100644 --- a/libraries/chain/include/eosio/chain/transaction_metadata.hpp +++ b/libraries/chain/include/eosio/chain/transaction_metadata.hpp @@ -83,10 +83,9 @@ class transaction_metadata { /// @returns constructed transaction_metadata with no key recovery (sig_cpu_usage=0, recovered_pub_keys=empty) static transaction_metadata_ptr - create_no_recover_keys( packed_transaction trx, trx_type t ) { - return std::make_shared( private_type(), - std::make_shared( std::move(trx) ), fc::microseconds(), flat_set(), - t == trx_type::implicit, t == trx_type::scheduled ); + create_no_recover_keys( packed_transaction_ptr trx, trx_type t ) { + return std::make_shared( private_type(), std::move(trx), + fc::microseconds(), flat_set(), t == trx_type::implicit, t == trx_type::scheduled ); } }; diff --git a/unittests/unapplied_transaction_queue_tests.cpp b/unittests/unapplied_transaction_queue_tests.cpp index 19b5b4fddad..08957edd8a2 100644 --- a/unittests/unapplied_transaction_queue_tests.cpp +++ b/unittests/unapplied_transaction_queue_tests.cpp @@ -18,7 +18,8 @@ auto unique_trx_meta_data( fc::time_point expire = fc::time_point::now() + fc::s trx.expiration = expire; trx.actions.emplace_back( vector{{creator,config::active_name}}, onerror{ nextid, "test", 4 }); - return transaction_metadata::create_no_recover_keys( packed_transaction( std::move(trx), true ), transaction_metadata::trx_type::input ); + return transaction_metadata::create_no_recover_keys( std::make_shared( std::move(trx), true ), + transaction_metadata::trx_type::input ); } auto next( unapplied_transaction_queue& q ) { From 8d66d200508fa5aa46036e8b6001c3c68f094182 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 7 Apr 2020 11:53:06 -0500 Subject: [PATCH 060/142] avoid trim block data in place --- unittests/restart_chain_tests.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index 42109c00ac3..3be6d0a8fb8 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -275,10 +275,16 @@ BOOST_AUTO_TEST_CASE(test_trim_blocklog_front) { chain.produce_blocks(20); chain.close(); + namespace bfs = boost::filesystem; + auto blocks_dir = chain.get_config().blocks_dir; - auto temp = boost::filesystem::unique_path(); - BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_front(blocks_dir, temp, 10)); - BOOST_REQUIRE_NO_THROW(block_log::smoke_test(blocks_dir)); + auto temp1 = bfs::unique_path(); + boost::filesystem::create_directory(temp1); + bfs::copy(blocks_dir / "blocks.log", temp1 / "blocks.log"); + bfs::copy(blocks_dir / "blocks.index", temp1 / "blocks.index"); + auto temp2 = bfs::unique_path(); + BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_front(temp1, temp2, 10)); + BOOST_REQUIRE_NO_THROW(block_log::smoke_test(temp1)); } BOOST_AUTO_TEST_SUITE_END() From 9504e39928e2d93975e3b7cb3715a6a8e3c668b2 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 9 Apr 2020 07:54:20 -0500 Subject: [PATCH 061/142] remove temp directory after test --- unittests/restart_chain_tests.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index 3be6d0a8fb8..733b437e6e2 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -285,6 +285,8 @@ BOOST_AUTO_TEST_CASE(test_trim_blocklog_front) { auto temp2 = bfs::unique_path(); BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_front(temp1, temp2, 10)); BOOST_REQUIRE_NO_THROW(block_log::smoke_test(temp1)); + bfs::remove_all(temp1); + bfs::remove_all(temp2); } BOOST_AUTO_TEST_SUITE_END() From 5dbe2fbef0ecbaa3f4e5d8195c343803a392ecaa Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 9 Apr 2020 08:09:03 -0500 Subject: [PATCH 062/142] revert cmake changes to avoid restart_chain unit tests running concurrently --- unittests/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index e2159d30d89..1849492e5d2 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -64,9 +64,6 @@ foreach(TEST_SUITE ${UNIT_TESTS}) # create an independent target for each test s string(APPEND ctest_tests "|") endif() string(APPEND ctest_tests ${TRIMMED_SUITE_NAME}_unit_test_$RUNTIME) - if(TRIMMED_SUITE_NAME STREQUAL "restart_chain") - set_tests_properties(${TRIMMED_SUITE_NAME}_unit_test_${RUNTIME} PROPERTIES RESOURCE_LOCK "restart_chain") - endif() endforeach() endif() endforeach(TEST_SUITE) From 1a340822cc2c82e011c9cf25cc53b9f9079032a1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 9 Apr 2020 09:57:12 -0500 Subject: [PATCH 063/142] Only log what is compared for invalid receipts --- libraries/chain/controller.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 22365baa0b9..acc62499624 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1937,7 +1937,7 @@ struct controller_impl { const transaction_receipt_header& r = trx_receipts.back(); EOS_ASSERT( r == static_cast(receipt), block_validate_exception, "receipt does not match", - ("producer_receipt", receipt)("validator_receipt", trx_receipts.back()) ); + ("producer_receipt", static_cast(receipt))("validator_receipt", r) ); } finalize_block(); From 2ead8bbdb65a930e11b6b9451f3d2569cdc72fbe Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 9 Apr 2020 10:22:58 -0500 Subject: [PATCH 064/142] Maintain compression when converting to packed_transaction_v0 --- libraries/chain/transaction.cpp | 2 +- unittests/misc_tests.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 0a47c10a3b5..84e5cf8279a 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -513,7 +513,7 @@ packed_transaction_v0_ptr packed_transaction::to_packed_transaction_v0() const { signed_transaction strx( transaction( get_transaction() ), sigs != nullptr ? *sigs : vector(), context_free_data != nullptr ? *context_free_data : vector() ); - return std::make_shared( std::move( strx ) ); + return std::make_shared( std::move( strx ), get_compression() ); } uint32_t packed_transaction::get_unprunable_size()const { diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index 52e5d9d417e..20736e205c8 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -752,6 +752,8 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { BOOST_CHECK_EQUAL(pkt.to_packed_transaction_v0()->get_signed_transaction().id(), pkt2.to_packed_transaction_v0()->get_signed_transaction().id()); BOOST_CHECK_EQUAL(pkt.to_packed_transaction_v0()->get_transaction().id(), pkt2.id()); BOOST_CHECK_EQUAL(pkt.get_transaction().id(), pkt2.id()); + BOOST_CHECK_EQUAL(pkt.to_packed_transaction_v0()->get_prunable_size(), pkt.get_prunable_size()); + BOOST_CHECK_EQUAL(pkt.to_packed_transaction_v0()->get_unprunable_size(), pkt.get_unprunable_size()); flat_set keys; auto cpu_time1 = pkt.to_packed_transaction_v0()->get_signed_transaction().get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); From 9bdedf6560d51808a633d3b056dee864f0b4b0fb Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 9 Apr 2020 11:22:25 -0500 Subject: [PATCH 065/142] refactor log_entry --- libraries/chain/block_log.cpp | 200 ++++++++++-------- .../chain/include/eosio/chain/block_log.hpp | 2 +- .../chain/include/eosio/chain/transaction.hpp | 1 + 3 files changed, 115 insertions(+), 88 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 52f49c58536..f783c79761e 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #define LOG_READ (std::ios::in | std::ios::binary) #define LOG_WRITE (std::ios::out | std::ios::binary | std::ios::app) @@ -40,21 +41,64 @@ namespace eosio { namespace chain { namespace detail { using unique_file = std::unique_ptr; - struct log_entry { - uint32_t offset; + /// calculate the offset from the start of serialized block entry to block start + int offset_to_block_start(uint32_t version) { + if (version < 4) return 0; + return sizeof(uint32_t) + 1; + } + + class log_entry_v4 : public pruned_block { + public: pruned_transaction::cf_compression_type compression; - pruned_block* block; + uint32_t offset; }; + template - void unpack(Stream& ds, log_entry& entry){ + void unpack(Stream& ds, log_entry_v4& entry){ fc::raw::unpack(ds, entry.offset); uint8_t compression; fc::raw::unpack(ds, compression); - EOS_ASSERT(compression == static_cast(pruned_transaction::cf_compression_type::none), block_log_exception, - "Only support compression_type none"); entry.compression = static_cast(compression); - fc::raw::unpack(ds, *entry.block); + EOS_ASSERT(entry.compression == pruned_transaction::cf_compression_type::none, block_log_exception, + "Only support compression_type none"); + fc::raw::unpack(ds, static_cast(entry)); + } + + std::vector pack(const pruned_block& block, pruned_transaction::cf_compression_type compression) { + // In version 4 of the irreversible blocks log format, these log entries consists of the following in order: + // 1. An uint32_t offset from the start of this log entry to the start of the next log entry. + // 2. An uint8_t indicating the compression status for the serialization of the pruned_block following this. + // 3. The serialization of a pruned_block representation of the block for the entry including padding. + + std::size_t padded_size = block.maximum_pruned_pack_size(compression); + std::vector buffer(padded_size + offset_to_block_start(4)); + fc::datastream stream(buffer.data(), buffer.size()); + + uint32_t offset = buffer.size(); + stream.write((char*)&offset, sizeof(offset)); + fc::raw::pack(stream, static_cast(compression)); + block.pack(stream, compression); + return buffer; + } + + std::vector pack(const log_entry_v4& entry) { + return pack(static_cast(entry), entry.compression); + } + + using log_entry = std::variant; + + template + void unpack(Stream& ds, log_entry& entry) { + std::visit( + overloaded{[&ds](signed_block& v) { fc::raw::unpack(ds, v); }, [&ds](log_entry_v4& v) { unpack(ds, v); }}, + entry); + } + + std::vector pack(const log_entry& entry) { + return std::visit(overloaded{[](const signed_block& v) { return fc::raw::pack(v); }, + [](const log_entry_v4& v) { return pack(v); }}, + entry); } class block_log_impl { @@ -96,22 +140,14 @@ namespace eosio { namespace chain { template static fc::optional extract_chain_context( const fc::path& data_dir, Lambda&& lambda ); - static std::vector pack_v4_log_entry(const pruned_block& b, - pruned_transaction::cf_compression_type segment_compression); - uint64_t write_log_entry(const pruned_block& b, pruned_transaction::cf_compression_type segment_compression); - void read_block_header(block_header& bh, uint64_t file_pos); - pruned_block_ptr read_block(uint64_t pos); - void read_head(); - void read_v4_log_entry(uint64_t pos, log_entry&); + void read_block_header(block_header& bh, uint64_t file_pos); + std::unique_ptr read_block(uint64_t pos); + void read_head(); - /// calculate the offset from the start of serialized block entry to block start - static int offset_to_block_start(uint32_t version) { - if (version < 4) return 0; - return sizeof(uint32_t) + 1; - } + static int blknum_offset_from_block_entry(uint32_t block_log_version) { @@ -122,7 +158,7 @@ namespace eosio { namespace chain { // block_id_type previous; //bytes 14:45, low 4 bytes is big endian block number of previous block int blknum_offset = 14; - blknum_offset += detail::block_log_impl::offset_to_block_start(block_log_version); + blknum_offset += detail::offset_to_block_start(block_log_version); return blknum_offset; } }; @@ -228,7 +264,11 @@ namespace eosio { namespace chain { FILE* const _file; const std::string _filename; }; - } +}}} // namespace eosio::chain::detail + +FC_REFLECT_DERIVED(eosio::chain::detail::log_entry_v4, (eosio::chain::pruned_block), (compression)(offset) ) + +namespace eosio { namespace chain { block_log::block_log(const fc::path& data_dir) :my(new detail::block_log_impl()) { @@ -330,30 +370,12 @@ namespace eosio { namespace chain { } } - std::vector detail::block_log_impl::pack_v4_log_entry(const pruned_block& b, pruned_transaction::cf_compression_type segment_compression) { - - // In version 4 of the irreversible blocks log format, these log entries consists of the following in order: - // 1. An uint32_t offset from the start of this log entry to the start of the next log entry. - // 2. An uint8_t indicating the compression status for the serialization of the pruned_block following this. - // 3. The serialization of a pruned_block representation of the block for the entry including padding. - - std::size_t padded_size = b.maximum_pruned_pack_size(segment_compression); - std::vector buffer(padded_size + offset_to_block_start(4)); - fc::datastream stream(buffer.data(), buffer.size()); - - uint32_t offset = buffer.size(); - stream.write((char*)&offset, sizeof(offset)); - fc::raw::pack(stream, segment_compression); - b.pack(stream, segment_compression); - return buffer; - } - uint64_t detail::block_log_impl::write_log_entry(const pruned_block& b, pruned_transaction::cf_compression_type segment_compression) { uint64_t pos = block_file.tellp(); std::vector buffer; if (version >= 4) { - buffer = detail::block_log_impl::pack_v4_log_entry(b, segment_compression); + buffer = detail::pack(b, segment_compression); } else { #warning: TODO avoid heap allocation auto block_ptr = b.to_signed_block(); @@ -477,28 +499,20 @@ namespace eosio { namespace chain { block_file << chain_id; } - pruned_block_ptr detail::block_log_impl::read_block(uint64_t pos) { + std::unique_ptr detail::block_log_impl::read_block(uint64_t pos) { block_file.seek(pos); auto ds = block_file.create_datastream(); if (version >= 4) { - log_entry entry; - pruned_block_ptr block = std::make_shared(); - entry.block = block.get(); - detail::unpack(ds, entry); - return block; + auto entry = std::make_unique(); + unpack(ds, *entry); + return entry; } else { signed_block block; fc::raw::unpack(ds, block); - return std::make_shared(std::move(block), true); + return std::make_unique(std::move(block), true); } } - void detail::block_log_impl::read_v4_log_entry(uint64_t pos, detail::log_entry& entry) { - block_file.seek(pos); - auto ds = block_file.create_datastream(); - unpack(ds, entry); - } - void detail::block_log_impl::read_block_header(block_header& bh, uint64_t pos) { block_file.seek(pos); auto ds = block_file.create_datastream(); @@ -518,9 +532,9 @@ namespace eosio { namespace chain { return r ? r->to_signed_block() : signed_block_ptr{}; } - pruned_block_ptr block_log::read_pruned_block_by_num(uint32_t block_num) const { + std::unique_ptr block_log::read_pruned_block_by_num(uint32_t block_num) const { try { - pruned_block_ptr b; + std::unique_ptr b; uint64_t pos = get_block_pos(block_num); if (pos != npos) { b = my->read_block(pos); @@ -713,19 +727,19 @@ namespace eosio { namespace chain { new_block_stream.write( (char*)&actual_totem, sizeof(actual_totem) ); } - std::exception_ptr except_ptr; - vector incomplete_block_data; - optional bad_block; - uint32_t block_num = 0; + std::exception_ptr except_ptr; + vector incomplete_block_data; + optional bad_block; + uint32_t block_num = 0; - block_id_type previous; + block_id_type expected_previous; uint64_t pos = old_block_stream.tellg(); - EOS_ASSERT(version == 4, block_log_exception, "Unsupported block log version"); - pruned_block tmp; detail::log_entry entry; - entry.block = &tmp; + if (version < 4) { + entry.emplace(); + } while( pos < end_pos ) { try { @@ -737,32 +751,33 @@ namespace eosio { namespace chain { break; } - auto id = tmp.id(); - if( block_header::num_from_id(previous) + 1 != block_header::num_from_id(id) ) { + auto id = std::visit([](const auto& v) { return v.id(); }, entry); + if( block_header::num_from_id(expected_previous) + 1 != block_header::num_from_id(id) ) { elog( "Block ${num} (${id}) skips blocks. Previous block in block log is block ${prev_num} (${previous})", ("num", block_header::num_from_id(id))("id", id) - ("prev_num", block_header::num_from_id(previous))("previous", previous) ); + ("prev_num", block_header::num_from_id(expected_previous))("previous", expected_previous) ); } - if( previous != tmp.previous ) { + auto actual_previous = std::visit([](const auto& v) { return v.previous; }, entry); + if( expected_previous != actual_previous ) { elog( "Block ${num} (${id}) does not link back to previous block. " "Expected previous: ${expected}. Actual previous: ${actual}.", - ("num", block_header::num_from_id(id))("id", id)("expected", previous)("actual", tmp.previous) ); + ("num", block_header::num_from_id(id))("id", id)("expected", expected_previous)("actual", actual_previous) ); } - previous = id; + expected_previous = id; uint64_t tmp_pos = std::numeric_limits::max(); if( (static_cast(old_block_stream.tellg()) + sizeof(pos)) <= end_pos ) { old_block_stream.read( reinterpret_cast(&tmp_pos), sizeof(tmp_pos) ); } if( pos != tmp_pos ) { - bad_block.emplace(std::move(tmp)); + bad_block.emplace(std::move(entry)); break; } - auto data = detail::block_log_impl::pack_v4_log_entry(tmp, entry.compression); + auto data = detail::pack(entry); new_block_stream.write( data.data(), data.size() ); new_block_stream.write( reinterpret_cast(&pos), sizeof(pos) ); - block_num = tmp.block_num(); + block_num = std::visit([](const auto& v) { return v.block_num(); }, entry); if(block_num % 1000 == 0) ilog( "Recovered block ${num}", ("num", block_num) ); pos = new_block_stream.tellp(); @@ -771,8 +786,16 @@ namespace eosio { namespace chain { } if( bad_block.valid() ) { - ilog( "Recovered only up to block number ${num}. Last block in block log was not properly committed:\n${last_block}", - ("num", block_num)("last_block", *bad_block) ); + fc::variant last_block = std::visit( + [](const auto& v) { + fc::variant r; + fc::to_variant(v, r); + return r; + }, + *bad_block); + ilog("Recovered only up to block number ${num}. Last block in block log was not properly " + "committed:\n${last_block}", + ("num", block_num)("last_block", last_block)); } else if( except_ptr ) { std::string error_msg; @@ -875,25 +898,28 @@ namespace eosio { namespace chain { EOS_ASSERT(pos != npos, block_log_exception, "Specified block_num ${block_num} does not exist in block log.", ("block_num", block_num)); - detail::log_entry entry; - pruned_block block; - entry.block = █ - my->read_v4_log_entry(pos, entry); + detail::log_entry_v4 entry; + my->block_file.seek(pos); + auto ds = my->block_file.create_datastream(); + unpack(ds, entry); - EOS_ASSERT(block.block_num() == block_num, block_log_exception, + EOS_ASSERT(entry.block_num() == block_num, block_log_exception, "Wrong block was read from block log."); auto pruner = overloaded{[](transaction_id_type&) { return false; }, [&ids](pruned_transaction& ptx) { return std::find(ids.begin(), ids.end(), ptx.id()) != ids.end() && prune(ptx); }}; - for (auto& trx : block.transactions) { - if (trx.trx.visit(pruner)) { - my->block_file.seek(pos); - std::vector buffer = detail::block_log_impl::pack_v4_log_entry(*entry.block, entry.compression); - EOS_ASSERT(buffer.size() <= entry.offset, block_log_exception, "Not enough space reserved in block log entry to serialize pruned block."); - my->block_file.write(buffer.data(), buffer.size()); - my->block_file.flush(); - } + bool pruned = false; + for (auto& trx : entry.transactions) { + pruned |= trx.trx.visit(pruner); + } + + if (pruned) { + my->block_file.seek(pos); + std::vector buffer = detail::pack(entry); + EOS_ASSERT(buffer.size() <= entry.offset, block_log_exception, "Not enough space reserved in block log entry to serialize pruned block."); + my->block_file.write(buffer.data(), buffer.size()); + my->block_file.flush(); } } FC_LOG_AND_RETHROW() diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index 9cbfa0d6eef..a7a43462258 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -50,7 +50,7 @@ namespace eosio { namespace chain { block_id_type read_block_id_by_num(uint32_t block_num)const; [[deprecated]] signed_block_ptr read_block_by_num(uint32_t block_num) const; - pruned_block_ptr read_pruned_block_by_num(uint32_t block_num) const; + std::unique_ptr read_pruned_block_by_num(uint32_t block_num) const; /** * Return offset of block in file, or block_log::npos if it does not exist. diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 5b31c7aa153..81db1a21336 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -336,6 +336,7 @@ FC_REFLECT( eosio::chain::transaction_header, (expiration)(ref_block_num)(ref_bl FC_REFLECT_DERIVED( eosio::chain::transaction, (eosio::chain::transaction_header), (context_free_actions)(actions)(transaction_extensions) ) FC_REFLECT_DERIVED( eosio::chain::signed_transaction, (eosio::chain::transaction), (signatures)(context_free_data) ) FC_REFLECT_ENUM( eosio::chain::packed_transaction::compression_type, (none)(zlib)) +FC_REFLECT_ENUM( eosio::chain::prunable_transaction_data::compression_type, (none)(zlib)) // @ignore unpacked_trx FC_REFLECT( eosio::chain::packed_transaction, (signatures)(compression)(packed_context_free_data)(packed_trx) ) FC_REFLECT( eosio::chain::pruned_transaction, (compression)(prunable_data)(packed_trx) ) From 644aa69fa682a274d040e23e6d0f01585341ed39 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 9 Apr 2020 17:14:33 -0500 Subject: [PATCH 066/142] Minor cleanup --- libraries/chain/apply_context.cpp | 1 - libraries/chain/controller.cpp | 2 +- .../chain/include/eosio/chain/abi_serializer.hpp | 15 ++++++++++----- libraries/chain/transaction.cpp | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 3aedb5109a5..d0b4888026a 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -654,7 +654,6 @@ int apply_context::get_action( uint32_t type, uint32_t index, char* buffer, size int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t buffer_size )const { const prunable_transaction_data::prunable_data_type& data = trx_context.packed_trx.get_prunable_data().prunable_data; - bool throw_exception = false; const bytes* cfd = nullptr; if( data.contains() || data.contains() ) { } else if( data.contains() ) { diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index acc62499624..884958fae14 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1882,7 +1882,7 @@ struct controller_impl { } else if( skip_auth_checks ) { packed_transaction_ptr ptrx( b, &pt ); // alias signed_block_ptr trx_metas.emplace_back( - transaction_metadata::create_no_recover_keys( ptrx, transaction_metadata::trx_type::input ), + transaction_metadata::create_no_recover_keys( std::move(ptrx), transaction_metadata::trx_type::input ), recover_keys_future{} ); } else { packed_transaction_ptr ptrx( b, &pt ); // alias signed_block_ptr diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index e436fde0f42..6ec9b8fd393 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -482,16 +482,21 @@ namespace impl { const auto& trx = ptrx.get_transaction(); mvo("id", trx.id()); const auto* sigs = ptrx.get_signatures(); - mvo("signatures", sigs != nullptr ? *sigs : vector()); + if( ptrx.get_prunable_data().prunable_data.contains() ) { + const auto& legacy = ptrx.get_prunable_data().prunable_data.get(); + mvo("signatures", legacy.signatures ); + } else { + mvo("signatures", vector()); + } mvo("compression", ptrx.get_compression()); if( ptrx.get_prunable_data().prunable_data.contains() ) { const auto& legacy = ptrx.get_prunable_data().prunable_data.get(); - mvo( "packed_context_free_data", legacy.packed_context_free_data ); + mvo("packed_context_free_data", legacy.packed_context_free_data); + mvo("context_free_data", legacy.context_free_segments); } else { - mvo( "packed_context_free_data", bytes() ); + mvo("packed_context_free_data", bytes()); + mvo("context_free_data", vector()); } - const auto* context_free_data = ptrx.get_context_free_data(); - mvo("context_free_data", context_free_data != nullptr ? *context_free_data : vector()); mvo("packed_trx", ptrx.get_packed_transaction()); add(mvo, "transaction", trx, resolver, ctx); out(name, std::move(mvo)); diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 84e5cf8279a..db2ab174bb6 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -577,7 +577,7 @@ uint32_t packed_transaction::calculate_estimated_size() const { } }; - return sizeof(*this) + packed_trx.size() * 2 + sizeof(prunable_data) + prunable_data.prunable_data.visit(visitor); + return sizeof(*this) + packed_trx.size() * 2 + prunable_data.prunable_data.visit(visitor); } digest_type packed_transaction::packed_digest()const { From cb53817a8752cbcecefe0c4ff1c56debfc09c8ba Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 10 Apr 2020 09:27:06 -0500 Subject: [PATCH 067/142] Use signed_block_v0 as the portable format of export_reversible_blocks/import_reversible_blocks --- plugins/chain_plugin/chain_plugin.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index dd12542b24e..b23c801fac7 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -1301,7 +1301,7 @@ bool chain_plugin::import_reversible_blocks( const fc::path& reversible_dir, new_reversible.add_index(); try { while( reversible_blocks.tellg() < end_pos ) { - signed_block tmp; + signed_block_v0 tmp; fc::raw::unpack(reversible_blocks, tmp); num = tmp.block_num(); @@ -1316,7 +1316,7 @@ bool chain_plugin::import_reversible_blocks( const fc::path& reversible_dir, new_reversible.create( [&]( auto& ubo ) { ubo.blocknum = num; - ubo.set_block( std::make_shared(std::move(tmp)) ); + ubo.set_block( std::make_shared(std::move(tmp), true) ); }); end = num; } @@ -1358,7 +1358,9 @@ bool chain_plugin::export_reversible_blocks( const fc::path& reversible_dir, signed_block tmp; fc::datastream ds( itr->packedblock.data(), itr->packedblock.size() ); fc::raw::unpack(ds, tmp); // Verify that packed block has not been corrupted. - reversible_blocks.write( itr->packedblock.data(), itr->packedblock.size() ); + signed_block_v0_ptr v0 = tmp.to_signed_block_v0(); // store in signed_block_v0 format + auto packed_v0 = fc::raw::pack(*v0); + reversible_blocks.write( packed_v0.data(), packed_v0.size() ); end = itr->blocknum; ++num; } From b8fd974a8499dbd4ccadce9330dd80eac919ee98 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 10 Apr 2020 09:44:45 -0500 Subject: [PATCH 068/142] Fixed the wrong offset size for block log entry encoding --- libraries/chain/block_log.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index f783c79761e..70f1a59f932 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -75,7 +75,7 @@ namespace eosio { namespace chain { std::vector buffer(padded_size + offset_to_block_start(4)); fc::datastream stream(buffer.data(), buffer.size()); - uint32_t offset = buffer.size(); + uint32_t offset = buffer.size() + sizeof(uint64_t); stream.write((char*)&offset, sizeof(offset)); fc::raw::pack(stream, static_cast(compression)); block.pack(stream, compression); From 13267dc66c67d28d9d4e86e5448fe93a61ef8e89 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 10 Apr 2020 10:44:29 -0500 Subject: [PATCH 069/142] Remove necessary bad block loggging --- libraries/chain/block_log.cpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 70f1a59f932..7b12b9072d1 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -786,16 +786,8 @@ namespace eosio { namespace chain { } if( bad_block.valid() ) { - fc::variant last_block = std::visit( - [](const auto& v) { - fc::variant r; - fc::to_variant(v, r); - return r; - }, - *bad_block); ilog("Recovered only up to block number ${num}. Last block in block log was not properly " - "committed:\n${last_block}", - ("num", block_num)("last_block", last_block)); + "committed",("num", block_num)); } else if( except_ptr ) { std::string error_msg; From f39322e86ead50608f31522ae10af234c52bf0ea Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 10 Apr 2020 10:56:50 -0500 Subject: [PATCH 070/142] Temporarily disable replay test since block_log changes are happening in PR #8891 --- tests/nodeos_multiple_version_protocol_feature_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nodeos_multiple_version_protocol_feature_test.py b/tests/nodeos_multiple_version_protocol_feature_test.py index e45ffe45d59..83cc2930fa9 100755 --- a/tests/nodeos_multiple_version_protocol_feature_test.py +++ b/tests/nodeos_multiple_version_protocol_feature_test.py @@ -195,7 +195,8 @@ def areNodesInSync(nodes:[Node]): oldNode.relaunch(oldNodeId, chainArg="--import-reversible-blocks {}".format(portableRevBlkPath), timeout=1, nodeosPath="programs/nodeos/nodeos") os.remove(portableRevBlkPath) - restartNode(oldNode, oldNodeId, chainArg="--replay", nodeosPath="programs/nodeos/nodeos") +# TODO re-enable in PR 8891 +# restartNode(oldNode, oldNodeId, chainArg="--replay", nodeosPath="programs/nodeos/nodeos") time.sleep(2) # Give some time to replay assert areNodesInSync(allNodes), "All nodes should be in sync" From 03dd9ea4745fc0e7df38efaa136b199d58737e99 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 10 Apr 2020 12:24:21 -0500 Subject: [PATCH 071/142] Add block log entry offset validation --- libraries/chain/block_log.cpp | 3 +++ libraries/fc | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 7b12b9072d1..a8d2e1e1767 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -56,6 +56,7 @@ namespace eosio { namespace chain { template void unpack(Stream& ds, log_entry_v4& entry){ + auto start_pos = ds.tellp(); fc::raw::unpack(ds, entry.offset); uint8_t compression; fc::raw::unpack(ds, compression); @@ -63,6 +64,8 @@ namespace eosio { namespace chain { EOS_ASSERT(entry.compression == pruned_transaction::cf_compression_type::none, block_log_exception, "Only support compression_type none"); fc::raw::unpack(ds, static_cast(entry)); + EOS_ASSERT(ds.tellp() - start_pos + sizeof(uint64_t) == entry.offset , block_log_exception, + "Invalid block log entry offset"); } std::vector pack(const pruned_block& block, pruned_transaction::cf_compression_type compression) { diff --git a/libraries/fc b/libraries/fc index dae9acc60f5..412ef3deb43 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit dae9acc60f594cf7c3a04f8de0315941c4bec3e9 +Subproject commit 412ef3deb435ed005a579b5f9b99024c52638915 From 2cf280eea33f01507011a49258660449eff3e02e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 10 Apr 2020 15:40:57 -0500 Subject: [PATCH 072/142] Temporarily disable replay test since block_log changes are happening in PR #8891 --- tests/nodeos_multiple_version_protocol_feature_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/nodeos_multiple_version_protocol_feature_test.py b/tests/nodeos_multiple_version_protocol_feature_test.py index 83cc2930fa9..121a265adf7 100755 --- a/tests/nodeos_multiple_version_protocol_feature_test.py +++ b/tests/nodeos_multiple_version_protocol_feature_test.py @@ -197,10 +197,10 @@ def areNodesInSync(nodes:[Node]): # TODO re-enable in PR 8891 # restartNode(oldNode, oldNodeId, chainArg="--replay", nodeosPath="programs/nodeos/nodeos") - time.sleep(2) # Give some time to replay - - assert areNodesInSync(allNodes), "All nodes should be in sync" - assert shouldNodeContainPreactivateFeature(oldNode), "4th node should contain PREACTIVATE_FEATURE" +# time.sleep(2) # Give some time to replay +# +# assert areNodesInSync(allNodes), "All nodes should be in sync" +# assert shouldNodeContainPreactivateFeature(oldNode), "4th node should contain PREACTIVATE_FEATURE" testSuccessful = True finally: From 60227616e3b9ee614dce1ecb6839b99840560a2b Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Sun, 12 Apr 2020 13:48:13 -0500 Subject: [PATCH 073/142] fix issue to repair block log with pruned blocks --- libraries/chain/block_log.cpp | 58 +++++++++++++++---- .../chain/include/eosio/chain/block_log.hpp | 6 +- programs/eosio-blocklog/main.cpp | 2 +- unittests/restart_chain_tests.cpp | 8 ++- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 161880421fc..84ad171f37f 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #define LOG_READ (std::ios::in | std::ios::binary) #define LOG_WRITE (std::ios::out | std::ios::binary | std::ios::app) @@ -53,10 +54,14 @@ namespace eosio { namespace chain { uint32_t offset = 0; }; + size_t get_stream_pos(fc::cfile_datastream& ds) { return ds.tellp(); } + size_t get_stream_pos(std::ifstream& ds) { return ds.tellg(); } + void skip_streamm_pos(fc::cfile_datastream& ds, size_t n) { ds.skip(n); } + void skip_streamm_pos(std::ifstream& ds, size_t n) { ds.seekg(n, ds.cur); } template void unpack(Stream& ds, log_entry_v4& entry){ - auto start_pos = ds.tellp(); + auto start_pos = get_stream_pos(ds); fc::raw::unpack(ds, entry.offset); uint8_t compression; fc::raw::unpack(ds, compression); @@ -64,8 +69,12 @@ namespace eosio { namespace chain { EOS_ASSERT(entry.compression == packed_transaction::cf_compression_type::none, block_log_exception, "Only support compression_type none"); fc::raw::unpack(ds, static_cast(entry)); - EOS_ASSERT(ds.tellp() - start_pos + sizeof(uint64_t) == entry.offset , block_log_exception, - "Invalid block log entry offset"); + uint64_t current_stream_offset = get_stream_pos(ds) - start_pos; + + int64_t bytes_to_skip = static_cast(entry.offset) - sizeof(uint64_t) - current_stream_offset; + EOS_ASSERT(bytes_to_skip >= 0, block_log_exception, + "Invalid block log entry offset"); + skip_streamm_pos(ds, bytes_to_skip); } std::vector pack(const signed_block& block, packed_transaction::cf_compression_type compression) { @@ -648,7 +657,7 @@ namespace eosio { namespace chain { ilog( "Reconstructing '${new_block_log}' from backed up block log", ("new_block_log", block_log_path) ); - std::fstream old_block_stream; + std::ifstream old_block_stream; std::fstream new_block_stream; old_block_stream.open( (backup_dir / "blocks.log").generic_string().c_str(), LOG_READ ); @@ -893,9 +902,14 @@ namespace eosio { namespace chain { } if (pruned) { - my->block_file.seek(pos); - std::vector buffer = detail::pack(entry); - EOS_ASSERT(buffer.size() <= entry.offset, block_log_exception, "Not enough space reserved in block log entry to serialize pruned block."); + // we don't want to rewrite entire entry, just the block data itself. + const auto block_offset = detail::offset_to_block_start(my->version); + my->block_file.seek(pos + block_offset); + const uint32_t max_block_size = entry.offset - block_offset - sizeof(uint64_t); + std::vector buffer(max_block_size); + fc::datastream stream(buffer.data(), buffer.size()); + static_cast(entry).pack(stream, entry.compression); + EOS_ASSERT(buffer.size() <= max_block_size, block_log_exception, "Not enough space reserved in block log entry to serialize pruned block."); my->block_file.write(buffer.data(), buffer.size()); my->block_file.flush(); } @@ -1120,6 +1134,7 @@ namespace eosio { namespace chain { FILE* ind_in = nullptr; //C style files for reading blocks.log and blocks.index //we use low level file IO because it is distinctly faster than C++ filebuf or iostream uint64_t first_block_pos = 0; //file position in blocks.log for the first block in the log + uint64_t block_file_size = 0; chain_id_type chain_id; }; @@ -1276,6 +1291,7 @@ namespace eosio { namespace chain { const auto expected_totem = block_log::npos; std::decay_t actual_totem; size = fread ( (char*)&actual_totem, sizeof(actual_totem), 1, blk_in); + block_file_size = fc::file_size(block_file_name); EOS_ASSERT(size == 1, block_log_exception, "Expected to read ${size} bytes, but did not read any bytes", ("size", sizeof(actual_totem))); @@ -1337,6 +1353,24 @@ namespace eosio { namespace chain { auto size = fread((void*)&block_n_pos, sizeof(block_n_pos), 1, ind_in); //filepos of block n EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", index_file_name.string())("b",n) ); + if (version >= 4) { + uint64_t block_n_plus_1_pos = block_file_size; + if (n < last_block) { + size = fread((void*)&block_n_plus_1_pos, sizeof(block_n_plus_1_pos), 1, ind_in); + EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", index_file_name.string())("b",n+1) ); + } + + uint32_t entry_offset = 0; + status = fseek(blk_in, block_n_pos, SEEK_SET); + EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file for block ${b}", ("file", block_file_name.string())("pos", block_n_pos)("b",n) ); + size = fread((void*)&entry_offset, sizeof(entry_offset), 1, blk_in); + EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", block_file_name.string())("b",n) ); + + EOS_ASSERT((entry_offset == (block_n_plus_1_pos - block_n_pos)), block_log_exception, + "The entry offset in block ${n} is ${entry_offset} does not match the difference of block positions in index file", + ("n", n)("entry_offset", entry_offset)("pos_diff", block_n_plus_1_pos - block_n_pos)); + } + //read blocks.log and verify block number n is found at the determined file position const auto calc_blknum_pos = block_n_pos + detail::block_log_impl::blknum_offset_from_block_entry(version); status = fseek(blk_in, calc_blknum_pos, SEEK_SET); @@ -1376,7 +1410,7 @@ namespace eosio { namespace chain { return 0; } - void block_log::smoke_test(fc::path block_dir) { + void block_log::smoke_test(fc::path block_dir, uint32_t interval) { trim_data td(block_dir); auto status = fseek(td.blk_in, -sizeof(uint64_t), SEEK_END); //get last_block from blocks.log, compare to from blocks.index EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", td.block_file_name.string())("pos", sizeof(uint64_t)) ); @@ -1392,10 +1426,10 @@ namespace eosio { namespace chain { bnum = fc::endian_reverse_u32(bnum) + 1; //convert from big endian to little endian and add 1 EOS_ASSERT( td.last_block == bnum, block_log_exception, "blocks.log says last block is ${lb} which disagrees with blocks.index", ("lb", bnum) ); ilog("blocks.log and blocks.index agree on number of blocks"); - uint32_t delta = (td.last_block + 8 - td.first_block) >> 3; - if (delta < 1) - delta = 1; - for (uint32_t n = td.first_block; ; n += delta) { + if (interval == 0) { + interval = std::max((td.last_block + 8 - td.first_block) >> 3, 1U); + } + for (uint32_t n = td.first_block; ; n += interval) { if (n > td.last_block) n = td.last_block; td.block_pos(n); //check block 'n' is where blocks.index says diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index 631e4736f35..b9de5787d62 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -79,7 +79,11 @@ namespace eosio { namespace chain { static bool trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block); static int trim_blocklog_end(fc::path block_dir, uint32_t n); - static void smoke_test(fc::path block_dir); + + /** + * @param n Only test 1 block out of every n blocks. If n is 0, it is maximum between 1 and the ceiling of the total number blocks divided by 8. + */ + static void smoke_test(fc::path block_dir, uint32_t n); private: void open(const fc::path& data_dir); diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp index 5b15e7a8ce1..702d03ea12f 100644 --- a/programs/eosio-blocklog/main.cpp +++ b/programs/eosio-blocklog/main.cpp @@ -228,7 +228,7 @@ bool trim_blocklog_front(bfs::path block_dir, uint32_t n) { //n is first void smoke_test(bfs::path block_dir) { using namespace std; cout << "\nSmoke test of blocks.log and blocks.index in directory " << block_dir << '\n'; - block_log::smoke_test(block_dir); + block_log::smoke_test(block_dir, 0); cout << "\nno problems found\n"; // if get here there were no exceptions } diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index ec91d304269..2050e5d0d57 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -250,7 +250,7 @@ BOOST_AUTO_TEST_CASE(test_restart_with_different_chain_id) { } BOOST_FIXTURE_TEST_CASE(test_restart_from_block_log, restart_from_block_log_test_fixture) { - BOOST_REQUIRE_NO_THROW(block_log::smoke_test(chain.get_config().blocks_dir)); + BOOST_REQUIRE_NO_THROW(block_log::smoke_test(chain.get_config().blocks_dir, 1)); } BOOST_FIXTURE_TEST_CASE(test_restart_from_trimed_block_log, restart_from_block_log_test_fixture) { @@ -264,8 +264,10 @@ BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log, light_vali } BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log_with_pruned_trx, light_validation_restart_from_block_log_test_fixture) { - block_log blog(chain.get_config().blocks_dir); + const auto& blocks_dir = chain.get_config().blocks_dir; + block_log blog(blocks_dir); BOOST_REQUIRE_NO_THROW(blog.prune_transactions(trace->block_num, std::vector{trace->id})); + BOOST_REQUIRE_NO_THROW(block_log::repair_log(blocks_dir)); } BOOST_AUTO_TEST_CASE(test_trim_blocklog_front) { @@ -283,7 +285,7 @@ BOOST_AUTO_TEST_CASE(test_trim_blocklog_front) { bfs::copy(blocks_dir / "blocks.index", temp1 / "blocks.index"); auto temp2 = bfs::unique_path(); BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_front(temp1, temp2, 10)); - BOOST_REQUIRE_NO_THROW(block_log::smoke_test(temp1)); + BOOST_REQUIRE_NO_THROW(block_log::smoke_test(temp1, 1)); bfs::remove_all(temp1); bfs::remove_all(temp2); } From e7686b3d3bc1d2bfd82ea99e84d0e68dba2558a6 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 13 Apr 2020 09:01:48 -0500 Subject: [PATCH 074/142] Fix a merge problem --- libraries/chain/block_log.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 84ad171f37f..97caec95f53 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -84,7 +84,7 @@ namespace eosio { namespace chain { // 3. The serialization of a pruned_block representation of the block for the entry including padding. std::size_t padded_size = block.maximum_pruned_pack_size(compression); - std::vector buffer(padded_size + offset_to_block_start(4)); + std::vector buffer(padded_size + offset_to_block_start(block_log::max_supported_version)); fc::datastream stream(buffer.data(), buffer.size()); uint32_t offset = buffer.size() + sizeof(uint64_t); @@ -98,17 +98,17 @@ namespace eosio { namespace chain { return pack(static_cast(entry), entry.compression); } - using log_entry = std::variant; + using log_entry = std::variant; template void unpack(Stream& ds, log_entry& entry) { std::visit( - overloaded{[&ds](signed_block& v) { fc::raw::unpack(ds, v); }, [&ds](log_entry_v4& v) { unpack(ds, v); }}, + overloaded{[&ds](signed_block_v0& v) { fc::raw::unpack(ds, v); }, [&ds](log_entry_v4& v) { unpack(ds, v); }}, entry); } std::vector pack(const log_entry& entry) { - return std::visit(overloaded{[](const signed_block& v) { return fc::raw::pack(v); }, + return std::visit(overloaded{[](const signed_block_v0& v) { return fc::raw::pack(v); }, [](const log_entry_v4& v) { return pack(v); }}, entry); } @@ -734,7 +734,7 @@ namespace eosio { namespace chain { detail::log_entry entry; if (version < 4) { - entry.emplace(); + entry.emplace(); } while( pos < end_pos ) { From 48cdf96380844551e746367ce6ddb31a7556d0e7 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 13 Apr 2020 09:07:34 -0500 Subject: [PATCH 075/142] remove unnecessary assertion --- libraries/chain/block_log.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 97caec95f53..d0f71e8b026 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -909,7 +909,6 @@ namespace eosio { namespace chain { std::vector buffer(max_block_size); fc::datastream stream(buffer.data(), buffer.size()); static_cast(entry).pack(stream, entry.compression); - EOS_ASSERT(buffer.size() <= max_block_size, block_log_exception, "Not enough space reserved in block log entry to serialize pruned block."); my->block_file.write(buffer.data(), buffer.size()); my->block_file.flush(); } From 1e0229ddf9739b59ad2a8f1016fc9ac73f8a5afb Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 13 Apr 2020 09:28:44 -0500 Subject: [PATCH 076/142] add a static assert --- libraries/chain/block_log.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index d0f71e8b026..b88f0c9b971 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -84,7 +84,9 @@ namespace eosio { namespace chain { // 3. The serialization of a pruned_block representation of the block for the entry including padding. std::size_t padded_size = block.maximum_pruned_pack_size(compression); - std::vector buffer(padded_size + offset_to_block_start(block_log::max_supported_version)); + static_assert( block_log::max_supported_version == 4, + "Code was written to support format of version 4, need to update this code for latest format." ); + std::vector buffer(padded_size + offset_to_block_start(block_log::max_supported_version)); fc::datastream stream(buffer.data(), buffer.size()); uint32_t offset = buffer.size() + sizeof(uint64_t); From 3cc92d87c285fd0cc5b458d938dcc146d0ab970a Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 13 Apr 2020 12:56:42 -0500 Subject: [PATCH 077/142] Fix PR comment --- libraries/chain/block_log.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index b88f0c9b971..534c018cfe0 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -68,7 +68,7 @@ namespace eosio { namespace chain { entry.compression = static_cast(compression); EOS_ASSERT(entry.compression == packed_transaction::cf_compression_type::none, block_log_exception, "Only support compression_type none"); - fc::raw::unpack(ds, static_cast(entry)); + static_cast(entry).unpack(ds, entry.compression); uint64_t current_stream_offset = get_stream_pos(ds) - start_pos; int64_t bytes_to_skip = static_cast(entry.offset) - sizeof(uint64_t) - current_stream_offset; From 49c6fe949ae70300fccee222d3a19f4c274f008e Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 13 Apr 2020 16:29:13 -0500 Subject: [PATCH 078/142] Change prune_transactions to return number of transactions been pruned. --- libraries/chain/block_log.cpp | 17 ++++++++--------- .../chain/include/eosio/chain/block_log.hpp | 5 ++++- unittests/restart_chain_tests.cpp | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 534c018cfe0..c33aae71893 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -879,7 +879,7 @@ namespace eosio { namespace chain { return true; } - void block_log::prune_transactions(uint32_t block_num, const std::vector& ids) { + size_t block_log::prune_transactions(uint32_t block_num, const std::vector& ids) { try { EOS_ASSERT( my->version >= 4, block_log_exception, "The block log version ${version} does not support transaction pruning.", ("version", my->version) ); @@ -895,15 +895,13 @@ namespace eosio { namespace chain { EOS_ASSERT(entry.block_num() == block_num, block_log_exception, "Wrong block was read from block log."); - auto pruner = overloaded{[](transaction_id_type&) { return false; }, - [&ids](packed_transaction& ptx) { return std::find(ids.begin(), ids.end(), ptx.id()) != ids.end() && prune(ptx); }}; + size_t num_trx_pruned = std::count_if(entry.transactions.begin(), entry.transactions.end(), [&ids] (auto& trx) { + auto pruner = overloaded{[](transaction_id_type&) { return false; }, + [&ids](packed_transaction& ptx) { return std::find(ids.begin(), ids.end(), ptx.id()) != ids.end() && prune(ptx); }}; + return trx.trx.visit(pruner); + }); - bool pruned = false; - for (auto& trx : entry.transactions) { - pruned |= trx.trx.visit(pruner); - } - - if (pruned) { + if (num_trx_pruned) { // we don't want to rewrite entire entry, just the block data itself. const auto block_offset = detail::offset_to_block_start(my->version); my->block_file.seek(pos + block_offset); @@ -914,6 +912,7 @@ namespace eosio { namespace chain { my->block_file.write(buffer.data(), buffer.size()); my->block_file.flush(); } + return num_trx_pruned; } FC_LOG_AND_RETHROW() } diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index b9de5787d62..3c86b8b5b01 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -56,7 +56,10 @@ namespace eosio { namespace chain { const signed_block_ptr& head() const; uint32_t first_block_num() const; - void prune_transactions(uint32_t block_num, const vector& ids); + /** + * @returns The number of transactions been pruned + **/ + size_t prune_transactions(uint32_t block_num, const vector& ids); static const uint64_t npos = std::numeric_limits::max(); diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index 2050e5d0d57..f61f4b63254 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -266,7 +266,7 @@ BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log, light_vali BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log_with_pruned_trx, light_validation_restart_from_block_log_test_fixture) { const auto& blocks_dir = chain.get_config().blocks_dir; block_log blog(blocks_dir); - BOOST_REQUIRE_NO_THROW(blog.prune_transactions(trace->block_num, std::vector{trace->id})); + BOOST_CHECK(blog.prune_transactions(trace->block_num, std::vector{trace->id}) == 1); BOOST_REQUIRE_NO_THROW(block_log::repair_log(blocks_dir)); } From 467a650d9b84ae56845d75469fee8aeb6a901c08 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 14 Apr 2020 07:32:38 -0500 Subject: [PATCH 079/142] Remove unused method --- libraries/chain/include/eosio/chain/transaction.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 381015c925b..1f5c21a3c1b 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -70,12 +70,6 @@ namespace eosio { namespace chain { uint8_t max_cpu_usage_ms = 0; /// upper limit on the total CPU time billed for this transaction fc::unsigned_int delay_sec = 0UL; /// number of seconds to delay this transaction for during which it may be canceled. - /** - * @return the absolute block number given the relative ref_block_num - */ - block_num_type get_ref_blocknum( block_num_type head_blocknum )const { - return ((head_blocknum/0xffff)*0xffff) + head_blocknum%0xffff; - } void set_reference_block( const block_id_type& reference_block ); bool verify_reference_block( const block_id_type& reference_block )const; void validate()const; From b114f69f7cc2c1b3db3416419b7b14147ed07e33 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 14 Apr 2020 08:40:03 -0500 Subject: [PATCH 080/142] Some more PR comments fix --- libraries/chain/block_log.cpp | 122 ++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 57 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index c33aae71893..d8db7ad971c 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -25,8 +25,6 @@ namespace eosio { namespace chain { - const uint32_t block_log::min_supported_version = 1; - /** * History: * Version 1: complete block log from genesis @@ -37,21 +35,33 @@ namespace eosio { namespace chain { * Version 4: changes the block entry from the serialization of signed_block to a tuple of offset to next entry, * compression_status and pruned_block. */ - const uint32_t block_log::max_supported_version = 4; + + enum Versions { + initial_version = 1, + block_x_start_version = 2, + genesis_state_or_chain_id_version = 3, + pruned_transaction_version = 4 + }; + + const uint32_t block_log::min_supported_version = initial_version; + const uint32_t block_log::max_supported_version = pruned_transaction_version; namespace detail { using unique_file = std::unique_ptr; /// calculate the offset from the start of serialized block entry to block start int offset_to_block_start(uint32_t version) { - if (version < 4) return 0; + if (version < pruned_transaction_version) return 0; return sizeof(uint32_t) + 1; } - class log_entry_v4 : public signed_block { - public: + struct log_entry_v4 : signed_block { + // In version 4 of the irreversible blocks log format, these log entries consists of the following in order: + // 1. An uint32_t size for number of bytes from the start of this log entry to the start of the next log entry. + // 2. An uint8_t indicating the compression status for the serialization of the pruned_block following this. + // 3. The serialization of a signed_block representation of the block for the entry including padding. packed_transaction::cf_compression_type compression = packed_transaction::cf_compression_type::none; - uint32_t offset = 0; + uint32_t size = 0; // the size of the log entry }; size_t get_stream_pos(fc::cfile_datastream& ds) { return ds.tellp(); } @@ -61,36 +71,34 @@ namespace eosio { namespace chain { template void unpack(Stream& ds, log_entry_v4& entry){ - auto start_pos = get_stream_pos(ds); - fc::raw::unpack(ds, entry.offset); + const auto start_pos = get_stream_pos(ds); + fc::raw::unpack(ds, entry.size); uint8_t compression; fc::raw::unpack(ds, compression); entry.compression = static_cast(compression); EOS_ASSERT(entry.compression == packed_transaction::cf_compression_type::none, block_log_exception, "Only support compression_type none"); static_cast(entry).unpack(ds, entry.compression); - uint64_t current_stream_offset = get_stream_pos(ds) - start_pos; - - int64_t bytes_to_skip = static_cast(entry.offset) - sizeof(uint64_t) - current_stream_offset; + const uint64_t current_stream_offset = get_stream_pos(ds) - start_pos; + // For a block which contains CFD (context free data) and the CFD is pruned afterwards, the entry.size may + // be the size before the CFD has been pruned while the actual serialized block does not have the CFD anymore. + // In this case, the serialized block has fewer bytes than what's indicated by entry.size. We need to + // skip over the extra bytes to allow ds to position to the last 8 bytes of the entry. + const int64_t bytes_to_skip = static_cast(entry.size) - sizeof(uint64_t) - current_stream_offset; EOS_ASSERT(bytes_to_skip >= 0, block_log_exception, - "Invalid block log entry offset"); + "Invalid block log entry size"); skip_streamm_pos(ds, bytes_to_skip); } std::vector pack(const signed_block& block, packed_transaction::cf_compression_type compression) { - // In version 4 of the irreversible blocks log format, these log entries consists of the following in order: - // 1. An uint32_t offset from the start of this log entry to the start of the next log entry. - // 2. An uint8_t indicating the compression status for the serialization of the pruned_block following this. - // 3. The serialization of a pruned_block representation of the block for the entry including padding. - - std::size_t padded_size = block.maximum_pruned_pack_size(compression); - static_assert( block_log::max_supported_version == 4, + const std::size_t padded_size = block.maximum_pruned_pack_size(compression); + static_assert( block_log::max_supported_version == pruned_transaction_version, "Code was written to support format of version 4, need to update this code for latest format." ); std::vector buffer(padded_size + offset_to_block_start(block_log::max_supported_version)); fc::datastream stream(buffer.data(), buffer.size()); - uint32_t offset = buffer.size() + sizeof(uint64_t); - stream.write((char*)&offset, sizeof(offset)); + const uint32_t size = buffer.size() + sizeof(uint64_t); + stream.write((char*)&size, sizeof(size)); fc::raw::pack(stream, static_cast(compression)); block.pack(stream, compression); return buffer; @@ -105,7 +113,8 @@ namespace eosio { namespace chain { template void unpack(Stream& ds, log_entry& entry) { std::visit( - overloaded{[&ds](signed_block_v0& v) { fc::raw::unpack(ds, v); }, [&ds](log_entry_v4& v) { unpack(ds, v); }}, + overloaded{[&ds](signed_block_v0& v) { fc::raw::unpack(ds, v); }, + [&ds](log_entry_v4& v) { unpack(ds, v); }}, entry); } @@ -281,7 +290,7 @@ namespace eosio { namespace chain { }; }}} // namespace eosio::chain::detail -FC_REFLECT_DERIVED(eosio::chain::detail::log_entry_v4, (eosio::chain::signed_block), (compression)(offset) ) +FC_REFLECT_DERIVED(eosio::chain::detail::log_entry_v4, (eosio::chain::signed_block), (compression)(size) ) namespace eosio { namespace chain { @@ -389,7 +398,7 @@ namespace eosio { namespace chain { uint64_t pos = block_file.tellp(); std::vector buffer; - if (version >= 4) { + if (version >= pruned_transaction_version) { buffer = detail::pack(b, segment_compression); } else { #warning: TODO avoid heap allocation @@ -508,7 +517,7 @@ namespace eosio { namespace chain { std::unique_ptr detail::block_log_impl::read_block(uint64_t pos) { block_file.seek(pos); auto ds = block_file.create_datastream(); - if (version >= 4) { + if (version >= pruned_transaction_version) { auto entry = std::make_unique(); unpack(ds, *entry); return entry; @@ -523,10 +532,10 @@ namespace eosio { namespace chain { block_file.seek(pos); auto ds = block_file.create_datastream(); - if (version >= 4 ) { - uint32_t offset; + if (version >= pruned_transaction_version ) { + uint32_t size; uint8_t compression; - fc::raw::unpack(ds, offset); + fc::raw::unpack(ds, size); fc::raw::unpack(ds, compression); EOS_ASSERT( compression == static_cast(packed_transaction::cf_compression_type::none), block_log_exception , "Only \"none\" compression type is supported."); @@ -679,7 +688,7 @@ namespace eosio { namespace chain { new_block_stream.write( (char*)&version, sizeof(version) ); uint32_t first_block_num = 1; - if (version != 1) { + if (version != initial_version) { old_block_stream.read ( (char*)&first_block_num, sizeof(first_block_num) ); // this assert is only here since repair_log is only used for --hard-replay-blockchain, which removes any @@ -713,7 +722,7 @@ namespace eosio { namespace chain { ("file", (backup_dir / "blocks.log").generic_string())("ver", version)("fbn", first_block_num)); } - if (version != 1) { + if (version != initial_version) { auto expected_totem = npos; std::decay_t actual_totem; old_block_stream.read ( (char*)&actual_totem, sizeof(actual_totem) ); @@ -735,7 +744,7 @@ namespace eosio { namespace chain { uint64_t pos = old_block_stream.tellg(); detail::log_entry entry; - if (version < 4) { + if (version < pruned_transaction_version) { entry.emplace(); } @@ -836,7 +845,7 @@ namespace eosio { namespace chain { ("version", version)("min", block_log::min_supported_version)("max", block_log::max_supported_version) ); uint32_t first_block_num = 1; - if (version != 1) { + if (version != initial_version) { block_stream.read ( (char*)&first_block_num, sizeof(first_block_num) ); } @@ -873,15 +882,10 @@ namespace eosio { namespace chain { return chain_id; })); } - - bool prune(packed_transaction& ptx) { - ptx.prune_all(); - return true; - } size_t block_log::prune_transactions(uint32_t block_num, const std::vector& ids) { try { - EOS_ASSERT( my->version >= 4, block_log_exception, + EOS_ASSERT( my->version >= pruned_transaction_version, block_log_exception, "The block log version ${version} does not support transaction pruning.", ("version", my->version) ); uint64_t pos = get_block_pos(block_num); EOS_ASSERT( pos != npos, block_log_exception, @@ -895,17 +899,21 @@ namespace eosio { namespace chain { EOS_ASSERT(entry.block_num() == block_num, block_log_exception, "Wrong block was read from block log."); - size_t num_trx_pruned = std::count_if(entry.transactions.begin(), entry.transactions.end(), [&ids] (auto& trx) { - auto pruner = overloaded{[](transaction_id_type&) { return false; }, - [&ids](packed_transaction& ptx) { return std::find(ids.begin(), ids.end(), ptx.id()) != ids.end() && prune(ptx); }}; - return trx.trx.visit(pruner); - }); + auto pruner = + overloaded{[](transaction_id_type&) { return false; }, + [&ids](packed_transaction& ptx) { + return std::find(ids.begin(), ids.end(), ptx.id()) != ids.end() && (ptx.prune_all(), true); + }}; + size_t num_trx_pruned = 0; + for (auto& trx : entry.transactions) { + num_trx_pruned += trx.trx.visit(pruner); + } if (num_trx_pruned) { // we don't want to rewrite entire entry, just the block data itself. const auto block_offset = detail::offset_to_block_start(my->version); my->block_file.seek(pos + block_offset); - const uint32_t max_block_size = entry.offset - block_offset - sizeof(uint64_t); + const uint32_t max_block_size = entry.size - block_offset - sizeof(uint64_t); std::vector buffer(max_block_size); fc::datastream stream(buffer.data(), buffer.size()); static_cast(entry).pack(stream, entry.compression); @@ -937,7 +945,7 @@ namespace eosio { namespace chain { EOS_ASSERT( size == 1, block_log_exception, "Block log file at '${blocks_log}' could not be read.", ("file", _block_file_name) ); EOS_ASSERT( block_log::is_supported_version(_version), block_log_unsupported_version, "block log version ${v} is not supported", ("v", _version)); - if (_version == 1) { + if (_version == initial_version) { _first_block_num = 1; } else { @@ -984,7 +992,7 @@ namespace eosio { namespace chain { block_log_exception, "Block log file at '${blocks_log}' first block already returned by former call to previous(), it is no longer valid to call this function.", ("blocks_log", _block_file_name) ); - if (_version == 1 && _blocks_found == _blocks_expected) { + if (_version == initial_version && _blocks_found == _blocks_expected) { _current_position_in_file = block_log::npos; return _current_position_in_file; } @@ -1110,11 +1118,11 @@ namespace eosio { namespace chain { } bool block_log::contains_genesis_state(uint32_t version, uint32_t first_block_num) { - return version <= 2 || first_block_num == 1; + return version < genesis_state_or_chain_id_version || first_block_num == 1; } bool block_log::contains_chain_id(uint32_t version, uint32_t first_block_num) { - return version >= 3 && first_block_num > 1; + return version >= genesis_state_or_chain_id_version && first_block_num > 1; } bool block_log::is_supported_version(uint32_t version) { @@ -1168,7 +1176,7 @@ namespace eosio { namespace chain { new_block_file.close(); new_block_file.open( LOG_RW_C ); - static_assert( block_log::max_supported_version == 4, + static_assert( block_log::max_supported_version == pruned_transaction_version, "Code was written to support format of version 4, need to update this code for latest format." ); uint32_t version = block_log::max_supported_version; new_block_file.seek(0); @@ -1263,7 +1271,7 @@ namespace eosio { namespace chain { EOS_ASSERT( block_log::is_supported_version(version), block_log_unsupported_version, "block log version ${v} is not supported", ("v",version)); detail::fileptr_datastream ds(blk_in, block_file_name.string()); - if (version == 1) { + if (version == initial_version) { first_block = 1; genesis_state gs; fc::raw::unpack(ds, gs); @@ -1353,22 +1361,22 @@ namespace eosio { namespace chain { auto size = fread((void*)&block_n_pos, sizeof(block_n_pos), 1, ind_in); //filepos of block n EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", index_file_name.string())("b",n) ); - if (version >= 4) { + if (version >= pruned_transaction_version) { uint64_t block_n_plus_1_pos = block_file_size; if (n < last_block) { size = fread((void*)&block_n_plus_1_pos, sizeof(block_n_plus_1_pos), 1, ind_in); EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", index_file_name.string())("b",n+1) ); } - uint32_t entry_offset = 0; + uint32_t entry_size = 0; status = fseek(blk_in, block_n_pos, SEEK_SET); EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file for block ${b}", ("file", block_file_name.string())("pos", block_n_pos)("b",n) ); - size = fread((void*)&entry_offset, sizeof(entry_offset), 1, blk_in); + size = fread((void*)&entry_size, sizeof(entry_size), 1, blk_in); EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", block_file_name.string())("b",n) ); - EOS_ASSERT((entry_offset == (block_n_plus_1_pos - block_n_pos)), block_log_exception, - "The entry offset in block ${n} is ${entry_offset} does not match the difference of block positions in index file", - ("n", n)("entry_offset", entry_offset)("pos_diff", block_n_plus_1_pos - block_n_pos)); + EOS_ASSERT((entry_size == (block_n_plus_1_pos - block_n_pos)), block_log_exception, + "The entry size in block ${n} is ${entry_size} does not match the difference of block positions in index file", + ("n", n)("entry_size", entry_size)("pos_diff", block_n_plus_1_pos - block_n_pos)); } //read blocks.log and verify block number n is found at the determined file position From 8f96b7a03f28b2ec1f03d37779bba294cded9ba7 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 14 Apr 2020 08:54:02 -0500 Subject: [PATCH 081/142] Change enum name to lowercase --- libraries/chain/block_log.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index d8db7ad971c..cceacd7ebf2 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -36,7 +36,7 @@ namespace eosio { namespace chain { * compression_status and pruned_block. */ - enum Versions { + enum versions { initial_version = 1, block_x_start_version = 2, genesis_state_or_chain_id_version = 3, From a1aaeb0bc39719e831d7560e76debd5b5a0b665f Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 14 Apr 2020 12:22:15 -0500 Subject: [PATCH 082/142] Update resume from state test for new version of state file --- pipeline.jsonc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline.jsonc b/pipeline.jsonc index 86cb5763571..ab926bfd714 100644 --- a/pipeline.jsonc +++ b/pipeline.jsonc @@ -40,8 +40,8 @@ "test": [ { - "commit": "17f1fc9c6b20d30b4febd9a69c61c7bee237f3b9" + "commit": "8f96b7a03f28b2ec1f03d37779bba294cded9ba7" } ] } -} \ No newline at end of file +} From 1f95e33cb226eb2bf91b9ac44705af02c57b83e1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 15 Apr 2020 14:10:29 -0500 Subject: [PATCH 083/142] gcc does not like members with same name as type --- plugins/state_history_plugin/state_history_plugin.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 9972432a047..e4d6ba06b0d 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -50,7 +50,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this acceptor; - trace_converter trace_converter; + trace_converter trace_convert; void get_log_entry(state_history_log& log, uint32_t block_num, fc::optional& result) { if (block_num < log.begin_block() || block_num >= log.end_block()) @@ -324,7 +324,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain().db(), trace_debug_mode, block_state)); + state_history::zlib_compress_bytes(trace_convert.pack(chain_plug->chain().db(), trace_debug_mode, block_state)); EOS_ASSERT(traces_bin.size() == (uint32_t)traces_bin.size(), plugin_exception, "traces is too big"); state_history_log_header header{.magic = ship_magic(ship_current_version), From 8c11fb111b479b2b2e761204228e94b78b42daeb Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 15 Apr 2020 14:33:21 -0500 Subject: [PATCH 084/142] Add ignore of estimated_size for reflection validation script --- libraries/chain/include/eosio/chain/transaction.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index eb839263b1b..a5f769f23ed 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -336,7 +336,7 @@ FC_REFLECT_DERIVED( eosio::chain::signed_transaction, (eosio::chain::transaction FC_REFLECT_ENUM( eosio::chain::packed_transaction_v0::compression_type, (none)(zlib)) // @ignore unpacked_trx trx_id FC_REFLECT( eosio::chain::packed_transaction_v0, (signatures)(compression)(packed_context_free_data)(packed_trx) ) -// @ignore unpacked_trx trx_id +// @ignore estimated_size unpacked_trx trx_id FC_REFLECT( eosio::chain::packed_transaction, (compression)(prunable_data)(packed_trx) ) FC_REFLECT( eosio::chain::prunable_transaction_data, (prunable_data)); FC_REFLECT( eosio::chain::prunable_transaction_data::none, (prunable_digest)) From b76c86320ec535c0d68442bcd9e113f8d82895f7 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 15 Apr 2020 15:56:29 -0500 Subject: [PATCH 085/142] Add an ignore for context_free_segments. The ignore is needed but this exposed a problem in the reflection that will be addressed in a future PR. --- libraries/chain/include/eosio/chain/transaction.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index a5f769f23ed..38440f50bc6 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -343,4 +343,5 @@ FC_REFLECT( eosio::chain::prunable_transaction_data::none, (prunable_digest)) FC_REFLECT( eosio::chain::prunable_transaction_data::signatures_only, (signatures)(context_free_mroot)) FC_REFLECT( eosio::chain::prunable_transaction_data::partial, (signatures)(context_free_segments)) FC_REFLECT( eosio::chain::prunable_transaction_data::full, (signatures)(context_free_segments)) +// @ignore context_free_segments FC_REFLECT( eosio::chain::prunable_transaction_data::full_legacy, (signatures)(packed_context_free_data)) From 32a0f587d37f63f8eb34785e5773a31255d28b41 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 15 Apr 2020 20:11:17 -0500 Subject: [PATCH 086/142] Fixed reflection issue with prunable_data_type::full_legacy context_free_data. Moved prunable_data_type into packed_transaction since it needs the reflector_init of packed_transaction to unpack correctly. --- libraries/chain/apply_context.cpp | 14 +-- libraries/chain/block.cpp | 2 +- .../include/eosio/chain/abi_serializer.hpp | 8 +- .../chain/include/eosio/chain/transaction.hpp | 109 +++++++++--------- libraries/chain/transaction.cpp | 98 ++++++++-------- unittests/misc_tests.cpp | 58 +++++----- 6 files changed, 150 insertions(+), 139 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 45916ae95e0..8d4e6a11a11 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -804,18 +804,18 @@ int apply_context::get_action( uint32_t type, uint32_t index, char* buffer, size int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t buffer_size )const { - const prunable_transaction_data::prunable_data_type& data = trx_context.packed_trx.get_prunable_data().prunable_data; + const packed_transaction::prunable_data_type::prunable_data_t& data = trx_context.packed_trx.get_prunable_data().prunable_data; const bytes* cfd = nullptr; - if( data.contains() || data.contains() ) { - } else if( data.contains() ) { - if( index >= data.get().context_free_segments.size() ) return -1; + if( data.contains() || data.contains() ) { + } else if( data.contains() ) { + if( index >= data.get().context_free_segments.size() ) return -1; cfd = trx_context.packed_trx.get_context_free_data(index); } else { const std::vector& context_free_data = - data.contains() ? - data.get().context_free_segments : - data.get().context_free_segments; + data.contains() ? + data.get().context_free_segments : + data.get().context_free_segments; if( index >= context_free_data.size() ) return -1; cfd = &context_free_data[index]; diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index b4bccd66113..00a278af2af 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -143,7 +143,7 @@ namespace eosio { namespace chain { auto visitor = overloaded{ [](const transaction_id_type &id) -> transaction_receipt_v0::trx_type { return id; }, [](const packed_transaction &trx) -> transaction_receipt_v0::trx_type { - const auto& legacy = trx.get_prunable_data().prunable_data.get(); + const auto& legacy = trx.get_prunable_data().prunable_data.get(); return packed_transaction_v0(trx.get_packed_transaction(), legacy.signatures, legacy.packed_context_free_data, trx.get_compression()); }}; diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 6ec9b8fd393..3a849820e3a 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -482,15 +482,15 @@ namespace impl { const auto& trx = ptrx.get_transaction(); mvo("id", trx.id()); const auto* sigs = ptrx.get_signatures(); - if( ptrx.get_prunable_data().prunable_data.contains() ) { - const auto& legacy = ptrx.get_prunable_data().prunable_data.get(); + if( ptrx.get_prunable_data().prunable_data.contains() ) { + const auto& legacy = ptrx.get_prunable_data().prunable_data.get(); mvo("signatures", legacy.signatures ); } else { mvo("signatures", vector()); } mvo("compression", ptrx.get_compression()); - if( ptrx.get_prunable_data().prunable_data.contains() ) { - const auto& legacy = ptrx.get_prunable_data().prunable_data.get(); + if( ptrx.get_prunable_data().prunable_data.contains() ) { + const auto& legacy = ptrx.get_prunable_data().prunable_data.get(); mvo("packed_context_free_data", legacy.packed_context_free_data); mvo("context_free_data", legacy.context_free_segments); } else { diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 38440f50bc6..b29a58c4380 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -198,7 +198,7 @@ namespace eosio { namespace chain { void reflector_init(); private: friend struct pruned_transaction; - friend struct prunable_transaction_data; + friend struct prunable_data; vector signatures; fc::enum_type compression; bytes packed_context_free_data; @@ -212,61 +212,62 @@ namespace eosio { namespace chain { using packed_transaction_v0_ptr = std::shared_ptr; - struct prunable_transaction_data { - enum class compression_type : uint8_t { - none = 0, - zlib = 1, - COMPRESSION_TYPE_COUNT - }; - // Do not exceed 127 as that will break compatibility in serialization format - static_assert( static_cast(compression_type::COMPRESSION_TYPE_COUNT) <= 127 ); - - struct none { - digest_type prunable_digest; - }; - - struct signatures_only { - std::vector signatures; - digest_type context_free_mroot; - }; - - using segment_type = fc::static_variant; - - struct partial { - std::vector signatures; - std::vector context_free_segments; - }; - - struct full { - std::vector signatures; - std::vector context_free_segments; - }; - - struct full_legacy { - std::vector signatures; - bytes packed_context_free_data; - vector context_free_segments; - }; + struct packed_transaction : fc::reflect_init { - using prunable_data_type = fc::static_variant< full_legacy, + struct prunable_data_type { + enum class compression_type : uint8_t { + none = 0, + zlib = 1, + COMPRESSION_TYPE_COUNT + }; + // Do not exceed 127 as that will break compatibility in serialization format + static_assert( static_cast(compression_type::COMPRESSION_TYPE_COUNT) <= 127 ); + + struct none { + digest_type prunable_digest; + }; + + struct signatures_only { + std::vector signatures; + digest_type context_free_mroot; + }; + + using segment_type = fc::static_variant; + + struct partial { + std::vector signatures; + std::vector context_free_segments; + }; + + struct full { + std::vector signatures; + std::vector context_free_segments; + }; + + struct full_legacy { + std::vector signatures; + bytes packed_context_free_data; + vector context_free_segments; + }; + + using prunable_data_t = fc::static_variant< full_legacy, none, signatures_only, partial, full >; - prunable_transaction_data prune_all() const; - digest_type digest() const; + prunable_data_type prune_all() const; + digest_type digest() const; - // Returns the maximum pack size of any prunable_transaction_data that is reachable - // by pruning this object. - std::size_t maximum_pruned_pack_size( compression_type segment_compression ) const; + // Returns the maximum pack size of any prunable_data that is reachable + // by pruning this object. + std::size_t maximum_pruned_pack_size( compression_type segment_compression ) const; - prunable_data_type prunable_data; - }; + prunable_data_t prunable_data; + }; - struct packed_transaction : fc::reflect_init { using compression_type = packed_transaction_v0::compression_type; - using cf_compression_type = prunable_transaction_data::compression_type; + using cf_compression_type = prunable_data_type::compression_type; packed_transaction() = default; packed_transaction(packed_transaction&&) = default; @@ -298,7 +299,7 @@ namespace eosio { namespace chain { const bytes* get_context_free_data(std::size_t segment_ordinal)const; const fc::enum_type& get_compression()const { return compression; } const bytes& get_packed_transaction()const { return packed_trx; } - const prunable_transaction_data& get_prunable_data() const { return prunable_data; } + const packed_transaction::prunable_data_type& get_prunable_data() const { return prunable_data; } void prune_all(); @@ -313,7 +314,7 @@ namespace eosio { namespace chain { private: uint32_t estimated_size = 0; fc::enum_type compression; - prunable_transaction_data prunable_data; + prunable_data_type prunable_data; bytes packed_trx; // packed and compressed (according to compression) transaction private: @@ -338,10 +339,10 @@ FC_REFLECT_ENUM( eosio::chain::packed_transaction_v0::compression_type, (none)(z FC_REFLECT( eosio::chain::packed_transaction_v0, (signatures)(compression)(packed_context_free_data)(packed_trx) ) // @ignore estimated_size unpacked_trx trx_id FC_REFLECT( eosio::chain::packed_transaction, (compression)(prunable_data)(packed_trx) ) -FC_REFLECT( eosio::chain::prunable_transaction_data, (prunable_data)); -FC_REFLECT( eosio::chain::prunable_transaction_data::none, (prunable_digest)) -FC_REFLECT( eosio::chain::prunable_transaction_data::signatures_only, (signatures)(context_free_mroot)) -FC_REFLECT( eosio::chain::prunable_transaction_data::partial, (signatures)(context_free_segments)) -FC_REFLECT( eosio::chain::prunable_transaction_data::full, (signatures)(context_free_segments)) +FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type, (prunable_data)); +FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::none, (prunable_digest)) +FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::signatures_only, (signatures)( context_free_mroot)) +FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::partial, (signatures)( context_free_segments)) +FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::full, (signatures)( context_free_segments)) // @ignore context_free_segments -FC_REFLECT( eosio::chain::prunable_transaction_data::full_legacy, (signatures)(packed_context_free_data)) +FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::full_legacy, (signatures)( packed_context_free_data)) diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index db2ab174bb6..aa6a5032428 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -390,54 +390,54 @@ void packed_transaction_v0::local_pack_context_free_data() packed_context_free_data = pack_context_free_data(unpacked_trx.context_free_data, compression); } -prunable_transaction_data prunable_transaction_data::prune_all() const { - return { prunable_transaction_data::none{ digest() } }; +packed_transaction::prunable_data_type packed_transaction::prunable_data_type::prune_all() const { + return {packed_transaction::prunable_data_type::none{digest() } }; } -static digest_type prunable_digest(const prunable_transaction_data::none& obj) { +static digest_type prunable_digest(const packed_transaction::prunable_data_type::none& obj) { return obj.prunable_digest; } -static digest_type prunable_digest(const prunable_transaction_data::partial& obj) { +static digest_type prunable_digest(const packed_transaction::prunable_data_type::partial& obj) { EOS_THROW(tx_prune_exception, "unimplemented"); } -static digest_type prunable_digest(const prunable_transaction_data::signatures_only& obj) { +static digest_type prunable_digest(const packed_transaction::prunable_data_type::signatures_only& obj) { EOS_THROW(tx_prune_exception, "unimplemented"); } -static digest_type prunable_digest(const prunable_transaction_data::full& obj) { +static digest_type prunable_digest(const packed_transaction::prunable_data_type::full& obj) { EOS_THROW(tx_prune_exception, "unimplemented"); } -static digest_type prunable_digest(const prunable_transaction_data::full_legacy& obj) { +static digest_type prunable_digest(const packed_transaction::prunable_data_type::full_legacy& obj) { digest_type::encoder prunable; fc::raw::pack( prunable, obj.signatures ); fc::raw::pack( prunable, obj.packed_context_free_data ); return prunable.result(); } -digest_type prunable_transaction_data::digest() const { +digest_type packed_transaction::prunable_data_type::digest() const { return prunable_data.visit( [](const auto& obj) { return prunable_digest(obj); } ); } static constexpr std::size_t digest_pack_size = 32; -static std::size_t padded_pack_size(const prunable_transaction_data::none& obj, prunable_transaction_data::compression_type) { +static std::size_t padded_pack_size( const packed_transaction::prunable_data_type::none& obj, packed_transaction::prunable_data_type::compression_type) { std::size_t result = fc::raw::pack_size(obj); EOS_ASSERT(result == digest_pack_size, packed_transaction_type_exception, "Unexpected size of packed digest"); return result; } -static std::size_t padded_pack_size(const prunable_transaction_data::signatures_only& obj, prunable_transaction_data::compression_type) { +static std::size_t padded_pack_size( const packed_transaction::prunable_data_type::signatures_only& obj, packed_transaction::prunable_data_type::compression_type) { return fc::raw::pack_size(obj); } -static std::size_t padded_pack_size(const prunable_transaction_data::partial& obj, prunable_transaction_data::compression_type segment_compression) { +static std::size_t padded_pack_size( const packed_transaction::prunable_data_type::partial& obj, packed_transaction::prunable_data_type::compression_type segment_compression) { EOS_THROW(tx_prune_exception, "unimplemented"); } -static std::size_t padded_pack_size(const prunable_transaction_data::full& obj, prunable_transaction_data::compression_type segment_compression) { +static std::size_t padded_pack_size( const packed_transaction::prunable_data_type::full& obj, packed_transaction::prunable_data_type::compression_type segment_compression) { EOS_THROW(tx_prune_exception, "unimplemented"); #if 0 std::size_t context_free_size = fc::raw::pack_size(fc::unsigned_int(obj.context_free_segments.size())); @@ -450,22 +450,22 @@ static std::size_t padded_pack_size(const prunable_transaction_data::full& obj, #endif } -static std::size_t padded_pack_size(const prunable_transaction_data::full_legacy& obj, prunable_transaction_data::compression_type) { +static std::size_t padded_pack_size( const packed_transaction::prunable_data_type::full_legacy& obj, packed_transaction::prunable_data_type::compression_type) { return std::max(fc::raw::pack_size(obj), digest_pack_size); } -std::size_t prunable_transaction_data::maximum_pruned_pack_size(prunable_transaction_data::compression_type compression) const { +std::size_t packed_transaction::prunable_data_type::maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type compression) const { return 1 + prunable_data.visit([&](const auto& t){ return padded_pack_size(t, compression); }); } -static prunable_transaction_data make_prunable_transaction_data( bool legacy, vector signatures, - vector context_free_data, - packed_transaction::compression_type _compression ) { +static packed_transaction::prunable_data_type make_prunable_transaction_data( bool legacy, vector signatures, + vector context_free_data, + packed_transaction::compression_type _compression ) { if(legacy) { bytes packed_cfd = pack_context_free_data( context_free_data, _compression ); - return { prunable_transaction_data::full_legacy{ std::move(signatures), std::move(packed_cfd), std::move(context_free_data) } }; + return {packed_transaction::prunable_data_type::full_legacy{std::move( signatures), std::move( packed_cfd), std::move( context_free_data) } }; } else { - return { prunable_transaction_data::full{ std::move(signatures), std::move(context_free_data) } }; + return {packed_transaction::prunable_data_type::full{std::move( signatures), std::move( context_free_data) } }; } } @@ -481,11 +481,11 @@ packed_transaction::packed_transaction(signed_transaction t, bool legacy, compre packed_transaction::packed_transaction(const packed_transaction_v0& other, bool legacy) : compression(other.compression), - prunable_data(legacy ? prunable_transaction_data{ prunable_transaction_data::full_legacy{ other.signatures, - other.packed_context_free_data, - other.unpacked_trx.context_free_data } } - : prunable_transaction_data{ prunable_transaction_data::full{ other.signatures, - other.unpacked_trx.context_free_data } }), + prunable_data(legacy ? prunable_data_type{prunable_data_type::full_legacy{other.signatures, + other.packed_context_free_data, + other.unpacked_trx.context_free_data } } + : prunable_data_type{prunable_data_type::full{other.signatures, + other.unpacked_trx.context_free_data } }), packed_trx(other.packed_trx), unpacked_trx(other.unpacked_trx), trx_id(other.id()) @@ -495,11 +495,11 @@ packed_transaction::packed_transaction(const packed_transaction_v0& other, bool packed_transaction::packed_transaction(packed_transaction_v0&& other, bool legacy) : compression(other.compression), - prunable_data(legacy ? prunable_transaction_data{ prunable_transaction_data::full_legacy{ std::move(other.signatures), - std::move(other.packed_context_free_data), - std::move(other.unpacked_trx.context_free_data) } } - : prunable_transaction_data{ prunable_transaction_data::full{ std::move(other.signatures), - std::move(other.unpacked_trx.context_free_data) } }), + prunable_data(legacy ? prunable_data_type{prunable_data_type::full_legacy{std::move( other.signatures), + std::move(other.packed_context_free_data), + std::move(other.unpacked_trx.context_free_data) } } + : prunable_data_type{prunable_data_type::full{std::move( other.signatures), + std::move(other.unpacked_trx.context_free_data) } }), packed_trx(std::move(other.packed_trx)), unpacked_trx(std::move(other.unpacked_trx)), trx_id(other.id()) @@ -523,14 +523,14 @@ uint32_t packed_transaction::get_unprunable_size()const { return static_cast(size); } -static uint32_t get_prunable_size_impl(const prunable_transaction_data::full_legacy& obj) { +static uint32_t get_prunable_size_impl(const packed_transaction::prunable_data_type::full_legacy& obj) { uint64_t size = fc::raw::pack_size(obj.signatures); size += obj.packed_context_free_data.size(); EOS_ASSERT( size <= std::numeric_limits::max(), tx_too_big, "packed_transaction is too big" ); return static_cast(size); } -static uint32_t get_prunable_size_impl(const prunable_transaction_data::full&) { +static uint32_t get_prunable_size_impl(const packed_transaction::prunable_data_type::full&) { EOS_THROW(tx_prune_exception, "unimplemented"); } @@ -550,7 +550,7 @@ uint32_t packed_transaction::calculate_estimated_size() const { for( const auto& v : vec ) s += v.size(); return s; }, - [](const std::vector& vec) { + [](const std::vector& vec) { uint32_t s = 0; for( const auto& v : vec ) { s += v.visit( overloaded{ @@ -562,17 +562,17 @@ uint32_t packed_transaction::calculate_estimated_size() const { } }; auto visitor = overloaded{ - [](const prunable_transaction_data::none& v) { return 0ul; }, - [](const prunable_transaction_data::signatures_only& v) { + [](const prunable_data_type::none& v) { return 0ul; }, + [](const prunable_data_type::signatures_only& v) { return v.signatures.size() * sizeof(signature_type); }, - [&](const prunable_transaction_data::partial& v) { + [&](const prunable_data_type::partial& v) { return v.signatures.size() * sizeof(signature_type) + est_size(v.context_free_segments); }, - [&](const prunable_transaction_data::full& v) { + [&](const prunable_data_type::full& v) { return v.signatures.size() * sizeof(signature_type) + est_size(v.context_free_segments); }, - [&](const prunable_transaction_data::full_legacy& v) { + [&](const prunable_data_type::full_legacy& v) { return v.signatures.size() * sizeof(signature_type) + v.packed_context_free_data.size() + est_size(v.context_free_segments); } }; @@ -590,24 +590,24 @@ digest_type packed_transaction::packed_digest()const { template static auto maybe_get_signatures(const T& obj) -> decltype(&obj.signatures) { return &obj.signatures; } -static auto maybe_get_signatures(const prunable_transaction_data::none&) -> const std::vector* { return nullptr; } +static auto maybe_get_signatures(const packed_transaction::prunable_data_type::none&) -> const std::vector* { return nullptr; } const vector* packed_transaction::get_signatures()const { return prunable_data.prunable_data.visit([](const auto& obj) { return maybe_get_signatures(obj); }); } const vector* packed_transaction::get_context_free_data()const { - if( prunable_data.prunable_data.contains() ) { - return &prunable_data.prunable_data.get().context_free_segments; - } else if( prunable_data.prunable_data.contains() ) { - return &prunable_data.prunable_data.get().context_free_segments; + if( prunable_data.prunable_data.contains() ) { + return &prunable_data.prunable_data.get().context_free_segments; + } else if( prunable_data.prunable_data.contains() ) { + return &prunable_data.prunable_data.get().context_free_segments; } else { return nullptr; } } -const bytes* maybe_get_context_free_data(const prunable_transaction_data::none&, std::size_t) { return nullptr; } -const bytes* maybe_get_context_free_data(const prunable_transaction_data::signatures_only&, std::size_t) { return nullptr; } -const bytes* maybe_get_context_free_data(const prunable_transaction_data::partial& p, std::size_t i) { +const bytes* maybe_get_context_free_data( const packed_transaction::prunable_data_type::none&, std::size_t) { return nullptr; } +const bytes* maybe_get_context_free_data( const packed_transaction::prunable_data_type::signatures_only&, std::size_t) { return nullptr; } +const bytes* maybe_get_context_free_data( const packed_transaction::prunable_data_type::partial& p, std::size_t i) { if( p.context_free_segments.size() <= i ) return nullptr; return p.context_free_segments[i].visit( overloaded{ @@ -615,11 +615,11 @@ const bytes* maybe_get_context_free_data(const prunable_transaction_data::partia []( const bytes& vec ) { return &vec; } } ); } -const bytes* maybe_get_context_free_data(const prunable_transaction_data::full_legacy& full_leg, std::size_t i) { +const bytes* maybe_get_context_free_data( const packed_transaction::prunable_data_type::full_legacy& full_leg, std::size_t i) { if( full_leg.context_free_segments.size() <= i ) return nullptr; return &full_leg.context_free_segments[i]; } -const bytes* maybe_get_context_free_data(const prunable_transaction_data::full& f, std::size_t i) { +const bytes* maybe_get_context_free_data( const packed_transaction::prunable_data_type::full& f, std::size_t i) { if( f.context_free_segments.size() <= i ) return nullptr; return &f.context_free_segments[i]; } @@ -644,6 +644,10 @@ void packed_transaction::reflector_init() EOS_ASSERT( unpacked_trx.expiration == time_point_sec(), tx_decompression_error, "packed_transaction already unpacked" ); unpacked_trx = unpack_transaction(packed_trx, compression); trx_id = unpacked_trx.id(); + if( prunable_data.prunable_data.contains() ) { + auto& legacy = prunable_data.prunable_data.get(); + legacy.context_free_segments = unpack_context_free_data( legacy.packed_context_free_data, compression ); + } estimated_size = calculate_estimated_size(); } diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index 20736e205c8..f3d48c94663 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -722,7 +722,8 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { ("name", "nonce") ("data", fc::raw::pack(std::string("dummy"))) }) - ); + ) + ("context_free_data", vector{{'d','u','m','m','y',' ','d','a','t','a'}}); abi_serializer::from_variant(pretty_trx, trx, test.get_resolver(), abi_serializer::create_yield_function( test.abi_serializer_max_time )); @@ -796,6 +797,10 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { BOOST_CHECK_EQUAL(pkt.id(), pkt4.get_transaction().id()); BOOST_CHECK_EQUAL(true, trx.expiration == pkt4.expiration()); BOOST_CHECK_EQUAL(true, trx.expiration == pkt4.get_transaction().expiration); + BOOST_REQUIRE(pkt.get_context_free_data() != nullptr); + BOOST_REQUIRE(pkt4.get_context_free_data() != nullptr); + BOOST_REQUIRE_EQUAL(pkt.get_context_free_data()->size(), pkt4.get_context_free_data()->size()); + BOOST_CHECK(std::equal(pkt.get_context_free_data()->begin(), pkt.get_context_free_data()->end(), pkt4.get_context_free_data()->begin())); keys.clear(); pkt4.to_packed_transaction_v0()->get_signed_transaction().get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); BOOST_CHECK_EQUAL(1u, keys.size()); @@ -866,7 +871,8 @@ BOOST_AUTO_TEST_CASE(transaction_metadata_test) { try { ("name", "nonce") ("data", fc::raw::pack(std::string("dummy data"))) }) - ); + ) + ("context_free_data", vector{{'d','u','m','m','y',' ','d','a','t','a'}}); abi_serializer::from_variant(pretty_trx, trx, test.get_resolver(), abi_serializer::create_yield_function( test.abi_serializer_max_time )); @@ -919,43 +925,43 @@ BOOST_AUTO_TEST_CASE(transaction_metadata_test) { try { BOOST_AUTO_TEST_CASE(prunable_transaction_data_test) { { - prunable_transaction_data basic{prunable_transaction_data::full_legacy{{}, {}}}; - prunable_transaction_data pruned = prunable_transaction_data(basic).prune_all(); - BOOST_TEST(basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none) >= - pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); - BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); - BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + packed_transaction::prunable_data_type basic{packed_transaction::prunable_data_type::full_legacy{{}, {}}}; + packed_transaction::prunable_data_type pruned = packed_transaction::prunable_data_type( basic).prune_all(); + BOOST_TEST( basic.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none) >= + pruned.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); BOOST_TEST(basic.digest().str() == pruned.digest().str()); } bytes large_bytes(48); { - prunable_transaction_data basic{prunable_transaction_data::full_legacy{{}, fc::raw::pack(std::vector(4, large_bytes))}}; - prunable_transaction_data pruned = prunable_transaction_data(basic).prune_all(); - BOOST_TEST(basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none) >= - pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); - BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); - BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + packed_transaction::prunable_data_type basic{packed_transaction::prunable_data_type::full_legacy{{}, fc::raw::pack( std::vector( 4, large_bytes))}}; + packed_transaction::prunable_data_type pruned = packed_transaction::prunable_data_type(basic).prune_all(); + BOOST_TEST( basic.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none) >= + pruned.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); BOOST_TEST(basic.digest().str() == pruned.digest().str()); } { - prunable_transaction_data basic{prunable_transaction_data::full_legacy{std::vector(4, signature_type()), fc::raw::pack(std::vector(4, large_bytes))}}; - prunable_transaction_data pruned = prunable_transaction_data(basic).prune_all(); - BOOST_TEST(basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none) >= - pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); - BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); - BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + packed_transaction::prunable_data_type basic{packed_transaction::prunable_data_type::full_legacy{std::vector( 4, signature_type()), fc::raw::pack( std::vector( 4, large_bytes))}}; + packed_transaction::prunable_data_type pruned = packed_transaction::prunable_data_type(basic).prune_all(); + BOOST_TEST( basic.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none) >= + pruned.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); BOOST_TEST(basic.digest().str() == pruned.digest().str()); } { - prunable_transaction_data basic{prunable_transaction_data::full_legacy{std::vector(4, signature_type()), {}}}; - prunable_transaction_data pruned = prunable_transaction_data(basic).prune_all(); - BOOST_TEST(basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none) >= - pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); - BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); - BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size(prunable_transaction_data::compression_type::none)); + packed_transaction::prunable_data_type basic{packed_transaction::prunable_data_type::full_legacy{std::vector( 4, signature_type()), {}}}; + packed_transaction::prunable_data_type pruned = packed_transaction::prunable_data_type(basic).prune_all(); + BOOST_TEST( basic.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none) >= + pruned.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(basic) <= basic.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); + BOOST_TEST(fc::raw::pack_size(pruned) <= pruned.maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type::none)); BOOST_TEST(basic.digest().str() == pruned.digest().str()); } } From 09a6adb8f6406d0c700843e34a2a6414e7011dc5 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 15 Apr 2020 20:30:14 -0500 Subject: [PATCH 087/142] Fix formating changed by refactor --- libraries/chain/transaction.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index aa6a5032428..b0bd2874bf6 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -256,7 +256,7 @@ static bytes zlib_compress_transaction(const transaction& t) { return out; } -packed_transaction_v0::packed_transaction_v0(const bytes& packed_txn, const vector& sigs, const bytes& packed_cfd, compression_type _compression ) +packed_transaction_v0::packed_transaction_v0(const bytes& packed_txn, const vector& sigs, const bytes& packed_cfd, compression_type _compression) :signatures(sigs) ,compression(_compression) ,packed_context_free_data(packed_cfd) @@ -423,21 +423,21 @@ digest_type packed_transaction::prunable_data_type::digest() const { static constexpr std::size_t digest_pack_size = 32; -static std::size_t padded_pack_size( const packed_transaction::prunable_data_type::none& obj, packed_transaction::prunable_data_type::compression_type) { +static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::none& obj, packed_transaction::prunable_data_type::compression_type) { std::size_t result = fc::raw::pack_size(obj); EOS_ASSERT(result == digest_pack_size, packed_transaction_type_exception, "Unexpected size of packed digest"); return result; } -static std::size_t padded_pack_size( const packed_transaction::prunable_data_type::signatures_only& obj, packed_transaction::prunable_data_type::compression_type) { +static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::signatures_only& obj, packed_transaction::prunable_data_type::compression_type) { return fc::raw::pack_size(obj); } -static std::size_t padded_pack_size( const packed_transaction::prunable_data_type::partial& obj, packed_transaction::prunable_data_type::compression_type segment_compression) { +static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::partial& obj, packed_transaction::prunable_data_type::compression_type segment_compression) { EOS_THROW(tx_prune_exception, "unimplemented"); } -static std::size_t padded_pack_size( const packed_transaction::prunable_data_type::full& obj, packed_transaction::prunable_data_type::compression_type segment_compression) { +static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::full& obj, packed_transaction::prunable_data_type::compression_type segment_compression) { EOS_THROW(tx_prune_exception, "unimplemented"); #if 0 std::size_t context_free_size = fc::raw::pack_size(fc::unsigned_int(obj.context_free_segments.size())); @@ -450,11 +450,11 @@ static std::size_t padded_pack_size( const packed_transaction::prunable_data_typ #endif } -static std::size_t padded_pack_size( const packed_transaction::prunable_data_type::full_legacy& obj, packed_transaction::prunable_data_type::compression_type) { +static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::full_legacy& obj, packed_transaction::prunable_data_type::compression_type) { return std::max(fc::raw::pack_size(obj), digest_pack_size); } -std::size_t packed_transaction::prunable_data_type::maximum_pruned_pack_size( packed_transaction::prunable_data_type::compression_type compression) const { +std::size_t packed_transaction::prunable_data_type::maximum_pruned_pack_size(packed_transaction::prunable_data_type::compression_type compression) const { return 1 + prunable_data.visit([&](const auto& t){ return padded_pack_size(t, compression); }); } @@ -495,10 +495,10 @@ packed_transaction::packed_transaction(const packed_transaction_v0& other, bool packed_transaction::packed_transaction(packed_transaction_v0&& other, bool legacy) : compression(other.compression), - prunable_data(legacy ? prunable_data_type{prunable_data_type::full_legacy{std::move( other.signatures), + prunable_data(legacy ? prunable_data_type{prunable_data_type::full_legacy{std::move(other.signatures), std::move(other.packed_context_free_data), std::move(other.unpacked_trx.context_free_data) } } - : prunable_data_type{prunable_data_type::full{std::move( other.signatures), + : prunable_data_type{prunable_data_type::full{std::move(other.signatures), std::move(other.unpacked_trx.context_free_data) } }), packed_trx(std::move(other.packed_trx)), unpacked_trx(std::move(other.unpacked_trx)), @@ -605,9 +605,9 @@ const vector* packed_transaction::get_context_free_data()const { } } -const bytes* maybe_get_context_free_data( const packed_transaction::prunable_data_type::none&, std::size_t) { return nullptr; } -const bytes* maybe_get_context_free_data( const packed_transaction::prunable_data_type::signatures_only&, std::size_t) { return nullptr; } -const bytes* maybe_get_context_free_data( const packed_transaction::prunable_data_type::partial& p, std::size_t i) { +const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::none&, std::size_t) { return nullptr; } +const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::signatures_only&, std::size_t) { return nullptr; } +const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::partial& p, std::size_t i) { if( p.context_free_segments.size() <= i ) return nullptr; return p.context_free_segments[i].visit( overloaded{ @@ -615,11 +615,11 @@ const bytes* maybe_get_context_free_data( const packed_transaction::prunable_dat []( const bytes& vec ) { return &vec; } } ); } -const bytes* maybe_get_context_free_data( const packed_transaction::prunable_data_type::full_legacy& full_leg, std::size_t i) { +const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::full_legacy& full_leg, std::size_t i) { if( full_leg.context_free_segments.size() <= i ) return nullptr; return &full_leg.context_free_segments[i]; } -const bytes* maybe_get_context_free_data( const packed_transaction::prunable_data_type::full& f, std::size_t i) { +const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::full& f, std::size_t i) { if( f.context_free_segments.size() <= i ) return nullptr; return &f.context_free_segments[i]; } From 90c0e870e55d25853fc49a08be2cd683fbc4fa31 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 16 Apr 2020 07:47:33 -0500 Subject: [PATCH 088/142] Remove unneeded friends --- libraries/chain/include/eosio/chain/transaction.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index b29a58c4380..5e5f550fad6 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -197,8 +197,6 @@ namespace eosio { namespace chain { friend struct packed_transaction; void reflector_init(); private: - friend struct pruned_transaction; - friend struct prunable_data; vector signatures; fc::enum_type compression; bytes packed_context_free_data; From e33aa23d921053db09a4ff7f54fd0274fb93ba96 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 16 Apr 2020 08:40:12 -0500 Subject: [PATCH 089/142] Update to new snapshot for resume from state test --- pipeline.jsonc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline.jsonc b/pipeline.jsonc index ab926bfd714..c6df402c67b 100644 --- a/pipeline.jsonc +++ b/pipeline.jsonc @@ -40,7 +40,7 @@ "test": [ { - "commit": "8f96b7a03f28b2ec1f03d37779bba294cded9ba7" + "commit": "90c0e870e55d25853fc49a08be2cd683fbc4fa31" } ] } From 9e3483874cc71fbcd1bc992b61b080d3cd3d9d06 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 17 Apr 2020 17:22:00 -0500 Subject: [PATCH 090/142] Change to_signed_block_v0 to return unique_ptr instead of shared_ptr --- libraries/chain/block.cpp | 6 +++--- libraries/chain/include/eosio/chain/block.hpp | 4 +++- plugins/chain_plugin/chain_plugin.cpp | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index 00a278af2af..288e7c4e455 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -133,11 +133,11 @@ namespace eosio { namespace chain { return validate_and_extract_block_extensions( block_extensions ); } - signed_block_v0_ptr signed_block::to_signed_block_v0() const { + signed_block_v0_uptr signed_block::to_signed_block_v0() const { if (prune_state != prune_state_type::complete_legacy) - return signed_block_v0_ptr{}; + return signed_block_v0_uptr{}; - auto result = std::make_shared(*static_cast(this)); + auto result = std::make_unique(*static_cast(this)); result->block_extensions = this->block_extensions; auto visitor = overloaded{ diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 35af9531050..81d61bf418c 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -102,7 +102,9 @@ namespace eosio { namespace chain { flat_multimap validate_and_extract_extensions()const; }; + using signed_block_v0_ptr = std::shared_ptr; + using signed_block_v0_uptr = std::unique_ptr; struct transaction_receipt : public transaction_receipt_header { @@ -143,7 +145,7 @@ namespace eosio { namespace chain { signed_block& operator=(const signed_block&) = delete; signed_block& operator=(signed_block&&) = default; signed_block clone() const { return *this; } - signed_block_v0_ptr to_signed_block_v0() const; + signed_block_v0_uptr to_signed_block_v0() const; fc::enum_type prune_state{prune_state_type::complete_legacy}; deque transactions; /// new or generated transactions diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index f2492cee807..a50dc3a8145 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -1423,7 +1423,7 @@ bool chain_plugin::export_reversible_blocks( const fc::path& reversible_dir, signed_block tmp; fc::datastream ds( itr->packedblock.data(), itr->packedblock.size() ); fc::raw::unpack(ds, tmp); // Verify that packed block has not been corrupted. - signed_block_v0_ptr v0 = tmp.to_signed_block_v0(); // store in signed_block_v0 format + signed_block_v0_uptr v0 = tmp.to_signed_block_v0(); // store in signed_block_v0 format auto packed_v0 = fc::raw::pack(*v0); reversible_blocks.write( packed_v0.data(), packed_v0.size() ); end = itr->blocknum; From b2f4c578a9ab67eb0088961f05af4a0611d93757 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 17 Apr 2020 17:24:55 -0500 Subject: [PATCH 091/142] Add new protocol version for pruned data. Send block type according to protocol_version of connection and if block can be converted. --- .../include/eosio/net_plugin/protocol.hpp | 23 +++++- plugins/net_plugin/net_plugin.cpp | 74 ++++++++++++------- 2 files changed, 70 insertions(+), 27 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp index 3658c933695..d52dfc68423 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp @@ -135,6 +135,19 @@ namespace eosio { uint32_t end_block{0}; }; + struct generic_message { + go_away_message tmp; // until merge of generic message support + }; + + struct generic_support_message { + go_away_message tmp; // until merge of generic message support + }; + + struct trx_message_v1 { + fc::optional trx_id; // only provided for large trx as trade-off for small trxs not worth it + packed_transaction trx; + }; + using net_message = static_variant; // which = 8 + packed_transaction_v0, // which = 8 + generic_message, // which = 9 + generic_support_message, // which = 10 + signed_block, // which = 11 + trx_message_v1>; // which = 12 } // namespace eosio @@ -162,6 +179,10 @@ FC_REFLECT( eosio::time_message, (org)(rec)(xmt)(dst) ) FC_REFLECT( eosio::notice_message, (known_trx)(known_blocks) ) FC_REFLECT( eosio::request_message, (req_trx)(req_blocks) ) FC_REFLECT( eosio::sync_request_message, (start_block)(end_block) ) +FC_REFLECT( eosio::generic_message, (tmp) ) +FC_REFLECT( eosio::generic_support_message, (tmp) ) +FC_REFLECT( eosio::trx_message_v1, (trx_id)(trx) ) + /** * diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index f549c38c3e5..ecd1f5ecd33 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -330,7 +330,7 @@ namespace eosio { */ chain::signature_type sign_compact(const chain::public_key_type& signer, const fc::sha256& digest) const; - constexpr uint16_t to_protocol_version(uint16_t v); + constexpr static uint16_t to_protocol_version(uint16_t v); connection_ptr find_connection(const string& host)const; // must call with held mutex }; @@ -392,6 +392,8 @@ namespace eosio { constexpr auto message_header_size = 4; constexpr uint32_t signed_block_v0_which = 7; // see protocol net_message constexpr uint32_t packed_transaction_v0_which = 8; // see protocol net_message + constexpr uint32_t signed_block_which = 11; // see protocol net_message + constexpr uint32_t trx_message_v1_which = 12; // see protocol net_message /** * For a while, network version was a 16 bit value equal to the second set of 16 bits @@ -412,10 +414,12 @@ namespace eosio { * the need for compatibility hooks */ constexpr uint16_t proto_base = 0; - constexpr uint16_t proto_explicit_sync = 1; // version at time of eosio 1.0 - constexpr uint16_t block_id_notify = 2; // reserved. feature was removed. next net_version should be 3 + constexpr uint16_t proto_explicit_sync = 1; // version at time of eosio 1.0 + constexpr uint16_t proto_block_id_notify = 2; // reserved. feature was removed. next net_version should be 3 + constexpr uint16_t proto_generic_messages = 3; // placeholder for generic messages + constexpr uint16_t proto_pruned_types = 4; // supports new signed_block & packed_transaction types - constexpr uint16_t net_version = proto_explicit_sync; + constexpr uint16_t net_version = proto_pruned_types; /** * Index by start_block_num @@ -574,7 +578,7 @@ namespace eosio { int16_t sent_handshake_count = 0; std::atomic connecting{true}; std::atomic syncing{false}; - uint16_t protocol_version = 0; + std::atomic protocol_version = 0; uint16_t consecutive_rejected_blocks = 0; std::atomic consecutive_immediate_connection_close = 0; @@ -1247,8 +1251,14 @@ namespace eosio { // this implementation is to avoid copy of signed_block to net_message // matches which of net_message for signed_block fc_dlog( logger, "sending block ${bn}", ("bn", sb->block_num()) ); - auto sb_v0 = sb->to_signed_block_v0(); - return create_send_buffer( signed_block_v0_which, *sb_v0 ); + return create_send_buffer( signed_block_which, *sb ); + } + + static std::shared_ptr> create_send_buffer( const signed_block_v0& sb_v0 ) { + // this implementation is to avoid copy of signed_block_v0 to net_message + // matches which of net_message for signed_block_v0 + fc_dlog( logger, "sending block ${bn}", ("bn", sb_v0.block_num()) ); + return create_send_buffer( signed_block_v0_which, sb_v0 ); } static std::shared_ptr> create_send_buffer( const packed_transaction& trx ) { @@ -1938,27 +1948,39 @@ namespace eosio { fc_dlog( logger, "bcast block ${b}", ("b", b->block_num()) ); if( my_impl->sync_master->syncing_with_peer() ) return; - - bool have_connection = false; - for_each_block_connection( [&have_connection]( auto& cp ) { - peer_dlog( cp, "socket_is_open ${s}, connecting ${c}, syncing ${ss}", - ("s", cp->socket_is_open())("c", cp->connecting.load())("ss", cp->syncing.load()) ); - if( !cp->current() ) { - return true; + std::shared_ptr> send_buffer, send_buffer_v0; + bool valid_v0_block = b->prune_state != signed_block::prune_state_type::incomplete; + auto get_send_buffer = [&](const connection_ptr& cp) { + const uint16_t v = cp->protocol_version.load(); + if( v >= proto_pruned_types ) { + if( !send_buffer ) { + send_buffer = create_send_buffer( b ); + } + return send_buffer; + } else { + if( !send_buffer_v0 ) { + const auto sb_v0 = valid_v0_block ? b->to_signed_block_v0() : signed_block_v0_uptr(); + if( !sb_v0 ) { + valid_v0_block = false; + // unable to convert to v0 signed block and client doesn't support proto_pruned_types, so tell it to go away + cp->enqueue( go_away_message( fatal_other ) ); + return send_buffer_v0; + } + send_buffer_v0 = create_send_buffer( *sb_v0 ); + } + return send_buffer_v0; } - have_connection = true; - return false; - } ); + }; - if( !have_connection ) return; - std::shared_ptr> send_buffer = create_send_buffer( b ); + for_each_block_connection( [this, &id, bnum = b->block_num(), &get_send_buffer]( auto& cp ) { + peer_dlog( cp, "socket_is_open ${s}, connecting ${c}, syncing ${ss}", + ("s", cp->socket_is_open())("c", cp->connecting.load())("ss", cp->syncing.load()) ); + if( !cp->current() ) return true; + std::shared_ptr> sb = get_send_buffer( cp ); + if( !sb ) return true; - for_each_block_connection( [this, &id, bnum = b->block_num(), &send_buffer]( auto& cp ) { - if( !cp->current() ) { - return true; - } - cp->strand.post( [this, cp, id, bnum, send_buffer]() { + cp->strand.post( [this, cp, id, bnum, sb]() { std::unique_lock g_conn( cp->conn_mtx ); bool has_block = cp->last_handshake_recv.last_irreversible_block_num >= bnum; g_conn.unlock(); @@ -1968,7 +1990,7 @@ namespace eosio { return; } fc_dlog( logger, "bcast block ${b} to ${p}", ("b", bnum)("p", cp->peer_name()) ); - cp->enqueue_buffer( send_buffer, no_reason ); + cp->enqueue_buffer( sb, no_reason ); } }); return true; @@ -2610,7 +2632,7 @@ namespace eosio { protocol_version = my_impl->to_protocol_version(msg.network_version); if( protocol_version != net_version ) { fc_ilog( logger, "Local network version: ${nv} Remote version: ${mnv}", - ("nv", net_version)( "mnv", protocol_version ) ); + ("nv", net_version)("mnv", protocol_version.load()) ); } g_conn.lock(); From 8ccc1096d9fe8655b591e2d896bbf3022b2dd527 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 17 Apr 2020 18:16:13 -0500 Subject: [PATCH 092/142] Add handling of block types to enqueue_block --- plugins/net_plugin/net_plugin.cpp | 70 ++++++++++++++++++------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index ecd1f5ecd33..e19412a6d98 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -181,7 +181,6 @@ namespace eosio { void bcast_transaction(const packed_transaction& trx); void rejected_transaction(const packed_transaction_ptr& trx, uint32_t head_blk_num); void bcast_block( const signed_block_ptr& b, const block_id_type& id ); - void bcast_notice( const block_id_type& id ); void rejected_block(const block_id_type& id); void recv_block(const connection_ptr& conn, const block_id_type& msg, uint32_t bnum); @@ -1257,7 +1256,7 @@ namespace eosio { static std::shared_ptr> create_send_buffer( const signed_block_v0& sb_v0 ) { // this implementation is to avoid copy of signed_block_v0 to net_message // matches which of net_message for signed_block_v0 - fc_dlog( logger, "sending block ${bn}", ("bn", sb_v0.block_num()) ); + fc_dlog( logger, "sending v0 block ${bn}", ("bn", sb_v0.block_num()) ); return create_send_buffer( signed_block_v0_which, sb_v0 ); } @@ -1268,10 +1267,42 @@ namespace eosio { return create_send_buffer( packed_transaction_v0_which, *pt_v0 ); } - void connection::enqueue_block( const signed_block_ptr& sb, bool to_sync_queue) { - fc_dlog( logger, "enqueue block ${num}", ("num", sb->block_num()) ); + static std::shared_ptr> get_send_buffer( connection& c, + const signed_block_ptr& b, + std::shared_ptr>& send_buffer, + std::shared_ptr>& send_buffer_v0, + bool& valid_v0_block ) + { + const uint16_t v = c.protocol_version.load(); + if( v >= proto_pruned_types ) { + if( !send_buffer ) { + send_buffer = create_send_buffer( b ); + } + return send_buffer; + } else { + if( !send_buffer_v0 ) { + const auto sb_v0 = valid_v0_block ? b->to_signed_block_v0() : signed_block_v0_uptr(); + if( !sb_v0 ) { + valid_v0_block = false; + // unable to convert to v0 signed block and client doesn't support proto_pruned_types, so tell it to go away + c.enqueue( go_away_message( fatal_other ) ); + return send_buffer_v0; + } + send_buffer_v0 = create_send_buffer( *sb_v0 ); + } + return send_buffer_v0; + } + } + + void connection::enqueue_block( const signed_block_ptr& b, bool to_sync_queue) { + fc_dlog( logger, "enqueue block ${num}", ("num", b->block_num()) ); verify_strand_in_this_thread( strand, __func__, __LINE__ ); - enqueue_buffer( create_send_buffer( sb ), no_reason, to_sync_queue); + + std::shared_ptr> send_buffer, send_buffer_v0; + bool valid_v0_block = b->prune_state != signed_block::prune_state_type::incomplete; + auto sb = get_send_buffer( *this, b, send_buffer, send_buffer_v0, valid_v0_block ); + if( !sb ) return; + enqueue_buffer( sb, no_reason, to_sync_queue); } void connection::enqueue_buffer( const std::shared_ptr>& send_buffer, @@ -1951,36 +1982,15 @@ namespace eosio { std::shared_ptr> send_buffer, send_buffer_v0; bool valid_v0_block = b->prune_state != signed_block::prune_state_type::incomplete; - auto get_send_buffer = [&](const connection_ptr& cp) { - const uint16_t v = cp->protocol_version.load(); - if( v >= proto_pruned_types ) { - if( !send_buffer ) { - send_buffer = create_send_buffer( b ); - } - return send_buffer; - } else { - if( !send_buffer_v0 ) { - const auto sb_v0 = valid_v0_block ? b->to_signed_block_v0() : signed_block_v0_uptr(); - if( !sb_v0 ) { - valid_v0_block = false; - // unable to convert to v0 signed block and client doesn't support proto_pruned_types, so tell it to go away - cp->enqueue( go_away_message( fatal_other ) ); - return send_buffer_v0; - } - send_buffer_v0 = create_send_buffer( *sb_v0 ); - } - return send_buffer_v0; - } - }; - - for_each_block_connection( [this, &id, bnum = b->block_num(), &get_send_buffer]( auto& cp ) { + const auto bnum = b->block_num(); + for_each_block_connection( [this, &id, &bnum, &b, &send_buffer, &send_buffer_v0, &valid_v0_block]( auto& cp ) { peer_dlog( cp, "socket_is_open ${s}, connecting ${c}, syncing ${ss}", ("s", cp->socket_is_open())("c", cp->connecting.load())("ss", cp->syncing.load()) ); if( !cp->current() ) return true; - std::shared_ptr> sb = get_send_buffer( cp ); + std::shared_ptr> sb = get_send_buffer( *cp, b, send_buffer, send_buffer_v0, valid_v0_block ); if( !sb ) return true; - cp->strand.post( [this, cp, id, bnum, sb]() { + cp->strand.post( [this, cp, id, bnum, sb{std::move(sb)}]() { std::unique_lock g_conn( cp->conn_mtx ); bool has_block = cp->last_handshake_recv.last_irreversible_block_num >= bnum; g_conn.unlock(); From 947f6cbb063421386566ef489642c4a0dc2cbae1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Sat, 18 Apr 2020 13:49:40 -0500 Subject: [PATCH 093/142] Support receiving signed_block and signed_block_v0 --- plugins/net_plugin/net_plugin.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index e19412a6d98..c15acb0a8c0 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2412,7 +2412,7 @@ namespace eosio { auto peek_ds = pending_message_buffer.create_peek_datastream(); unsigned_int which{}; fc::raw::unpack( peek_ds, which ); - if( which == signed_block_v0_which ) { + if( which == signed_block_which || which == signed_block_v0_which ) { block_header bh; fc::raw::unpack( peek_ds, bh ); @@ -2453,10 +2453,16 @@ namespace eosio { } auto ds = pending_message_buffer.create_datastream(); - fc::raw::unpack( ds, which ); // throw away - signed_block_v0 sb_v0; - fc::raw::unpack( ds, sb_v0 ); - shared_ptr ptr = std::make_shared( std::move( sb_v0 ), true ); + fc::raw::unpack( ds, which ); + shared_ptr ptr; + if( which == signed_block_which ) { + ptr = std::make_shared(); + fc::raw::unpack( ds, *ptr ); + } else { + signed_block_v0 sb_v0; + fc::raw::unpack( ds, sb_v0 ); + ptr = std::make_shared( std::move( sb_v0 ), true ); + } auto is_webauthn_sig = []( const fc::crypto::signature& s ) { return s.which() == fc::crypto::signature::storage_type::position(); From 98201b4df24de76c896f7a97f950f115467369ea Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Sat, 18 Apr 2020 15:35:14 -0500 Subject: [PATCH 094/142] Add support for trx_message_v1 both send/receive --- .../include/eosio/net_plugin/protocol.hpp | 2 +- plugins/net_plugin/net_plugin.cpp | 312 +++++++++++------- 2 files changed, 197 insertions(+), 117 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp index d52dfc68423..3acec84fcbf 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp @@ -145,7 +145,7 @@ namespace eosio { struct trx_message_v1 { fc::optional trx_id; // only provided for large trx as trade-off for small trxs not worth it - packed_transaction trx; + std::shared_ptr trx; }; using net_message = static_variant> create_send_buffer( const packed_transaction& trx ) { - // this implementation is to avoid copy of packed_transaction to net_message - // matches which of net_message for packed_transaction - auto pt_v0 = trx.to_packed_transaction_v0(); - return create_send_buffer( packed_transaction_v0_which, *pt_v0 ); + static std::shared_ptr> create_send_buffer( const packed_transaction_ptr& trx ) { + // const cast required, trx_message_v1 has non-const shared_ptr because FC_REFLECT does not work with const types + fc::optional trx_id; + if( trx->get_estimated_size() > 7 * sizeof(transaction_id_type) ) { + fc_dlog( logger, "including trx id, est size: ${es}", ("es", trx->get_estimated_size()) ); + trx_id = trx->id(); + } + trx_message_v1 v1{ std::move(trx_id), std::const_pointer_cast(trx) }; + return create_send_buffer( trx_message_v1_which, v1 ); + } + + static std::shared_ptr> create_send_buffer( const packed_transaction_v0& trx ) { + // this implementation is to avoid copy of packed_transaction_v0 to net_message + // matches which of net_message for packed_transaction_v0 + return create_send_buffer( packed_transaction_v0_which, trx ); } static std::shared_ptr> get_send_buffer( connection& c, @@ -1912,6 +1926,22 @@ namespace eosio { return added; } + bool dispatch_manager::add_peer_txn( const transaction_id_type& tid, uint32_t connection_id ) { + std::lock_guard g( local_txns_mtx ); + auto tptr = local_txns.get().find( tid ); + if( tptr == local_txns.end() ) return false; + const auto expiration = tptr->expires; + + tptr = local_txns.get().find( std::make_tuple( std::ref( tid ), connection_id ) ); + bool added = (tptr == local_txns.end()); + if( added ) { + node_transaction_state nts = {tid, expiration, 0, connection_id}; + local_txns.insert( std::move( nts ) ); + } + return added; + } + + // thread safe void dispatch_manager::update_txns_block_num( const signed_block_ptr& sb ) { update_block_num ubn( sb->block_num() ); @@ -2028,13 +2058,13 @@ namespace eosio { fc_dlog( logger, "rejected block ${id}", ("id", id) ); } - void dispatch_manager::bcast_transaction(const packed_transaction& trx) { - const auto& id = trx.id(); - time_point_sec trx_expiration = trx.expiration(); + void dispatch_manager::bcast_transaction(const packed_transaction_ptr& trx) { + const auto& id = trx->id(); + time_point_sec trx_expiration = trx->expiration(); node_transaction_state nts = {id, trx_expiration, 0, 0}; - std::shared_ptr> send_buffer; - for_each_connection( [this, &trx, &nts, &send_buffer]( auto& cp ) { + std::shared_ptr> send_buffer, send_buffer_v0; + for_each_connection( [this, &trx, &nts, &send_buffer, &send_buffer_v0]( auto& cp ) { if( cp->is_blocks_only_connection() || !cp->current() ) { return true; } @@ -2042,13 +2072,24 @@ namespace eosio { if( !add_peer_txn(nts) ) { return true; } - if( !send_buffer ) { - send_buffer = create_send_buffer( trx ); - } - cp->strand.post( [cp, send_buffer]() { + std::shared_ptr> sb; + const uint16_t v = cp->protocol_version.load(); + if( v >= proto_pruned_types ) { + if( !send_buffer ) { + send_buffer = create_send_buffer( trx ); + } + sb = send_buffer; + } else { + if( !send_buffer_v0 ) { + packed_transaction_v0_ptr v0 = trx->to_packed_transaction_v0(); + send_buffer_v0 = create_send_buffer( *v0 ); + } + sb = send_buffer_v0; + } + cp->strand.post( [cp, sb]() { fc_dlog( logger, "sending trx to ${n}", ("n", cp->peer_name()) ); - cp->enqueue_buffer( send_buffer, no_reason ); + cp->enqueue_buffer( sb, no_reason ); } ); return true; } ); @@ -2413,90 +2454,10 @@ namespace eosio { unsigned_int which{}; fc::raw::unpack( peek_ds, which ); if( which == signed_block_which || which == signed_block_v0_which ) { - block_header bh; - fc::raw::unpack( peek_ds, bh ); - - const block_id_type blk_id = bh.calculate_id(); - const uint32_t blk_num = bh.block_num(); - if( my_impl->dispatcher->have_block( blk_id ) ) { - fc_dlog( logger, "canceling wait on ${p}, already received block ${num}, id ${id}...", - ("p", peer_name())("num", blk_num)("id", blk_id.str().substr(8,16)) ); - my_impl->sync_master->sync_recv_block( shared_from_this(), blk_id, blk_num, false ); - cancel_wait(); - - pending_message_buffer.advance_read_ptr( message_length ); - return true; - } - fc_dlog( logger, "${p} received block ${num}, id ${id}..., latency: ${latency}", - ("p", peer_name())("num", bh.block_num())("id", blk_id.str().substr(8,16)) - ("latency", (fc::time_point::now() - bh.timestamp).count()/1000) ); - if( !my_impl->sync_master->syncing_with_peer() ) { // guard against peer thinking it needs to send us old blocks - uint32_t lib = 0; - std::tie( lib, std::ignore, std::ignore, std::ignore, std::ignore, std::ignore ) = my_impl->get_chain_info(); - if( blk_num < lib ) { - std::unique_lock g( conn_mtx ); - const auto last_sent_lib = last_handshake_sent.last_irreversible_block_num; - g.unlock(); - if( blk_num < last_sent_lib ) { - fc_ilog( logger, "received block ${n} less than sent lib ${lib}", ("n", blk_num)("lib", last_sent_lib) ); - close(); - } else { - fc_ilog( logger, "received block ${n} less than lib ${lib}", ("n", blk_num)("lib", lib) ); - enqueue( (sync_request_message) {0, 0} ); - send_handshake(); - cancel_wait(); - } + return process_next_block_message( message_length ); - pending_message_buffer.advance_read_ptr( message_length ); - return true; - } - } - - auto ds = pending_message_buffer.create_datastream(); - fc::raw::unpack( ds, which ); - shared_ptr ptr; - if( which == signed_block_which ) { - ptr = std::make_shared(); - fc::raw::unpack( ds, *ptr ); - } else { - signed_block_v0 sb_v0; - fc::raw::unpack( ds, sb_v0 ); - ptr = std::make_shared( std::move( sb_v0 ), true ); - } - - auto is_webauthn_sig = []( const fc::crypto::signature& s ) { - return s.which() == fc::crypto::signature::storage_type::position(); - }; - bool has_webauthn_sig = is_webauthn_sig( ptr->producer_signature ); - - constexpr auto additional_sigs_eid = additional_block_signatures_extension::extension_id(); - auto exts = ptr->validate_and_extract_extensions(); - if( exts.count( additional_sigs_eid ) ) { - const auto &additional_sigs = exts.lower_bound( additional_sigs_eid )->second.get().signatures; - has_webauthn_sig |= std::any_of( additional_sigs.begin(), additional_sigs.end(), is_webauthn_sig ); - } - - if( has_webauthn_sig ) { - fc_dlog( logger, "WebAuthn signed block received from ${p}, closing connection", ("p", peer_name())); - close(); - return false; - } - - handle_message( blk_id, std::move( ptr ) ); - - } else if( which == packed_transaction_v0_which ) { - if( !my_impl->p2p_accept_transactions ) { - fc_dlog( logger, "p2p-accept-transaction=false - dropping txn" ); - pending_message_buffer.advance_read_ptr( message_length ); - return true; - } - - auto ds = pending_message_buffer.create_datastream(); - fc::raw::unpack( ds, which ); // throw away - packed_transaction_v0 pt_v0; - fc::raw::unpack( ds, pt_v0 ); - shared_ptr ptr = std::make_shared( pt_v0, true ); - handle_message( std::move( ptr ) ); + } else if( which == trx_message_v1_which || which == packed_transaction_v0_which ) { + return process_next_trx_message( message_length ); } else { auto ds = pending_message_buffer.create_datastream(); @@ -2515,6 +2476,141 @@ namespace eosio { return true; } + // called from connection strand + bool connection::process_next_block_message(uint32_t message_length) { + auto peek_ds = pending_message_buffer.create_peek_datastream(); + unsigned_int which{}; + fc::raw::unpack( peek_ds, which ); // throw away + block_header bh; + fc::raw::unpack( peek_ds, bh ); + + const block_id_type blk_id = bh.calculate_id(); + const uint32_t blk_num = bh.block_num(); + if( my_impl->dispatcher->have_block( blk_id ) ) { + fc_dlog( logger, "canceling wait on ${p}, already received block ${num}, id ${id}...", + ("p", peer_name())("num", blk_num)("id", blk_id.str().substr(8,16)) ); + my_impl->sync_master->sync_recv_block( shared_from_this(), blk_id, blk_num, false ); + cancel_wait(); + + pending_message_buffer.advance_read_ptr( message_length ); + return true; + } + fc_dlog( logger, "${p} received block ${num}, id ${id}..., latency: ${latency}", + ("p", peer_name())("num", bh.block_num())("id", blk_id.str().substr(8,16)) + ("latency", (fc::time_point::now() - bh.timestamp).count()/1000) ); + if( !my_impl->sync_master->syncing_with_peer() ) { // guard against peer thinking it needs to send us old blocks + uint32_t lib = 0; + std::tie( lib, std::ignore, std::ignore, std::ignore, std::ignore, std::ignore ) = my_impl->get_chain_info(); + if( blk_num < lib ) { + std::unique_lock g( conn_mtx ); + const auto last_sent_lib = last_handshake_sent.last_irreversible_block_num; + g.unlock(); + if( blk_num < last_sent_lib ) { + fc_ilog( logger, "received block ${n} less than sent lib ${lib}", ("n", blk_num)("lib", last_sent_lib) ); + close(); + } else { + fc_ilog( logger, "received block ${n} less than lib ${lib}", ("n", blk_num)("lib", lib) ); + enqueue( (sync_request_message) {0, 0} ); + send_handshake(); + cancel_wait(); + } + + pending_message_buffer.advance_read_ptr( message_length ); + return true; + } + } + + auto ds = pending_message_buffer.create_datastream(); + fc::raw::unpack( ds, which ); + shared_ptr ptr; + if( which == signed_block_which ) { + ptr = std::make_shared(); + fc::raw::unpack( ds, *ptr ); + } else { + signed_block_v0 sb_v0; + fc::raw::unpack( ds, sb_v0 ); + ptr = std::make_shared( std::move( sb_v0 ), true ); + } + + auto is_webauthn_sig = []( const fc::crypto::signature& s ) { + return s.which() == fc::crypto::signature::storage_type::position(); + }; + bool has_webauthn_sig = is_webauthn_sig( ptr->producer_signature ); + + constexpr auto additional_sigs_eid = additional_block_signatures_extension::extension_id(); + auto exts = ptr->validate_and_extract_extensions(); + if( exts.count( additional_sigs_eid ) ) { + const auto &additional_sigs = exts.lower_bound( additional_sigs_eid )->second.get().signatures; + has_webauthn_sig |= std::any_of( additional_sigs.begin(), additional_sigs.end(), is_webauthn_sig ); + } + + if( has_webauthn_sig ) { + fc_dlog( logger, "WebAuthn signed block received from ${p}, closing connection", ("p", peer_name())); + close(); + return false; + } + + handle_message( blk_id, std::move( ptr ) ); + return true; + } + + // called from connection strand + bool connection::process_next_trx_message(uint32_t message_length) { + if( !my_impl->p2p_accept_transactions ) { + fc_dlog( logger, "p2p-accept-transaction=false - dropping txn" ); + pending_message_buffer.advance_read_ptr( message_length ); + return true; + } + + uint32_t trx_in_progress_sz = this->trx_in_progress_size.load(); + if( trx_in_progress_sz > def_max_trx_in_progress_size ) { + fc_wlog( logger, "Dropping trx, too many trx in progress ${s} bytes", ("s", trx_in_progress_sz) ); + return true; + } + + bool have_trx = false; + shared_ptr ptr; + auto ds = pending_message_buffer.create_datastream(); + unsigned_int which{}; + fc::raw::unpack( ds, which ); + if( which == trx_message_v1_which ) { + fc::optional trx_id; + fc::raw::unpack( ds, trx_id ); + if( trx_id ) { + have_trx = my_impl->dispatcher->have_txn( *trx_id ); + } + if( have_trx ) { + my_impl->dispatcher->add_peer_txn( *trx_id, connection_id ); + } else { + std::shared_ptr trx; + fc::raw::unpack( ds, trx ); + ptr = std::move( trx ); + EOS_ASSERT( !trx_id || *trx_id == ptr->id(), transaction_id_type_exception, + "Provided trx_id does not match provided packed_transaction" ); + node_transaction_state nts = {ptr->id(), ptr->expiration(), 0, connection_id}; + my_impl->dispatcher->add_peer_txn( nts ); + } + + } else { + packed_transaction_v0 pt_v0; + fc::raw::unpack( ds, pt_v0 ); + have_trx = my_impl->dispatcher->have_txn( pt_v0.id() ); + node_transaction_state nts = {pt_v0.id(), pt_v0.expiration(), 0, connection_id}; + my_impl->dispatcher->add_peer_txn( nts ); + if ( !have_trx ) { + ptr = std::make_shared( pt_v0, true ); + } + } + + if( have_trx ) { + fc_dlog( logger, "got a duplicate transaction - dropping trx" ); + return true; + } + + handle_message( std::move( ptr ) ); + return true; + } + // call only from main application thread void net_plugin_impl::update_chain_info() { controller& cc = chain_plug->chain(); @@ -2884,22 +2980,6 @@ namespace eosio { const auto& tid = trx->id(); peer_dlog( this, "received packed_transaction ${id}", ("id", tid) ); - uint32_t trx_in_progress_sz = this->trx_in_progress_size.load(); - if( trx_in_progress_sz > def_max_trx_in_progress_size ) { - fc_wlog( logger, "Dropping trx ${id}, too many trx in progress ${s} bytes", - ("id", tid)("s", trx_in_progress_sz) ); - return; - } - - bool have_trx = my_impl->dispatcher->have_txn( tid ); - node_transaction_state nts = {tid, trx->expiration(), 0, connection_id}; - my_impl->dispatcher->add_peer_txn( nts ); - - if( have_trx ) { - fc_dlog( logger, "got a duplicate transaction - dropping ${id}", ("id", tid) ); - return; - } - trx_in_progress_size += calc_trx_size( trx ); app().post( priority::low, [trx{std::move(trx)}, weak = weak_from_this()]() { my_impl->chain_plug->accept_transaction( trx, @@ -3169,7 +3249,7 @@ namespace eosio { dispatcher->rejected_transaction(results.second->packed_trx(), head_blk_num); } else { fc_dlog( logger, "signaled ACK, trx-id = ${id}", ("id", id) ); - dispatcher->bcast_transaction(*results.second->packed_trx()); + dispatcher->bcast_transaction(results.second->packed_trx()); } }); } From de97b3c8a9e873e5d006ad2b918ad79e9d2a34c3 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Sat, 18 Apr 2020 16:44:27 -0500 Subject: [PATCH 095/142] Advance read ptr on already received trx --- plugins/net_plugin/net_plugin.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 754dffd17c2..3f66f6c9bc6 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -2565,12 +2565,14 @@ namespace eosio { uint32_t trx_in_progress_sz = this->trx_in_progress_size.load(); if( trx_in_progress_sz > def_max_trx_in_progress_size ) { fc_wlog( logger, "Dropping trx, too many trx in progress ${s} bytes", ("s", trx_in_progress_sz) ); + pending_message_buffer.advance_read_ptr( message_length ); return true; } bool have_trx = false; shared_ptr ptr; auto ds = pending_message_buffer.create_datastream(); + const auto buff_size_start = pending_message_buffer.bytes_to_read(); unsigned_int which{}; fc::raw::unpack( ds, which ); if( which == trx_message_v1_which ) { @@ -2581,6 +2583,8 @@ namespace eosio { } if( have_trx ) { my_impl->dispatcher->add_peer_txn( *trx_id, connection_id ); + const auto buff_size_current = pending_message_buffer.bytes_to_read(); + pending_message_buffer.advance_read_ptr( message_length - (buff_size_start - buff_size_current) ); } else { std::shared_ptr trx; fc::raw::unpack( ds, trx ); @@ -2603,7 +2607,7 @@ namespace eosio { } if( have_trx ) { - fc_dlog( logger, "got a duplicate transaction - dropping trx" ); + fc_dlog( logger, "got a duplicate transaction - dropping" ); return true; } From da2d33e6e25264ce6a05a537ac012b7d4578f9c5 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 20 Apr 2020 08:26:49 -0500 Subject: [PATCH 096/142] Add p2p-reject-incomplete-blocks option. --- plugins/net_plugin/net_plugin.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 3f66f6c9bc6..67355b1e900 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -238,6 +238,7 @@ namespace eosio { uint32_t max_client_count = 0; uint32_t max_nodes_per_host = 1; bool p2p_accept_transactions = true; + bool p2p_reject_incomplete_blocks = true; /// Peer clock may be no more than 1 second skewed from our clock, including network latency. const std::chrono::system_clock::duration peer_authentication_interval{std::chrono::seconds{1}}; @@ -1297,6 +1298,8 @@ namespace eosio { if( !send_buffer_v0 ) { const auto sb_v0 = valid_v0_block ? b->to_signed_block_v0() : signed_block_v0_uptr(); if( !sb_v0 ) { + peer_wlog( (&c), "Sending go away for incomplete block #${n} ${id}...", + ("n", b->block_num())("id", b->calculate_id().str().substr(8,16)) ); valid_v0_block = false; // unable to convert to v0 signed block and client doesn't support proto_pruned_types, so tell it to go away c.enqueue( go_away_message( fatal_other ) ); @@ -3010,6 +3013,14 @@ namespace eosio { // called from connection strand void connection::handle_message( const block_id_type& id, signed_block_ptr ptr ) { peer_dlog( this, "received signed_block ${id}", ("id", ptr->block_num() ) ); + if( my_impl->p2p_reject_incomplete_blocks ) { + if( ptr->prune_state == signed_block::prune_state_type::incomplete ) { + peer_wlog( this, "Sending go away for incomplete block #${n} ${id}...", + ("n", ptr->block_num())("id", id.str().substr(8,16)) ); + enqueue( go_away_message( fatal_other ) ); + return; + } + } auto priority = my_impl->sync_master->syncing_with_peer() ? priority::medium : priority::high; app().post(priority, [ptr{std::move(ptr)}, id, c = shared_from_this()]() mutable { c->process_signed_block( id, std::move( ptr ) ); @@ -3398,6 +3409,7 @@ namespace eosio { " p2p.blk.eos.io:9876:blk\n") ( "p2p-max-nodes-per-host", bpo::value()->default_value(def_max_nodes_per_host), "Maximum number of client nodes from any single IP address") ( "p2p-accept-transactions", bpo::value()->default_value(true), "Allow transactions received over p2p network to be evaluated and relayed if valid.") + ( "p2p-reject-incomplete-blocks", bpo::value()->default_value(true), "Reject pruned signed_blocks even in light validation") ( "agent-name", bpo::value()->default_value("EOS Test Agent"), "The name supplied to identify this node amongst the peers.") ( "allowed-connection", bpo::value>()->multitoken()->default_value({"any"}, "any"), "Can be 'any' or 'producers' or 'specified' or 'none'. If 'specified', peer-key must be specified at least once. If only 'producers', peer-key is not required. 'producers' and 'specified' may be combined.") ( "peer-key", bpo::value>()->composing()->multitoken(), "Optional public key of peer allowed to connect. May be used multiple times.") From af64b09566b90d2f28ec66945d746692412a2b37 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 20 Apr 2020 11:57:22 -0500 Subject: [PATCH 097/142] Protect agaisnt null packed_transaction_ptr in trx_message_v1. Add more resonable limit for adding trx id to trx_message_v1. --- plugins/net_plugin/net_plugin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 67355b1e900..e00b019c2be 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -1268,7 +1268,7 @@ namespace eosio { static std::shared_ptr> create_send_buffer( const packed_transaction_ptr& trx ) { // const cast required, trx_message_v1 has non-const shared_ptr because FC_REFLECT does not work with const types fc::optional trx_id; - if( trx->get_estimated_size() > 7 * sizeof(transaction_id_type) ) { + if( trx->get_estimated_size() > 1024 ) { // simple guess on threshold fc_dlog( logger, "including trx id, est size: ${es}", ("es", trx->get_estimated_size()) ); trx_id = trx->id(); } @@ -2592,7 +2592,7 @@ namespace eosio { std::shared_ptr trx; fc::raw::unpack( ds, trx ); ptr = std::move( trx ); - EOS_ASSERT( !trx_id || *trx_id == ptr->id(), transaction_id_type_exception, + EOS_ASSERT( !trx_id || !ptr || *trx_id == ptr->id(), transaction_id_type_exception, "Provided trx_id does not match provided packed_transaction" ); node_transaction_state nts = {ptr->id(), ptr->expiration(), 0, connection_id}; my_impl->dispatcher->add_peer_txn( nts ); From 57906da3098f960b255b89e2ef74c6d56d307ee2 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 20 Apr 2020 14:24:50 -0500 Subject: [PATCH 098/142] Remove generic message placeholders as they will come in after not before this change. --- .../include/eosio/net_plugin/protocol.hpp | 16 ++-------------- plugins/net_plugin/net_plugin.cpp | 14 +++++++++----- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp index 3acec84fcbf..85e1420aa50 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp @@ -135,14 +135,6 @@ namespace eosio { uint32_t end_block{0}; }; - struct generic_message { - go_away_message tmp; // until merge of generic message support - }; - - struct generic_support_message { - go_away_message tmp; // until merge of generic message support - }; - struct trx_message_v1 { fc::optional trx_id; // only provided for large trx as trade-off for small trxs not worth it std::shared_ptr trx; @@ -157,10 +149,8 @@ namespace eosio { sync_request_message, signed_block_v0, // which = 7 packed_transaction_v0, // which = 8 - generic_message, // which = 9 - generic_support_message, // which = 10 - signed_block, // which = 11 - trx_message_v1>; // which = 12 + signed_block, // which = 9 + trx_message_v1>; // which = 10 } // namespace eosio @@ -179,8 +169,6 @@ FC_REFLECT( eosio::time_message, (org)(rec)(xmt)(dst) ) FC_REFLECT( eosio::notice_message, (known_trx)(known_blocks) ) FC_REFLECT( eosio::request_message, (req_trx)(req_blocks) ) FC_REFLECT( eosio::sync_request_message, (start_block)(end_block) ) -FC_REFLECT( eosio::generic_message, (tmp) ) -FC_REFLECT( eosio::generic_support_message, (tmp) ) FC_REFLECT( eosio::trx_message_v1, (trx_id)(trx) ) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index e00b019c2be..eed80cfd67e 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -393,8 +393,8 @@ namespace eosio { constexpr auto message_header_size = 4; constexpr uint32_t signed_block_v0_which = 7; // see protocol net_message constexpr uint32_t packed_transaction_v0_which = 8; // see protocol net_message - constexpr uint32_t signed_block_which = 11; // see protocol net_message - constexpr uint32_t trx_message_v1_which = 12; // see protocol net_message + constexpr uint32_t signed_block_which = 9; // see protocol net_message + constexpr uint32_t trx_message_v1_which = 10; // see protocol net_message /** * For a while, network version was a 16 bit value equal to the second set of 16 bits @@ -417,8 +417,8 @@ namespace eosio { constexpr uint16_t proto_base = 0; constexpr uint16_t proto_explicit_sync = 1; // version at time of eosio 1.0 constexpr uint16_t proto_block_id_notify = 2; // reserved. feature was removed. next net_version should be 3 - constexpr uint16_t proto_generic_messages = 3; // placeholder for generic messages - constexpr uint16_t proto_pruned_types = 4; // supports new signed_block & packed_transaction types + constexpr uint16_t proto_pruned_types = 3; // supports new signed_block & packed_transaction types + constexpr uint16_t proto_generic_messages = 4; // placeholder for generic messages constexpr uint16_t net_version = proto_pruned_types; @@ -1252,6 +1252,7 @@ namespace eosio { } static std::shared_ptr> create_send_buffer( const signed_block_ptr& sb ) { + static_assert( signed_block_which == net_message::position() ); // this implementation is to avoid copy of signed_block to net_message // matches which of net_message for signed_block fc_dlog( logger, "sending block ${bn}", ("bn", sb->block_num()) ); @@ -1259,6 +1260,7 @@ namespace eosio { } static std::shared_ptr> create_send_buffer( const signed_block_v0& sb_v0 ) { + static_assert( signed_block_v0_which == net_message::position() ); // this implementation is to avoid copy of signed_block_v0 to net_message // matches which of net_message for signed_block_v0 fc_dlog( logger, "sending v0 block ${bn}", ("bn", sb_v0.block_num()) ); @@ -1266,17 +1268,19 @@ namespace eosio { } static std::shared_ptr> create_send_buffer( const packed_transaction_ptr& trx ) { - // const cast required, trx_message_v1 has non-const shared_ptr because FC_REFLECT does not work with const types + static_assert( trx_message_v1_which == net_message::position() ); fc::optional trx_id; if( trx->get_estimated_size() > 1024 ) { // simple guess on threshold fc_dlog( logger, "including trx id, est size: ${es}", ("es", trx->get_estimated_size()) ); trx_id = trx->id(); } + // const cast required, trx_message_v1 has non-const shared_ptr because FC_REFLECT does not work with const types trx_message_v1 v1{ std::move(trx_id), std::const_pointer_cast(trx) }; return create_send_buffer( trx_message_v1_which, v1 ); } static std::shared_ptr> create_send_buffer( const packed_transaction_v0& trx ) { + static_assert( packed_transaction_v0_which == net_message::position() ); // this implementation is to avoid copy of packed_transaction_v0 to net_message // matches which of net_message for packed_transaction_v0 return create_send_buffer( packed_transaction_v0_which, trx ); From aa5203969508a44f3e268e02fbcb4bc0253771c8 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 16 Apr 2020 12:05:31 -0500 Subject: [PATCH 099/142] Implement SHiP v1 --- .../chain/include/eosio/chain/exceptions.hpp | 4 + .../chain/include/eosio/chain/transaction.hpp | 1 + libraries/chain/transaction.cpp | 3 + libraries/state_history/CMakeLists.txt | 1 + .../include/eosio/state_history/log.hpp | 283 ++++------------ .../eosio/state_history/serialization.hpp | 35 +- .../eosio/state_history/trace_converter.hpp | 23 +- .../include/eosio/state_history/types.hpp | 118 +++++++ libraries/state_history/log.cpp | 312 ++++++++++++++++++ libraries/state_history/trace_converter.cpp | 274 ++++++++++++++- .../state_history_plugin.cpp | 61 +--- unittests/CMakeLists.txt | 2 +- unittests/state_history_tests.cpp | 203 ++++++++++++ 13 files changed, 1026 insertions(+), 294 deletions(-) create mode 100644 libraries/state_history/log.cpp create mode 100644 unittests/state_history_tests.cpp diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index eef2238eb7d..a4ec430de4a 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -161,6 +161,7 @@ namespace eosio { namespace chain { * |- resource_limit_exception * |- mongo_db_exception * |- contract_api_exception + * |- history_state_exception */ FC_DECLARE_DERIVED_EXCEPTION( chain_type_exception, chain_exception, @@ -656,5 +657,8 @@ namespace eosio { namespace chain { 3270002, "Protocol feature exception (invalid block)" ) FC_DECLARE_DERIVED_EXCEPTION( pruned_context_free_data_bad_block_exception, objective_block_validation_exception, 3270003, "Context free data pruned (invalid block)" ) + + FC_DECLARE_DERIVED_EXCEPTION( state_history_exception, chain_exception, + 3280000, "History state exception" ) } } // eosio::chain diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 5e5f550fad6..c7dd65617fb 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -246,6 +246,7 @@ namespace eosio { namespace chain { std::vector signatures; bytes packed_context_free_data; vector context_free_segments; + void unpack_context_free_data(packed_transaction_v0::compression_type compression); }; using prunable_data_t = fc::static_variant< full_legacy, diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index b0bd2874bf6..7f6ee018dbb 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -651,4 +651,7 @@ void packed_transaction::reflector_init() estimated_size = calculate_estimated_size(); } +void packed_transaction::prunable_data_type::full_legacy::unpack_context_free_data(packed_transaction_v0::compression_type compression) { + context_free_segments = chain::unpack_context_free_data(packed_context_free_data, compression); +} } } // eosio::chain diff --git a/libraries/state_history/CMakeLists.txt b/libraries/state_history/CMakeLists.txt index 5a27b819f19..a0ceda47fd5 100644 --- a/libraries/state_history/CMakeLists.txt +++ b/libraries/state_history/CMakeLists.txt @@ -4,6 +4,7 @@ add_library( state_history abi.cpp compression.cpp create_deltas.cpp + log.cpp trace_converter.cpp ${HEADERS} ) diff --git a/libraries/state_history/include/eosio/state_history/log.hpp b/libraries/state_history/include/eosio/state_history/log.hpp index a14bdaaafc4..5f8da607b96 100644 --- a/libraries/state_history/include/eosio/state_history/log.hpp +++ b/libraries/state_history/include/eosio/state_history/log.hpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace eosio { @@ -31,8 +32,8 @@ namespace eosio { inline uint64_t ship_magic(uint32_t version) { return N(ship).to_uint64_t() | version; } inline bool is_ship(uint64_t magic) { return (magic & 0xffff'ffff'0000'0000) == N(ship).to_uint64_t(); } inline uint32_t get_ship_version(uint64_t magic) { return magic; } -inline bool is_ship_supported_version(uint64_t magic) { return get_ship_version(magic) == 0; } -static const uint32_t ship_current_version = 0; +inline bool is_ship_supported_version(uint64_t magic) { return get_ship_version(magic) <= 1; } +static const uint32_t ship_current_version = 1; struct state_history_log_header { uint64_t magic = ship_magic(ship_current_version); @@ -53,244 +54,88 @@ class state_history_log { uint32_t _begin_block = 0; uint32_t _end_block = 0; chain::block_id_type last_block_id; - + uint32_t version = ship_current_version; public: - state_history_log(const char* const name, std::string log_filename, std::string index_filename) - : name(name) - , log_filename(std::move(log_filename)) - , index_filename(std::move(index_filename)) { - open_log(); - open_index(); - } + // The type aliases below help to make it obvious about the meanings of member function return values. + using block_num_type = uint32_t; + using version_type = uint32_t; + using file_position_type = uint64_t; - uint32_t begin_block() const { return _begin_block; } - uint32_t end_block() const { return _end_block; } + state_history_log(const char* const name, std::string log_filename, std::string index_filename); - void read_header(state_history_log_header& header, bool assert_version = true) { - char bytes[state_history_log_header_serial_size]; - log.read(bytes, sizeof(bytes)); - fc::datastream ds(bytes, sizeof(bytes)); - fc::raw::unpack(ds, header); - EOS_ASSERT(!ds.remaining(), chain::plugin_exception, "state_history_log_header_serial_size mismatch"); - if (assert_version) - EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic), chain::plugin_exception, - "corrupt ${name}.log (0)", ("name", name)); - } + block_num_type begin_block() const { return _begin_block; } + block_num_type end_block() const { return _end_block; } - void write_header(const state_history_log_header& header) { - char bytes[state_history_log_header_serial_size]; - fc::datastream ds(bytes, sizeof(bytes)); - fc::raw::pack(ds, header); - EOS_ASSERT(!ds.remaining(), chain::plugin_exception, "state_history_log_header_serial_size mismatch"); - log.write(bytes, sizeof(bytes)); - } + void read_header(state_history_log_header& header, bool assert_version = true); + void write_header(const state_history_log_header& header); template void write_entry(const state_history_log_header& header, const chain::block_id_type& prev_id, F write_payload) { - auto block_num = chain::block_header::num_from_id(header.block_id); - EOS_ASSERT(_begin_block == _end_block || block_num <= _end_block, chain::plugin_exception, - "missed a block in ${name}.log", ("name", name)); - - if (_begin_block != _end_block && block_num > _begin_block) { - if (block_num == _end_block) { - EOS_ASSERT(prev_id == last_block_id, chain::plugin_exception, "missed a fork change in ${name}.log", - ("name", name)); - } else { - state_history_log_header prev; - get_entry(block_num - 1, prev); - EOS_ASSERT(prev_id == prev.block_id, chain::plugin_exception, "missed a fork change in ${name}.log", - ("name", name)); - } - } - - if (block_num < _end_block) - truncate(block_num); - log.seek_end(0); - uint64_t pos = log.tellp(); - write_header(header); + auto [pos, block_num] = write_entry_header(header, prev_id); write_payload(log); - uint64_t end = log.tellp(); - EOS_ASSERT(end == pos + state_history_log_header_serial_size + header.payload_size, chain::plugin_exception, - "wrote payload with incorrect size to ${name}.log", ("name", name)); - log.write((char*)&pos, sizeof(pos)); - - index.seek_end(0); - index.write((char*)&pos, sizeof(pos)); - if (_begin_block == _end_block) - _begin_block = block_num; - _end_block = block_num + 1; - last_block_id = header.block_id; + write_entry_position(header, pos, block_num); } - // returns cfile positioned at payload - fc::cfile& get_entry(uint32_t block_num, state_history_log_header& header) { - EOS_ASSERT(block_num >= _begin_block && block_num < _end_block, chain::plugin_exception, - "read non-existing block in ${name}.log", ("name", name)); - log.seek(get_pos(block_num)); - read_header(header); - return log; - } - - chain::block_id_type get_block_id(uint32_t block_num) { - state_history_log_header header; - get_entry(block_num, header); - return header.block_id; + /// @returns cfile positioned at payload + fc::cfile& get_entry_header(block_num_type block_num, state_history_log_header& header); + chain::block_id_type get_block_id(block_num_type block_num); + + /** + * @returns the entry payload and its version if existed + **/ + fc::optional> get_entry(block_num_type block_num); + + template + fc::optional get_entry(uint32_t block_num, Converter conv) { + auto entry = this->get_entry(block_num); + if (entry) + return conv(entry->first, entry->second); + return {}; } + protected: + fc::cfile& get_log() { return log; } private: - bool get_last_block(uint64_t size) { - state_history_log_header header; - uint64_t suffix; - log.seek(size - sizeof(suffix)); - log.read((char*)&suffix, sizeof(suffix)); - if (suffix > size || suffix + state_history_log_header_serial_size > size) { - elog("corrupt ${name}.log (2)", ("name", name)); - return false; - } - log.seek(suffix); - read_header(header, false); - if (!is_ship(header.magic) || !is_ship_supported_version(header.magic) || - suffix + state_history_log_header_serial_size + header.payload_size + sizeof(suffix) != size) { - elog("corrupt ${name}.log (3)", ("name", name)); - return false; - } - _end_block = chain::block_header::num_from_id(header.block_id) + 1; - last_block_id = header.block_id; - if (_begin_block >= _end_block) { - elog("corrupt ${name}.log (4)", ("name", name)); - return false; - } - return true; - } + bool get_last_block(uint64_t size); + void recover_blocks(uint64_t size); + void open_log(); + void open_index(); + file_position_type get_pos(block_num_type block_num); + void truncate(block_num_type block_num); + + /** + * @returns the file position of the written entry and its block num + **/ + std::pair write_entry_header(const state_history_log_header& header, + const chain::block_id_type& prev_id); + void write_entry_position(const state_history_log_header& header, file_position_type pos, block_num_type block_num); +}; // state_history_log - void recover_blocks(uint64_t size) { - ilog("recover ${name}.log", ("name", name)); - uint64_t pos = 0; - uint32_t num_found = 0; - while (true) { - state_history_log_header header; - if (pos + state_history_log_header_serial_size > size) - break; - log.seek(pos); - read_header(header, false); - uint64_t suffix; - if (!is_ship(header.magic) || !is_ship_supported_version(header.magic) || header.payload_size > size || - pos + state_history_log_header_serial_size + header.payload_size + sizeof(suffix) > size) { - EOS_ASSERT(!is_ship(header.magic) || is_ship_supported_version(header.magic), chain::plugin_exception, - "${name}.log has an unsupported version", ("name", name)); - break; - } - log.seek(pos + state_history_log_header_serial_size + header.payload_size); - log.read((char*)&suffix, sizeof(suffix)); - if (suffix != pos) - break; - pos = pos + state_history_log_header_serial_size + header.payload_size + sizeof(suffix); - if (!(++num_found % 10000)) { - printf("%10u blocks found, log pos=%12llu\r", (unsigned)num_found, (unsigned long long)pos); - fflush(stdout); - } - } - log.flush(); - boost::filesystem::resize_file(log_filename, pos); - log.flush(); - EOS_ASSERT(get_last_block(pos), chain::plugin_exception, "recover ${name}.log failed", ("name", name)); - } +class state_history_traces_log : public state_history_log { + state_history::trace_converter trace_convert; - void open_log() { - log.set_file_path(log_filename); - log.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app - log.seek_end(0); - uint64_t size = log.tellp(); - if (size >= state_history_log_header_serial_size) { - state_history_log_header header; - log.seek(0); - read_header(header, false); - EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic) && - state_history_log_header_serial_size + header.payload_size + sizeof(uint64_t) <= size, - chain::plugin_exception, "corrupt ${name}.log (1)", ("name", name)); - _begin_block = chain::block_header::num_from_id(header.block_id); - last_block_id = header.block_id; - if (!get_last_block(size)) - recover_blocks(size); - ilog("${name}.log has blocks ${b}-${e}", ("name", name)("b", _begin_block)("e", _end_block - 1)); - } else { - EOS_ASSERT(!size, chain::plugin_exception, "corrupt ${name}.log (5)", ("name", name)); - ilog("${name}.log is empty", ("name", name)); - } - } + public: + using compression_type = chain::packed_transaction::prunable_data_type::compression_type; + bool trace_debug_mode = false; + compression_type compression = compression_type::none; - void open_index() { - index.set_file_path(index_filename); - index.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app - index.seek_end(0); - if (index.tellp() == (static_cast(_end_block) - _begin_block) * sizeof(uint64_t)) - return; - ilog("Regenerate ${name}.index", ("name", name)); - index.close(); - index.open("w+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc + state_history_traces_log(fc::path state_history_dir); - log.seek_end(0); - uint64_t size = log.tellp(); - uint64_t pos = 0; - uint32_t num_found = 0; - while (pos < size) { - state_history_log_header header; - EOS_ASSERT(pos + state_history_log_header_serial_size <= size, chain::plugin_exception, - "corrupt ${name}.log (6)", ("name", name)); - log.seek(pos); - read_header(header, false); - uint64_t suffix_pos = pos + state_history_log_header_serial_size + header.payload_size; - uint64_t suffix; - EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic) && - suffix_pos + sizeof(suffix) <= size, - chain::plugin_exception, "corrupt ${name}.log (7)", ("name", name)); - log.seek(suffix_pos); - log.read((char*)&suffix, sizeof(suffix)); - // ilog("block ${b} at ${pos}-${end} suffix=${suffix} file_size=${fs}", - // ("b", header.block_num)("pos", pos)("end", suffix_pos + sizeof(suffix))("suffix", suffix)("fs", size)); - EOS_ASSERT(suffix == pos, chain::plugin_exception, "corrupt ${name}.log (8)", ("name", name)); + static bool exists(fc::path state_history_dir); - index.write((char*)&pos, sizeof(pos)); - pos = suffix_pos + sizeof(suffix); - if (!(++num_found % 10000)) { - printf("%10u blocks found, log pos=%12llu\r", (unsigned)num_found, (unsigned long long)pos); - fflush(stdout); - } - } + void add_transaction(const chain::transaction_trace_ptr& trace, const chain::packed_transaction_ptr& transaction) { + trace_convert.add_transaction(trace, transaction); } - uint64_t get_pos(uint32_t block_num) { - uint64_t pos; - index.seek((block_num - _begin_block) * sizeof(pos)); - index.read((char*)&pos, sizeof(pos)); - return pos; - } + fc::optional get_log_entry(block_num_type block_num); - void truncate(uint32_t block_num) { - log.flush(); - index.flush(); - uint64_t num_removed = 0; - if (block_num <= _begin_block) { - num_removed = _end_block - _begin_block; - log.seek(0); - index.seek(0); - boost::filesystem::resize_file(log_filename, 0); - boost::filesystem::resize_file(index_filename, 0); - _begin_block = _end_block = 0; - } else { - num_removed = _end_block - block_num; - uint64_t pos = get_pos(block_num); - log.seek(0); - index.seek(0); - boost::filesystem::resize_file(log_filename, pos); - boost::filesystem::resize_file(index_filename, (block_num - _begin_block) * sizeof(uint64_t)); - _end_block = block_num; - } - log.flush(); - index.flush(); - ilog("fork or replay: removed ${n} blocks from ${name}.log", ("n", num_removed)("name", name)); - } -}; // state_history_log + void store_traces(const chainbase::database& db, const chain::block_state_ptr& block_state); + + /** + * @param[in,out] ids The ids to been pruned and returns the ids not found in the specified block + **/ + void prune_traces(block_num_type block_num, std::vector& ids); +}; } // namespace eosio diff --git a/libraries/state_history/include/eosio/state_history/serialization.hpp b/libraries/state_history/include/eosio/state_history/serialization.hpp index 7b0ad094be5..c15b62f423e 100644 --- a/libraries/state_history/include/eosio/state_history/serialization.hpp +++ b/libraries/state_history/include/eosio/state_history/serialization.hpp @@ -41,6 +41,12 @@ history_context_wrapper, std::decay_t> make_history_context_w return {db, context, obj}; } +struct trace_receipt_context { + uint8_t failed_status = eosio::chain::transaction_receipt_header::hard_fail; + bool debug_mode; + uint32_t version; +}; + namespace fc { template @@ -615,10 +621,10 @@ datastream& operator<<(datastream& ds, const history_context_wrapper datastream& operator<<( - datastream& ds, - const history_context_wrapper, eosio::state_history::augmented_transaction_trace>& obj) { + datastream& ds, + const history_context_wrapper& obj) { auto& trace = *obj.obj.trace; - bool debug_mode = obj.context.second; + bool debug_mode = obj.context.debug_mode; fc::raw::pack(ds, fc::unsigned_int(0)); fc::raw::pack(ds, as_type(trace.id)); if (trace.receipt) { @@ -629,13 +635,14 @@ datastream& operator<<( fc::raw::pack(ds, as_type(trace.receipt->cpu_usage_us)); fc::raw::pack(ds, as_type(trace.receipt->net_usage_words)); } else { - fc::raw::pack(ds, uint8_t(obj.context.first)); + fc::raw::pack(ds, uint8_t(obj.context.failed_status)); fc::raw::pack(ds, uint32_t(0)); fc::raw::pack(ds, fc::unsigned_int(0)); } fc::raw::pack(ds, as_type(debug_mode ? trace.elapsed.count() : 0)); fc::raw::pack(ds, as_type(trace.net_usage)); fc::raw::pack(ds, as_type(trace.scheduled)); + history_context_serialize_container(ds, obj.db, debug_mode, as_type>(trace.action_traces)); @@ -657,10 +664,10 @@ datastream& operator<<( fc::raw::pack(ds, bool(trace.failed_dtrx_trace)); if (trace.failed_dtrx_trace) { - uint8_t stat = eosio::chain::transaction_receipt_header::hard_fail; + trace_receipt_context context = obj.context; + context.failed_status = eosio::chain::transaction_receipt_header::hard_fail; if (trace.receipt && trace.receipt->status.value == eosio::chain::transaction_receipt_header::soft_fail) - stat = eosio::chain::transaction_receipt_header::soft_fail; - std::pair context = std::make_pair(stat, debug_mode); + context.failed_status = eosio::chain::transaction_receipt_header::soft_fail; fc::raw::pack( // ds, make_history_context_wrapper( obj.db, context, @@ -680,8 +687,9 @@ datastream& operator<<( fc::raw::pack(ds, as_type(trx.max_cpu_usage_ms)); fc::raw::pack(ds, as_type(trx.delay_sec)); fc::raw::pack(ds, as_type(trx.transaction_extensions)); - const auto* sigs = pt.get_signatures(); - const auto* cfd = pt.get_context_free_data(); + + const auto* sigs = obj.context.version == 0 ? pt.get_signatures() : nullptr; + const auto* cfd = obj.context.version == 0 ? pt.get_context_free_data() : nullptr; fc::raw::pack(ds, as_type>(sigs ? *sigs : std::vector{})); fc::raw::pack(ds, as_type>(cfd ? *cfd : std::vector{})); } @@ -689,15 +697,6 @@ datastream& operator<<( return ds; } -template -datastream& -operator<<(datastream& ds, - const history_context_wrapper& obj) { - std::pair context = std::make_pair(eosio::chain::transaction_receipt_header::hard_fail, obj.context); - ds << make_history_context_wrapper(obj.db, context, obj.obj); - return ds; -} - template datastream& operator<<(datastream& ds, const eosio::state_history::get_blocks_result_v0& obj) { fc::raw::pack(ds, obj.head); diff --git a/libraries/state_history/include/eosio/state_history/trace_converter.hpp b/libraries/state_history/include/eosio/state_history/trace_converter.hpp index 72d375ec570..bd1cb3e3d0c 100644 --- a/libraries/state_history/include/eosio/state_history/trace_converter.hpp +++ b/libraries/state_history/include/eosio/state_history/trace_converter.hpp @@ -2,6 +2,7 @@ #include #include +#include namespace eosio { namespace state_history { @@ -9,13 +10,31 @@ namespace state_history { using chain::block_state_ptr; using chain::transaction_id_type; using chain::packed_transaction_ptr; +using compression_type = chain::packed_transaction::prunable_data_type::compression_type; struct trace_converter { std::map cached_traces; fc::optional onblock_trace; - + void add_transaction(const transaction_trace_ptr& trace, const packed_transaction_ptr& transaction); - bytes pack(const chainbase::database& db, bool trace_debug_mode, const block_state_ptr& block_state); + + bytes pack(const chainbase::database& db, bool trace_debug_mode, const block_state_ptr& block_state, + uint32_t version, compression_type compression); + static bytes to_traces_bin_v0(const bytes& entry_payload, uint32_t version); + + /** + * Prune the signatures and context free data in a v1 log entry payload + * + * @param[in] entry_payload The entry_payload + * @param[in] version The version of the entry + * @param[in, out] ids The list of transaction ids to be pruned. After the member function returns, + * it would be modified to contain the list of transaction ids that do not exists + * in the log entry. + * + * @returns The serialized prunable_data section + **/ + static bytes prune_traces(const bytes& entry_payload, uint32_t version, std::vector& ids); + }; } // namespace state_history diff --git a/libraries/state_history/include/eosio/state_history/types.hpp b/libraries/state_history/include/eosio/state_history/types.hpp index 3bcdb904347..df2a9b2830d 100644 --- a/libraries/state_history/include/eosio/state_history/types.hpp +++ b/libraries/state_history/include/eosio/state_history/types.hpp @@ -88,6 +88,113 @@ struct get_blocks_result_v0 { using state_request = fc::static_variant; using state_result = fc::static_variant; +struct account_auth_sequence { + uint64_t account = {}; + uint64_t sequence = {}; +}; + +struct account_delta { + uint64_t account = {}; + uint64_t delta = {}; +}; + +struct permission_level { + uint64_t actor = {}; + uint64_t permission = {}; +}; + +struct action_receipt_v0 { + uint64_t receiver = {}; + eosio::chain::digest_type act_digest = {}; + uint64_t global_sequence = {}; + uint64_t recv_sequence = {}; + std::vector auth_sequence = {}; + fc::unsigned_int code_sequence = {}; + fc::unsigned_int abi_sequence = {}; +}; + +using action_receipt = fc::static_variant; + +struct action { + uint64_t account = {}; + uint64_t name = {}; + std::vector authorization = {}; + bytes data = {}; +}; + +struct action_trace_v0 { + fc::unsigned_int action_ordinal = {}; + fc::unsigned_int creator_action_ordinal = {}; + fc::optional receipt = {}; + uint64_t receiver = {}; + action act = {}; + bool context_free = {}; + int64_t elapsed = {}; + std::string console = {}; + std::vector account_ram_deltas = {}; + fc::optional except = {}; + fc::optional error_code = {}; +}; + +struct action_trace_v1 { + fc::unsigned_int action_ordinal = {}; + fc::unsigned_int creator_action_ordinal = {}; + fc::optional receipt = {}; + uint64_t receiver = {}; + action act = {}; + bool context_free = {}; + int64_t elapsed = {}; + std::string console = {}; + std::vector account_ram_deltas = {}; + std::vector account_disk_deltas = {}; + fc::optional except = {}; + fc::optional error_code = {}; + bytes return_value = {}; +}; + +using action_trace = fc::static_variant; + +struct partial_transaction_v0 { + eosio::chain::time_point_sec expiration = {}; + uint16_t ref_block_num = {}; + uint32_t ref_block_prefix = {}; + fc::unsigned_int max_net_usage_words = {}; + uint8_t max_cpu_usage_ms = {}; + fc::unsigned_int delay_sec = {}; + std::vector transaction_extensions = {}; + std::vector signatures = {}; + std::vector context_free_data = {}; +}; + +using partial_transaction = fc::static_variant; + +struct transaction_trace_recurse; + +struct transaction_trace_v0 { + using transaction_trace = fc::static_variant; + eosio::chain::digest_type id = {}; + uint8_t status = {}; + uint32_t cpu_usage_us = {}; + fc::unsigned_int net_usage_words = {}; + int64_t elapsed = {}; + uint64_t net_usage = {}; + bool scheduled = {}; + std::vector action_traces = {}; + fc::optional account_ram_delta = {}; + fc::optional except = {}; + fc::optional error_code = {}; + + // semantically, this should be optional; + // it is represented as vector because optional cannot be used for incomplete type + std::vector failed_dtrx_trace = {}; + fc::optional partial = {}; +}; + +using transaction_trace = fc::static_variant; +struct transaction_trace_recurse { + transaction_trace recurse; +}; + } // namespace state_history } // namespace eosio @@ -98,4 +205,15 @@ FC_REFLECT_EMPTY(eosio::state_history::get_status_request_v0); FC_REFLECT(eosio::state_history::get_status_result_v0, (head)(last_irreversible)(trace_begin_block)(trace_end_block)(chain_state_begin_block)(chain_state_end_block)(chain_id)); FC_REFLECT(eosio::state_history::get_blocks_request_v0, (start_block_num)(end_block_num)(max_messages_in_flight)(have_positions)(irreversible_only)(fetch_block)(fetch_traces)(fetch_deltas)); FC_REFLECT(eosio::state_history::get_blocks_ack_request_v0, (num_messages)); + +FC_REFLECT(eosio::state_history::account_auth_sequence, (account)(sequence)); +FC_REFLECT(eosio::state_history::account_delta, (account)(delta)); +FC_REFLECT(eosio::state_history::permission_level,(actor)(permission)); +FC_REFLECT(eosio::state_history::action_receipt_v0, (receiver)(act_digest)(global_sequence)(recv_sequence)(auth_sequence)(code_sequence)(abi_sequence)); +FC_REFLECT(eosio::state_history::action, (account)(name)(authorization)(data)); +FC_REFLECT(eosio::state_history::action_trace_v0, (action_ordinal)(creator_action_ordinal)(receipt)(receiver)(act)(context_free)(elapsed)(console)(account_ram_deltas)(except)(error_code)); +FC_REFLECT(eosio::state_history::action_trace_v1, (action_ordinal)(creator_action_ordinal)(receipt)(receiver)(act)(context_free)(elapsed)(console)(account_ram_deltas)(account_disk_deltas)(except)(error_code)(return_value)); +FC_REFLECT(eosio::state_history::partial_transaction_v0, (expiration)(ref_block_num)(ref_block_prefix)(max_net_usage_words)(max_cpu_usage_ms)(delay_sec)(transaction_extensions)(signatures)(context_free_data)); +FC_REFLECT(eosio::state_history::transaction_trace_v0,(id)(status)(cpu_usage_us)(net_usage_words)(elapsed)(net_usage)(scheduled)(action_traces)(account_ram_delta)(except)(error_code)(failed_dtrx_trace)(partial)); +FC_REFLECT(eosio::state_history::transaction_trace_recurse, (recurse)); // clang-format on diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp new file mode 100644 index 00000000000..71da9e5b13e --- /dev/null +++ b/libraries/state_history/log.cpp @@ -0,0 +1,312 @@ +#include +#include + +namespace eosio { + +state_history_log::state_history_log(const char* const name, std::string log_filename, std::string index_filename) + : name(name) + , log_filename(std::move(log_filename)) + , index_filename(std::move(index_filename)) { + open_log(); + open_index(); +} + +void state_history_log::read_header(state_history_log_header& header, bool assert_version) { + char bytes[state_history_log_header_serial_size]; + log.read(bytes, sizeof(bytes)); + fc::datastream ds(bytes, sizeof(bytes)); + fc::raw::unpack(ds, header); + EOS_ASSERT(!ds.remaining(), chain::state_history_exception, "state_history_log_header_serial_size mismatch"); + version = get_ship_version(header.magic); + if (assert_version) + EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic), chain::state_history_exception, + "corrupt ${name}.log (0)", ("name", name)); +} + +void state_history_log::write_header(const state_history_log_header& header) { + char bytes[state_history_log_header_serial_size]; + fc::datastream ds(bytes, sizeof(bytes)); + fc::raw::pack(ds, header); + EOS_ASSERT(!ds.remaining(), chain::state_history_exception, "state_history_log_header_serial_size mismatch"); + log.write(bytes, sizeof(bytes)); +} + +// returns cfile positioned at payload +fc::cfile& state_history_log::get_entry_header(state_history_log::block_num_type block_num, + state_history_log_header& header) { + EOS_ASSERT(block_num >= _begin_block && block_num < _end_block, chain::state_history_exception, + "read non-existing block in ${name}.log", ("name", name)); + log.seek(get_pos(block_num)); + read_header(header); + return log; +} + +chain::block_id_type state_history_log::get_block_id(state_history_log::block_num_type block_num) { + state_history_log_header header; + get_entry_header(block_num, header); + return header.block_id; +} + +bool state_history_log::get_last_block(uint64_t size) { + state_history_log_header header; + uint64_t suffix; + log.seek(size - sizeof(suffix)); + log.read((char*)&suffix, sizeof(suffix)); + if (suffix > size || suffix + state_history_log_header_serial_size > size) { + elog("corrupt ${name}.log (2)", ("name", name)); + return false; + } + log.seek(suffix); + read_header(header, false); + if (!is_ship(header.magic) || !is_ship_supported_version(header.magic) || + suffix + state_history_log_header_serial_size + header.payload_size + sizeof(suffix) != size) { + elog("corrupt ${name}.log (3)", ("name", name)); + return false; + } + _end_block = chain::block_header::num_from_id(header.block_id) + 1; + last_block_id = header.block_id; + if (_begin_block >= _end_block) { + elog("corrupt ${name}.log (4)", ("name", name)); + return false; + } + return true; +} + +void state_history_log::recover_blocks(uint64_t size) { + ilog("recover ${name}.log", ("name", name)); + uint64_t pos = 0; + uint32_t num_found = 0; + while (true) { + state_history_log_header header; + if (pos + state_history_log_header_serial_size > size) + break; + log.seek(pos); + read_header(header, false); + uint64_t suffix; + if (!is_ship(header.magic) || !is_ship_supported_version(header.magic) || header.payload_size > size || + pos + state_history_log_header_serial_size + header.payload_size + sizeof(suffix) > size) { + EOS_ASSERT(!is_ship(header.magic) || is_ship_supported_version(header.magic), chain::state_history_exception, + "${name}.log has an unsupported version", ("name", name)); + break; + } + log.seek(pos + state_history_log_header_serial_size + header.payload_size); + log.read((char*)&suffix, sizeof(suffix)); + if (suffix != pos) + break; + pos = pos + state_history_log_header_serial_size + header.payload_size + sizeof(suffix); + if (!(++num_found % 10000)) { + printf("%10u blocks found, log pos=%12llu\r", (unsigned)num_found, (unsigned long long)pos); + fflush(stdout); + } + } + log.flush(); + boost::filesystem::resize_file(log_filename, pos); + log.flush(); + EOS_ASSERT(get_last_block(pos), chain::state_history_exception, "recover ${name}.log failed", ("name", name)); +} + +void state_history_log::open_log() { + log.set_file_path(log_filename); + log.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app + log.seek_end(0); + uint64_t size = log.tellp(); + if (size >= state_history_log_header_serial_size) { + state_history_log_header header; + log.seek(0); + read_header(header, false); + EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic) && + state_history_log_header_serial_size + header.payload_size + sizeof(uint64_t) <= size, + chain::state_history_exception, "corrupt ${name}.log (1)", ("name", name)); + _begin_block = chain::block_header::num_from_id(header.block_id); + last_block_id = header.block_id; + if (!get_last_block(size)) + recover_blocks(size); + ilog("${name}.log has blocks ${b}-${e}", ("name", name)("b", _begin_block)("e", _end_block - 1)); + } else { + EOS_ASSERT(!size, chain::state_history_exception, "corrupt ${name}.log (5)", ("name", name)); + ilog("${name}.log is empty", ("name", name)); + } +} + +void state_history_log::open_index() { + index.set_file_path(index_filename); + index.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app + index.seek_end(0); + if (index.tellp() == (static_cast(_end_block) - _begin_block) * sizeof(uint64_t)) + return; + ilog("Regenerate ${name}.index", ("name", name)); + index.close(); + index.open("w+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc + + log.seek_end(0); + uint64_t size = log.tellp(); + uint64_t pos = 0; + uint32_t num_found = 0; + while (pos < size) { + state_history_log_header header; + EOS_ASSERT(pos + state_history_log_header_serial_size <= size, chain::state_history_exception, + "corrupt ${name}.log (6)", ("name", name)); + log.seek(pos); + read_header(header, false); + uint64_t suffix_pos = pos + state_history_log_header_serial_size + header.payload_size; + uint64_t suffix; + EOS_ASSERT(is_ship(header.magic) && is_ship_supported_version(header.magic) && + suffix_pos + sizeof(suffix) <= size, + chain::state_history_exception, "corrupt ${name}.log (7)", ("name", name)); + log.seek(suffix_pos); + log.read((char*)&suffix, sizeof(suffix)); + // ilog("block ${b} at ${pos}-${end} suffix=${suffix} file_size=${fs}", + // ("b", header.block_num)("pos", pos)("end", suffix_pos + sizeof(suffix))("suffix", suffix)("fs", size)); + EOS_ASSERT(suffix == pos, chain::state_history_exception, "corrupt ${name}.log (8)", ("name", name)); + + index.write((char*)&pos, sizeof(pos)); + pos = suffix_pos + sizeof(suffix); + if (!(++num_found % 10000)) { + printf("%10u blocks found, log pos=%12llu\r", (unsigned)num_found, (unsigned long long)pos); + fflush(stdout); + } + } +} + +state_history_log::file_position_type state_history_log::get_pos(state_history_log::block_num_type block_num) { + uint64_t pos; + index.seek((block_num - _begin_block) * sizeof(pos)); + index.read((char*)&pos, sizeof(pos)); + return pos; +} + +void state_history_log::truncate(state_history_log::block_num_type block_num) { + log.flush(); + index.flush(); + uint64_t num_removed = 0; + if (block_num <= _begin_block) { + num_removed = _end_block - _begin_block; + log.seek(0); + index.seek(0); + boost::filesystem::resize_file(log_filename, 0); + boost::filesystem::resize_file(index_filename, 0); + _begin_block = _end_block = 0; + } else { + num_removed = _end_block - block_num; + uint64_t pos = get_pos(block_num); + log.seek(0); + index.seek(0); + boost::filesystem::resize_file(log_filename, pos); + boost::filesystem::resize_file(index_filename, (block_num - _begin_block) * sizeof(uint64_t)); + _end_block = block_num; + } + log.flush(); + index.flush(); + ilog("fork or replay: removed ${n} blocks from ${name}.log", ("n", num_removed)("name", name)); +} + +std::pair +state_history_log::write_entry_header(const state_history_log_header& header, const chain::block_id_type& prev_id) { + auto block_num = chain::block_header::num_from_id(header.block_id); + EOS_ASSERT(_begin_block == _end_block || block_num <= _end_block, chain::state_history_exception, + "missed a block in ${name}.log", ("name", name)); + + if (_begin_block != _end_block && block_num > _begin_block) { + if (block_num == _end_block) { + EOS_ASSERT(prev_id == last_block_id, chain::state_history_exception, "missed a fork change in ${name}.log", + ("name", name)); + } else { + state_history_log_header prev; + get_entry_header(block_num - 1, prev); + EOS_ASSERT(prev_id == prev.block_id, chain::state_history_exception, "missed a fork change in ${name}.log", + ("name", name)); + } + } + + if (block_num < _end_block) + truncate(block_num); + log.seek_end(0); + uint64_t pos = log.tellp(); + write_header(header); + return std::make_pair(pos, block_num); +} + +void state_history_log::write_entry_position(const state_history_log_header& header, + state_history_log::file_position_type pos, + state_history_log::block_num_type block_num) { + uint64_t end = log.tellp(); + EOS_ASSERT(end == pos + state_history_log_header_serial_size + header.payload_size, chain::state_history_exception, + "wrote payload with incorrect size to ${name}.log", ("name", name)); + log.write((char*)&pos, sizeof(pos)); + + index.seek_end(0); + index.write((char*)&pos, sizeof(pos)); + if (_begin_block == _end_block) + _begin_block = block_num; + _end_block = block_num + 1; + last_block_id = header.block_id; +} + +fc::optional> +state_history_log::get_entry(state_history_log::block_num_type block_num) { + if (block_num < begin_block() || block_num >= end_block()) + return {}; + state_history_log_header header; + auto& stream = get_entry_header(block_num, header); + uint32_t s; + stream.read((char*)&s, sizeof(s)); + chain::bytes data(s); + if (s) + stream.read(data.data(), s); + return std::make_pair(data, get_ship_version(header.magic)); +} + +state_history_traces_log::state_history_traces_log(fc::path state_history_dir) + : state_history_log("trace_history", (state_history_dir / "trace_history.log").string(), + (state_history_dir / "trace_history.index").string()) {} + +fc::optional state_history_traces_log::get_log_entry(block_num_type block_num) { + return this->get_entry(block_num, [](const chain::bytes& data, uint32_t version) { + return state_history::trace_converter::to_traces_bin_v0(data, version); + }); +} + +void state_history_traces_log::prune_traces(state_history_log::block_num_type block_num, + std::vector& ids) { + auto entry_result = get_entry(block_num); + EOS_ASSERT(entry_result, chain::state_history_exception, "nonexistant block num ${block_num}", + ("block_num", block_num)); + + auto pos = get_log().tellp(); + + auto [entry_payload, version] = *entry_result; + auto pruned_section = state_history::trace_converter::prune_traces(entry_payload, version, ids); + auto bytes_to_write = pruned_section.size(); + + if (bytes_to_write > 0) { + // the log object is in append mode, which cannot be used to update written content, + // we need to open the file in the update mode + + fc::cfile update_log; + update_log.set_file_path(get_log().get_file_path()); + update_log.open(fc::cfile::update_rw_mode); + update_log.seek(pos - bytes_to_write); + update_log.write(pruned_section.data(), bytes_to_write); + } +} + +void state_history_traces_log::store_traces(const chainbase::database& db, const chain::block_state_ptr& block_state) { + + auto traces_bin = trace_convert.pack(db, trace_debug_mode, block_state, ship_current_version, compression); + + state_history_log_header header{.magic = ship_magic(ship_current_version), + .block_id = block_state->id, + .payload_size = sizeof(uint32_t) + traces_bin.size()}; + this->write_entry(header, block_state->block->previous, [&](auto& stream) { + uint32_t s = (uint32_t)traces_bin.size(); + stream.write((char*)&s, sizeof(s)); + if (!traces_bin.empty()) + stream.write(traces_bin.data(), traces_bin.size()); + }); +} + +bool state_history_traces_log::exists(fc::path state_history_dir) { + return fc::exists(state_history_dir / "trace_history.log") && fc::exists(state_history_dir / "trace_history.index"); +} + +} // namespace eosio diff --git a/libraries/state_history/trace_converter.cpp b/libraries/state_history/trace_converter.cpp index 8cc4300ecc7..70698e1a4f2 100644 --- a/libraries/state_history/trace_converter.cpp +++ b/libraries/state_history/trace_converter.cpp @@ -1,11 +1,19 @@ +#include +#include +#include +#include #include #include +extern const char* state_history_plugin_abi; + +namespace bio = boost::iostreams; namespace eosio { namespace state_history { using eosio::chain::packed_transaction; -using eosio::chain::plugin_exception; +using eosio::chain::state_history_exception; +using prunable_data_type = packed_transaction::prunable_data_type; bool is_onblock(const transaction_trace_ptr& p) { if (p->action_traces.size() != 1) @@ -30,25 +38,273 @@ void trace_converter::add_transaction(const transaction_trace_ptr& trace, const } } -bytes trace_converter::pack(const chainbase::database& db, bool trace_debug_mode, const block_state_ptr& block_state) { +namespace { + +bytes decompress_buffer(const char* buffer, uint32_t len, compression_type compression) { + bytes decompressed; + bio::filtering_ostreambuf strm; + EOS_ASSERT(compression == compression_type::zlib, state_history_exception, "unspported compression format"); + strm.push(bio::zlib_decompressor()); + strm.push(bio::back_inserter(decompressed)); + bio::write(strm, buffer, len); + bio::close(strm); + return decompressed; +} + +std::vector prepare_traces(trace_converter& converter, + const block_state_ptr& block_state) { std::vector traces; - if (onblock_trace) - traces.push_back(*onblock_trace); + if (converter.onblock_trace) + traces.push_back(*converter.onblock_trace); for (auto& r : block_state->block->transactions) { transaction_id_type id; if (r.trx.contains()) id = r.trx.get(); else id = r.trx.get().id(); - auto it = cached_traces.find(id); - EOS_ASSERT(it != cached_traces.end() && it->second.trace->receipt, plugin_exception, + auto it = converter.cached_traces.find(id); + EOS_ASSERT(it != converter.cached_traces.end() && it->second.trace->receipt, state_history_exception, "missing trace for transaction ${id}", ("id", id)); traces.push_back(it->second); } - cached_traces.clear(); - onblock_trace.reset(); + converter.cached_traces.clear(); + converter.onblock_trace.reset(); + return traces; +} + +template +void for_each_packed_transaction(const eosio::state_history::augmented_transaction_trace& obj, const Lambda& lambda) { + auto& trace = *obj.trace; + if (trace.failed_dtrx_trace) { + for_each_packed_transaction( + eosio::state_history::augmented_transaction_trace{trace.failed_dtrx_trace, obj.packed_trx}, lambda); + } + bool include_packed_trx = obj.packed_trx && !trace.failed_dtrx_trace; + if (include_packed_trx) { + lambda(*obj.packed_trx); + } +} + +template +void for_each_packed_transaction(const std::vector& traces, + const Lambda& lambda) { + for (const auto& v : traces) { + for_each_packed_transaction(v, lambda); + } +} + +void pack(fc::datastream& ds, const prunable_data_type& data, compression_type compression) { + if (compression == compression_type::none) { + fc::raw::pack(ds, data); + } + else if (compression == compression_type::zlib) { + bytes uncompressed = fc::raw::pack(data); + fc::raw::pack(ds, zlib_compress_bytes(uncompressed)); + } +} + +void unpack(fc::datastream& ds, prunable_data_type& data, compression_type compression) { + if (compression == compression_type::none) { + fc::raw::unpack(ds, data); + } + else if (compression == compression_type::zlib) { + bytes compressed; + fc::raw::unpack(ds, compressed); + bytes uncompressed = zlib_decompress(compressed); + fc::datastream uncompressed_ds(uncompressed.data(), uncompressed.size()); + fc::raw::unpack(uncompressed_ds, data); + } +} + + +// each individual pruable data in a trasaction is packed as an optional prunable_data_type::full +// or an optional bytes to a compressed prunable_data_type::full +size_t pack_pruable(bytes& buffer, const packed_transaction& trx, + compression_type compression) { + + const auto& data = trx.get_prunable_data(); + + int max_size = data.maximum_pruned_pack_size(compression); + if (data.prunable_data.contains()) { + max_size += sizeof(uint8_t); + } + uint64_t start_buffer_position = buffer.size(); + buffer.resize(buffer.size() + max_size); + fc::datastream ds(buffer.data() + start_buffer_position, max_size); + pack(ds, data, compression); + + if (data.prunable_data.contains()) { + fc::raw::pack(ds, static_cast(trx.get_compression())); + } + buffer.resize(start_buffer_position + ds.tellp()); + return max_size; +} + + +void unpack_prunable(const char* read_buffer, fc::datastream& ds, prunable_data_type& data, + compression_type compression) { + + unpack(ds, data, compression); + if (data.prunable_data.contains()) { + uint8_t local_comp; + fc::raw::unpack(ds, local_comp); + auto& full_legacy = data.prunable_data.get(); + full_legacy.unpack_context_free_data(static_cast(local_comp)); + } +} + +bytes pack_prunables(const std::vector& traces, compression_type compression) { + + std::vector out; + out.reserve(1024); + + int size_with_paddings = 0; + for_each_packed_transaction( + traces, [&](const chain::packed_transaction& pt) { size_with_paddings += pack_pruable(out, pt, compression); }); + + out.resize(size_with_paddings); + return out; +} + +std::pair, compression_type> +traces_from_v1_entry_payload(const char* read_buffer, fc::datastream& ds) { + uint8_t compr; + fc::raw::unpack(ds, compr); + fc::unsigned_int len; + fc::raw::unpack(ds, len); + EOS_ASSERT(ds.remaining() >= static_cast(len), fc::out_of_range_exception, "read datastream over by ${v}", + ("v", ds.remaining() - len)); + + auto unprunable_section = decompress_buffer(read_buffer + ds.tellp(), len, compression_type::zlib); + ds.skip(len); + + std::vector traces; + fc::datastream traces_ds(unprunable_section.data(), unprunable_section.size()); + fc::raw::unpack(traces_ds, traces); + return std::make_pair(traces, static_cast(compr)); +} + +template +void do_visit_trace(const char* read_buffer, fc::datastream& strm, transaction_trace& trace, + compression_type compression, Visitor&& visitor) { + auto& trace_v0 = trace.get(); + if (trace_v0.failed_dtrx_trace.size()) { + // failed_dtrx_trace have at most one element because it is encoded as an optional + do_visit_trace(read_buffer, strm, trace_v0.failed_dtrx_trace[0].recurse, compression, visitor); + } + if (trace_v0.partial) { + prunable_data_type prunable; + unpack_prunable(read_buffer, strm, prunable, compression); + visitor(trace_v0, prunable); + } +} +} // namespace + +bytes trace_converter::pack(const chainbase::database& db, bool trace_debug_mode, const block_state_ptr& block_state, + uint32_t version, compression_type compression) { + + auto traces = prepare_traces(*this, block_state); + auto unprunable = zlib_compress_bytes(fc::raw::pack(make_history_context_wrapper( + db, trace_receipt_context{.debug_mode = trace_debug_mode, .version = version}, traces))); + + if (version == 0) { + return unprunable; + } else { + // In version 1 of ShiP traces log disk format, it log entry consists of 3 parts. + // 1. a unit8_t compression tag to indicate the compression level of prunables + // 2. an zlib compressed unprunable section contains the serialization of the vector of traces excluding + // the prunable_data data (i.e. signatures and context free data) + // 3. a prunable section contains the serialization of the vector of optional (possibly compressed) + // prunable_data + + auto prunables = pack_prunables(traces, compression); + fc::datastream unprunable_size_strm; + fc::raw::pack(unprunable_size_strm, unprunable); + uint8_t compression_tag = static_cast(compression); + bytes buffer(sizeof(compression_tag) + unprunable_size_strm.tellp() + prunables.size()); + + fc::datastream strm(buffer.data(), buffer.size()); + fc::raw::pack(strm, compression_tag); + fc::raw::pack(strm, unprunable); + strm.write(prunables.data(), prunables.size()); + return buffer; + } +} - return fc::raw::pack(make_history_context_wrapper(db, trace_debug_mode, traces)); +struct restore_partial { + partial_transaction_v0& ptrx; + void operator()(prunable_data_type::full_legacy& data) const { + ptrx.signatures = std::move(data.signatures); + ptrx.context_free_data = std::move(data.context_free_segments); + } + void operator()(prunable_data_type::none& data) const {} + void operator()(prunable_data_type::signatures_only& data) const { + ptrx.signatures = std::move(data.signatures); + } + void operator()(prunable_data_type::partial& data) const { + ptrx.signatures = std::move(data.signatures); + } + void operator()(prunable_data_type::full& data) const { + ptrx.signatures = std::move(data.signatures); + ptrx.context_free_data = std::move(data.context_free_segments); + } +}; + +bytes trace_converter::to_traces_bin_v0(const bytes& entry_payload, uint32_t version) { + if (version == 0) + return zlib_decompress(entry_payload); + else { + fc::datastream strm(entry_payload.data(), entry_payload.size()); + + auto [traces, compression] = traces_from_v1_entry_payload(entry_payload.data(), strm); + for (auto& trace : traces) { + do_visit_trace(entry_payload.data(), strm, trace, compression, + [](transaction_trace_v0& trace, prunable_data_type& prunable_data) { + auto& ptrx = trace.partial->get(); + prunable_data.prunable_data.visit(restore_partial{ptrx}); + }); + } + + return fc::raw::pack(traces); + } +} + +bytes trace_converter::prune_traces(const bytes& entry_payload, uint32_t version, std::vector& ids) { + EOS_ASSERT(version > 0, state_history_exception, "state history log version 0 does not support trace pruning"); + fc::datastream read_strm(entry_payload.data(), entry_payload.size()); + + auto [traces, compression] = traces_from_v1_entry_payload(entry_payload.data(), read_strm); + uint64_t last_read_pos = read_strm.tellp(); + uint64_t change_offset = entry_payload.size(); + bool modified = false; + + bytes write_buffer(entry_payload.size() - last_read_pos); + fc::datastream write_strm(write_buffer.data(), write_buffer.size()); + + auto prune_trace = [&, compression=compression](transaction_trace_v0& trace, auto& incoming_prunable) { + auto itr = std::find(ids.begin(), ids.end(), trace.id); + if (itr != ids.end()) { + // the incoming trace matches the one of ids to be pruned + state_history::pack(write_strm, incoming_prunable.prune_all(), compression); + modified = true; + + // delete the found id from the ids vector + if (itr != ids.end() - 1) { + *itr = ids.back(); + } + ids.resize(ids.size() - 1); + } else { + // the incoming trace should not be pruned + auto bytes_read = read_strm.tellp() - last_read_pos; + write_strm.write(entry_payload.data() + last_read_pos, bytes_read); + } + last_read_pos = read_strm.tellp(); + }; + + for (auto& trace : traces) { + do_visit_trace(entry_payload.data(), read_strm, trace, compression, prune_trace); + } + return modified? write_buffer : bytes{}; } } // namespace state_history diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index e4d6ba06b0d..ac931322f22 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -41,29 +41,14 @@ auto catch_and_log(F f) { struct state_history_plugin_impl : std::enable_shared_from_this { chain_plugin* chain_plug = nullptr; - fc::optional trace_log; + fc::optional trace_log; fc::optional chain_state_log; - bool trace_debug_mode = false; bool stopping = false; fc::optional applied_transaction_connection; fc::optional accepted_block_connection; string endpoint_address = "0.0.0.0"; uint16_t endpoint_port = 8080; std::unique_ptr acceptor; - trace_converter trace_convert; - - void get_log_entry(state_history_log& log, uint32_t block_num, fc::optional& result) { - if (block_num < log.begin_block() || block_num >= log.end_block()) - return; - state_history_log_header header; - auto& stream = log.get_entry(block_num, header); - uint32_t s; - stream.read((char*)&s, sizeof(s)); - bytes compressed(s); - if (s) - stream.read(compressed.data(), s); - result = state_history::zlib_decompress(compressed); - } void get_block(uint32_t block_num, fc::optional& result) { chain::signed_block_ptr p; @@ -223,10 +208,14 @@ struct state_history_plugin_impl : std::enable_shared_from_thisstart_block_num - 1, *prev_block_id}; if (current_request->fetch_block) plugin->get_block(current_request->start_block_num, result.block); - if (current_request->fetch_traces && plugin->trace_log) - plugin->get_log_entry(*plugin->trace_log, current_request->start_block_num, result.traces); - if (current_request->fetch_deltas && plugin->chain_state_log) - plugin->get_log_entry(*plugin->chain_state_log, current_request->start_block_num, result.deltas); + if (current_request->fetch_traces && plugin->trace_log) { + result.traces = plugin->trace_log->get_log_entry(current_request->start_block_num); + } + if (current_request->fetch_deltas && plugin->chain_state_log) { + result.deltas = plugin->chain_state_log->get_entry( + current_request->start_block_num, + [](const bytes& data, uint32_t) { return state_history::zlib_decompress(data); }); + } } ++current_request->start_block_num; } @@ -324,11 +313,12 @@ struct state_history_plugin_impl : std::enable_shared_from_thisadd_transaction(p, t); } void on_accepted_block(const block_state_ptr& block_state) { - store_traces(block_state); + if (trace_log) + trace_log->store_traces(chain_plug->chain().db(), block_state); store_chain_state(block_state); for (auto& s : sessions) { auto& p = s.second; @@ -340,24 +330,6 @@ struct state_history_plugin_impl : std::enable_shared_from_thischain().db(), trace_debug_mode, block_state)); - EOS_ASSERT(traces_bin.size() == (uint32_t)traces_bin.size(), plugin_exception, "traces is too big"); - - state_history_log_header header{.magic = ship_magic(ship_current_version), - .block_id = block_state->id, - .payload_size = sizeof(uint32_t) + traces_bin.size()}; - trace_log->write_entry(header, block_state->block->previous, [&](auto& stream) { - uint32_t s = (uint32_t)traces_bin.size(); - stream.write((char*)&s, sizeof(s)); - if (!traces_bin.empty()) - stream.write(traces_bin.data(), traces_bin.size()); - }); - } - void store_chain_state(const block_state_ptr& block_state) { if (!chain_state_log) return; @@ -434,13 +406,12 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { } boost::filesystem::create_directories(state_history_dir); - if (options.at("trace-history-debug-mode").as()) { - my->trace_debug_mode = true; + if (options.at("trace-history").as()) { + my->trace_log.emplace(state_history_dir); + if (options.at("trace-history-debug-mode").as()) + my->trace_log->trace_debug_mode = true; } - if (options.at("trace-history").as()) - my->trace_log.emplace("trace_history", (state_history_dir / "trace_history.log").string(), - (state_history_dir / "trace_history.index").string()); if (options.at("chain-state-history").as()) my->chain_state_log.emplace("chain_state_history", (state_history_dir / "chain_state_history.log").string(), (state_history_dir / "chain_state_history.index").string()); diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 8675c1e69a8..172b64a796c 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -38,7 +38,7 @@ add_custom_command( file(GLOB UNIT_TESTS "*.cpp") # find all unit test suites add_executable( unit_test ${UNIT_TESTS} protocol_feature_digest_tests.cpp) # build unit tests as one executable -target_link_libraries( unit_test eosio_chain_wrap chainbase eosio_testing fc appbase ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( unit_test eosio_chain_wrap chainbase eosio_testing fc appbase state_history ${PLATFORM_SPECIFIC_LIBS} ) target_compile_options(unit_test PUBLIC -DDISABLE_EOSLIB_SERIALIZE) target_include_directories( unit_test PUBLIC diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp new file mode 100644 index 00000000000..c62b0d7da77 --- /dev/null +++ b/unittests/state_history_tests.cpp @@ -0,0 +1,203 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace eosio; +using namespace testing; +using namespace chain; + +struct dummy_action { + static eosio::chain::name get_name() { return N(dummyaction); } + static eosio::chain::name get_account() { return N(testapi); } + + char a; // 1 + uint64_t b; // 8 + int32_t c; // 4 +}; + +struct cf_action { + static eosio::chain::name get_name() { return N(cfaction); } + static eosio::chain::name get_account() { return N(testapi); } + + uint32_t payload = 100; + uint32_t cfd_idx = 0; // context free data index +}; + +FC_REFLECT(dummy_action, (a)(b)(c)) +FC_REFLECT(cf_action, (payload)(cfd_idx)) + +#define DUMMY_ACTION_DEFAULT_A 0x45 +#define DUMMY_ACTION_DEFAULT_B 0xab11cd1244556677 +#define DUMMY_ACTION_DEFAULT_C 0x7451ae12 + +std::vector deploy_test_api(eosio::testing::tester& chain) { + std::vector result; + chain.create_account(N(testapi)); + chain.create_account(N(dummy)); + result.push_back(chain.produce_block()); + chain.set_code(N(testapi), contracts::test_api_wasm()); + result.push_back(chain.produce_block()); + return result; +} + +eosio::chain::transaction_trace_ptr push_test_cfd_transaction(eosio::testing::tester& chain) { + cf_action cfa; + signed_transaction trx; + action act({}, cfa); + trx.context_free_actions.push_back(act); + trx.context_free_data.emplace_back(fc::raw::pack(100)); + trx.context_free_data.emplace_back(fc::raw::pack(200)); + // add a normal action along with cfa + dummy_action da = {DUMMY_ACTION_DEFAULT_A, DUMMY_ACTION_DEFAULT_B, DUMMY_ACTION_DEFAULT_C}; + action act1(vector{{N(testapi), config::active_name}}, da); + trx.actions.push_back(act1); + chain.set_transaction_headers(trx); + // run normal passing case + auto sigs = trx.sign(chain.get_private_key(N(testapi), "active"), chain.control->get_chain_id()); + return chain.push_transaction(trx); +} + +state_history::partial_transaction_v0 get_partial_from_traces_bin(const bytes& traces_bin, + const transaction_id_type& id) { + fc::datastream strm(traces_bin.data(), traces_bin.size()); + std::vector traces; + fc::raw::unpack(strm, traces); + + auto cfd_trace_itr = std::find_if(traces.begin(), traces.end(), [id](const state_history::transaction_trace& v) { + return v.get().id == id; + }); + + // make sure the trace with cfd can be found + BOOST_REQUIRE(cfd_trace_itr != traces.end()); + BOOST_REQUIRE(cfd_trace_itr->contains()); + auto trace_v0 = cfd_trace_itr->get(); + BOOST_REQUIRE(trace_v0.partial); + BOOST_REQUIRE(trace_v0.partial->contains()); + return trace_v0.partial->get(); +} + +struct scoped_temp_path { + boost::filesystem::path path; + scoped_temp_path() { path = boost::filesystem::unique_path(); } + ~scoped_temp_path() { + boost::filesystem::remove_all(path); + } +}; + +struct trace_converter_test_fixture { + packed_transaction::prunable_data_type::compression_type compression; + + void start() { + tester chain; + + state_history::trace_converter converter_v0, converter_v1; + std::map on_disk_log_entries_v0; + std::map on_disk_log_entries_v1; + + chain.control->applied_transaction.connect( + [&](std::tuple t) { + converter_v0.add_transaction(std::get<0>(t), std::get<1>(t)); + converter_v1.add_transaction(std::get<0>(t), std::get<1>(t)); + }); + + chain.control->accepted_block.connect([&](const block_state_ptr& bs) { + on_disk_log_entries_v0[bs->block_num] = converter_v0.pack(chain.control->db(), true, bs, 0, compression); + on_disk_log_entries_v1[bs->block_num] = converter_v1.pack(chain.control->db(), true, bs, 1, compression); + }); + + deploy_test_api(chain); + auto cfd_trace = push_test_cfd_transaction(chain); + chain.produce_blocks(10); + + BOOST_CHECK(on_disk_log_entries_v0.size()); + BOOST_CHECK(on_disk_log_entries_v1.size()); + + // make sure v0 and v1 are identifical when transform to traces bin + BOOST_CHECK(std::equal(on_disk_log_entries_v0.begin(), on_disk_log_entries_v0.end(), + on_disk_log_entries_v1.begin(), [&](const auto& entry_v0, const auto& entry_v1) { + return state_history::trace_converter::to_traces_bin_v0(entry_v0.second, 0) == + state_history::trace_converter::to_traces_bin_v0(entry_v1.second, 1); + })); + + // Now deserialize the on disk trace log and make sure that the cfd exists + auto& cfd_entry_v1 = on_disk_log_entries_v1[cfd_trace->block_num]; + auto partial = + get_partial_from_traces_bin(state_history::trace_converter::to_traces_bin_v0(cfd_entry_v1, 1), cfd_trace->id); + BOOST_REQUIRE(partial.context_free_data.size()); + BOOST_REQUIRE(partial.signatures.size()); + + // prune the cfd for the block + std::vector ids{cfd_trace->id}; + auto pruned = state_history::trace_converter::prune_traces(cfd_entry_v1, 1, ids); + BOOST_CHECK(pruned.size() < cfd_entry_v1.size()); + BOOST_CHECK(ids.size() == 0); + + std::copy(pruned.begin(), pruned.end(), cfd_entry_v1.end() - pruned.size()); + + // read the pruned trace and make sure the signature/cfd are empty + auto pruned_partial = get_partial_from_traces_bin(state_history::trace_converter::to_traces_bin_v0(cfd_entry_v1, 1), + cfd_trace->id); + BOOST_CHECK(pruned_partial.context_free_data.size() == 0); + BOOST_CHECK(pruned_partial.signatures.size() == 0); + } +}; + +BOOST_AUTO_TEST_SUITE(test_state_history) + +BOOST_FIXTURE_TEST_CASE(test_trace_converter_test_no_compression, trace_converter_test_fixture) { + compression = packed_transaction::prunable_data_type::compression_type::none; + start(); +} + +// BOOST_FIXTURE_TEST_CASE(test_trace_converter_test_zlib_compression, trace_converter_test_fixture) { +// compression = packed_transaction::prunable_data_type::compression_type::zlib; +// start(); +// } + +BOOST_AUTO_TEST_CASE(test_trace_log) { + tester chain; + + scoped_temp_path state_history_dir; + fc::create_directories(state_history_dir.path); + state_history_traces_log log(state_history_dir.path); + + chain.control->applied_transaction.connect( + [&](std::tuple t) { + log.add_transaction(std::get<0>(t), std::get<1>(t)); + }); + + chain.control->accepted_block.connect( + [&](const block_state_ptr& bs) { log.store_traces(chain.control->db(), bs); }); + + + deploy_test_api(chain); + auto cfd_trace = push_test_cfd_transaction(chain); + chain.produce_blocks(10); + + auto traces_bin = log.get_log_entry(cfd_trace->block_num); + BOOST_REQUIRE(traces_bin); + + auto partial = get_partial_from_traces_bin(*traces_bin, cfd_trace->id); + BOOST_REQUIRE(partial.context_free_data.size()); + BOOST_REQUIRE(partial.signatures.size()); + + std::vector ids{cfd_trace->id}; + log.prune_traces(cfd_trace->block_num, ids); + BOOST_REQUIRE(ids.empty()); + + auto pruned_traces_bin = log.get_log_entry(cfd_trace->block_num); + BOOST_REQUIRE(pruned_traces_bin); + + auto pruned_partial = get_partial_from_traces_bin(*pruned_traces_bin, cfd_trace->id); + BOOST_CHECK(pruned_partial.context_free_data.empty()); + BOOST_CHECK(pruned_partial.signatures.empty()); +} + +BOOST_AUTO_TEST_SUITE_END() From 2061e51003b8f79a54251588e6ccd41dd63c5200 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 22 Apr 2020 10:02:07 -0500 Subject: [PATCH 100/142] Convert full_legacy to full for prunable packing --- .../chain/include/eosio/chain/transaction.hpp | 1 - libraries/chain/transaction.cpp | 9 ++-- libraries/state_history/trace_converter.cpp | 50 ++++++++----------- 3 files changed, 26 insertions(+), 34 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index c7dd65617fb..5e5f550fad6 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -246,7 +246,6 @@ namespace eosio { namespace chain { std::vector signatures; bytes packed_context_free_data; vector context_free_segments; - void unpack_context_free_data(packed_transaction_v0::compression_type compression); }; using prunable_data_t = fc::static_variant< full_legacy, diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 7f6ee018dbb..411e8cca11c 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -407,7 +407,10 @@ static digest_type prunable_digest(const packed_transaction::prunable_data_type: } static digest_type prunable_digest(const packed_transaction::prunable_data_type::full& obj) { - EOS_THROW(tx_prune_exception, "unimplemented"); + digest_type::encoder prunable; + fc::raw::pack( prunable, obj.signatures ); + fc::raw::pack( prunable, obj.context_free_segments ); + return prunable.result(); } static digest_type prunable_digest(const packed_transaction::prunable_data_type::full_legacy& obj) { @@ -650,8 +653,4 @@ void packed_transaction::reflector_init() } estimated_size = calculate_estimated_size(); } - -void packed_transaction::prunable_data_type::full_legacy::unpack_context_free_data(packed_transaction_v0::compression_type compression) { - context_free_segments = chain::unpack_context_free_data(packed_context_free_data, compression); -} } } // eosio::chain diff --git a/libraries/state_history/trace_converter.cpp b/libraries/state_history/trace_converter.cpp index 70698e1a4f2..34a5c754766 100644 --- a/libraries/state_history/trace_converter.cpp +++ b/libraries/state_history/trace_converter.cpp @@ -103,20 +103,6 @@ void pack(fc::datastream& ds, const prunable_data_type& data, compressio } } -void unpack(fc::datastream& ds, prunable_data_type& data, compression_type compression) { - if (compression == compression_type::none) { - fc::raw::unpack(ds, data); - } - else if (compression == compression_type::zlib) { - bytes compressed; - fc::raw::unpack(ds, compressed); - bytes uncompressed = zlib_decompress(compressed); - fc::datastream uncompressed_ds(uncompressed.data(), uncompressed.size()); - fc::raw::unpack(uncompressed_ds, data); - } -} - - // each individual pruable data in a trasaction is packed as an optional prunable_data_type::full // or an optional bytes to a compressed prunable_data_type::full size_t pack_pruable(bytes& buffer, const packed_transaction& trx, @@ -125,17 +111,20 @@ size_t pack_pruable(bytes& buffer, const packed_transaction& trx, const auto& data = trx.get_prunable_data(); int max_size = data.maximum_pruned_pack_size(compression); - if (data.prunable_data.contains()) { - max_size += sizeof(uint8_t); - } uint64_t start_buffer_position = buffer.size(); buffer.resize(buffer.size() + max_size); fc::datastream ds(buffer.data() + start_buffer_position, max_size); - pack(ds, data, compression); - + if (data.prunable_data.contains()) { - fc::raw::pack(ds, static_cast(trx.get_compression())); + // convert full_legacy to full before serialization + const auto& full_legacy = data.prunable_data.get(); + prunable_data_type pd = {prunable_data_type::full{full_legacy.signatures, full_legacy.context_free_segments}}; + pack(ds, pd, compression); } + else { + pack(ds, data, compression); + } + buffer.resize(start_buffer_position + ds.tellp()); return max_size; } @@ -144,12 +133,17 @@ size_t pack_pruable(bytes& buffer, const packed_transaction& trx, void unpack_prunable(const char* read_buffer, fc::datastream& ds, prunable_data_type& data, compression_type compression) { - unpack(ds, data, compression); - if (data.prunable_data.contains()) { - uint8_t local_comp; - fc::raw::unpack(ds, local_comp); - auto& full_legacy = data.prunable_data.get(); - full_legacy.unpack_context_free_data(static_cast(local_comp)); + if (compression == compression_type::none) { + fc::raw::unpack(ds, data); + } + else { + fc::unsigned_int len; + fc::raw::unpack(ds, len); + bytes uncompressed = decompress_buffer(read_buffer + ds.tellp(), len, compression); + ds.skip(len); + + fc::datastream uncompressed_ds(uncompressed.data(), uncompressed.size()); + fc::raw::unpack(uncompressed_ds, data); } } @@ -234,8 +228,8 @@ bytes trace_converter::pack(const chainbase::database& db, bool trace_debug_mode struct restore_partial { partial_transaction_v0& ptrx; void operator()(prunable_data_type::full_legacy& data) const { - ptrx.signatures = std::move(data.signatures); - ptrx.context_free_data = std::move(data.context_free_segments); + // this should never happen because we convert full_legacy to full before serialization + assert(false); } void operator()(prunable_data_type::none& data) const {} void operator()(prunable_data_type::signatures_only& data) const { From 0918c873b6238b3d0e30165814ccf5db6a1943fc Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 22 Apr 2020 10:26:34 -0500 Subject: [PATCH 101/142] trying to fix CI issue --- unittests/state_history_tests.cpp | 57 +++++++++++++++++-------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index c62b0d7da77..b752277eac3 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -162,42 +162,47 @@ BOOST_FIXTURE_TEST_CASE(test_trace_converter_test_no_compression, trace_converte // } BOOST_AUTO_TEST_CASE(test_trace_log) { - tester chain; + namespace bfs = boost::filesystem; + tester chain; - scoped_temp_path state_history_dir; - fc::create_directories(state_history_dir.path); - state_history_traces_log log(state_history_dir.path); + scoped_temp_path state_history_dir; + fc::create_directories(state_history_dir.path); + state_history_traces_log log(state_history_dir.path); - chain.control->applied_transaction.connect( - [&](std::tuple t) { - log.add_transaction(std::get<0>(t), std::get<1>(t)); - }); + chain.control->applied_transaction.connect( + [&](std::tuple t) { + log.add_transaction(std::get<0>(t), std::get<1>(t)); + }); - chain.control->accepted_block.connect( - [&](const block_state_ptr& bs) { log.store_traces(chain.control->db(), bs); }); + chain.control->accepted_block.connect([&](const block_state_ptr& bs) { log.store_traces(chain.control->db(), bs); }); + deploy_test_api(chain); + auto cfd_trace = push_test_cfd_transaction(chain); + chain.produce_blocks(10); - deploy_test_api(chain); - auto cfd_trace = push_test_cfd_transaction(chain); - chain.produce_blocks(10); + auto traces_bin = log.get_log_entry(cfd_trace->block_num); + BOOST_REQUIRE(traces_bin); - auto traces_bin = log.get_log_entry(cfd_trace->block_num); - BOOST_REQUIRE(traces_bin); + auto partial = get_partial_from_traces_bin(*traces_bin, cfd_trace->id); + BOOST_REQUIRE(partial.context_free_data.size()); + BOOST_REQUIRE(partial.signatures.size()); - auto partial = get_partial_from_traces_bin(*traces_bin, cfd_trace->id); - BOOST_REQUIRE(partial.context_free_data.size()); - BOOST_REQUIRE(partial.signatures.size()); + scoped_temp_path temp_dir; + fc::create_directories(temp_dir.path); + bfs::copy(state_history_dir.path / "trace_history.log", temp_dir.path / "trace_history.log"); + bfs::copy(state_history_dir.path / "trace_history.index", temp_dir.path / "trace_history.index"); - std::vector ids{cfd_trace->id}; - log.prune_traces(cfd_trace->block_num, ids); - BOOST_REQUIRE(ids.empty()); + state_history_traces_log temp_log(temp_dir.path); + std::vector ids{cfd_trace->id}; + temp_log.prune_traces(cfd_trace->block_num, ids); + BOOST_REQUIRE(ids.empty()); - auto pruned_traces_bin = log.get_log_entry(cfd_trace->block_num); - BOOST_REQUIRE(pruned_traces_bin); + auto pruned_traces_bin = temp_log.get_log_entry(cfd_trace->block_num); + BOOST_REQUIRE(pruned_traces_bin); - auto pruned_partial = get_partial_from_traces_bin(*pruned_traces_bin, cfd_trace->id); - BOOST_CHECK(pruned_partial.context_free_data.empty()); - BOOST_CHECK(pruned_partial.signatures.empty()); + auto pruned_partial = get_partial_from_traces_bin(*pruned_traces_bin, cfd_trace->id); + BOOST_CHECK(pruned_partial.context_free_data.empty()); + BOOST_CHECK(pruned_partial.signatures.empty()); } BOOST_AUTO_TEST_SUITE_END() From 82c55847feb33f7d606cfcb1d15299374b108875 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 22 Apr 2020 11:28:29 -0500 Subject: [PATCH 102/142] Resolve CI issue --- .../include/eosio/state_history/log.hpp | 2 +- libraries/state_history/log.cpp | 2 +- unittests/state_history_tests.cpp | 17 ++++++++--------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/libraries/state_history/include/eosio/state_history/log.hpp b/libraries/state_history/include/eosio/state_history/log.hpp index 5f8da607b96..1cb51180e94 100644 --- a/libraries/state_history/include/eosio/state_history/log.hpp +++ b/libraries/state_history/include/eosio/state_history/log.hpp @@ -134,7 +134,7 @@ class state_history_traces_log : public state_history_log { /** * @param[in,out] ids The ids to been pruned and returns the ids not found in the specified block **/ - void prune_traces(block_num_type block_num, std::vector& ids); + void prune_transactions(block_num_type block_num, std::vector& ids); }; } // namespace eosio diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp index 71da9e5b13e..d265a9bbb7b 100644 --- a/libraries/state_history/log.cpp +++ b/libraries/state_history/log.cpp @@ -266,7 +266,7 @@ fc::optional state_history_traces_log::get_log_entry(block_num_typ }); } -void state_history_traces_log::prune_traces(state_history_log::block_num_type block_num, +void state_history_traces_log::prune_transactions(state_history_log::block_num_type block_num, std::vector& ids) { auto entry_result = get_entry(block_num); EOS_ASSERT(entry_result, chain::state_history_exception, "nonexistant block num ${block_num}", diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index b752277eac3..a38dba45ddf 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -85,7 +85,12 @@ state_history::partial_transaction_v0 get_partial_from_traces_bin(const bytes& struct scoped_temp_path { boost::filesystem::path path; - scoped_temp_path() { path = boost::filesystem::unique_path(); } + scoped_temp_path() { + path = boost::filesystem::unique_path(); + if (boost::unit_test::framework::master_test_suite().argc >= 2) { + path += boost::unit_test::framework::master_test_suite().argv[1]; + } + } ~scoped_temp_path() { boost::filesystem::remove_all(path); } @@ -187,17 +192,11 @@ BOOST_AUTO_TEST_CASE(test_trace_log) { BOOST_REQUIRE(partial.context_free_data.size()); BOOST_REQUIRE(partial.signatures.size()); - scoped_temp_path temp_dir; - fc::create_directories(temp_dir.path); - bfs::copy(state_history_dir.path / "trace_history.log", temp_dir.path / "trace_history.log"); - bfs::copy(state_history_dir.path / "trace_history.index", temp_dir.path / "trace_history.index"); - - state_history_traces_log temp_log(temp_dir.path); std::vector ids{cfd_trace->id}; - temp_log.prune_traces(cfd_trace->block_num, ids); + log.prune_transactions(cfd_trace->block_num, ids); BOOST_REQUIRE(ids.empty()); - auto pruned_traces_bin = temp_log.get_log_entry(cfd_trace->block_num); + auto pruned_traces_bin = log.get_log_entry(cfd_trace->block_num); BOOST_REQUIRE(pruned_traces_bin); auto pruned_partial = get_partial_from_traces_bin(*pruned_traces_bin, cfd_trace->id); From a7d03869beea520328bae39338241459cbfb4750 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 22 Apr 2020 16:01:15 -0500 Subject: [PATCH 103/142] fix get_log_entry reading issue --- libraries/state_history/log.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp index d265a9bbb7b..20620be39f2 100644 --- a/libraries/state_history/log.cpp +++ b/libraries/state_history/log.cpp @@ -107,7 +107,11 @@ void state_history_log::recover_blocks(uint64_t size) { void state_history_log::open_log() { log.set_file_path(log_filename); - log.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app + if (!fc::exists(log_filename)) { + log.open("a+b"); + log.close(); + } + log.open("rb+"); // we need to use "rb+" mode; otherwise, transaction pruning changes on disk won't be available for reading log.seek_end(0); uint64_t size = log.tellp(); if (size >= state_history_log_header_serial_size) { @@ -272,21 +276,16 @@ void state_history_traces_log::prune_transactions(state_history_log::block_num_t EOS_ASSERT(entry_result, chain::state_history_exception, "nonexistant block num ${block_num}", ("block_num", block_num)); - auto pos = get_log().tellp(); + auto& log = get_log(); + auto pos = log.tellp(); auto [entry_payload, version] = *entry_result; auto pruned_section = state_history::trace_converter::prune_traces(entry_payload, version, ids); auto bytes_to_write = pruned_section.size(); if (bytes_to_write > 0) { - // the log object is in append mode, which cannot be used to update written content, - // we need to open the file in the update mode - - fc::cfile update_log; - update_log.set_file_path(get_log().get_file_path()); - update_log.open(fc::cfile::update_rw_mode); - update_log.seek(pos - bytes_to_write); - update_log.write(pruned_section.data(), bytes_to_write); + log.seek(pos - bytes_to_write); + log.write(pruned_section.data(), bytes_to_write); } } From afdba27698b648b0a8b474907bf81bced2303f91 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 22 Apr 2020 16:10:14 -0500 Subject: [PATCH 104/142] Revert "fix get_log_entry reading issue" This reverts commit a7d03869beea520328bae39338241459cbfb4750. --- libraries/state_history/log.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp index 20620be39f2..d265a9bbb7b 100644 --- a/libraries/state_history/log.cpp +++ b/libraries/state_history/log.cpp @@ -107,11 +107,7 @@ void state_history_log::recover_blocks(uint64_t size) { void state_history_log::open_log() { log.set_file_path(log_filename); - if (!fc::exists(log_filename)) { - log.open("a+b"); - log.close(); - } - log.open("rb+"); // we need to use "rb+" mode; otherwise, transaction pruning changes on disk won't be available for reading + log.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app log.seek_end(0); uint64_t size = log.tellp(); if (size >= state_history_log_header_serial_size) { @@ -276,16 +272,21 @@ void state_history_traces_log::prune_transactions(state_history_log::block_num_t EOS_ASSERT(entry_result, chain::state_history_exception, "nonexistant block num ${block_num}", ("block_num", block_num)); - auto& log = get_log(); - auto pos = log.tellp(); + auto pos = get_log().tellp(); auto [entry_payload, version] = *entry_result; auto pruned_section = state_history::trace_converter::prune_traces(entry_payload, version, ids); auto bytes_to_write = pruned_section.size(); if (bytes_to_write > 0) { - log.seek(pos - bytes_to_write); - log.write(pruned_section.data(), bytes_to_write); + // the log object is in append mode, which cannot be used to update written content, + // we need to open the file in the update mode + + fc::cfile update_log; + update_log.set_file_path(get_log().get_file_path()); + update_log.open(fc::cfile::update_rw_mode); + update_log.seek(pos - bytes_to_write); + update_log.write(pruned_section.data(), bytes_to_write); } } From 2d361c811e908e2c8cfff2e301e4a837514fcc8c Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 22 Apr 2020 16:17:20 -0500 Subject: [PATCH 105/142] fix test failure --- unittests/state_history_tests.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index a38dba45ddf..20a10a6b73c 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -196,7 +196,10 @@ BOOST_AUTO_TEST_CASE(test_trace_log) { log.prune_transactions(cfd_trace->block_num, ids); BOOST_REQUIRE(ids.empty()); - auto pruned_traces_bin = log.get_log_entry(cfd_trace->block_num); + // we assume the nodeos has to be stopped while running, it can only be read + // correctly with restart + state_history_traces_log new_log(state_history_dir.path); + auto pruned_traces_bin = new_log.get_log_entry(cfd_trace->block_num); BOOST_REQUIRE(pruned_traces_bin); auto pruned_partial = get_partial_from_traces_bin(*pruned_traces_bin, cfd_trace->id); From 53ad85c81a2fdc4d5310cd26beee49607aff03c2 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 21 Apr 2020 16:55:46 -0500 Subject: [PATCH 106/142] Change block log prune interface --- libraries/chain/block_log.cpp | 26 ++++++++++++++----- .../chain/include/eosio/chain/block_log.hpp | 6 ++++- unittests/restart_chain_tests.cpp | 3 ++- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index cceacd7ebf2..ba9a63ac537 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -883,7 +883,7 @@ namespace eosio { namespace chain { })); } - size_t block_log::prune_transactions(uint32_t block_num, const std::vector& ids) { + size_t block_log::prune_transactions(uint32_t block_num, std::vector& ids) { try { EOS_ASSERT( my->version >= pruned_transaction_version, block_log_exception, "The block log version ${version} does not support transaction pruning.", ("version", my->version) ); @@ -899,11 +899,21 @@ namespace eosio { namespace chain { EOS_ASSERT(entry.block_num() == block_num, block_log_exception, "Wrong block was read from block log."); - auto pruner = - overloaded{[](transaction_id_type&) { return false; }, - [&ids](packed_transaction& ptx) { - return std::find(ids.begin(), ids.end(), ptx.id()) != ids.end() && (ptx.prune_all(), true); - }}; + auto pruner = overloaded{[](transaction_id_type&) { return false; }, + [&ids](packed_transaction& ptx) { + auto it = std::find(ids.begin(), ids.end(), ptx.id()); + if (it != ids.end()) { + ptx.prune_all(); + // remove the found entry from ids + if (it != ids.end() - 1) { + *it = ids.back(); + } + ids.resize(ids.size() - 1); + return true; + } + return false; + }}; + size_t num_trx_pruned = 0; for (auto& trx : entry.transactions) { num_trx_pruned += trx.trx.visit(pruner); @@ -1445,4 +1455,8 @@ namespace eosio { namespace chain { break; } } + + bool block_log::exists(const fc::path& data_dir) { + return fc::exists(data_dir / "blocks.log") && fc::exists(data_dir / "blocks.index"); + } }} /// eosio::chain diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index 3c86b8b5b01..8f9ed72426b 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -56,10 +56,14 @@ namespace eosio { namespace chain { const signed_block_ptr& head() const; uint32_t first_block_num() const; + static bool exists(const fc::path& data_dir); /** + * @param ids[in,out] The list of transaction ids to be pruned. After the member function returns, + * it would be modified to contain the list of transaction ids that do not + * exists in the specified block. * @returns The number of transactions been pruned **/ - size_t prune_transactions(uint32_t block_num, const vector& ids); + size_t prune_transactions(uint32_t block_num, vector& ids); static const uint64_t npos = std::numeric_limits::max(); diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index f61f4b63254..0ee90a670e4 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -266,7 +266,8 @@ BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log, light_vali BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log_with_pruned_trx, light_validation_restart_from_block_log_test_fixture) { const auto& blocks_dir = chain.get_config().blocks_dir; block_log blog(blocks_dir); - BOOST_CHECK(blog.prune_transactions(trace->block_num, std::vector{trace->id}) == 1); + std::vector ids{trace->id}; + BOOST_CHECK(blog.prune_transactions(trace->block_num, ids) == 1); BOOST_REQUIRE_NO_THROW(block_log::repair_log(blocks_dir)); } From e3a8078c1bce2be47dda79b6e225ac893a11a4f3 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 22 Apr 2020 07:32:12 -0500 Subject: [PATCH 107/142] Add prune transaction to eosio-blocklog working --- libraries/state_history/log.cpp | 4 +- programs/eosio-blocklog/CMakeLists.txt | 2 +- programs/eosio-blocklog/main.cpp | 53 ++++++++++- tests/CMakeLists.txt | 1 + tests/Cluster.py | 6 +- tests/eosio_blocklog_prune_test.py | 120 +++++++++++++++++++++++++ tests/testUtils.py | 12 ++- 7 files changed, 187 insertions(+), 11 deletions(-) create mode 100755 tests/eosio_blocklog_prune_test.py diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp index d265a9bbb7b..e4b6bf09a8d 100644 --- a/libraries/state_history/log.cpp +++ b/libraries/state_history/log.cpp @@ -267,7 +267,7 @@ fc::optional state_history_traces_log::get_log_entry(block_num_typ } void state_history_traces_log::prune_transactions(state_history_log::block_num_type block_num, - std::vector& ids) { + std::vector& ids) { auto entry_result = get_entry(block_num); EOS_ASSERT(entry_result, chain::state_history_exception, "nonexistant block num ${block_num}", ("block_num", block_num)); @@ -276,7 +276,7 @@ void state_history_traces_log::prune_transactions(state_history_log::block_num_t auto [entry_payload, version] = *entry_result; auto pruned_section = state_history::trace_converter::prune_traces(entry_payload, version, ids); - auto bytes_to_write = pruned_section.size(); + auto bytes_to_write = pruned_section.size(); if (bytes_to_write > 0) { // the log object is in append mode, which cannot be used to update written content, diff --git a/programs/eosio-blocklog/CMakeLists.txt b/programs/eosio-blocklog/CMakeLists.txt index 548a6c5be25..213772e86ba 100644 --- a/programs/eosio-blocklog/CMakeLists.txt +++ b/programs/eosio-blocklog/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories(eosio-blocklog PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) target_link_libraries( eosio-blocklog PRIVATE appbase - PRIVATE eosio_chain fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) + PRIVATE eosio_chain fc state_history ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) copy_bin( eosio-blocklog ) install( TARGETS diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp index 702d03ea12f..e8c7ef3a57f 100644 --- a/programs/eosio-blocklog/main.cpp +++ b/programs/eosio-blocklog/main.cpp @@ -4,6 +4,8 @@ #include #include +#include + #include #include #include @@ -47,7 +49,8 @@ struct blocklog { bool make_index = false; bool trim_log = false; bool smoke_test = false; - bool help = false; + bool prune_transactions = false; + bool help = false; }; struct report_time { @@ -171,6 +174,8 @@ void blocklog::set_program_options(options_description& cli) cli.add_options() ("blocks-dir", bpo::value()->default_value("blocks"), "the location of the blocks directory (absolute path or relative to the current directory)") + ("state-history-dir", bpo::value()->default_value("state-history"), + "the location of the state-history directory (absolute path or relative to application data dir)") ("output-file,o", bpo::value(), "the file to write the output to (absolute or relative path). If not specified then output is to stdout.") ("first,f", bpo::value(&first_block)->default_value(0), @@ -187,6 +192,10 @@ void blocklog::set_program_options(options_description& cli) "Trim blocks.log and blocks.index. Must give 'blocks-dir' and 'first and/or 'last'.") ("smoke-test", bpo::bool_switch(&smoke_test)->default_value(false), "Quick test that blocks.log and blocks.index are well formed and agree with each other.") + ("block-num", bpo::value()->default_value(0), "The block number which contains the transactions to be pruned") + ("transaction,t", bpo::value >()->multitoken(), "The transaction id to be pruned") + ("prune-transactions", bpo::bool_switch(&prune_transactions)->default_value(false), + "Prune the context free data and signatures from specified transactions of specified block-num.") ("help,h", bpo::bool_switch(&help)->default_value(false), "Print this help message and exit.") ; } @@ -232,6 +241,37 @@ void smoke_test(bfs::path block_dir) { cout << "\nno problems found\n"; // if get here there were no exceptions } +template +int prune_transactions(const char* type, bfs::path dir, uint32_t block_num, + std::vector unpruned_ids) { + using namespace std; + if (Log::exists(dir)) { + Log log(dir); + log.prune_transactions(block_num, unpruned_ids); + if (unpruned_ids.size()) { + cerr << "block " << block_num << " in " << type << " does not contain the following transactions: "; + copy(unpruned_ids.begin(), unpruned_ids.end(), ostream_iterator(cerr, " ")); + cerr << "\n"; + } + } else { + cerr << "No " << type << " is found in " << dir.native() << "\n"; + } + return unpruned_ids.size(); +} + +int prune_transactions(bfs::path block_dir, bfs::path state_history_dir, uint32_t block_num, + const std::vector& ids) { + + if (block_num == 0 || ids.size() == 0) { + std::cerr << "prune-transaction does nothing unless specify block-num and transaction\n"; + return -1; + } + + using eosio::state_history_traces_log; + return prune_transactions("block log", block_dir, block_num, ids) + + prune_transactions("state history traces log", state_history_dir, block_num, ids); +} + int main(int argc, char** argv) { std::ios::sync_with_stdio(false); // for potential performance boost for large block log files options_description cli ("eosio-blocklog command line options"); @@ -280,6 +320,17 @@ int main(int argc, char** argv) { rt.report(); return 0; } + if (blog.prune_transactions) { + const auto blocks_dir = vmap.at("blocks-dir").as(); + const auto state_history_dir = vmap.at("state-history-dir").as(); + const auto block_num = vmap.at("block-num").as(); + const auto ids = vmap.at("transaction").as>(); + + report_time rt("prune transactions"); + int ret = prune_transactions(blocks_dir, state_history_dir, block_num, {ids.begin(), ids.end()}); + rt.report(); + return ret; + } //else print blocks.log as JSON blog.initialize(vmap); blog.read_log(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b0e1bf6cf30..a870178fcee 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -50,6 +50,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ship_test.py ${CMAKE_CURRENT_BINARY_D configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ship_client.js ${CMAKE_CURRENT_BINARY_DIR}/ship_client.js COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_high_transaction_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_high_transaction_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/light_validation_sync_test.py ${CMAKE_CURRENT_BINARY_DIR}/light_validation_sync_test.py COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/eosio_blocklog_prune_test.py ${CMAKE_CURRENT_BINARY_DIR}/eosio_blocklog_prune_test.py COPYONLY) #To run plugin_test with all log from blockchain displayed, put --verbose after --, i.e. plugin_test -- --verbose add_test(NAME plugin_test COMMAND plugin_test --report_level=detailed --color_output) diff --git a/tests/Cluster.py b/tests/Cluster.py index 2b06300a83e..d2341d91994 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -1639,9 +1639,9 @@ def printBlockLogIfNeeded(self): self.printBlockLog() - def getBlockLog(self, nodeExtension, blockLogAction=BlockLogAction.return_blocks, outputFile=None, first=None, last=None, throwException=False, silentErrors=False, exitOnError=False): - blockLogDir=Utils.getNodeDataDir(nodeExtension, "blocks") - return Utils.getBlockLog(blockLogDir, blockLogAction=blockLogAction, outputFile=outputFile, first=first, last=last, throwException=throwException, silentErrors=silentErrors, exitOnError=exitOnError) + def getBlockLog(self, nodeExtension, blockLogAction=BlockLogAction.return_blocks, outputFile=None, first=None, last=None, extraArgs="", throwException=False, silentErrors=False, exitOnError=False): + nodeDataDir=Utils.getNodeDataDir(nodeExtension) + return Utils.getBlockLog(nodeDataDir, blockLogAction=blockLogAction, outputFile=outputFile, first=first, last=last, extraArgs=extraArgs, throwException=throwException, silentErrors=silentErrors, exitOnError=exitOnError) def printBlockLog(self): blockLogBios=self.getBlockLog("bios") diff --git a/tests/eosio_blocklog_prune_test.py b/tests/eosio_blocklog_prune_test.py new file mode 100755 index 00000000000..6bbde27e06e --- /dev/null +++ b/tests/eosio_blocklog_prune_test.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +from testUtils import Account +from testUtils import Utils +from Cluster import Cluster +from WalletMgr import WalletMgr +from Node import Node +from Node import ReturnType +from TestHelper import TestHelper +from TestHelper import AppArgs +from testUtils import BlockLogAction +import json + +############################################################### +# eosio_blocklog_prune_test.py +# +# Test eosio-blocklog prune-transaction +# +# This test creates a producer node and a verification node. +# The verification node is configured with state history plugin. +# A transaction with context free data is pushed to the producer node. +# Afterwards, it uses eosio-blocklog to prune the transaction with context free +# data and check if the context free data in the block log has actually been pruned. +# Notice that there's no verification on whether the pruning on the +# state history traces log has actually been pruned. We rely on the +# unittests/state_history_tests.cpp to perform this test. +# +############################################################### + +# Parse command line arguments +args = TestHelper.parse_args({"-v","--clean-run","--dump-error-details","--leave-running","--keep-logs"}) +Utils.Debug = args.v +killAll=args.clean_run +dumpErrorDetails=args.dump_error_details +dontKill=args.leave_running +killEosInstances=not dontKill +killWallet=not dontKill +keepLogs=args.keep_logs + +walletMgr=WalletMgr(True) +cluster=Cluster(walletd=True) +cluster.setWalletMgr(walletMgr) + +testSuccessful = False +try: + TestHelper.printSystemInfo("BEGIN") + cluster.killall(allInstances=killAll) + cluster.cleanup() + + assert cluster.launch( + pnodes=1, + prodCount=1, + totalProducers=1, + totalNodes=2, + useBiosBootFile=False, + loadSystemContract=False, + specificExtraNodeosArgs={ + 1:"--plugin eosio::state_history_plugin --trace-history --disable-replay-opts --sync-fetch-span 200 --plugin eosio::net_api_plugin "}) + + producerNode = cluster.getNode(0) + validationNode = cluster.getNode(1) + + # Create a transaction to create an account + Utils.Print("create a new account payloadless from the producer node") + payloadlessAcc = Account("payloadless") + payloadlessAcc.ownerPublicKey = "EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV" + payloadlessAcc.activePublicKey = "EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV" + producerNode.createAccount(payloadlessAcc, cluster.eosioAccount) + + + contractDir="unittests/test-contracts/payloadless" + wasmFile="payloadless.wasm" + abiFile="payloadless.abi" + Utils.Print("Publish payloadless contract") + trans = producerNode.publishContract(payloadlessAcc, contractDir, wasmFile, abiFile, waitForTransBlock=True) + + trx = { + "actions": [{"account": "payloadless", "name": "doit", "authorization": [{ + "actor": "payloadless", "permission": "active"}], "data": ""}], + "context_free_actions": [{"account": "payloadless", "name": "doit", "data": ""}], + "context_free_data": ["a1b2c3", "1a2b3c"], + } + + cmd = "push transaction '{}' -p payloadless".format(json.dumps(trx)) + trans = producerNode.processCleosCmd(cmd, cmd, silentErrors=False) + assert trans, "Failed to push transaction with context free data" + + cfTrxBlockNum = int(trans["processed"]["block_num"]) + cfTrxId = trans["transaction_id"] + + # Wait until the block where create account is executed to become irreversible + def isBlockNumIrr(): + return validationNode.getIrreversibleBlockNum() >= cfTrxBlockNum + Utils.waitForBool(isBlockNumIrr, timeout=30, sleepTime=0.1) + + Utils.Print("verify the account payloadless from validation node") + cmd = "get account -j payloadless" + trans = validationNode.processCleosCmd(cmd, cmd, silentErrors=False) + assert trans["account_name"], "Failed to get the account payloadless" + + Utils.Print("verify the context free transaction from validation node") + cmd = "get transaction " + cfTrxId + trans = validationNode.processCleosCmd(cmd, cmd, silentErrors=False) + assert trans, "Failed to get the transaction with context free data from the light validation node" + + cluster.getBlockLog(1, blockLogAction=BlockLogAction.prune_transactions, extraArgs=" --block-num {} --transaction {}".format(trans["block_num"], cfTrxId), exitOnError=True) + + trans = validationNode.processCleosCmd(cmd, cmd, silentErrors=False) + assert trans, "Failed to get the transaction with context free data from the light validation node" + assert trans["trx"]["receipt"]["trx"][1]["prunable_data"]["prunable_data"][0] == 1, "the the transaction with context free data has not been pruned" + + + # output = cluster.getBlockLog(1, blockLogAction=BlockLogAction.prune_transactions, extraArgs=" --block-num {} --transaction {}".format(cfTrxBlockNum-1, cfTrxId)) + + testSuccessful = True +finally: + TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails) + +exitCode = 0 if testSuccessful else 1 +exit(exitCode) diff --git a/tests/testUtils.py b/tests/testUtils.py index c3224120000..ad75b39f169 100755 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -47,6 +47,7 @@ class BlockLogAction(EnumType): addEnum(BlockLogAction, "trim") addEnum(BlockLogAction, "smoke_test") addEnum(BlockLogAction, "return_blocks") +addEnum(BlockLogAction, "prune_transactions") ########################################################################################### class Utils: @@ -327,11 +328,12 @@ def pgrepCmd(serverName): return "pgrep %s %s" % (pgrepOpts, serverName) @staticmethod - def getBlockLog(blockLogLocation, blockLogAction=BlockLogAction.return_blocks, outputFile=None, first=None, last=None, throwException=False, silentErrors=False, exitOnError=False): + def getBlockLog(nodeDataDir, blockLogAction=BlockLogAction.return_blocks, outputFile=None, first=None, last=None, extraArgs="", throwException=False, silentErrors=False, exitOnError=False): + blockLogLocation = os.path.join(nodeDataDir, "blocks") assert(isinstance(blockLogLocation, str)) outputFileStr=" --output-file %s " % (outputFile) if outputFile is not None else "" firstStr=" --first %s " % (first) if first is not None else "" - lastStr=" --last %s " % (last) if last is not None else "" + lastStr = " --last %s " % (last) if last is not None else "" blockLogActionStr=None returnType=ReturnType.raw @@ -343,11 +345,13 @@ def getBlockLog(blockLogLocation, blockLogAction=BlockLogAction.return_blocks, o elif blockLogAction==BlockLogAction.trim: blockLogActionStr=" --trim " elif blockLogAction==BlockLogAction.smoke_test: - blockLogActionStr=" --smoke-test " + blockLogActionStr = " --smoke-test " + elif blockLogAction == BlockLogAction.prune_transactions: + blockLogActionStr = " --state-history-dir {}/state-history --prune-transactions ".format(nodeDataDir) else: unhandledEnumType(blockLogAction) - cmd="%s --blocks-dir %s --as-json-array %s%s%s%s" % (Utils.EosBlockLogPath, blockLogLocation, outputFileStr, firstStr, lastStr, blockLogActionStr) + cmd="%s --blocks-dir %s --as-json-array %s%s%s%s %s" % (Utils.EosBlockLogPath, blockLogLocation, outputFileStr, firstStr, lastStr, blockLogActionStr, extraArgs) if Utils.Debug: Utils.Print("cmd: %s" % (cmd)) rtn=None try: From 552ff210f55621f98fb41707b0818f74f7ca5c7c Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 22 Apr 2020 17:30:10 -0500 Subject: [PATCH 108/142] Fix some typo and PR comments --- .../include/eosio/state_history/serialization.hpp | 4 ++-- libraries/state_history/trace_converter.cpp | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/libraries/state_history/include/eosio/state_history/serialization.hpp b/libraries/state_history/include/eosio/state_history/serialization.hpp index c15b62f423e..b0f56923595 100644 --- a/libraries/state_history/include/eosio/state_history/serialization.hpp +++ b/libraries/state_history/include/eosio/state_history/serialization.hpp @@ -43,8 +43,8 @@ history_context_wrapper, std::decay_t> make_history_context_w struct trace_receipt_context { uint8_t failed_status = eosio::chain::transaction_receipt_header::hard_fail; - bool debug_mode; - uint32_t version; + bool debug_mode = false; + uint32_t version = 0; }; namespace fc { diff --git a/libraries/state_history/trace_converter.cpp b/libraries/state_history/trace_converter.cpp index 34a5c754766..c97ff3d951a 100644 --- a/libraries/state_history/trace_converter.cpp +++ b/libraries/state_history/trace_converter.cpp @@ -43,7 +43,7 @@ namespace { bytes decompress_buffer(const char* buffer, uint32_t len, compression_type compression) { bytes decompressed; bio::filtering_ostreambuf strm; - EOS_ASSERT(compression == compression_type::zlib, state_history_exception, "unspported compression format"); + EOS_ASSERT(compression == compression_type::zlib, state_history_exception, "unsupported compression format"); strm.push(bio::zlib_decompressor()); strm.push(bio::back_inserter(decompressed)); bio::write(strm, buffer, len); @@ -103,9 +103,9 @@ void pack(fc::datastream& ds, const prunable_data_type& data, compressio } } -// each individual pruable data in a trasaction is packed as an optional prunable_data_type::full +// each individual prunable data in a transaction is packed as an optional prunable_data_type::full // or an optional bytes to a compressed prunable_data_type::full -size_t pack_pruable(bytes& buffer, const packed_transaction& trx, +size_t pack_prunable(bytes& buffer, const packed_transaction& trx, compression_type compression) { const auto& data = trx.get_prunable_data(); @@ -154,8 +154,9 @@ bytes pack_prunables(const std::vector& traces, com int size_with_paddings = 0; for_each_packed_transaction( - traces, [&](const chain::packed_transaction& pt) { size_with_paddings += pack_pruable(out, pt, compression); }); + traces, [&](const chain::packed_transaction& pt) { size_with_paddings += pack_prunable(out, pt, compression); }); + EOS_ASSERT(size_with_paddings >= out.size(), state_history_exception, "The estimated max size is small than the actual size"); out.resize(size_with_paddings); return out; } From 367bec7fd983571a109a72704e0cc265b4b38eb6 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 22 Apr 2020 19:32:22 -0500 Subject: [PATCH 109/142] remove outdated comments --- libraries/state_history/trace_converter.cpp | 41 ++++++++++----------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/libraries/state_history/trace_converter.cpp b/libraries/state_history/trace_converter.cpp index c97ff3d951a..53059306bac 100644 --- a/libraries/state_history/trace_converter.cpp +++ b/libraries/state_history/trace_converter.cpp @@ -103,8 +103,6 @@ void pack(fc::datastream& ds, const prunable_data_type& data, compressio } } -// each individual prunable data in a transaction is packed as an optional prunable_data_type::full -// or an optional bytes to a compressed prunable_data_type::full size_t pack_prunable(bytes& buffer, const packed_transaction& trx, compression_type compression) { @@ -193,6 +191,26 @@ void do_visit_trace(const char* read_buffer, fc::datastream& strm, visitor(trace_v0, prunable); } } + +struct restore_partial { + partial_transaction_v0& ptrx; + void operator()(prunable_data_type::full_legacy& data) const { + // this should never happen because we convert full_legacy to full before serialization + assert(false); + } + void operator()(prunable_data_type::none& data) const {} + void operator()(prunable_data_type::signatures_only& data) const { + ptrx.signatures = std::move(data.signatures); + } + void operator()(prunable_data_type::partial& data) const { + ptrx.signatures = std::move(data.signatures); + } + void operator()(prunable_data_type::full& data) const { + ptrx.signatures = std::move(data.signatures); + ptrx.context_free_data = std::move(data.context_free_segments); + } +}; + } // namespace bytes trace_converter::pack(const chainbase::database& db, bool trace_debug_mode, const block_state_ptr& block_state, @@ -226,25 +244,6 @@ bytes trace_converter::pack(const chainbase::database& db, bool trace_debug_mode } } -struct restore_partial { - partial_transaction_v0& ptrx; - void operator()(prunable_data_type::full_legacy& data) const { - // this should never happen because we convert full_legacy to full before serialization - assert(false); - } - void operator()(prunable_data_type::none& data) const {} - void operator()(prunable_data_type::signatures_only& data) const { - ptrx.signatures = std::move(data.signatures); - } - void operator()(prunable_data_type::partial& data) const { - ptrx.signatures = std::move(data.signatures); - } - void operator()(prunable_data_type::full& data) const { - ptrx.signatures = std::move(data.signatures); - ptrx.context_free_data = std::move(data.context_free_segments); - } -}; - bytes trace_converter::to_traces_bin_v0(const bytes& entry_payload, uint32_t version) { if (version == 0) return zlib_decompress(entry_payload); From b6c000c4ed3d608963207e6a764876ac80bbedcc Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 23 Apr 2020 08:33:44 -0500 Subject: [PATCH 110/142] Change comments for history_state_exception --- libraries/chain/include/eosio/chain/exceptions.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index a4ec430de4a..94fdcc19786 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -161,7 +161,7 @@ namespace eosio { namespace chain { * |- resource_limit_exception * |- mongo_db_exception * |- contract_api_exception - * |- history_state_exception + * |- state_history_exception */ FC_DECLARE_DERIVED_EXCEPTION( chain_type_exception, chain_exception, @@ -659,6 +659,6 @@ namespace eosio { namespace chain { 3270003, "Context free data pruned (invalid block)" ) FC_DECLARE_DERIVED_EXCEPTION( state_history_exception, chain_exception, - 3280000, "History state exception" ) + 3280000, "State history exception" ) } } // eosio::chain From bc36faf419d53a55b53b4d3f1c40e9e7cfbb178e Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 23 Apr 2020 10:08:04 -0500 Subject: [PATCH 111/142] unittest code refactor --- unittests/restart_chain_tests.cpp | 67 ++++++------------------------ unittests/state_history_tests.cpp | 64 +--------------------------- unittests/test_cfd_transaction.cpp | 54 ++++++++++++++++++++++++ unittests/test_cfd_transaction.hpp | 18 ++++++++ 4 files changed, 86 insertions(+), 117 deletions(-) create mode 100644 unittests/test_cfd_transaction.cpp create mode 100644 unittests/test_cfd_transaction.hpp diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index 0ee90a670e4..22ed170ecab 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -11,6 +11,7 @@ #include #include #include +#include "test_cfd_transaction.hpp" using namespace eosio; using namespace testing; @@ -29,30 +30,6 @@ void remove_existing_states(controller::config& config) { fc::create_directories(state_path); } -struct dummy_action { - static eosio::chain::name get_name() { return N(dummyaction); } - static eosio::chain::name get_account() { return N(testapi); } - - char a; // 1 - uint64_t b; // 8 - int32_t c; // 4 -}; - -struct cf_action { - static eosio::chain::name get_name() { return N(cfaction); } - static eosio::chain::name get_account() { return N(testapi); } - - uint32_t payload = 100; - uint32_t cfd_idx = 0; // context free data index -}; - -FC_REFLECT(dummy_action, (a)(b)(c)) -FC_REFLECT(cf_action, (payload)(cfd_idx)) - -#define DUMMY_ACTION_DEFAULT_A 0x45 -#define DUMMY_ACTION_DEFAULT_B 0xab11cd1244556677 -#define DUMMY_ACTION_DEFAULT_C 0x7451ae12 - class replay_tester : public base_tester { public: template @@ -119,26 +96,9 @@ struct light_validation_restart_from_block_log_test_fixture { eosio::chain::transaction_trace_ptr trace; light_validation_restart_from_block_log_test_fixture() : chain(setup_policy::full) { - chain.create_account(N(testapi)); - chain.create_account(N(dummy)); - chain.produce_block(); - chain.set_code(N(testapi), contracts::test_api_wasm()); - chain.produce_block(); - - cf_action cfa; - signed_transaction trx; - action act({}, cfa); - trx.context_free_actions.push_back(act); - trx.context_free_data.emplace_back(fc::raw::pack(100)); // verify payload matches context free data - trx.context_free_data.emplace_back(fc::raw::pack(200)); - // add a normal action along with cfa - dummy_action da = {DUMMY_ACTION_DEFAULT_A, DUMMY_ACTION_DEFAULT_B, DUMMY_ACTION_DEFAULT_C}; - action act1(vector{{N(testapi), config::active_name}}, da); - trx.actions.push_back(act1); - chain.set_transaction_headers(trx); - // run normal passing case - auto sigs = trx.sign(chain.get_private_key(N(testapi), "active"), chain.control->get_chain_id()); - trace = chain.push_transaction(trx); + + deploy_test_api(chain); + trace = push_test_cfd_transaction(chain); chain.produce_blocks(10); BOOST_REQUIRE(trace->receipt); @@ -253,7 +213,7 @@ BOOST_FIXTURE_TEST_CASE(test_restart_from_block_log, restart_from_block_log_test BOOST_REQUIRE_NO_THROW(block_log::smoke_test(chain.get_config().blocks_dir, 1)); } -BOOST_FIXTURE_TEST_CASE(test_restart_from_trimed_block_log, restart_from_block_log_test_fixture) { +BOOST_FIXTURE_TEST_CASE(test_restart_from_trimmed_block_log, restart_from_block_log_test_fixture) { auto& config = chain.get_config(); auto blocks_path = config.blocks_dir; remove_all(blocks_path/"reversible"); @@ -268,6 +228,7 @@ BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log_with_pruned block_log blog(blocks_dir); std::vector ids{trace->id}; BOOST_CHECK(blog.prune_transactions(trace->block_num, ids) == 1); + BOOST_CHECK(ids.empty()); BOOST_REQUIRE_NO_THROW(block_log::repair_log(blocks_dir)); } @@ -280,15 +241,13 @@ BOOST_AUTO_TEST_CASE(test_trim_blocklog_front) { namespace bfs = boost::filesystem; auto blocks_dir = chain.get_config().blocks_dir; - auto temp1 = bfs::unique_path(); - boost::filesystem::create_directory(temp1); - bfs::copy(blocks_dir / "blocks.log", temp1 / "blocks.log"); - bfs::copy(blocks_dir / "blocks.index", temp1 / "blocks.index"); - auto temp2 = bfs::unique_path(); - BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_front(temp1, temp2, 10)); - BOOST_REQUIRE_NO_THROW(block_log::smoke_test(temp1, 1)); - bfs::remove_all(temp1); - bfs::remove_all(temp2); + + scoped_temp_path temp1, temp2; + boost::filesystem::create_directory(temp1.path); + bfs::copy(blocks_dir / "blocks.log", temp1.path / "blocks.log"); + bfs::copy(blocks_dir / "blocks.index", temp1.path / "blocks.index"); + BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_front(temp1.path, temp2.path, 10)); + BOOST_REQUIRE_NO_THROW(block_log::smoke_test(temp1.path, 1)); } BOOST_AUTO_TEST_SUITE_END() diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index 20a10a6b73c..6c9e5ac24f3 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -8,61 +8,12 @@ #include #include #include +#include "test_cfd_transaction.hpp" using namespace eosio; using namespace testing; using namespace chain; -struct dummy_action { - static eosio::chain::name get_name() { return N(dummyaction); } - static eosio::chain::name get_account() { return N(testapi); } - - char a; // 1 - uint64_t b; // 8 - int32_t c; // 4 -}; - -struct cf_action { - static eosio::chain::name get_name() { return N(cfaction); } - static eosio::chain::name get_account() { return N(testapi); } - - uint32_t payload = 100; - uint32_t cfd_idx = 0; // context free data index -}; - -FC_REFLECT(dummy_action, (a)(b)(c)) -FC_REFLECT(cf_action, (payload)(cfd_idx)) - -#define DUMMY_ACTION_DEFAULT_A 0x45 -#define DUMMY_ACTION_DEFAULT_B 0xab11cd1244556677 -#define DUMMY_ACTION_DEFAULT_C 0x7451ae12 - -std::vector deploy_test_api(eosio::testing::tester& chain) { - std::vector result; - chain.create_account(N(testapi)); - chain.create_account(N(dummy)); - result.push_back(chain.produce_block()); - chain.set_code(N(testapi), contracts::test_api_wasm()); - result.push_back(chain.produce_block()); - return result; -} - -eosio::chain::transaction_trace_ptr push_test_cfd_transaction(eosio::testing::tester& chain) { - cf_action cfa; - signed_transaction trx; - action act({}, cfa); - trx.context_free_actions.push_back(act); - trx.context_free_data.emplace_back(fc::raw::pack(100)); - trx.context_free_data.emplace_back(fc::raw::pack(200)); - // add a normal action along with cfa - dummy_action da = {DUMMY_ACTION_DEFAULT_A, DUMMY_ACTION_DEFAULT_B, DUMMY_ACTION_DEFAULT_C}; - action act1(vector{{N(testapi), config::active_name}}, da); - trx.actions.push_back(act1); - chain.set_transaction_headers(trx); - // run normal passing case - auto sigs = trx.sign(chain.get_private_key(N(testapi), "active"), chain.control->get_chain_id()); - return chain.push_transaction(trx); -} state_history::partial_transaction_v0 get_partial_from_traces_bin(const bytes& traces_bin, const transaction_id_type& id) { @@ -83,19 +34,6 @@ state_history::partial_transaction_v0 get_partial_from_traces_bin(const bytes& return trace_v0.partial->get(); } -struct scoped_temp_path { - boost::filesystem::path path; - scoped_temp_path() { - path = boost::filesystem::unique_path(); - if (boost::unit_test::framework::master_test_suite().argc >= 2) { - path += boost::unit_test::framework::master_test_suite().argv[1]; - } - } - ~scoped_temp_path() { - boost::filesystem::remove_all(path); - } -}; - struct trace_converter_test_fixture { packed_transaction::prunable_data_type::compression_type compression; diff --git a/unittests/test_cfd_transaction.cpp b/unittests/test_cfd_transaction.cpp new file mode 100644 index 00000000000..24f578cbf33 --- /dev/null +++ b/unittests/test_cfd_transaction.cpp @@ -0,0 +1,54 @@ +#include "test_cfd_transaction.hpp" +#include + +struct dummy_action { + static eosio::chain::name get_name() { return N(dummyaction); } + static eosio::chain::name get_account() { return N(testapi); } + + char a; // 1 + uint64_t b; // 8 + int32_t c; // 4 +}; + +struct cf_action { + static eosio::chain::name get_name() { return N(cfaction); } + static eosio::chain::name get_account() { return N(testapi); } + + uint32_t payload = 100; + uint32_t cfd_idx = 0; // context free data index +}; + +FC_REFLECT(dummy_action, (a)(b)(c)) +FC_REFLECT(cf_action, (payload)(cfd_idx)) + +#define DUMMY_ACTION_DEFAULT_A 0x45 +#define DUMMY_ACTION_DEFAULT_B 0xab11cd1244556677 +#define DUMMY_ACTION_DEFAULT_C 0x7451ae12 + +std::vector deploy_test_api(eosio::testing::tester& chain) { + std::vector result; + chain.create_account(N(testapi)); + chain.create_account(N(dummy)); + result.push_back(chain.produce_block()); + chain.set_code(N(testapi), eosio::testing::contracts::test_api_wasm()); + result.push_back(chain.produce_block()); + return result; +} + +eosio::chain::transaction_trace_ptr push_test_cfd_transaction(eosio::testing::tester& chain) { + cf_action cfa; + eosio::chain::signed_transaction trx; + eosio::chain::action act({}, cfa); + trx.context_free_actions.push_back(act); + trx.context_free_data.emplace_back(fc::raw::pack(100)); + trx.context_free_data.emplace_back(fc::raw::pack(200)); + // add a normal action along with cfa + dummy_action da = {DUMMY_ACTION_DEFAULT_A, DUMMY_ACTION_DEFAULT_B, DUMMY_ACTION_DEFAULT_C}; + eosio::chain::action act1( + std::vector{{N(testapi), eosio::chain::config::active_name}}, da); + trx.actions.push_back(act1); + chain.set_transaction_headers(trx); + // run normal passing case + auto sigs = trx.sign(chain.get_private_key(N(testapi), "active"), chain.control->get_chain_id()); + return chain.push_transaction(trx); +} \ No newline at end of file diff --git a/unittests/test_cfd_transaction.hpp b/unittests/test_cfd_transaction.hpp new file mode 100644 index 00000000000..75d986e551d --- /dev/null +++ b/unittests/test_cfd_transaction.hpp @@ -0,0 +1,18 @@ +#pragma once +#include + +std::vector deploy_test_api(eosio::testing::tester& chain); +eosio::chain::transaction_trace_ptr push_test_cfd_transaction(eosio::testing::tester& chain); + +struct scoped_temp_path { + boost::filesystem::path path; + scoped_temp_path() { + path = boost::filesystem::unique_path(); + if (boost::unit_test::framework::master_test_suite().argc >= 2) { + path += boost::unit_test::framework::master_test_suite().argv[1]; + } + } + ~scoped_temp_path() { + boost::filesystem::remove_all(path); + } +}; \ No newline at end of file From 82175f312b15ea4257cde1fa3c5763418a828c82 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 23 Apr 2020 10:21:46 -0500 Subject: [PATCH 112/142] Change on PR comment --- libraries/state_history/trace_converter.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/libraries/state_history/trace_converter.cpp b/libraries/state_history/trace_converter.cpp index 53059306bac..6256f73d079 100644 --- a/libraries/state_history/trace_converter.cpp +++ b/libraries/state_history/trace_converter.cpp @@ -281,12 +281,7 @@ bytes trace_converter::prune_traces(const bytes& entry_payload, uint32_t version // the incoming trace matches the one of ids to be pruned state_history::pack(write_strm, incoming_prunable.prune_all(), compression); modified = true; - - // delete the found id from the ids vector - if (itr != ids.end() - 1) { - *itr = ids.back(); - } - ids.resize(ids.size() - 1); + ids.erase(it); } else { // the incoming trace should not be pruned auto bytes_read = read_strm.tellp() - last_read_pos; From 7a435b89977837ee92ba52e2c447199d02bcd0dc Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 23 Apr 2020 10:52:27 -0500 Subject: [PATCH 113/142] fix typo --- libraries/state_history/trace_converter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/state_history/trace_converter.cpp b/libraries/state_history/trace_converter.cpp index 6256f73d079..b30b68b5c18 100644 --- a/libraries/state_history/trace_converter.cpp +++ b/libraries/state_history/trace_converter.cpp @@ -281,7 +281,7 @@ bytes trace_converter::prune_traces(const bytes& entry_payload, uint32_t version // the incoming trace matches the one of ids to be pruned state_history::pack(write_strm, incoming_prunable.prune_all(), compression); modified = true; - ids.erase(it); + ids.erase(itr); } else { // the incoming trace should not be pruned auto bytes_read = read_strm.tellp() - last_read_pos; From 7d30848c7ccaac13fd200021c610d11a4a793c74 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 23 Apr 2020 10:52:43 -0500 Subject: [PATCH 114/142] add more check --- unittests/restart_chain_tests.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/unittests/restart_chain_tests.cpp b/unittests/restart_chain_tests.cpp index 22ed170ecab..f1964872e53 100644 --- a/unittests/restart_chain_tests.cpp +++ b/unittests/restart_chain_tests.cpp @@ -248,6 +248,11 @@ BOOST_AUTO_TEST_CASE(test_trim_blocklog_front) { bfs::copy(blocks_dir / "blocks.index", temp1.path / "blocks.index"); BOOST_REQUIRE_NO_THROW(block_log::trim_blocklog_front(temp1.path, temp2.path, 10)); BOOST_REQUIRE_NO_THROW(block_log::smoke_test(temp1.path, 1)); + + block_log old_log(blocks_dir); + block_log new_log(temp1.path); + BOOST_CHECK(new_log.first_block_num() == 10); + BOOST_CHECK(new_log.head()->block_num() == old_log.head()->block_num()); } BOOST_AUTO_TEST_SUITE_END() From 1b0c6d594e89966ada35f3f74902ca734957c910 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 23 Apr 2020 12:26:37 -0500 Subject: [PATCH 115/142] extract chain state log from plugin to library --- .../include/eosio/state_history/log.hpp | 11 +++++- .../eosio/state_history/trace_converter.hpp | 1 - libraries/state_history/log.cpp | 36 +++++++++++++++++- .../state_history_plugin.cpp | 38 +++---------------- unittests/state_history_tests.cpp | 2 +- 5 files changed, 52 insertions(+), 36 deletions(-) diff --git a/libraries/state_history/include/eosio/state_history/log.hpp b/libraries/state_history/include/eosio/state_history/log.hpp index 1cb51180e94..5816e1fcb0c 100644 --- a/libraries/state_history/include/eosio/state_history/log.hpp +++ b/libraries/state_history/include/eosio/state_history/log.hpp @@ -129,7 +129,7 @@ class state_history_traces_log : public state_history_log { fc::optional get_log_entry(block_num_type block_num); - void store_traces(const chainbase::database& db, const chain::block_state_ptr& block_state); + void store(const chainbase::database& db, const chain::block_state_ptr& block_state); /** * @param[in,out] ids The ids to been pruned and returns the ids not found in the specified block @@ -137,6 +137,15 @@ class state_history_traces_log : public state_history_log { void prune_transactions(block_num_type block_num, std::vector& ids); }; +class state_history_chain_state_log : public state_history_log { + public: + state_history_chain_state_log(fc::path state_history_dir); + + fc::optional get_log_entry(block_num_type block_num); + + void store(const chainbase::database& db, const chain::block_state_ptr& block_state); +}; + } // namespace eosio FC_REFLECT(eosio::state_history_log_header, (magic)(block_id)(payload_size)) diff --git a/libraries/state_history/include/eosio/state_history/trace_converter.hpp b/libraries/state_history/include/eosio/state_history/trace_converter.hpp index bd1cb3e3d0c..6ccd40d66a8 100644 --- a/libraries/state_history/include/eosio/state_history/trace_converter.hpp +++ b/libraries/state_history/include/eosio/state_history/trace_converter.hpp @@ -2,7 +2,6 @@ #include #include -#include namespace eosio { namespace state_history { diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp index e4b6bf09a8d..5d704dd3821 100644 --- a/libraries/state_history/log.cpp +++ b/libraries/state_history/log.cpp @@ -1,5 +1,8 @@ +#include +#include #include #include +#include namespace eosio { @@ -290,7 +293,7 @@ void state_history_traces_log::prune_transactions(state_history_log::block_num_t } } -void state_history_traces_log::store_traces(const chainbase::database& db, const chain::block_state_ptr& block_state) { +void state_history_traces_log::store(const chainbase::database& db, const chain::block_state_ptr& block_state) { auto traces_bin = trace_convert.pack(db, trace_debug_mode, block_state, ship_current_version, compression); @@ -309,4 +312,35 @@ bool state_history_traces_log::exists(fc::path state_history_dir) { return fc::exists(state_history_dir / "trace_history.log") && fc::exists(state_history_dir / "trace_history.index"); } +state_history_chain_state_log::state_history_chain_state_log(fc::path state_history_dir) + : state_history_log("chain_state_history", (state_history_dir / "chain_state_history.log").string(), + (state_history_dir / "chain_state_history.index").string()) {} + +fc::optional state_history_chain_state_log::get_log_entry(block_num_type block_num) { + this->get_entry(block_num, [](const chain::bytes& data, uint32_t) { return state_history::zlib_decompress(data); }); +} + +void state_history_chain_state_log::store(const chainbase::database& db, const chain::block_state_ptr& block_state) { + bool fresh = this->begin_block() == this->end_block(); + if (fresh) + ilog("Placing initial state in block ${n}", ("n", block_state->block->block_num())); + + using namespace state_history; + + std::vector deltas = create_deltas(db, fresh); + auto deltas_bin = zlib_compress_bytes(fc::raw::pack(deltas)); + EOS_ASSERT(deltas_bin.size() == (uint32_t)deltas_bin.size(), chain::state_history_exception, "deltas is too big"); + + state_history_log_header header{.magic = ship_magic(ship_current_version), + .block_id = block_state->id, + .payload_size = sizeof(uint32_t) + deltas_bin.size()}; + + this->write_entry(header, block_state->block->previous, [&](auto& stream) { + uint32_t s = (uint32_t)deltas_bin.size(); + stream.write((char*)&s, sizeof(s)); + if (!deltas_bin.empty()) + stream.write(deltas_bin.data(), deltas_bin.size()); + }); +} + } // namespace eosio diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index ac931322f22..ecaaf65f759 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -1,9 +1,6 @@ #include -#include -#include #include #include -#include #include #include @@ -42,7 +39,7 @@ auto catch_and_log(F f) { struct state_history_plugin_impl : std::enable_shared_from_this { chain_plugin* chain_plug = nullptr; fc::optional trace_log; - fc::optional chain_state_log; + fc::optional chain_state_log; bool stopping = false; fc::optional applied_transaction_connection; fc::optional accepted_block_connection; @@ -212,9 +209,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thistrace_log->get_log_entry(current_request->start_block_num); } if (current_request->fetch_deltas && plugin->chain_state_log) { - result.deltas = plugin->chain_state_log->get_entry( - current_request->start_block_num, - [](const bytes& data, uint32_t) { return state_history::zlib_decompress(data); }); + result.deltas = plugin->chain_state_log->get_log_entry(current_request->start_block_num); } } ++current_request->start_block_num; @@ -318,8 +313,9 @@ struct state_history_plugin_impl : std::enable_shared_from_thisstore_traces(chain_plug->chain().db(), block_state); - store_chain_state(block_state); + trace_log->store(chain_plug->chain().db(), block_state); + if (chain_state_log) + chain_state_log->store(chain_plug->chain().db(), block_state); for (auto& s : sessions) { auto& p = s.second; if (p) { @@ -329,27 +325,6 @@ struct state_history_plugin_impl : std::enable_shared_from_thisbegin_block() == chain_state_log->end_block(); - if (fresh) - ilog("Placing initial state in block ${n}", ("n", block_state->block->block_num())); - - std::vector deltas = state_history::create_deltas(chain_plug->chain().db(), fresh); - auto deltas_bin = state_history::zlib_compress_bytes(fc::raw::pack(deltas)); - EOS_ASSERT(deltas_bin.size() == (uint32_t)deltas_bin.size(), plugin_exception, "deltas is too big"); - state_history_log_header header{.magic = ship_magic(ship_current_version), - .block_id = block_state->id, - .payload_size = sizeof(uint32_t) + deltas_bin.size()}; - chain_state_log->write_entry(header, block_state->block->previous, [&](auto& stream) { - uint32_t s = (uint32_t)deltas_bin.size(); - stream.write((char*)&s, sizeof(s)); - if (!deltas_bin.empty()) - stream.write(deltas_bin.data(), deltas_bin.size()); - }); - } // store_chain_state }; // state_history_plugin_impl state_history_plugin::state_history_plugin() @@ -413,8 +388,7 @@ void state_history_plugin::plugin_initialize(const variables_map& options) { } if (options.at("chain-state-history").as()) - my->chain_state_log.emplace("chain_state_history", (state_history_dir / "chain_state_history.log").string(), - (state_history_dir / "chain_state_history.index").string()); + my->chain_state_log.emplace(state_history_dir); } FC_LOG_AND_RETHROW() } // state_history_plugin::plugin_initialize diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index 6c9e5ac24f3..932b1c2ec31 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -117,7 +117,7 @@ BOOST_AUTO_TEST_CASE(test_trace_log) { log.add_transaction(std::get<0>(t), std::get<1>(t)); }); - chain.control->accepted_block.connect([&](const block_state_ptr& bs) { log.store_traces(chain.control->db(), bs); }); + chain.control->accepted_block.connect([&](const block_state_ptr& bs) { log.store(chain.control->db(), bs); }); deploy_test_api(chain); auto cfd_trace = push_test_cfd_transaction(chain); From f11667d81a2952fca119cd79f71523543a6176d7 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 23 Apr 2020 13:48:13 -0500 Subject: [PATCH 116/142] some more PR review changes --- .../include/eosio/state_history/trace_converter.hpp | 2 +- libraries/state_history/log.cpp | 3 ++- tests/eosio_blocklog_prune_test.py | 13 +++++++++++-- tests/testUtils.py | 10 +++++----- unittests/state_history_tests.cpp | 5 ----- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/libraries/state_history/include/eosio/state_history/trace_converter.hpp b/libraries/state_history/include/eosio/state_history/trace_converter.hpp index 6ccd40d66a8..058106a7215 100644 --- a/libraries/state_history/include/eosio/state_history/trace_converter.hpp +++ b/libraries/state_history/include/eosio/state_history/trace_converter.hpp @@ -27,7 +27,7 @@ struct trace_converter { * @param[in] entry_payload The entry_payload * @param[in] version The version of the entry * @param[in, out] ids The list of transaction ids to be pruned. After the member function returns, - * it would be modified to contain the list of transaction ids that do not exists + * it would be modified to contain the list of transaction ids that do not exist * in the log entry. * * @returns The serialized prunable_data section diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp index 5d704dd3821..d12055aafc4 100644 --- a/libraries/state_history/log.cpp +++ b/libraries/state_history/log.cpp @@ -317,7 +317,8 @@ state_history_chain_state_log::state_history_chain_state_log(fc::path state_hist (state_history_dir / "chain_state_history.index").string()) {} fc::optional state_history_chain_state_log::get_log_entry(block_num_type block_num) { - this->get_entry(block_num, [](const chain::bytes& data, uint32_t) { return state_history::zlib_decompress(data); }); + return this->get_entry(block_num, + [](const chain::bytes& data, uint32_t) { return state_history::zlib_decompress(data); }); } void state_history_chain_state_log::store(const chainbase::database& db, const chain::block_state_ptr& block_state) { diff --git a/tests/eosio_blocklog_prune_test.py b/tests/eosio_blocklog_prune_test.py index 6bbde27e06e..dfd55500c20 100755 --- a/tests/eosio_blocklog_prune_test.py +++ b/tests/eosio_blocklog_prune_test.py @@ -10,6 +10,7 @@ from TestHelper import AppArgs from testUtils import BlockLogAction import json +import sys ############################################################### # eosio_blocklog_prune_test.py @@ -103,14 +104,22 @@ def isBlockNumIrr(): trans = validationNode.processCleosCmd(cmd, cmd, silentErrors=False) assert trans, "Failed to get the transaction with context free data from the light validation node" + + # prune the transaction with block-num=trans["block_num"], id=cfTrxId cluster.getBlockLog(1, blockLogAction=BlockLogAction.prune_transactions, extraArgs=" --block-num {} --transaction {}".format(trans["block_num"], cfTrxId), exitOnError=True) trans = validationNode.processCleosCmd(cmd, cmd, silentErrors=False) assert trans, "Failed to get the transaction with context free data from the light validation node" + # check whether the transaction has been pruned based on the tag of prunable_data, if the tag is 1, when it's prunable_data_t::none assert trans["trx"]["receipt"]["trx"][1]["prunable_data"]["prunable_data"][0] == 1, "the the transaction with context free data has not been pruned" - - # output = cluster.getBlockLog(1, blockLogAction=BlockLogAction.prune_transactions, extraArgs=" --block-num {} --transaction {}".format(cfTrxBlockNum-1, cfTrxId)) + # try to prune the transaction where it doesn't belong + try: + cluster.getBlockLog(1, blockLogAction=BlockLogAction.prune_transactions, extraArgs=" --block-num {} --transaction {}".format(cfTrxBlockNum - 1, cfTrxId), throwException=True, silentErrors=True) + except: + ex = sys.exc_info()[0] + msg=ex.output.decode("utf-8") + assert "does not contain the following transactions: " + cfTrxId in msg, "The transaction id is not displayed in the console when it cannot be found" testSuccessful = True finally: diff --git a/tests/testUtils.py b/tests/testUtils.py index ad75b39f169..71a281a9cbf 100755 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -269,13 +269,13 @@ def runCmdArrReturnJson(cmdArr, trace=False, silentErrors=True): raise @staticmethod - def runCmdReturnStr(cmd, trace=False): + def runCmdReturnStr(cmd, trace=False, silentErrors=False): cmdArr=shlex.split(cmd) - return Utils.runCmdArrReturnStr(cmdArr) + return Utils.runCmdArrReturnStr(cmdArr, trace=trace, silentErrors=silentErrors) @staticmethod - def runCmdArrReturnStr(cmdArr, trace=False): - retStr=Utils.checkOutput(cmdArr) + def runCmdArrReturnStr(cmdArr, trace=False, silentErrors=False): + retStr=Utils.checkOutput(cmdArr,ignoreError=silentErrors) if trace: Utils.Print ("RAW > %s" % (retStr)) return retStr @@ -358,7 +358,7 @@ def getBlockLog(nodeDataDir, blockLogAction=BlockLogAction.return_blocks, output if returnType==ReturnType.json: rtn=Utils.runCmdReturnJson(cmd, silentErrors=silentErrors) else: - rtn=Utils.runCmdReturnStr(cmd) + rtn=Utils.runCmdReturnStr(cmd, silentErrors=silentErrors) except subprocess.CalledProcessError as ex: if throwException: raise diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index 932b1c2ec31..26b03002fd3 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -99,11 +99,6 @@ BOOST_FIXTURE_TEST_CASE(test_trace_converter_test_no_compression, trace_converte start(); } -// BOOST_FIXTURE_TEST_CASE(test_trace_converter_test_zlib_compression, trace_converter_test_fixture) { -// compression = packed_transaction::prunable_data_type::compression_type::zlib; -// start(); -// } - BOOST_AUTO_TEST_CASE(test_trace_log) { namespace bfs = boost::filesystem; tester chain; From 39fc444579736e8afea16a2d8e2954ff2acc7c86 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 24 Apr 2020 17:12:12 -0500 Subject: [PATCH 117/142] Remove unused proto_generic_messages --- plugins/net_plugin/net_plugin.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index eed80cfd67e..efaa60e3c77 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -418,7 +418,6 @@ namespace eosio { constexpr uint16_t proto_explicit_sync = 1; // version at time of eosio 1.0 constexpr uint16_t proto_block_id_notify = 2; // reserved. feature was removed. next net_version should be 3 constexpr uint16_t proto_pruned_types = 3; // supports new signed_block & packed_transaction types - constexpr uint16_t proto_generic_messages = 4; // placeholder for generic messages constexpr uint16_t net_version = proto_pruned_types; From 6d2048976c55cd1fd41c0786dae831729e0aacbf Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Sun, 26 Apr 2020 11:02:56 -0500 Subject: [PATCH 118/142] Modify prune_all not to convert to none when both signature & cfd are emtpy --- libraries/chain/transaction.cpp | 19 +- .../include/eosio/state_history/log.hpp | 1 - .../eosio/state_history/trace_converter.hpp | 10 +- libraries/state_history/log.cpp | 15 +- libraries/state_history/trace_converter.cpp | 287 +++++++++--------- programs/eosio-blocklog/main.cpp | 8 +- tests/eosio_blocklog_prune_test.py | 2 +- unittests/state_history_tests.cpp | 122 ++++---- 8 files changed, 235 insertions(+), 229 deletions(-) diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 411e8cca11c..ea15a2ffa5a 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -390,9 +390,6 @@ void packed_transaction_v0::local_pack_context_free_data() packed_context_free_data = pack_context_free_data(unpacked_trx.context_free_data, compression); } -packed_transaction::prunable_data_type packed_transaction::prunable_data_type::prune_all() const { - return {packed_transaction::prunable_data_type::none{digest() } }; -} static digest_type prunable_digest(const packed_transaction::prunable_data_type::none& obj) { return obj.prunable_digest; @@ -424,6 +421,22 @@ digest_type packed_transaction::prunable_data_type::digest() const { return prunable_data.visit( [](const auto& obj) { return prunable_digest(obj); } ); } +packed_transaction::prunable_data_type packed_transaction::prunable_data_type::prune_all() const { + return prunable_data.visit(overloaded{ + [](const none& obj) -> packed_transaction::prunable_data_type { + return {obj}; + }, + [](const packed_transaction::prunable_data_type::signatures_only& obj) + -> packed_transaction::prunable_data_type { + return {none{prunable_digest(obj)}}; + }, + [](const auto& obj) -> packed_transaction::prunable_data_type { + if (obj.signatures.empty() && obj.context_free_segments.empty()) + return {obj}; + return {none{prunable_digest(obj)}}; + }}); +} + static constexpr std::size_t digest_pack_size = 32; static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::none& obj, packed_transaction::prunable_data_type::compression_type) { diff --git a/libraries/state_history/include/eosio/state_history/log.hpp b/libraries/state_history/include/eosio/state_history/log.hpp index 5816e1fcb0c..bf0dadee348 100644 --- a/libraries/state_history/include/eosio/state_history/log.hpp +++ b/libraries/state_history/include/eosio/state_history/log.hpp @@ -117,7 +117,6 @@ class state_history_traces_log : public state_history_log { public: using compression_type = chain::packed_transaction::prunable_data_type::compression_type; bool trace_debug_mode = false; - compression_type compression = compression_type::none; state_history_traces_log(fc::path state_history_dir); diff --git a/libraries/state_history/include/eosio/state_history/trace_converter.hpp b/libraries/state_history/include/eosio/state_history/trace_converter.hpp index 058106a7215..cd3d8799723 100644 --- a/libraries/state_history/include/eosio/state_history/trace_converter.hpp +++ b/libraries/state_history/include/eosio/state_history/trace_converter.hpp @@ -18,21 +18,21 @@ struct trace_converter { void add_transaction(const transaction_trace_ptr& trace, const packed_transaction_ptr& transaction); bytes pack(const chainbase::database& db, bool trace_debug_mode, const block_state_ptr& block_state, - uint32_t version, compression_type compression); + uint32_t version); static bytes to_traces_bin_v0(const bytes& entry_payload, uint32_t version); /** - * Prune the signatures and context free data in a v1 log entry payload + * Prune the signatures and context free data in a v1 log entry payload for the specified transactions. * - * @param[in] entry_payload The entry_payload + * @param[in,out] entry_payload The entry_payload. * @param[in] version The version of the entry * @param[in, out] ids The list of transaction ids to be pruned. After the member function returns, * it would be modified to contain the list of transaction ids that do not exist * in the log entry. * - * @returns The serialized prunable_data section + * @returns The pair of offsets within entry_payload where data has been modified. **/ - static bytes prune_traces(const bytes& entry_payload, uint32_t version, std::vector& ids); + static std::pair prune_traces(bytes& entry_payload, uint32_t version, std::vector& ids); }; diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp index d12055aafc4..98dde337e50 100644 --- a/libraries/state_history/log.cpp +++ b/libraries/state_history/log.cpp @@ -256,7 +256,7 @@ state_history_log::get_entry(state_history_log::block_num_type block_num) { chain::bytes data(s); if (s) stream.read(data.data(), s); - return std::make_pair(data, get_ship_version(header.magic)); + return std::make_pair(std::move(data), get_ship_version(header.magic)); } state_history_traces_log::state_history_traces_log(fc::path state_history_dir) @@ -277,25 +277,24 @@ void state_history_traces_log::prune_transactions(state_history_log::block_num_t auto pos = get_log().tellp(); - auto [entry_payload, version] = *entry_result; - auto pruned_section = state_history::trace_converter::prune_traces(entry_payload, version, ids); - auto bytes_to_write = pruned_section.size(); + auto& [entry_payload, version] = *entry_result; + auto [start_offset, end_offset] = state_history::trace_converter::prune_traces(entry_payload, version, ids); - if (bytes_to_write > 0) { + if (end_offset > 0) { // the log object is in append mode, which cannot be used to update written content, // we need to open the file in the update mode fc::cfile update_log; update_log.set_file_path(get_log().get_file_path()); update_log.open(fc::cfile::update_rw_mode); - update_log.seek(pos - bytes_to_write); - update_log.write(pruned_section.data(), bytes_to_write); + update_log.seek(pos - entry_payload.size() + start_offset); + update_log.write(entry_payload.data() + start_offset, end_offset-start_offset); } } void state_history_traces_log::store(const chainbase::database& db, const chain::block_state_ptr& block_state) { - auto traces_bin = trace_convert.pack(db, trace_debug_mode, block_state, ship_current_version, compression); + auto traces_bin = trace_convert.pack(db, trace_debug_mode, block_state, ship_current_version); state_history_log_header header{.magic = ship_magic(ship_current_version), .block_id = block_state->id, diff --git a/libraries/state_history/trace_converter.cpp b/libraries/state_history/trace_converter.cpp index b30b68b5c18..c94320576cf 100644 --- a/libraries/state_history/trace_converter.cpp +++ b/libraries/state_history/trace_converter.cpp @@ -1,10 +1,10 @@ +#include #include #include #include #include #include #include - extern const char* state_history_plugin_abi; namespace bio = boost::iostreams; @@ -40,15 +40,27 @@ void trace_converter::add_transaction(const transaction_trace_ptr& trace, const namespace { -bytes decompress_buffer(const char* buffer, uint32_t len, compression_type compression) { +template +Object unpack_zlib_compressed(const char* buffer, fc::datastream& ds) { + fc::unsigned_int len; + fc::raw::unpack(ds, len); + + if (len.value == 0) + return Object{}; + + EOS_ASSERT(ds.remaining() >= static_cast(len), fc::out_of_range_exception, "read datastream over by ${v}", + ("v", ds.remaining() - len)); + bytes decompressed; - bio::filtering_ostreambuf strm; - EOS_ASSERT(compression == compression_type::zlib, state_history_exception, "unsupported compression format"); - strm.push(bio::zlib_decompressor()); - strm.push(bio::back_inserter(decompressed)); - bio::write(strm, buffer, len); + bio::filtering_ostreambuf strm(bio::zlib_decompressor() | bio::back_inserter(decompressed)); + bio::write(strm, buffer + ds.tellp(), len); bio::close(strm); - return decompressed; + ds.skip(len); + + fc::datastream decompressed_ds(decompressed.data(), decompressed.size()); + Object obj; + fc::raw::unpack(decompressed_ds, obj); + return obj; } std::vector prepare_traces(trace_converter& converter, @@ -85,6 +97,7 @@ void for_each_packed_transaction(const eosio::state_history::augmented_transacti } } +/// used to traverse every packed_transaction inside traces before the pruned_data has been serialized template void for_each_packed_transaction(const std::vector& traces, const Lambda& lambda) { @@ -93,153 +106,163 @@ void for_each_packed_transaction(const std::vector& ds, const prunable_data_type& data, compression_type compression) { - if (compression == compression_type::none) { - fc::raw::pack(ds, data); - } - else if (compression == compression_type::zlib) { - bytes uncompressed = fc::raw::pack(data); - fc::raw::pack(ds, zlib_compress_bytes(uncompressed)); - } +BOOST_DECLARE_HAS_MEMBER(has_context_free_segments, context_free_segments); + +template ::value, int> = 0> +void pack(bytes& buffer, const T& obj) { + fc::datastream ss; + fc::raw::pack(ss, obj); + const auto len = ss.tellp(); + const auto pos = buffer.size(); + buffer.resize(pos + len); + fc::datastream ds(buffer.data() + pos, len); + fc::raw::pack(ds, obj); } -size_t pack_prunable(bytes& buffer, const packed_transaction& trx, - compression_type compression) { - - const auto& data = trx.get_prunable_data(); - - int max_size = data.maximum_pruned_pack_size(compression); - uint64_t start_buffer_position = buffer.size(); - buffer.resize(buffer.size() + max_size); - fc::datastream ds(buffer.data() + start_buffer_position, max_size); - - if (data.prunable_data.contains()) { - // convert full_legacy to full before serialization - const auto& full_legacy = data.prunable_data.get(); - prunable_data_type pd = {prunable_data_type::full{full_legacy.signatures, full_legacy.context_free_segments}}; - pack(ds, pd, compression); - } - else { - pack(ds, data, compression); - } - - buffer.resize(start_buffer_position + ds.tellp()); - return max_size; +template ::value, int> = 0> +void pack(fc::datastream& ds, const T& obj) { + fc::raw::pack(ds, obj); } - -void unpack_prunable(const char* read_buffer, fc::datastream& ds, prunable_data_type& data, - compression_type compression) { - - if (compression == compression_type::none) { - fc::raw::unpack(ds, data); - } - else { - fc::unsigned_int len; - fc::raw::unpack(ds, len); - bytes uncompressed = decompress_buffer(read_buffer + ds.tellp(), len, compression); - ds.skip(len); - - fc::datastream uncompressed_ds(uncompressed.data(), uncompressed.size()); - fc::raw::unpack(uncompressed_ds, data); - } +template ::value, int> = 0> +void pack(Buffer& buffer, const T& obj) { + const auto comopressed_context_free_segments = + obj.context_free_segments.size() ? zlib_compress_bytes(fc::raw::pack(obj.context_free_segments)) : bytes{}; + pack(buffer, std::make_pair(std::ref(obj.signatures), std::ref(comopressed_context_free_segments))); } -bytes pack_prunables(const std::vector& traces, compression_type compression) { - - std::vector out; - out.reserve(1024); - - int size_with_paddings = 0; - for_each_packed_transaction( - traces, [&](const chain::packed_transaction& pt) { size_with_paddings += pack_prunable(out, pt, compression); }); +void pack(bytes& buffer, const prunable_data_type& obj) { + buffer.push_back(obj.prunable_data.which()); + obj.prunable_data.visit([&buffer](const auto& obj) { pack(buffer, obj); }); +} - EOS_ASSERT(size_with_paddings >= out.size(), state_history_exception, "The estimated max size is small than the actual size"); - out.resize(size_with_paddings); - return out; +void pack(fc::datastream& ds, const prunable_data_type& obj) { + fc::raw::pack(ds, static_cast(obj.prunable_data.which())); + obj.prunable_data.visit([&ds](const auto& obj) { pack(ds, obj); }); } -std::pair, compression_type> -traces_from_v1_entry_payload(const char* read_buffer, fc::datastream& ds) { - uint8_t compr; - fc::raw::unpack(ds, compr); - fc::unsigned_int len; - fc::raw::unpack(ds, len); - EOS_ASSERT(ds.remaining() >= static_cast(len), fc::out_of_range_exception, "read datastream over by ${v}", - ("v", ds.remaining() - len)); +template ::value, int> = 0> +void unpack(const char*, fc::datastream& ds, T& obj) { + fc::raw::unpack(ds, obj); +} - auto unprunable_section = decompress_buffer(read_buffer + ds.tellp(), len, compression_type::zlib); - ds.skip(len); +template ::value, int> = 0> +void unpack(const char* read_buffer, fc::datastream& ds, T& t) { + fc::raw::unpack(ds, t.signatures); + t.context_free_segments = unpack_zlib_compressed(read_buffer, ds); +} - std::vector traces; - fc::datastream traces_ds(unprunable_section.data(), unprunable_section.size()); - fc::raw::unpack(traces_ds, traces); - return std::make_pair(traces, static_cast(compr)); +void unpack(const char* read_buffer, fc::datastream& ds, prunable_data_type& prunable) { + uint8_t tag; + fc::raw::unpack(ds, tag); + prunable.prunable_data.set_which(tag); + prunable.prunable_data.visit([read_buffer, &ds](auto& v) { unpack(read_buffer, ds, v); }); } +/// used to traverse each trace along with its associated unpacked prunable_data template -void do_visit_trace(const char* read_buffer, fc::datastream& strm, transaction_trace& trace, - compression_type compression, Visitor&& visitor) { +void visit_deserialized_trace(const char* read_buffer, fc::datastream& ds, transaction_trace& trace, + Visitor&& visitor) { auto& trace_v0 = trace.get(); if (trace_v0.failed_dtrx_trace.size()) { // failed_dtrx_trace have at most one element because it is encoded as an optional - do_visit_trace(read_buffer, strm, trace_v0.failed_dtrx_trace[0].recurse, compression, visitor); + visit_deserialized_trace(read_buffer, ds, trace_v0.failed_dtrx_trace[0].recurse, std::forward(visitor)); } if (trace_v0.partial) { prunable_data_type prunable; - unpack_prunable(read_buffer, strm, prunable, compression); + unpack(read_buffer, ds, prunable); visitor(trace_v0, prunable); } } struct restore_partial { partial_transaction_v0& ptrx; - void operator()(prunable_data_type::full_legacy& data) const { - // this should never happen because we convert full_legacy to full before serialization - assert(false); - } - void operator()(prunable_data_type::none& data) const {} - void operator()(prunable_data_type::signatures_only& data) const { - ptrx.signatures = std::move(data.signatures); - } - void operator()(prunable_data_type::partial& data) const { + void operator()(prunable_data_type::full_legacy& data) const { ptrx.signatures = std::move(data.signatures); + ptrx.context_free_data = std::move(data.context_free_segments); } + void operator()(prunable_data_type::none& data) const {} + void operator()(prunable_data_type::signatures_only& data) const { ptrx.signatures = std::move(data.signatures); } + void operator()(prunable_data_type::partial& data) const { ptrx.signatures = std::move(data.signatures); } void operator()(prunable_data_type::full& data) const { ptrx.signatures = std::move(data.signatures); ptrx.context_free_data = std::move(data.context_free_segments); } }; +struct trace_pruner { + char* buffer; /// the address to the traces entry payload, data == read_strm._start == write_strm._start + uint64_t last_read_pos; + std::vector& ids; /// the transaction ids to be pruned + fc::datastream& read_strm; /// read_strm and write_strm share the same underlying buffer + fc::datastream write_strm; + uint64_t change_position = 0; /// when it's nonzero, represents the offset to data where the content has been changed + + trace_pruner(bytes& entry_payload, fc::datastream& rds, std::vector& ids) + : buffer(entry_payload.data()) + , last_read_pos(rds.tellp()) + , ids(ids) + , read_strm(rds) + , write_strm(buffer, entry_payload.size()) { + write_strm.skip(last_read_pos); + } + + /// This member function prunes each trace by overriding the input buffer with the pruned content. + /// It relies on the fact that the serialized pruned data won't be larger than its un-pruned counterpart + /// which is subsequently based on: + /// 1) the presence of context free data requires the presence of signatures; + /// 2) prunable_data::prune_all() would never convert it to prunable_data::none when both the context free data and + /// signature are empty. + void operator()(transaction_trace_v0& trace, prunable_data_type& incoming_prunable) { + auto itr = std::find(ids.begin(), ids.end(), trace.id); + if (itr != ids.end()) { + // the incoming trace matches the one of ids to be pruned + if (change_position == 0) + change_position = write_strm.tellp(); + pack(write_strm, incoming_prunable.prune_all()); + ids.erase(itr); + } else if (change_position == 0) { + // no change so far, skip the original serialized prunable_data bytes + write_strm.skip(read_strm.tellp() - last_read_pos); + } else { + // change detected, shift the original serialized prunable_data bytes to new location + write_strm.write(buffer + last_read_pos, read_strm.tellp() - last_read_pos); + } + last_read_pos = read_strm.tellp(); + } + + /// @returns the pair of start and end offset to buffer where the content has been changed. + std::pair changed_region() { + if (change_position == 0) + return {0, 0}; + return {change_position, write_strm.tellp()}; + } +}; + } // namespace bytes trace_converter::pack(const chainbase::database& db, bool trace_debug_mode, const block_state_ptr& block_state, - uint32_t version, compression_type compression) { + uint32_t version) { - auto traces = prepare_traces(*this, block_state); - auto unprunable = zlib_compress_bytes(fc::raw::pack(make_history_context_wrapper( + auto traces = prepare_traces(*this, block_state); + const auto unprunable = zlib_compress_bytes(fc::raw::pack(make_history_context_wrapper( db, trace_receipt_context{.debug_mode = trace_debug_mode, .version = version}, traces))); if (version == 0) { return unprunable; } else { // In version 1 of ShiP traces log disk format, it log entry consists of 3 parts. - // 1. a unit8_t compression tag to indicate the compression level of prunables - // 2. an zlib compressed unprunable section contains the serialization of the vector of traces excluding + // 1. an zlib compressed unprunable section contains the serialization of the vector of traces excluding // the prunable_data data (i.e. signatures and context free data) - // 3. a prunable section contains the serialization of the vector of optional (possibly compressed) - // prunable_data - - auto prunables = pack_prunables(traces, compression); - fc::datastream unprunable_size_strm; - fc::raw::pack(unprunable_size_strm, unprunable); - uint8_t compression_tag = static_cast(compression); - bytes buffer(sizeof(compression_tag) + unprunable_size_strm.tellp() + prunables.size()); - - fc::datastream strm(buffer.data(), buffer.size()); - fc::raw::pack(strm, compression_tag); - fc::raw::pack(strm, unprunable); - strm.write(prunables.data(), prunables.size()); + // 2. a prunable section contains the serialization of the vector of prunable_data, where all the contained + // context_free_segments are zlib compressed. + bytes buffer; + state_history::pack(buffer, unprunable); + + for_each_packed_transaction(traces, [&buffer](const chain::packed_transaction& pt) { + state_history::pack(buffer, pt.get_prunable_data()); + }); + return buffer; } } @@ -250,50 +273,30 @@ bytes trace_converter::to_traces_bin_v0(const bytes& entry_payload, uint32_t ver else { fc::datastream strm(entry_payload.data(), entry_payload.size()); - auto [traces, compression] = traces_from_v1_entry_payload(entry_payload.data(), strm); + auto traces = unpack_zlib_compressed>(entry_payload.data(), strm); for (auto& trace : traces) { - do_visit_trace(entry_payload.data(), strm, trace, compression, - [](transaction_trace_v0& trace, prunable_data_type& prunable_data) { - auto& ptrx = trace.partial->get(); - prunable_data.prunable_data.visit(restore_partial{ptrx}); - }); + visit_deserialized_trace(entry_payload.data(), strm, trace, + [](transaction_trace_v0& trace, prunable_data_type& prunable_data) { + auto& ptrx = trace.partial->get(); + prunable_data.prunable_data.visit(restore_partial{ptrx}); + }); } return fc::raw::pack(traces); } } -bytes trace_converter::prune_traces(const bytes& entry_payload, uint32_t version, std::vector& ids) { +std::pair trace_converter::prune_traces(bytes& entry_payload, uint32_t version, + std::vector& ids) { EOS_ASSERT(version > 0, state_history_exception, "state history log version 0 does not support trace pruning"); fc::datastream read_strm(entry_payload.data(), entry_payload.size()); + auto traces = unpack_zlib_compressed>(entry_payload.data(), read_strm); - auto [traces, compression] = traces_from_v1_entry_payload(entry_payload.data(), read_strm); - uint64_t last_read_pos = read_strm.tellp(); - uint64_t change_offset = entry_payload.size(); - bool modified = false; - - bytes write_buffer(entry_payload.size() - last_read_pos); - fc::datastream write_strm(write_buffer.data(), write_buffer.size()); - - auto prune_trace = [&, compression=compression](transaction_trace_v0& trace, auto& incoming_prunable) { - auto itr = std::find(ids.begin(), ids.end(), trace.id); - if (itr != ids.end()) { - // the incoming trace matches the one of ids to be pruned - state_history::pack(write_strm, incoming_prunable.prune_all(), compression); - modified = true; - ids.erase(itr); - } else { - // the incoming trace should not be pruned - auto bytes_read = read_strm.tellp() - last_read_pos; - write_strm.write(entry_payload.data() + last_read_pos, bytes_read); - } - last_read_pos = read_strm.tellp(); - }; - + auto prune_trace = trace_pruner(entry_payload, read_strm, ids); for (auto& trace : traces) { - do_visit_trace(entry_payload.data(), read_strm, trace, compression, prune_trace); + visit_deserialized_trace(entry_payload.data(), read_strm, trace, prune_trace); } - return modified? write_buffer : bytes{}; + return prune_trace.changed_region(); } } // namespace state_history diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp index e8c7ef3a57f..7f3d52590aa 100644 --- a/programs/eosio-blocklog/main.cpp +++ b/programs/eosio-blocklog/main.cpp @@ -321,10 +321,10 @@ int main(int argc, char** argv) { return 0; } if (blog.prune_transactions) { - const auto blocks_dir = vmap.at("blocks-dir").as(); - const auto state_history_dir = vmap.at("state-history-dir").as(); - const auto block_num = vmap.at("block-num").as(); - const auto ids = vmap.at("transaction").as>(); + const auto blocks_dir = vmap["blocks-dir"].as(); + const auto state_history_dir = vmap["state-history-dir"].as(); + const auto block_num = vmap["block-num"].as(); + const auto ids = vmap.count("transaction") ? vmap["transaction"].as>() : std::vector{}; report_time rt("prune transactions"); int ret = prune_transactions(blocks_dir, state_history_dir, block_num, {ids.begin(), ids.end()}); diff --git a/tests/eosio_blocklog_prune_test.py b/tests/eosio_blocklog_prune_test.py index dfd55500c20..7a916ea03c6 100755 --- a/tests/eosio_blocklog_prune_test.py +++ b/tests/eosio_blocklog_prune_test.py @@ -110,7 +110,7 @@ def isBlockNumIrr(): trans = validationNode.processCleosCmd(cmd, cmd, silentErrors=False) assert trans, "Failed to get the transaction with context free data from the light validation node" - # check whether the transaction has been pruned based on the tag of prunable_data, if the tag is 1, when it's prunable_data_t::none + # check whether the transaction has been pruned based on the tag of prunable_data, if the tag is 1, then it's a prunable_data_t::none assert trans["trx"]["receipt"]["trx"][1]["prunable_data"]["prunable_data"][0] == 1, "the the transaction with context free data has not been pruned" # try to prune the transaction where it doesn't belong diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index 26b03002fd3..380640e574a 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -4,17 +4,16 @@ #include #include +#include "test_cfd_transaction.hpp" +#include #include #include #include -#include -#include "test_cfd_transaction.hpp" using namespace eosio; using namespace testing; using namespace chain; - state_history::partial_transaction_v0 get_partial_from_traces_bin(const bytes& traces_bin, const transaction_id_type& id) { fc::datastream strm(traces_bin.data(), traces_bin.size()); @@ -34,69 +33,59 @@ state_history::partial_transaction_v0 get_partial_from_traces_bin(const bytes& return trace_v0.partial->get(); } -struct trace_converter_test_fixture { - packed_transaction::prunable_data_type::compression_type compression; - - void start() { - tester chain; - - state_history::trace_converter converter_v0, converter_v1; - std::map on_disk_log_entries_v0; - std::map on_disk_log_entries_v1; - - chain.control->applied_transaction.connect( - [&](std::tuple t) { - converter_v0.add_transaction(std::get<0>(t), std::get<1>(t)); - converter_v1.add_transaction(std::get<0>(t), std::get<1>(t)); - }); - - chain.control->accepted_block.connect([&](const block_state_ptr& bs) { - on_disk_log_entries_v0[bs->block_num] = converter_v0.pack(chain.control->db(), true, bs, 0, compression); - on_disk_log_entries_v1[bs->block_num] = converter_v1.pack(chain.control->db(), true, bs, 1, compression); - }); - - deploy_test_api(chain); - auto cfd_trace = push_test_cfd_transaction(chain); - chain.produce_blocks(10); - - BOOST_CHECK(on_disk_log_entries_v0.size()); - BOOST_CHECK(on_disk_log_entries_v1.size()); - - // make sure v0 and v1 are identifical when transform to traces bin - BOOST_CHECK(std::equal(on_disk_log_entries_v0.begin(), on_disk_log_entries_v0.end(), - on_disk_log_entries_v1.begin(), [&](const auto& entry_v0, const auto& entry_v1) { - return state_history::trace_converter::to_traces_bin_v0(entry_v0.second, 0) == - state_history::trace_converter::to_traces_bin_v0(entry_v1.second, 1); - })); - - // Now deserialize the on disk trace log and make sure that the cfd exists - auto& cfd_entry_v1 = on_disk_log_entries_v1[cfd_trace->block_num]; - auto partial = - get_partial_from_traces_bin(state_history::trace_converter::to_traces_bin_v0(cfd_entry_v1, 1), cfd_trace->id); - BOOST_REQUIRE(partial.context_free_data.size()); - BOOST_REQUIRE(partial.signatures.size()); - - // prune the cfd for the block - std::vector ids{cfd_trace->id}; - auto pruned = state_history::trace_converter::prune_traces(cfd_entry_v1, 1, ids); - BOOST_CHECK(pruned.size() < cfd_entry_v1.size()); - BOOST_CHECK(ids.size() == 0); - - std::copy(pruned.begin(), pruned.end(), cfd_entry_v1.end() - pruned.size()); - - // read the pruned trace and make sure the signature/cfd are empty - auto pruned_partial = get_partial_from_traces_bin(state_history::trace_converter::to_traces_bin_v0(cfd_entry_v1, 1), - cfd_trace->id); - BOOST_CHECK(pruned_partial.context_free_data.size() == 0); - BOOST_CHECK(pruned_partial.signatures.size() == 0); - } -}; - BOOST_AUTO_TEST_SUITE(test_state_history) -BOOST_FIXTURE_TEST_CASE(test_trace_converter_test_no_compression, trace_converter_test_fixture) { - compression = packed_transaction::prunable_data_type::compression_type::none; - start(); +BOOST_AUTO_TEST_CASE(test_trace_converter_test) { + + tester chain; + + state_history::trace_converter converter_v0, converter_v1; + std::map on_disk_log_entries_v0; + std::map on_disk_log_entries_v1; + + chain.control->applied_transaction.connect( + [&](std::tuple t) { + converter_v0.add_transaction(std::get<0>(t), std::get<1>(t)); + converter_v1.add_transaction(std::get<0>(t), std::get<1>(t)); + }); + + chain.control->accepted_block.connect([&](const block_state_ptr& bs) { + on_disk_log_entries_v0[bs->block_num] = converter_v0.pack(chain.control->db(), true, bs, 0); + on_disk_log_entries_v1[bs->block_num] = converter_v1.pack(chain.control->db(), true, bs, 1); + }); + + deploy_test_api(chain); + auto cfd_trace = push_test_cfd_transaction(chain); + chain.produce_blocks(1); + + BOOST_CHECK(on_disk_log_entries_v0.size()); + BOOST_CHECK(on_disk_log_entries_v1.size()); + + // make sure v0 and v1 are identifical when transform to traces bin + BOOST_CHECK(std::equal(on_disk_log_entries_v0.begin(), on_disk_log_entries_v0.end(), on_disk_log_entries_v1.begin(), + [&](const auto& entry_v0, const auto& entry_v1) { + return state_history::trace_converter::to_traces_bin_v0(entry_v0.second, 0) == + state_history::trace_converter::to_traces_bin_v0(entry_v1.second, 1); + })); + + // Now deserialize the on disk trace log and make sure that the cfd exists + auto& cfd_entry_v1 = on_disk_log_entries_v1.at(cfd_trace->block_num); + auto partial = + get_partial_from_traces_bin(state_history::trace_converter::to_traces_bin_v0(cfd_entry_v1, 1), cfd_trace->id); + BOOST_REQUIRE(partial.context_free_data.size()); + BOOST_REQUIRE(partial.signatures.size()); + + // prune the cfd for the block + std::vector ids{cfd_trace->id}; + auto offsets = state_history::trace_converter::prune_traces(cfd_entry_v1, 1, ids); + BOOST_CHECK(offsets.first > 0 && offsets.second > 0); + BOOST_CHECK(ids.size() == 0); + + // read the pruned trace and make sure the signature/cfd are empty + auto pruned_partial = + get_partial_from_traces_bin(state_history::trace_converter::to_traces_bin_v0(cfd_entry_v1, 1), cfd_trace->id); + BOOST_CHECK(pruned_partial.context_free_data.size() == 0); + BOOST_CHECK(pruned_partial.signatures.size() == 0); } BOOST_AUTO_TEST_CASE(test_trace_log) { @@ -118,6 +107,9 @@ BOOST_AUTO_TEST_CASE(test_trace_log) { auto cfd_trace = push_test_cfd_transaction(chain); chain.produce_blocks(10); + packed_transaction::prunable_data_type prunable; + auto x = prunable.prune_all(); + auto traces_bin = log.get_log_entry(cfd_trace->block_num); BOOST_REQUIRE(traces_bin); @@ -132,7 +124,7 @@ BOOST_AUTO_TEST_CASE(test_trace_log) { // we assume the nodeos has to be stopped while running, it can only be read // correctly with restart state_history_traces_log new_log(state_history_dir.path); - auto pruned_traces_bin = new_log.get_log_entry(cfd_trace->block_num); + auto pruned_traces_bin = new_log.get_log_entry(cfd_trace->block_num); BOOST_REQUIRE(pruned_traces_bin); auto pruned_partial = get_partial_from_traces_bin(*pruned_traces_bin, cfd_trace->id); From a686c409b3a4716fe7ea28b578410d6393defb01 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 27 Apr 2020 12:58:40 -0500 Subject: [PATCH 119/142] remove signature_only type --- libraries/chain/apply_context.cpp | 2 +- .../chain/include/eosio/chain/transaction.hpp | 7 ----- libraries/chain/transaction.cpp | 26 ++----------------- libraries/state_history/trace_converter.cpp | 17 +++++++++--- 4 files changed, 16 insertions(+), 36 deletions(-) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 8d4e6a11a11..4774d8be377 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -806,7 +806,7 @@ int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t b { const packed_transaction::prunable_data_type::prunable_data_t& data = trx_context.packed_trx.get_prunable_data().prunable_data; const bytes* cfd = nullptr; - if( data.contains() || data.contains() ) { + if( data.contains() ) { } else if( data.contains() ) { if( index >= data.get().context_free_segments.size() ) return -1; diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 5e5f550fad6..7ab7df96707 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -225,11 +225,6 @@ namespace eosio { namespace chain { digest_type prunable_digest; }; - struct signatures_only { - std::vector signatures; - digest_type context_free_mroot; - }; - using segment_type = fc::static_variant; struct partial { @@ -250,7 +245,6 @@ namespace eosio { namespace chain { using prunable_data_t = fc::static_variant< full_legacy, none, - signatures_only, partial, full >; @@ -339,7 +333,6 @@ FC_REFLECT( eosio::chain::packed_transaction_v0, (signatures)(compression)(packe FC_REFLECT( eosio::chain::packed_transaction, (compression)(prunable_data)(packed_trx) ) FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type, (prunable_data)); FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::none, (prunable_digest)) -FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::signatures_only, (signatures)( context_free_mroot)) FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::partial, (signatures)( context_free_segments)) FC_REFLECT( eosio::chain::packed_transaction::prunable_data_type::full, (signatures)( context_free_segments)) // @ignore context_free_segments diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index ea15a2ffa5a..64af37e6cbf 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -399,10 +399,6 @@ static digest_type prunable_digest(const packed_transaction::prunable_data_type: EOS_THROW(tx_prune_exception, "unimplemented"); } -static digest_type prunable_digest(const packed_transaction::prunable_data_type::signatures_only& obj) { - EOS_THROW(tx_prune_exception, "unimplemented"); -} - static digest_type prunable_digest(const packed_transaction::prunable_data_type::full& obj) { digest_type::encoder prunable; fc::raw::pack( prunable, obj.signatures ); @@ -422,19 +418,8 @@ digest_type packed_transaction::prunable_data_type::digest() const { } packed_transaction::prunable_data_type packed_transaction::prunable_data_type::prune_all() const { - return prunable_data.visit(overloaded{ - [](const none& obj) -> packed_transaction::prunable_data_type { - return {obj}; - }, - [](const packed_transaction::prunable_data_type::signatures_only& obj) - -> packed_transaction::prunable_data_type { - return {none{prunable_digest(obj)}}; - }, - [](const auto& obj) -> packed_transaction::prunable_data_type { - if (obj.signatures.empty() && obj.context_free_segments.empty()) - return {obj}; - return {none{prunable_digest(obj)}}; - }}); + return prunable_data.visit( + [](const auto& obj) -> packed_transaction::prunable_data_type { return {none{prunable_digest(obj)}}; }); } static constexpr std::size_t digest_pack_size = 32; @@ -445,9 +430,6 @@ static std::size_t padded_pack_size(const packed_transaction::prunable_data_type return result; } -static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::signatures_only& obj, packed_transaction::prunable_data_type::compression_type) { - return fc::raw::pack_size(obj); -} static std::size_t padded_pack_size(const packed_transaction::prunable_data_type::partial& obj, packed_transaction::prunable_data_type::compression_type segment_compression) { EOS_THROW(tx_prune_exception, "unimplemented"); @@ -579,9 +561,6 @@ uint32_t packed_transaction::calculate_estimated_size() const { }; auto visitor = overloaded{ [](const prunable_data_type::none& v) { return 0ul; }, - [](const prunable_data_type::signatures_only& v) { - return v.signatures.size() * sizeof(signature_type); - }, [&](const prunable_data_type::partial& v) { return v.signatures.size() * sizeof(signature_type) + est_size(v.context_free_segments); }, @@ -622,7 +601,6 @@ const vector* packed_transaction::get_context_free_data()const { } const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::none&, std::size_t) { return nullptr; } -const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::signatures_only&, std::size_t) { return nullptr; } const bytes* maybe_get_context_free_data(const packed_transaction::prunable_data_type::partial& p, std::size_t i) { if( p.context_free_segments.size() <= i ) return nullptr; return p.context_free_segments[i].visit( diff --git a/libraries/state_history/trace_converter.cpp b/libraries/state_history/trace_converter.cpp index c94320576cf..04f171cc9c5 100644 --- a/libraries/state_history/trace_converter.cpp +++ b/libraries/state_history/trace_converter.cpp @@ -106,6 +106,16 @@ void for_each_packed_transaction(const std::vector prunable_data_type { return {elem}; }, + [&obj](const auto& elem) -> prunable_data_type { + if (elem.signatures.empty() && elem.context_free_segments.empty()) + return {elem}; + return obj.prune_all(); + }}); +} + BOOST_DECLARE_HAS_MEMBER(has_context_free_segments, context_free_segments); template ::value, int> = 0> @@ -182,7 +192,6 @@ struct restore_partial { ptrx.context_free_data = std::move(data.context_free_segments); } void operator()(prunable_data_type::none& data) const {} - void operator()(prunable_data_type::signatures_only& data) const { ptrx.signatures = std::move(data.signatures); } void operator()(prunable_data_type::partial& data) const { ptrx.signatures = std::move(data.signatures); } void operator()(prunable_data_type::full& data) const { ptrx.signatures = std::move(data.signatures); @@ -211,15 +220,15 @@ struct trace_pruner { /// It relies on the fact that the serialized pruned data won't be larger than its un-pruned counterpart /// which is subsequently based on: /// 1) the presence of context free data requires the presence of signatures; - /// 2) prunable_data::prune_all() would never convert it to prunable_data::none when both the context free data and - /// signature are empty. + /// 2) prune() would never convert it to prunable_data::none when both the context free data and + /// signature are empty, which is different from the behavior in prunable_data::prune_all(). void operator()(transaction_trace_v0& trace, prunable_data_type& incoming_prunable) { auto itr = std::find(ids.begin(), ids.end(), trace.id); if (itr != ids.end()) { // the incoming trace matches the one of ids to be pruned if (change_position == 0) change_position = write_strm.tellp(); - pack(write_strm, incoming_prunable.prune_all()); + pack(write_strm, prune(incoming_prunable)); ids.erase(itr); } else if (change_position == 0) { // no change so far, skip the original serialized prunable_data bytes From 7cb192187386ea9d1fabfdf94c2fa8b7587ab075 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 27 Apr 2020 14:04:29 -0500 Subject: [PATCH 120/142] Factor out send buffer creation into buffer_factory --- plugins/net_plugin/net_plugin.cpp | 273 ++++++++++++++++++------------ 1 file changed, 161 insertions(+), 112 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index efaa60e3c77..e4a456339f3 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -1208,120 +1208,176 @@ namespace eosio { return true; } - void connection::enqueue( const net_message& m ) { - verify_strand_in_this_thread( strand, __func__, __LINE__ ); - go_away_reason close_after_send = no_reason; - if (m.contains()) { - close_after_send = m.get().reason; - } + //------------------------------------------------------------------------ - const uint32_t payload_size = fc::raw::pack_size( m ); + using send_buffer_type = std::shared_ptr>; - const char* const header = reinterpret_cast(&payload_size); // avoid variable size encoding of uint32_t - constexpr size_t header_size = sizeof(payload_size); - static_assert( header_size == message_header_size, "invalid message_header_size" ); - const size_t buffer_size = header_size + payload_size; + struct buffer_factory { - auto send_buffer = std::make_shared>(buffer_size); - fc::datastream ds( send_buffer->data(), buffer_size); - ds.write( header, header_size ); - fc::raw::pack( ds, m ); + /// caches result for subsequent calls, only provide same net_message instance for each invocation + const send_buffer_type& get_send_buffer( const net_message& m ) { + if( !send_buffer ) { + send_buffer = create_send_buffer( m ); + } + return send_buffer; + } - enqueue_buffer( send_buffer, close_after_send ); - } + protected: + send_buffer_type send_buffer; - template< typename T> - static std::shared_ptr> create_send_buffer( uint32_t which, const T& v ) { - // match net_message static_variant pack - const uint32_t which_size = fc::raw::pack_size( unsigned_int( which ) ); - const uint32_t payload_size = which_size + fc::raw::pack_size( v ); + protected: + static send_buffer_type create_send_buffer( const net_message& m ) { + const uint32_t payload_size = fc::raw::pack_size( m ); - const char* const header = reinterpret_cast(&payload_size); // avoid variable size encoding of uint32_t - constexpr size_t header_size = sizeof( payload_size ); - static_assert( header_size == message_header_size, "invalid message_header_size" ); - const size_t buffer_size = header_size + payload_size; + const char* const header = reinterpret_cast(&payload_size); // avoid variable size encoding of uint32_t + constexpr size_t header_size = sizeof(payload_size); + static_assert( header_size == message_header_size, "invalid message_header_size" ); + const size_t buffer_size = header_size + payload_size; - auto send_buffer = std::make_shared>( buffer_size ); - fc::datastream ds( send_buffer->data(), buffer_size ); - ds.write( header, header_size ); - fc::raw::pack( ds, unsigned_int( which ) ); - fc::raw::pack( ds, v ); + auto send_buffer = std::make_shared>(buffer_size); + fc::datastream ds( send_buffer->data(), buffer_size); + ds.write( header, header_size ); + fc::raw::pack( ds, m ); - return send_buffer; - } + return send_buffer; + } - static std::shared_ptr> create_send_buffer( const signed_block_ptr& sb ) { - static_assert( signed_block_which == net_message::position() ); - // this implementation is to avoid copy of signed_block to net_message - // matches which of net_message for signed_block - fc_dlog( logger, "sending block ${bn}", ("bn", sb->block_num()) ); - return create_send_buffer( signed_block_which, *sb ); - } + template< typename T> + static send_buffer_type create_send_buffer( uint32_t which, const T& v ) { + // match net_message static_variant pack + const uint32_t which_size = fc::raw::pack_size( unsigned_int( which ) ); + const uint32_t payload_size = which_size + fc::raw::pack_size( v ); - static std::shared_ptr> create_send_buffer( const signed_block_v0& sb_v0 ) { - static_assert( signed_block_v0_which == net_message::position() ); - // this implementation is to avoid copy of signed_block_v0 to net_message - // matches which of net_message for signed_block_v0 - fc_dlog( logger, "sending v0 block ${bn}", ("bn", sb_v0.block_num()) ); - return create_send_buffer( signed_block_v0_which, sb_v0 ); - } + const char* const header = reinterpret_cast(&payload_size); // avoid variable size encoding of uint32_t + constexpr size_t header_size = sizeof( payload_size ); + static_assert( header_size == message_header_size, "invalid message_header_size" ); + const size_t buffer_size = header_size + payload_size; - static std::shared_ptr> create_send_buffer( const packed_transaction_ptr& trx ) { - static_assert( trx_message_v1_which == net_message::position() ); - fc::optional trx_id; - if( trx->get_estimated_size() > 1024 ) { // simple guess on threshold - fc_dlog( logger, "including trx id, est size: ${es}", ("es", trx->get_estimated_size()) ); - trx_id = trx->id(); + auto send_buffer = std::make_shared>( buffer_size ); + fc::datastream ds( send_buffer->data(), buffer_size ); + ds.write( header, header_size ); + fc::raw::pack( ds, unsigned_int( which ) ); + fc::raw::pack( ds, v ); + + return send_buffer; } - // const cast required, trx_message_v1 has non-const shared_ptr because FC_REFLECT does not work with const types - trx_message_v1 v1{ std::move(trx_id), std::const_pointer_cast(trx) }; - return create_send_buffer( trx_message_v1_which, v1 ); - } - static std::shared_ptr> create_send_buffer( const packed_transaction_v0& trx ) { - static_assert( packed_transaction_v0_which == net_message::position() ); - // this implementation is to avoid copy of packed_transaction_v0 to net_message - // matches which of net_message for packed_transaction_v0 - return create_send_buffer( packed_transaction_v0_which, trx ); - } + }; - static std::shared_ptr> get_send_buffer( connection& c, - const signed_block_ptr& b, - std::shared_ptr>& send_buffer, - std::shared_ptr>& send_buffer_v0, - bool& valid_v0_block ) - { - const uint16_t v = c.protocol_version.load(); - if( v >= proto_pruned_types ) { - if( !send_buffer ) { - send_buffer = create_send_buffer( b ); + struct block_buffer_factory : public buffer_factory { + + /// caches result for subsequent calls, only provide same signed_block_ptr instance for each invocation. + /// protocol_version can differ per invocation as buffer_factory potentially caches multiple send buffers. + const send_buffer_type& get_send_buffer( const signed_block_ptr& sb, uint16_t protocol_version ) { + if( protocol_version >= proto_pruned_types ) { + if( !send_buffer ) { + send_buffer = create_send_buffer( sb ); + } + return send_buffer; + } else { + if( !send_buffer_v0 ) { + const auto v0 = sb->to_signed_block_v0(); + if( !v0 ) return send_buffer_v0; + send_buffer_v0 = create_send_buffer( *v0 ); + } + return send_buffer_v0; } - return send_buffer; - } else { - if( !send_buffer_v0 ) { - const auto sb_v0 = valid_v0_block ? b->to_signed_block_v0() : signed_block_v0_uptr(); - if( !sb_v0 ) { - peer_wlog( (&c), "Sending go away for incomplete block #${n} ${id}...", - ("n", b->block_num())("id", b->calculate_id().str().substr(8,16)) ); - valid_v0_block = false; - // unable to convert to v0 signed block and client doesn't support proto_pruned_types, so tell it to go away - c.enqueue( go_away_message( fatal_other ) ); - return send_buffer_v0; + } + + private: + send_buffer_type send_buffer_v0; + + private: + + static std::shared_ptr> create_send_buffer( const signed_block_ptr& sb ) { + static_assert( signed_block_which == net_message::position() ); + // this implementation is to avoid copy of signed_block to net_message + // matches which of net_message for signed_block + fc_dlog( logger, "sending block ${bn}", ("bn", sb->block_num()) ); + return buffer_factory::create_send_buffer( signed_block_which, *sb ); + } + + static std::shared_ptr> create_send_buffer( const signed_block_v0& sb_v0 ) { + static_assert( signed_block_v0_which == net_message::position() ); + // this implementation is to avoid copy of signed_block_v0 to net_message + // matches which of net_message for signed_block_v0 + fc_dlog( logger, "sending v0 block ${bn}", ("bn", sb_v0.block_num()) ); + return buffer_factory::create_send_buffer( signed_block_v0_which, sb_v0 ); + } + }; + + struct trx_buffer_factory : public buffer_factory { + + /// caches result for subsequent calls, only provide same packed_transaction_ptr instance for each invocation. + /// protocol_version can differ per invocation as buffer_factory potentially caches multiple send buffers. + const send_buffer_type& get_send_buffer( const packed_transaction_ptr& trx, uint16_t protocol_version ) { + if( protocol_version >= proto_pruned_types ) { + if( !send_buffer ) { + send_buffer = create_send_buffer( trx ); } - send_buffer_v0 = create_send_buffer( *sb_v0 ); + return send_buffer; + } else { + if( !send_buffer_v0 ) { + const auto v0 = trx->to_packed_transaction_v0(); + if( !v0 ) return send_buffer_v0; + send_buffer_v0 = create_send_buffer( *v0 ); + } + return send_buffer_v0; + } + } + + private: + send_buffer_type send_buffer_v0; + + private: + + static std::shared_ptr> create_send_buffer( const packed_transaction_ptr& trx ) { + static_assert( trx_message_v1_which == net_message::position() ); + fc::optional trx_id; + if( trx->get_estimated_size() > 1024 ) { // simple guess on threshold + fc_dlog( logger, "including trx id, est size: ${es}", ("es", trx->get_estimated_size()) ); + trx_id = trx->id(); } - return send_buffer_v0; + // const cast required, trx_message_v1 has non-const shared_ptr because FC_REFLECT does not work with const types + trx_message_v1 v1{std::move( trx_id ), std::const_pointer_cast( trx )}; + return buffer_factory::create_send_buffer( trx_message_v1_which, v1 ); + } + + static std::shared_ptr> create_send_buffer( const packed_transaction_v0& trx ) { + static_assert( packed_transaction_v0_which == net_message::position() ); + // this implementation is to avoid copy of packed_transaction_v0 to net_message + // matches which of net_message for packed_transaction_v0 + return buffer_factory::create_send_buffer( packed_transaction_v0_which, trx ); + } + }; + + //------------------------------------------------------------------------ + + void connection::enqueue( const net_message& m ) { + verify_strand_in_this_thread( strand, __func__, __LINE__ ); + go_away_reason close_after_send = no_reason; + if (m.contains()) { + close_after_send = m.get().reason; } + + buffer_factory buff_factory; + auto send_buffer = buff_factory.get_send_buffer( m ); + enqueue_buffer( send_buffer, close_after_send ); } void connection::enqueue_block( const signed_block_ptr& b, bool to_sync_queue) { fc_dlog( logger, "enqueue block ${num}", ("num", b->block_num()) ); verify_strand_in_this_thread( strand, __func__, __LINE__ ); - std::shared_ptr> send_buffer, send_buffer_v0; - bool valid_v0_block = b->prune_state != signed_block::prune_state_type::incomplete; - auto sb = get_send_buffer( *this, b, send_buffer, send_buffer_v0, valid_v0_block ); - if( !sb ) return; + block_buffer_factory buff_factory; + auto sb = buff_factory.get_send_buffer( b, protocol_version.load() ); + if( !sb ) { + peer_wlog( this, "Sending go away for incomplete block #${n} ${id}...", + ("n", b->block_num())("id", b->calculate_id().str().substr(8,16)) ); + // unable to convert to v0 signed block and client doesn't support proto_pruned_types, so tell it to go away + enqueue( go_away_message( fatal_other ) ); + return; + } enqueue_buffer( sb, no_reason, to_sync_queue); } @@ -2016,15 +2072,20 @@ namespace eosio { if( my_impl->sync_master->syncing_with_peer() ) return; - std::shared_ptr> send_buffer, send_buffer_v0; - bool valid_v0_block = b->prune_state != signed_block::prune_state_type::incomplete; + block_buffer_factory buff_factory; const auto bnum = b->block_num(); - for_each_block_connection( [this, &id, &bnum, &b, &send_buffer, &send_buffer_v0, &valid_v0_block]( auto& cp ) { + for_each_block_connection( [this, &id, &bnum, &b, &buff_factory]( auto& cp ) { peer_dlog( cp, "socket_is_open ${s}, connecting ${c}, syncing ${ss}", ("s", cp->socket_is_open())("c", cp->connecting.load())("ss", cp->syncing.load()) ); if( !cp->current() ) return true; - std::shared_ptr> sb = get_send_buffer( *cp, b, send_buffer, send_buffer_v0, valid_v0_block ); - if( !sb ) return true; + send_buffer_type sb = buff_factory.get_send_buffer( b, cp->protocol_version.load() ); + if( !sb ) { + peer_wlog( cp, "Sending go away for incomplete block #${n} ${id}...", + ("n", b->block_num())("id", b->calculate_id().str().substr(8,16)) ); + // unable to convert to v0 signed block and client doesn't support proto_pruned_types, so tell it to go away + cp->enqueue( go_away_message( fatal_other ) ); + return true; + } cp->strand.post( [this, cp, id, bnum, sb{std::move(sb)}]() { std::unique_lock g_conn( cp->conn_mtx ); @@ -2069,8 +2130,8 @@ namespace eosio { time_point_sec trx_expiration = trx->expiration(); node_transaction_state nts = {id, trx_expiration, 0, 0}; - std::shared_ptr> send_buffer, send_buffer_v0; - for_each_connection( [this, &trx, &nts, &send_buffer, &send_buffer_v0]( auto& cp ) { + trx_buffer_factory buff_factory; + for_each_connection( [this, &trx, &nts, &buff_factory]( auto& cp ) { if( cp->is_blocks_only_connection() || !cp->current() ) { return true; } @@ -2079,21 +2140,9 @@ namespace eosio { return true; } - std::shared_ptr> sb; - const uint16_t v = cp->protocol_version.load(); - if( v >= proto_pruned_types ) { - if( !send_buffer ) { - send_buffer = create_send_buffer( trx ); - } - sb = send_buffer; - } else { - if( !send_buffer_v0 ) { - packed_transaction_v0_ptr v0 = trx->to_packed_transaction_v0(); - send_buffer_v0 = create_send_buffer( *v0 ); - } - sb = send_buffer_v0; - } - cp->strand.post( [cp, sb]() { + send_buffer_type sb = buff_factory.get_send_buffer( trx, cp->protocol_version.load() ); + if( !sb ) return true; + cp->strand.post( [cp, sb{std::move(sb)}]() { fc_dlog( logger, "sending trx to ${n}", ("n", cp->peer_name()) ); cp->enqueue_buffer( sb, no_reason ); } ); From 1239ee8552b382650afe34f23e4207ffe71d7592 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 28 Apr 2020 08:55:39 -0500 Subject: [PATCH 121/142] PR comment change --- libraries/chain/block_log.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index ba9a63ac537..aa8fac239ee 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -905,10 +905,7 @@ namespace eosio { namespace chain { if (it != ids.end()) { ptx.prune_all(); // remove the found entry from ids - if (it != ids.end() - 1) { - *it = ids.back(); - } - ids.resize(ids.size() - 1); + ids.erase(it); return true; } return false; From 113beeecd84b78c2ee09fda313971fc31f263fff Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 28 Apr 2020 11:12:04 -0500 Subject: [PATCH 122/142] More PR comment change --- libraries/state_history/trace_converter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/state_history/trace_converter.cpp b/libraries/state_history/trace_converter.cpp index 04f171cc9c5..16d9184f477 100644 --- a/libraries/state_history/trace_converter.cpp +++ b/libraries/state_history/trace_converter.cpp @@ -192,7 +192,7 @@ struct restore_partial { ptrx.context_free_data = std::move(data.context_free_segments); } void operator()(prunable_data_type::none& data) const {} - void operator()(prunable_data_type::partial& data) const { ptrx.signatures = std::move(data.signatures); } + void operator()(prunable_data_type::partial& data) const { EOS_ASSERT(false, state_history_exception, "Not implemented"); } void operator()(prunable_data_type::full& data) const { ptrx.signatures = std::move(data.signatures); ptrx.context_free_data = std::move(data.context_free_segments); From e65ebb03ab57dff04e09986917ec594b50873159 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 29 Apr 2020 11:12:37 -0500 Subject: [PATCH 123/142] Minor changes from review comments --- libraries/chain/block.cpp | 2 +- libraries/chain/block_log.cpp | 73 +++++++++++++++++++++----------- libraries/chain/transaction.cpp | 5 +-- programs/eosio-blocklog/main.cpp | 2 +- 4 files changed, 51 insertions(+), 31 deletions(-) diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index 00a278af2af..cc9cac94a5b 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -36,7 +36,7 @@ namespace eosio { namespace chain { transaction_receipt::transaction_receipt(const transaction_receipt_v0& other, bool legacy) : transaction_receipt_header(static_cast(other)), - trx( std::move(other.trx).visit(transaction_receipt_translator{legacy})) + trx( other.trx.visit(transaction_receipt_translator{legacy})) {} transaction_receipt::transaction_receipt(transaction_receipt_v0&& other, bool legacy) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index aa8fac239ee..39278d79ebc 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -55,13 +55,20 @@ namespace eosio { namespace chain { return sizeof(uint32_t) + 1; } - struct log_entry_v4 : signed_block { + + struct log_entry_v4 { // In version 4 of the irreversible blocks log format, these log entries consists of the following in order: // 1. An uint32_t size for number of bytes from the start of this log entry to the start of the next log entry. // 2. An uint8_t indicating the compression status for the serialization of the pruned_block following this. // 3. The serialization of a signed_block representation of the block for the entry including padding. - packed_transaction::cf_compression_type compression = packed_transaction::cf_compression_type::none; - uint32_t size = 0; // the size of the log entry + + struct metadata_type { + packed_transaction::cf_compression_type compression = packed_transaction::cf_compression_type::none; + uint32_t size = 0; // the size of the log entry + }; + + metadata_type meta; + signed_block block; }; size_t get_stream_pos(fc::cfile_datastream& ds) { return ds.tellp(); } @@ -70,24 +77,33 @@ namespace eosio { namespace chain { void skip_streamm_pos(std::ifstream& ds, size_t n) { ds.seekg(n, ds.cur); } template - void unpack(Stream& ds, log_entry_v4& entry){ - const auto start_pos = get_stream_pos(ds); - fc::raw::unpack(ds, entry.size); + log_entry_v4::metadata_type unpack(Stream& ds, signed_block& block){ + log_entry_v4::metadata_type meta; + const auto start_pos = get_stream_pos(ds); + fc::raw::unpack(ds, meta.size); uint8_t compression; fc::raw::unpack(ds, compression); - entry.compression = static_cast(compression); - EOS_ASSERT(entry.compression == packed_transaction::cf_compression_type::none, block_log_exception, + meta.compression = static_cast(compression); + EOS_ASSERT(meta.compression == packed_transaction::cf_compression_type::none, block_log_exception, "Only support compression_type none"); - static_cast(entry).unpack(ds, entry.compression); + EOS_ASSERT(meta.compression < packed_transaction::cf_compression_type::COMPRESSION_TYPE_COUNT, block_log_exception, + "Unknown compression_type"); + block.unpack(ds, meta.compression); const uint64_t current_stream_offset = get_stream_pos(ds) - start_pos; // For a block which contains CFD (context free data) and the CFD is pruned afterwards, the entry.size may // be the size before the CFD has been pruned while the actual serialized block does not have the CFD anymore. // In this case, the serialized block has fewer bytes than what's indicated by entry.size. We need to // skip over the extra bytes to allow ds to position to the last 8 bytes of the entry. - const int64_t bytes_to_skip = static_cast(entry.size) - sizeof(uint64_t) - current_stream_offset; + const int64_t bytes_to_skip = static_cast(meta.size) - sizeof(uint64_t) - current_stream_offset; EOS_ASSERT(bytes_to_skip >= 0, block_log_exception, "Invalid block log entry size"); skip_streamm_pos(ds, bytes_to_skip); + return meta; + } + + template + void unpack(Stream& ds, log_entry_v4& entry){ + entry.meta = eosio::chain::detail::unpack(ds, entry.block); } std::vector pack(const signed_block& block, packed_transaction::cf_compression_type compression) { @@ -105,7 +121,7 @@ namespace eosio { namespace chain { } std::vector pack(const log_entry_v4& entry) { - return pack(static_cast(entry), entry.compression); + return pack(entry.block, entry.meta.compression); } using log_entry = std::variant; @@ -114,7 +130,7 @@ namespace eosio { namespace chain { void unpack(Stream& ds, log_entry& entry) { std::visit( overloaded{[&ds](signed_block_v0& v) { fc::raw::unpack(ds, v); }, - [&ds](log_entry_v4& v) { unpack(ds, v); }}, + [&ds](log_entry_v4& v) { eosio::chain::detail::unpack(ds, v); }}, entry); } @@ -290,7 +306,8 @@ namespace eosio { namespace chain { }; }}} // namespace eosio::chain::detail -FC_REFLECT_DERIVED(eosio::chain::detail::log_entry_v4, (eosio::chain::signed_block), (compression)(size) ) +FC_REFLECT(eosio::chain::detail::log_entry_v4::metadata_type, (compression)(size) ) +FC_REFLECT(eosio::chain::detail::log_entry_v4, (meta)(block) ) namespace eosio { namespace chain { @@ -518,9 +535,9 @@ namespace eosio { namespace chain { block_file.seek(pos); auto ds = block_file.create_datastream(); if (version >= pruned_transaction_version) { - auto entry = std::make_unique(); - unpack(ds, *entry); - return entry; + auto block = std::make_unique(); + eosio::chain::detail::unpack(ds, *block); + return block; } else { signed_block_v0 block; fc::raw::unpack(ds, block); @@ -750,7 +767,7 @@ namespace eosio { namespace chain { while( pos < end_pos ) { try { - unpack(old_block_stream, entry); + eosio::chain::detail::unpack(old_block_stream, entry); } catch (...) { except_ptr = std::current_exception(); incomplete_block_data.resize( end_pos - pos ); @@ -758,13 +775,19 @@ namespace eosio { namespace chain { break; } - auto id = std::visit([](const auto& v) { return v.calculate_id(); }, entry); + const block_header& header = + std::visit(overloaded{[](const signed_block_v0& v) -> const block_header& { return v; }, + [](const detail::log_entry_v4& v) -> const block_header& { return v.block; }}, + entry); + + + auto id = header.calculate_id(); if( block_header::num_from_id(expected_previous) + 1 != block_header::num_from_id(id) ) { elog( "Block ${num} (${id}) skips blocks. Previous block in block log is block ${prev_num} (${previous})", ("num", block_header::num_from_id(id))("id", id) ("prev_num", block_header::num_from_id(expected_previous))("previous", expected_previous) ); } - auto actual_previous = std::visit([](const auto& v) { return v.previous; }, entry); + auto actual_previous = header.previous; if( expected_previous != actual_previous ) { elog( "Block ${num} (${id}) does not link back to previous block. " "Expected previous: ${expected}. Actual previous: ${actual}.", @@ -784,7 +807,7 @@ namespace eosio { namespace chain { auto data = detail::pack(entry); new_block_stream.write( data.data(), data.size() ); new_block_stream.write( reinterpret_cast(&pos), sizeof(pos) ); - block_num = std::visit([](const auto& v) { return v.block_num(); }, entry); + block_num = header.block_num(); if(block_num % 1000 == 0) ilog( "Recovered block ${num}", ("num", block_num) ); pos = new_block_stream.tellp(); @@ -894,9 +917,9 @@ namespace eosio { namespace chain { detail::log_entry_v4 entry; my->block_file.seek(pos); auto ds = my->block_file.create_datastream(); - unpack(ds, entry); + eosio::chain::detail::unpack(ds, entry); - EOS_ASSERT(entry.block_num() == block_num, block_log_exception, + EOS_ASSERT(entry.block.block_num() == block_num, block_log_exception, "Wrong block was read from block log."); auto pruner = overloaded{[](transaction_id_type&) { return false; }, @@ -912,7 +935,7 @@ namespace eosio { namespace chain { }}; size_t num_trx_pruned = 0; - for (auto& trx : entry.transactions) { + for (auto& trx : entry.block.transactions) { num_trx_pruned += trx.trx.visit(pruner); } @@ -920,10 +943,10 @@ namespace eosio { namespace chain { // we don't want to rewrite entire entry, just the block data itself. const auto block_offset = detail::offset_to_block_start(my->version); my->block_file.seek(pos + block_offset); - const uint32_t max_block_size = entry.size - block_offset - sizeof(uint64_t); + const uint32_t max_block_size = entry.meta.size - block_offset - sizeof(uint64_t); std::vector buffer(max_block_size); fc::datastream stream(buffer.data(), buffer.size()); - static_cast(entry).pack(stream, entry.compression); + entry.block.pack(stream, entry.meta.compression); my->block_file.write(buffer.data(), buffer.size()); my->block_file.flush(); } diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 64af37e6cbf..f93b01957d4 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -400,10 +400,7 @@ static digest_type prunable_digest(const packed_transaction::prunable_data_type: } static digest_type prunable_digest(const packed_transaction::prunable_data_type::full& obj) { - digest_type::encoder prunable; - fc::raw::pack( prunable, obj.signatures ); - fc::raw::pack( prunable, obj.context_free_segments ); - return prunable.result(); + EOS_THROW(tx_prune_exception, "unimplemented"); } static digest_type prunable_digest(const packed_transaction::prunable_data_type::full_legacy& obj) { diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp index 7f3d52590aa..65edfbd78f7 100644 --- a/programs/eosio-blocklog/main.cpp +++ b/programs/eosio-blocklog/main.cpp @@ -175,7 +175,7 @@ void blocklog::set_program_options(options_description& cli) ("blocks-dir", bpo::value()->default_value("blocks"), "the location of the blocks directory (absolute path or relative to the current directory)") ("state-history-dir", bpo::value()->default_value("state-history"), - "the location of the state-history directory (absolute path or relative to application data dir)") + "the location of the state-history directory (absolute path or relative to the current dir)") ("output-file,o", bpo::value(), "the file to write the output to (absolute or relative path). If not specified then output is to stdout.") ("first,f", bpo::value(&first_block)->default_value(0), From 510dc806449ac227bc33bc8d06213451c6ab5fdc Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 29 Apr 2020 12:28:47 -0500 Subject: [PATCH 124/142] remove unnecessary code --- libraries/chain/block_log.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 39278d79ebc..4c0e8fdbc64 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -306,9 +306,6 @@ namespace eosio { namespace chain { }; }}} // namespace eosio::chain::detail -FC_REFLECT(eosio::chain::detail::log_entry_v4::metadata_type, (compression)(size) ) -FC_REFLECT(eosio::chain::detail::log_entry_v4, (meta)(block) ) - namespace eosio { namespace chain { block_log::block_log(const fc::path& data_dir) From a168004a284533993396467f1b1e6ace08382e57 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 29 Apr 2020 12:38:48 -0500 Subject: [PATCH 125/142] some more changes --- libraries/chain/block_log.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 4c0e8fdbc64..da7a393dc8b 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -84,10 +84,10 @@ namespace eosio { namespace chain { uint8_t compression; fc::raw::unpack(ds, compression); meta.compression = static_cast(compression); - EOS_ASSERT(meta.compression == packed_transaction::cf_compression_type::none, block_log_exception, - "Only support compression_type none"); EOS_ASSERT(meta.compression < packed_transaction::cf_compression_type::COMPRESSION_TYPE_COUNT, block_log_exception, "Unknown compression_type"); + EOS_ASSERT(meta.compression == packed_transaction::cf_compression_type::none, block_log_exception, + "Only support compression_type none"); block.unpack(ds, meta.compression); const uint64_t current_stream_offset = get_stream_pos(ds) - start_pos; // For a block which contains CFD (context free data) and the CFD is pruned afterwards, the entry.size may @@ -109,7 +109,7 @@ namespace eosio { namespace chain { std::vector pack(const signed_block& block, packed_transaction::cf_compression_type compression) { const std::size_t padded_size = block.maximum_pruned_pack_size(compression); static_assert( block_log::max_supported_version == pruned_transaction_version, - "Code was written to support format of version 4, need to update this code for latest format." ); + "Code was written to support format of version 4 or lower, need to update this code for latest format." ); std::vector buffer(padded_size + offset_to_block_start(block_log::max_supported_version)); fc::datastream stream(buffer.data(), buffer.size()); @@ -304,9 +304,8 @@ namespace eosio { namespace chain { FILE* const _file; const std::string _filename; }; -}}} // namespace eosio::chain::detail + } // namespace eosio::chain::detail -namespace eosio { namespace chain { block_log::block_log(const fc::path& data_dir) :my(new detail::block_log_impl()) { From 1256d0813f8d7d92c9ce18d9f249c611ecbb8021 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 29 Apr 2020 13:15:23 -0500 Subject: [PATCH 126/142] some more changes --- libraries/chain/block_log.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index da7a393dc8b..19b4ce053fa 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -83,9 +83,9 @@ namespace eosio { namespace chain { fc::raw::unpack(ds, meta.size); uint8_t compression; fc::raw::unpack(ds, compression); - meta.compression = static_cast(compression); - EOS_ASSERT(meta.compression < packed_transaction::cf_compression_type::COMPRESSION_TYPE_COUNT, block_log_exception, + EOS_ASSERT(compression < static_cast(packed_transaction::cf_compression_type::COMPRESSION_TYPE_COUNT), block_log_exception, "Unknown compression_type"); + meta.compression = static_cast(compression); EOS_ASSERT(meta.compression == packed_transaction::cf_compression_type::none, block_log_exception, "Only support compression_type none"); block.unpack(ds, meta.compression); @@ -109,7 +109,7 @@ namespace eosio { namespace chain { std::vector pack(const signed_block& block, packed_transaction::cf_compression_type compression) { const std::size_t padded_size = block.maximum_pruned_pack_size(compression); static_assert( block_log::max_supported_version == pruned_transaction_version, - "Code was written to support format of version 4 or lower, need to update this code for latest format." ); + "Code was written to support format of version 4, need to update this code for latest format." ); std::vector buffer(padded_size + offset_to_block_start(block_log::max_supported_version)); fc::datastream stream(buffer.data(), buffer.size()); @@ -1203,7 +1203,7 @@ namespace eosio { namespace chain { new_block_file.open( LOG_RW_C ); static_assert( block_log::max_supported_version == pruned_transaction_version, - "Code was written to support format of version 4, need to update this code for latest format." ); + "Code was written to support format of version 4 or lower, need to update this code for latest format." ); uint32_t version = block_log::max_supported_version; new_block_file.seek(0); new_block_file.write((char*)&version, sizeof(version)); From 6f1821a288aca57b35e2dc5117eadbf97f7fa799 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 29 Apr 2020 14:31:49 -0500 Subject: [PATCH 127/142] add missing test --- tests/CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a870178fcee..c17aa233811 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -103,6 +103,8 @@ set_tests_properties(db_modes_test PROPERTIES COST 6000) add_test(NAME release-build-test COMMAND tests/release-build.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) add_test(NAME validate-reflection-test COMMAND build/tests/validate-reflection.py plugins/ programs/ libraries/ --recurse --extension "cpp" --extension "hpp" -e WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/..) add_test(NAME version-label-test COMMAND tests/version-label.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME light_validation_sync_test COMMAND tests/light_validation_sync_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) # Long running tests add_test(NAME nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity-test --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) @@ -156,9 +158,6 @@ set_property(TEST nodeos_high_transaction_lr_test PROPERTY LABELS long_running_t add_test(NAME nodeos_repeat_transaction_lr_test COMMAND tests/nodeos_high_transaction_test.py -v --clean-run --dump-error-detail -p 4 -n 8 --num-transactions 1000 --max-transactions-per-second 500 --send-duplicates WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST nodeos_repeat_transaction_lr_test PROPERTY LABELS long_running_tests) -add_test(NAME light_validation_sync_test COMMAND tests/light_validation_sync_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -set_property(TEST light_validation_sync_test PROPERTY LABELS long_running_tests) - if(ENABLE_COVERAGE_TESTING) set(Coverage_NAME ${PROJECT_NAME}_coverage) From 0efbfd3e3f9f221799f7897bfe55d63f9f5c2e6c Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 29 Apr 2020 15:13:57 -0500 Subject: [PATCH 128/142] label light_validation_sync_test & eosio_blocklog_prune_test as nonparallelizable_tests --- tests/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c17aa233811..8be52e5b436 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -104,7 +104,9 @@ add_test(NAME release-build-test COMMAND tests/release-build.sh WORKING_DIRECTOR add_test(NAME validate-reflection-test COMMAND build/tests/validate-reflection.py plugins/ programs/ libraries/ --recurse --extension "cpp" --extension "hpp" -e WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/..) add_test(NAME version-label-test COMMAND tests/version-label.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) add_test(NAME light_validation_sync_test COMMAND tests/light_validation_sync_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST light_validation_sync_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS nonparallelizable_tests) # Long running tests add_test(NAME nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity-test --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) From efd8030ab89fc0114cd7420618d1692b3f5a3e97 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 30 Apr 2020 08:00:41 -0500 Subject: [PATCH 129/142] try to run light_validation_sync_test & eosio_blocklog_prune_test as long_running_tests --- tests/CMakeLists.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8be52e5b436..4f848dfb0e5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -103,10 +103,6 @@ set_tests_properties(db_modes_test PROPERTIES COST 6000) add_test(NAME release-build-test COMMAND tests/release-build.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) add_test(NAME validate-reflection-test COMMAND build/tests/validate-reflection.py plugins/ programs/ libraries/ --recurse --extension "cpp" --extension "hpp" -e WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/..) add_test(NAME version-label-test COMMAND tests/version-label.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME light_validation_sync_test COMMAND tests/light_validation_sync_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -set_property(TEST light_validation_sync_test PROPERTY LABELS nonparallelizable_tests) -add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS nonparallelizable_tests) # Long running tests add_test(NAME nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity-test --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) @@ -160,6 +156,13 @@ set_property(TEST nodeos_high_transaction_lr_test PROPERTY LABELS long_running_t add_test(NAME nodeos_repeat_transaction_lr_test COMMAND tests/nodeos_high_transaction_test.py -v --clean-run --dump-error-detail -p 4 -n 8 --num-transactions 1000 --max-transactions-per-second 500 --send-duplicates WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST nodeos_repeat_transaction_lr_test PROPERTY LABELS long_running_tests) +add_test(NAME light_validation_sync_test COMMAND tests/light_validation_sync_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST light_validation_sync_test PROPERTY LABELS long_running_tests) + +add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS long_running_tests) + + if(ENABLE_COVERAGE_TESTING) set(Coverage_NAME ${PROJECT_NAME}_coverage) From bc34e5e1df9ff616f83e94254c25432f11ec81fd Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 30 Apr 2020 16:26:40 -0500 Subject: [PATCH 130/142] Revert "try to run light_validation_sync_test & eosio_blocklog_prune_test as long_running_tests" This reverts commit efd8030ab89fc0114cd7420618d1692b3f5a3e97. --- tests/CMakeLists.txt | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4f848dfb0e5..8be52e5b436 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -103,6 +103,10 @@ set_tests_properties(db_modes_test PROPERTIES COST 6000) add_test(NAME release-build-test COMMAND tests/release-build.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) add_test(NAME validate-reflection-test COMMAND build/tests/validate-reflection.py plugins/ programs/ libraries/ --recurse --extension "cpp" --extension "hpp" -e WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/..) add_test(NAME version-label-test COMMAND tests/version-label.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME light_validation_sync_test COMMAND tests/light_validation_sync_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST light_validation_sync_test PROPERTY LABELS nonparallelizable_tests) +add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS nonparallelizable_tests) # Long running tests add_test(NAME nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity-test --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) @@ -156,13 +160,6 @@ set_property(TEST nodeos_high_transaction_lr_test PROPERTY LABELS long_running_t add_test(NAME nodeos_repeat_transaction_lr_test COMMAND tests/nodeos_high_transaction_test.py -v --clean-run --dump-error-detail -p 4 -n 8 --num-transactions 1000 --max-transactions-per-second 500 --send-duplicates WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST nodeos_repeat_transaction_lr_test PROPERTY LABELS long_running_tests) -add_test(NAME light_validation_sync_test COMMAND tests/light_validation_sync_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -set_property(TEST light_validation_sync_test PROPERTY LABELS long_running_tests) - -add_test(NAME eosio_blocklog_prune_test COMMAND tests/eosio_blocklog_prune_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -set_property(TEST eosio_blocklog_prune_test PROPERTY LABELS long_running_tests) - - if(ENABLE_COVERAGE_TESTING) set(Coverage_NAME ${PROJECT_NAME}_coverage) From 08330f86bcd2cdd3cf5690196d496912edc96704 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 30 Apr 2020 21:23:19 -0500 Subject: [PATCH 131/142] restart node before check prune result --- tests/eosio_blocklog_prune_test.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/eosio_blocklog_prune_test.py b/tests/eosio_blocklog_prune_test.py index 7a916ea03c6..4179dc0996e 100755 --- a/tests/eosio_blocklog_prune_test.py +++ b/tests/eosio_blocklog_prune_test.py @@ -11,6 +11,7 @@ from testUtils import BlockLogAction import json import sys +import signal ############################################################### # eosio_blocklog_prune_test.py @@ -104,15 +105,11 @@ def isBlockNumIrr(): trans = validationNode.processCleosCmd(cmd, cmd, silentErrors=False) assert trans, "Failed to get the transaction with context free data from the light validation node" + validationNode.kill(signal.SIGTERM) # prune the transaction with block-num=trans["block_num"], id=cfTrxId cluster.getBlockLog(1, blockLogAction=BlockLogAction.prune_transactions, extraArgs=" --block-num {} --transaction {}".format(trans["block_num"], cfTrxId), exitOnError=True) - trans = validationNode.processCleosCmd(cmd, cmd, silentErrors=False) - assert trans, "Failed to get the transaction with context free data from the light validation node" - # check whether the transaction has been pruned based on the tag of prunable_data, if the tag is 1, then it's a prunable_data_t::none - assert trans["trx"]["receipt"]["trx"][1]["prunable_data"]["prunable_data"][0] == 1, "the the transaction with context free data has not been pruned" - # try to prune the transaction where it doesn't belong try: cluster.getBlockLog(1, blockLogAction=BlockLogAction.prune_transactions, extraArgs=" --block-num {} --transaction {}".format(cfTrxBlockNum - 1, cfTrxId), throwException=True, silentErrors=True) @@ -121,6 +118,16 @@ def isBlockNumIrr(): msg=ex.output.decode("utf-8") assert "does not contain the following transactions: " + cfTrxId in msg, "The transaction id is not displayed in the console when it cannot be found" + # For Linux, the pruned result won't be immediately visible unless the node is restarted. For MacOS, the result is immediately visible even without restart. + + isRelaunchSuccess = validationNode.relaunch(1) + assert isRelaunchSuccess, "Fail to relaunch verification node" + + trans = validationNode.processCleosCmd(cmd, cmd, silentErrors=False) + assert trans, "Failed to get the transaction with context free data from the light validation node" + # check whether the transaction has been pruned based on the tag of prunable_data, if the tag is 1, then it's a prunable_data_t::none + assert trans["trx"]["receipt"]["trx"][1]["prunable_data"]["prunable_data"][0] == 1, "the the transaction with context free data has not been pruned" + testSuccessful = True finally: TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails) From 85668c6ddde24746b17acfff7800ff4ed1a7dafd Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 4 May 2020 11:53:43 -0500 Subject: [PATCH 132/142] Use optional instead of unique_ptr since we never have to convert to shared_ptr --- libraries/chain/block.cpp | 10 +++++----- libraries/chain/include/eosio/chain/block.hpp | 3 +-- plugins/chain_plugin/chain_plugin.cpp | 2 +- unittests/misc_tests.cpp | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/libraries/chain/block.cpp b/libraries/chain/block.cpp index 288e7c4e455..535f7a32025 100644 --- a/libraries/chain/block.cpp +++ b/libraries/chain/block.cpp @@ -133,12 +133,12 @@ namespace eosio { namespace chain { return validate_and_extract_block_extensions( block_extensions ); } - signed_block_v0_uptr signed_block::to_signed_block_v0() const { + fc::optional signed_block::to_signed_block_v0() const { if (prune_state != prune_state_type::complete_legacy) - return signed_block_v0_uptr{}; + return {}; - auto result = std::make_unique(*static_cast(this)); - result->block_extensions = this->block_extensions; + signed_block_v0 result(*static_cast(this)); + result.block_extensions = this->block_extensions; auto visitor = overloaded{ [](const transaction_id_type &id) -> transaction_receipt_v0::trx_type { return id; }, @@ -148,7 +148,7 @@ namespace eosio { namespace chain { }}; for (const transaction_receipt &r : transactions){ - result->transactions.emplace_back(transaction_receipt_v0{r, r.trx.visit(visitor)}); + result.transactions.emplace_back(transaction_receipt_v0{r, r.trx.visit(visitor)}); } return result; } diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 81d61bf418c..7053993a99f 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -104,7 +104,6 @@ namespace eosio { namespace chain { }; using signed_block_v0_ptr = std::shared_ptr; - using signed_block_v0_uptr = std::unique_ptr; struct transaction_receipt : public transaction_receipt_header { @@ -145,7 +144,7 @@ namespace eosio { namespace chain { signed_block& operator=(const signed_block&) = delete; signed_block& operator=(signed_block&&) = default; signed_block clone() const { return *this; } - signed_block_v0_uptr to_signed_block_v0() const; + fc::optional to_signed_block_v0() const; fc::enum_type prune_state{prune_state_type::complete_legacy}; deque transactions; /// new or generated transactions diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index a50dc3a8145..48ed6bc31ad 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -1423,7 +1423,7 @@ bool chain_plugin::export_reversible_blocks( const fc::path& reversible_dir, signed_block tmp; fc::datastream ds( itr->packedblock.data(), itr->packedblock.size() ); fc::raw::unpack(ds, tmp); // Verify that packed block has not been corrupted. - signed_block_v0_uptr v0 = tmp.to_signed_block_v0(); // store in signed_block_v0 format + const auto v0 = tmp.to_signed_block_v0(); // store in signed_block_v0 format auto packed_v0 = fc::raw::pack(*v0); reversible_blocks.write( packed_v0.data(), packed_v0.size() ); end = itr->blocknum; diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index f3d48c94663..00f0a97d6f1 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -1020,14 +1020,14 @@ BOOST_AUTO_TEST_CASE(pruned_block_test) { t.push_transaction(trx); signed_block_ptr produced = t.produce_block(); - signed_block_v0_ptr original = produced->to_signed_block_v0(); + auto original = produced->to_signed_block_v0(); BOOST_REQUIRE(original); signed_block basic(*original, true); BOOST_TEST(basic.transaction_mroot.str() == original->transaction_mroot.str()); BOOST_TEST(basic.transaction_mroot.str() == calculate_trx_merkle(basic.transactions).str()); - signed_block_v0_ptr recovered = basic.to_signed_block_v0(); + auto recovered = basic.to_signed_block_v0(); BOOST_REQUIRE(recovered); BOOST_TEST(fc::raw::pack(*original) == fc::raw::pack(*recovered)); From 3e2327a183fe5fce776e5346f785215f3f8b3efc Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 4 May 2020 11:59:55 -0500 Subject: [PATCH 133/142] Use static_variant tag value instead of just hardcoding the value. --- plugins/net_plugin/net_plugin.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index e4a456339f3..a022c02fc7a 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -391,10 +391,10 @@ namespace eosio { constexpr auto def_sync_fetch_span = 100; constexpr auto message_header_size = 4; - constexpr uint32_t signed_block_v0_which = 7; // see protocol net_message - constexpr uint32_t packed_transaction_v0_which = 8; // see protocol net_message - constexpr uint32_t signed_block_which = 9; // see protocol net_message - constexpr uint32_t trx_message_v1_which = 10; // see protocol net_message + constexpr uint32_t signed_block_v0_which = net_message::tag::value; // see protocol net_message + constexpr uint32_t packed_transaction_v0_which = net_message::tag::value; // see protocol net_message + constexpr uint32_t signed_block_which = net_message::tag::value; // see protocol net_message + constexpr uint32_t trx_message_v1_which = net_message::tag::value; // see protocol net_message /** * For a while, network version was a 16 bit value equal to the second set of 16 bits From e067c048d48a617cff353a75822801193215a3ed Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 5 May 2020 14:02:50 -0500 Subject: [PATCH 134/142] remove unneeded comment --- tests/eosio_blocklog_prune_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/eosio_blocklog_prune_test.py b/tests/eosio_blocklog_prune_test.py index 4179dc0996e..495fbc8b278 100755 --- a/tests/eosio_blocklog_prune_test.py +++ b/tests/eosio_blocklog_prune_test.py @@ -118,8 +118,6 @@ def isBlockNumIrr(): msg=ex.output.decode("utf-8") assert "does not contain the following transactions: " + cfTrxId in msg, "The transaction id is not displayed in the console when it cannot be found" - # For Linux, the pruned result won't be immediately visible unless the node is restarted. For MacOS, the result is immediately visible even without restart. - isRelaunchSuccess = validationNode.relaunch(1) assert isRelaunchSuccess, "Fail to relaunch verification node" From af07e007cfa9c66e6760f95e52ab5eedebad1460 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 7 May 2020 08:40:05 -0500 Subject: [PATCH 135/142] remove deleted copy constructor used by trace_api_plugin --- libraries/chain/include/eosio/chain/transaction.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 7ab7df96707..dc8afdc677b 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -60,7 +60,7 @@ namespace eosio { namespace chain { transaction_header() = default; explicit transaction_header( const transaction_header& ) = default; transaction_header( transaction_header&& ) = default; - transaction_header& operator=(const transaction_header&) = delete; + // transaction_header& operator=(const transaction_header&) = delete; transaction_header& operator=(transaction_header&&) = default; time_point_sec expiration; ///< the time at which a transaction expires From 8c54b44c3dfd642e72ed03fab7ef918fcf3545c2 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 7 May 2020 09:21:36 -0500 Subject: [PATCH 136/142] Fix merge issue, copy code from prune-cfd-stage-1 branch --- .../chain/webassembly/cf_transaction.cpp | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/libraries/chain/webassembly/cf_transaction.cpp b/libraries/chain/webassembly/cf_transaction.cpp index eb49eb26f52..17c5952d88d 100644 --- a/libraries/chain/webassembly/cf_transaction.cpp +++ b/libraries/chain/webassembly/cf_transaction.cpp @@ -3,30 +3,39 @@ namespace eosio { namespace chain { namespace webassembly { int32_t interface::read_transaction( legacy_span data ) const { - bytes trx = context.get_packed_transaction(); + if( data.size() == 0 ) return transaction_size(); - auto s = trx.size(); - if (data.size() == 0) return s; + const packed_transaction& packed_trx = context.trx_context.packed_trx; + const bytes& trx = + packed_trx.get_compression() == packed_transaction::compression_type::none ? + packed_trx.get_packed_transaction() : + fc::raw::pack( static_cast( packed_trx.get_transaction() ) ); - auto copy_size = std::min( static_cast(data.size()), s ); + size_t copy_size = std::min( static_cast(data.size()), trx.size() ); std::memcpy( data.data(), trx.data(), copy_size ); return copy_size; } int32_t interface::transaction_size() const { - return context.get_packed_transaction().size(); + const packed_transaction& packed_trx = context.trx_context.packed_trx; + if( packed_trx.get_compression() == packed_transaction::compression_type::none) { + return packed_trx.get_packed_transaction().size(); + } else { + return fc::raw::pack_size( static_cast( packed_trx.get_transaction() ) ); + } } int32_t interface::expiration() const { - return context.trx_context.trx.expiration.sec_since_epoch(); + return context.trx_context.packed_trx.get_transaction().expiration.sec_since_epoch(); } int32_t interface::tapos_block_num() const { - return context.trx_context.trx.ref_block_num; + return context.trx_context.packed_trx.get_transaction().ref_block_num; } + int32_t interface::tapos_block_prefix() const { - return context.trx_context.trx.ref_block_prefix; + return context.trx_context.packed_trx.get_transaction().ref_block_prefix; } int32_t interface::get_action( uint32_t type, uint32_t index, legacy_span buffer ) const { From 8d4249ad0c663b664543a2761065f8785d6a3799 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 7 May 2020 10:10:51 -0500 Subject: [PATCH 137/142] Remove constructors so object can be created via initilization list --- libraries/chain/include/eosio/chain/transaction.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index dc8afdc677b..a8afcd59776 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -57,12 +57,6 @@ namespace eosio { namespace chain { * region. */ struct transaction_header { - transaction_header() = default; - explicit transaction_header( const transaction_header& ) = default; - transaction_header( transaction_header&& ) = default; - // transaction_header& operator=(const transaction_header&) = delete; - transaction_header& operator=(transaction_header&&) = default; - time_point_sec expiration; ///< the time at which a transaction expires uint16_t ref_block_num = 0U; ///< specifies a block num in the last 2^16 blocks. uint32_t ref_block_prefix = 0UL; ///< specifies the lower 32 bits of the blockid at get_ref_blocknum From 1a3520fc2815b6252272f74d8034952b81ed8755 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 7 May 2020 10:11:22 -0500 Subject: [PATCH 138/142] Change to better use new signaled packed_transaction_ptr --- .../include/eosio/trace_api/chain_extraction.hpp | 4 ++-- .../include/eosio/trace_api/extract_util.hpp | 5 +++-- .../trace_api_plugin/include/eosio/trace_api/trace.hpp | 4 +--- plugins/trace_api_plugin/test/test_extraction.cpp | 10 +++++----- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp index a66f8e42ae8..d1644b6aeeb 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/chain_extraction.hpp @@ -63,9 +63,9 @@ class chain_extraction_impl_type { if( is_onblock( trace )) { onblock_trace.emplace( trace ); } else if( trace->failed_dtrx_trace ) { - cached_traces[trace->failed_dtrx_trace->id] = {trace, static_cast(t), t.signatures}; + cached_traces[trace->failed_dtrx_trace->id] = {trace, t}; } else { - cached_traces[trace->id] = {trace, static_cast(t), t.signatures}; + cached_traces[trace->id] = {trace, t}; } } diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/extract_util.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/extract_util.hpp index 0e98af8b87a..d8e6b6a019b 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/extract_util.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/extract_util.hpp @@ -51,8 +51,9 @@ inline transaction_trace_v1 to_transaction_trace_v1( const cache_trace& t ) { r.cpu_usage_us = t.trace->receipt->cpu_usage_us; r.net_usage_words = t.trace->receipt->net_usage_words; } - r.signatures = t.trx_signatures; - r.trx_header = t.trx_header; + auto sigs = t.trx->get_signatures(); + if( sigs ) r.signatures = *sigs; + r.trx_header = static_cast( t.trx->get_transaction() ); r.actions.reserve( t.trace->action_traces.size()); for( const auto& at : t.trace->action_traces ) { if( !at.context_free ) { // not including CFA at this time diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/trace.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/trace.hpp index 3e0937fa626..cbef542e1ad 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/trace.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/trace.hpp @@ -54,8 +54,7 @@ namespace eosio { namespace trace_api { struct cache_trace { chain::transaction_trace_ptr trace; - chain::transaction_header trx_header; - std::vector trx_signatures; + chain::packed_transaction_ptr trx; }; } } @@ -66,4 +65,3 @@ FC_REFLECT(eosio::trace_api::transaction_trace_v0, (id)(actions)) FC_REFLECT_DERIVED(eosio::trace_api::transaction_trace_v1, (eosio::trace_api::transaction_trace_v0), (status)(cpu_usage_us)(net_usage_words)(signatures)(trx_header)) FC_REFLECT(eosio::trace_api::block_trace_v0, (id)(number)(previous_id)(timestamp)(producer)(transactions)) FC_REFLECT_DERIVED(eosio::trace_api::block_trace_v1, (eosio::trace_api::block_trace_v0), (transaction_mroot)(action_mroot)(schedule_version)(transactions_v1)) -FC_REFLECT(eosio::trace_api::cache_trace, (trace)(trx_header)(trx_signatures)) \ No newline at end of file diff --git a/plugins/trace_api_plugin/test/test_extraction.cpp b/plugins/trace_api_plugin/test/test_extraction.cpp index f9e8bd32b8e..63c078b0629 100644 --- a/plugins/trace_api_plugin/test/test_extraction.cpp +++ b/plugins/trace_api_plugin/test/test_extraction.cpp @@ -274,7 +274,7 @@ BOOST_AUTO_TEST_SUITE(block_extraction) fc::enum_type{bsp1->block->transactions[0].status}, bsp1->block->transactions[0].cpu_usage_us, bsp1->block->transactions[0].net_usage_words, - ptrx1.get_signatures(), + *ptrx1.get_signatures(), make_trx_header(ptrx1.get_transaction()) } } @@ -344,7 +344,7 @@ BOOST_AUTO_TEST_SUITE(block_extraction) fc::enum_type{bsp1->block->transactions[0].status}, bsp1->block->transactions[0].cpu_usage_us, bsp1->block->transactions[0].net_usage_words, - ptrx1.get_signatures(), + *ptrx1.get_signatures(), make_trx_header(ptrx1.get_transaction()) } , @@ -363,7 +363,7 @@ BOOST_AUTO_TEST_SUITE(block_extraction) fc::enum_type{bsp1->block->transactions[1].status}, bsp1->block->transactions[1].cpu_usage_us, bsp1->block->transactions[1].net_usage_words, - ptrx2.get_signatures(), + *ptrx2.get_signatures(), make_trx_header(ptrx2.get_transaction()) } , @@ -382,7 +382,7 @@ BOOST_AUTO_TEST_SUITE(block_extraction) fc::enum_type{bsp1->block->transactions[2].status}, bsp1->block->transactions[2].cpu_usage_us, bsp1->block->transactions[2].net_usage_words, - ptrx3.get_signatures(), + *ptrx3.get_signatures(), make_trx_header(ptrx3.get_transaction()) } } @@ -444,7 +444,7 @@ BOOST_AUTO_TEST_SUITE(block_extraction) fc::enum_type{bsp1->block->transactions[0].status}, bsp1->block->transactions[0].cpu_usage_us, bsp1->block->transactions[0].net_usage_words, - transfer_trx.get_signatures(), + *transfer_trx.get_signatures(), make_trx_header(transfer_trx.get_transaction()) } } From 509c8a3fce9998b86eab8f8a7845b59f3fdcf109 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 7 May 2020 10:30:56 -0500 Subject: [PATCH 139/142] Changes for new packed_transaction --- programs/eosio-tester/main.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/programs/eosio-tester/main.cpp b/programs/eosio-tester/main.cpp index f6f5b39bef6..b65bdbd5296 100644 --- a/programs/eosio-tester/main.cpp +++ b/programs/eosio-tester/main.cpp @@ -42,6 +42,7 @@ using eosio::chain::protocol_feature_exception; using eosio::chain::protocol_feature_set; using eosio::chain::signed_transaction; using eosio::chain::transaction_trace_ptr; +using eosio::chain::packed_transaction_ptr; using eosio::state_history::block_position; using eosio::state_history::create_deltas; using eosio::state_history::get_blocks_result_v0; @@ -112,17 +113,19 @@ struct transaction_checktime_factory { struct intrinsic_context { eosio::chain::controller& control; - eosio::chain::signed_transaction trx; + eosio::chain::packed_transaction trx; std::unique_ptr trx_ctx; std::unique_ptr apply_context; intrinsic_context(eosio::chain::controller& control) : control{ control } { static transaction_checktime_factory xxx_timer; - trx.actions.emplace_back(); - trx.actions.back().account = eosio::chain::name{ "eosio.null" }; - trx.actions.back().authorization.push_back({ eosio::chain::name{ "eosio" }, eosio::chain::name{ "active" } }); - trx_ctx = std::make_unique(control, trx, trx.id(), xxx_timer.get(), + eosio::chain::signed_transaction strx; + strx.actions.emplace_back(); + strx.actions.back().account = eosio::chain::name{ "eosio.null" }; + strx.actions.back().authorization.push_back({ eosio::chain::name{ "eosio" }, eosio::chain::name{ "active" } }); + trx = eosio::chain::packed_transaction( std::move(strx), true ); + trx_ctx = std::make_unique(control, trx, xxx_timer.get(), fc::time_point::now()); trx_ctx->init_for_implicit_trx(0); trx_ctx->exec(); @@ -257,7 +260,7 @@ struct test_chain { message.head = block_position{ control->head_block_num(), control->head_block_id() }; message.last_irreversible = block_position{ control->last_irreversible_block_num(), control->last_irreversible_block_id() }; - message.this_block = block_position{ block_state->block->block_num(), block_state->block->id() }; + message.this_block = block_position{ block_state->block->block_num(), block_state->id }; message.prev_block = prev_block; message.block = std::move(block_bin); message.traces = std::move(traces_bin); @@ -790,7 +793,7 @@ struct callbacks { chain.start_if_needed(); for (auto& key : args.keys) signed_trx.sign(key, chain.control->get_chain_id()); auto ptrx = std::make_shared( - signed_trx, eosio::chain::packed_transaction::compression_type::none); + std::move(signed_trx), true, eosio::chain::packed_transaction::compression_type::none); auto fut = eosio::chain::transaction_metadata::start_recover_keys( ptrx, chain.control->get_thread_pool(), chain.control->get_chain_id(), fc::microseconds::maximum()); auto start_time = std::chrono::steady_clock::now(); @@ -919,7 +922,7 @@ struct callbacks { eosio::chain::signed_transaction signed_trx{ std::move(transaction), std::move(args.signatures), std::move(args.context_free_data) }; for (auto& key : args.keys) signed_trx.sign(key, chain.control->get_chain_id()); - eosio::chain::packed_transaction ptrx{ signed_trx, eosio::chain::packed_transaction::compression_type::none }; + eosio::chain::packed_transaction ptrx{ std::move(signed_trx), true, eosio::chain::packed_transaction::compression_type::none }; auto data = fc::raw::pack(ptrx); auto start_time = std::chrono::steady_clock::now(); auto result = r.query_handler->query_transaction(*r.write_snapshot, data.data(), data.size()); From d438f91ec204d46d2d5db530721e79a8b610e881 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Thu, 7 May 2020 11:09:29 -0500 Subject: [PATCH 140/142] fix tester merge issue --- programs/eosio-tester/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/eosio-tester/main.cpp b/programs/eosio-tester/main.cpp index b65bdbd5296..d94ab507576 100644 --- a/programs/eosio-tester/main.cpp +++ b/programs/eosio-tester/main.cpp @@ -253,7 +253,7 @@ struct test_chain { void on_accepted_block(const block_state_ptr& block_state) { auto block_bin = fc::raw::pack(*block_state->block); - auto traces_bin = trace_converter.pack(control->db(), false, block_state); + auto traces_bin = trace_converter.pack(control->db(), false, block_state, 1); // hard code version for now, will be changed in later commit auto deltas_bin = fc::raw::pack(create_deltas(control->db(), !prev_block)); get_blocks_result_v0 message; From 6de603529d962193eb64b3b47b2bfee9fbe185b4 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 7 May 2020 14:17:32 -0500 Subject: [PATCH 141/142] Fix merge conflict --- pipeline.jsonc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline.jsonc b/pipeline.jsonc index 3d1b3cdb52f..c6df402c67b 100644 --- a/pipeline.jsonc +++ b/pipeline.jsonc @@ -40,7 +40,7 @@ "test": [ { - "commit": "833ed8276ca5576394eebe875adaacf6e40f5720" + "commit": "90c0e870e55d25853fc49a08be2cd683fbc4fa31" } ] } From 7f525b23b5d85c68d24e7f421b780a0da1dacbe8 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 7 May 2020 16:33:28 -0500 Subject: [PATCH 142/142] Update to new state which is combination of develop (wasm constraint changes) and prune-cfd-stage-1 --- pipeline.jsonc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline.jsonc b/pipeline.jsonc index c6df402c67b..486d52b20b8 100644 --- a/pipeline.jsonc +++ b/pipeline.jsonc @@ -40,7 +40,7 @@ "test": [ { - "commit": "90c0e870e55d25853fc49a08be2cd683fbc4fa31" + "commit": "6de603529d962193eb64b3b47b2bfee9fbe185b4" } ] }