CFD: Initial support for pruned_block

libraries/chain/block.cpp

@@ -20,7 +20,19 @@ namespace eosio { namespace chain {
       }
    }
 
-   flat_multimap<uint16_t, block_extension> signed_block::validate_and_extract_extensions()const {
+   static fc::static_variant<transaction_id_type, pruned_transaction> translate_transaction_receipt(const transaction_id_type& tid, bool) {
+      return tid;
+   }
+   static fc::static_variant<transaction_id_type, pruned_transaction> translate_transaction_receipt(const packed_transaction& ptrx, bool legacy) {
+      return pruned_transaction(ptrx, legacy);
+   }
+
+   pruned_transaction_receipt::pruned_transaction_receipt(const transaction_receipt& other, bool legacy)
+     : transaction_receipt_header(static_cast<const transaction_receipt_header&>(other)),
+       trx(other.trx.visit([&](const auto& obj) { return translate_transaction_receipt(obj, legacy); }))
+   {}
+
+   static flat_multimap<uint16_t, block_extension> validate_and_extract_block_extensions(const extensions_type& block_extensions) {
       using decompose_t = block_extension_types::decompose_t;
 
       flat_multimap<uint16_t, block_extension> results;
@@ -61,4 +73,42 @@ namespace eosio { namespace chain {
 
    }
 
-} } /// namespace eosio::chain
+   flat_multimap<uint16_t, block_extension> signed_block::validate_and_extract_extensions()const {
+      return validate_and_extract_block_extensions( block_extensions );
+   }
+
+   pruned_block::pruned_block( const signed_block& other, bool legacy )
+     : signed_block_header(static_cast<const signed_block_header&>(other)),
+       prune_state(legacy ? prune_state_type::complete_legacy : prune_state_type::complete),
+       block_extensions(other.block_extensions)
+   {
+      for(const auto& trx : other.transactions) {
+         transactions.emplace_back(trx, legacy);
+      }
+   }
+
+   static std::size_t pruned_trx_receipt_packed_size(const pruned_transaction& obj, pruned_transaction::cf_compression_type segment_compression) {
+      return obj.maximum_pruned_pack_size(segment_compression);
+   }
+   static std::size_t pruned_trx_receipt_packed_size(const transaction_id_type& obj, pruned_transaction::cf_compression_type) {
+      return fc::raw::pack_size(obj);
+   }
+
+   std::size_t pruned_transaction_receipt::maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const {
+      return fc::raw::pack_size(*static_cast<const transaction_receipt_header*>(this)) + 1 +
+         trx.visit([&](const auto& obj){ return pruned_trx_receipt_packed_size(obj, segment_compression); });
+   }
+
+   std::size_t pruned_block::maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const {
+      std::size_t result = fc::raw::pack_size(fc::unsigned_int(transactions.size()));
+      for(const pruned_transaction_receipt& r: transactions) {
+         result += r.maximum_pruned_pack_size( segment_compression );
+      }
+      return fc::raw::pack_size(*static_cast<const signed_block_header*>(this)) + fc::raw::pack_size(prune_state) + result + fc::raw::pack_size(block_extensions);
+   }
+
+   flat_multimap<uint16_t, block_extension> pruned_block::validate_and_extract_extensions()const {
+      return validate_and_extract_block_extensions( block_extensions );
+   }
+
+} } /// namespace eosio::chain

libraries/chain/include/eosio/chain/block.hpp

@@ -103,6 +103,67 @@ namespace eosio { namespace chain {
    };
    using signed_block_ptr = std::shared_ptr<signed_block>;
 
+   struct pruned_transaction_receipt : public transaction_receipt_header {
+
+      pruned_transaction_receipt():transaction_receipt_header(){}
+      pruned_transaction_receipt(const transaction_receipt&, bool legacy);
+      explicit pruned_transaction_receipt( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){}
+      explicit pruned_transaction_receipt( const pruned_transaction& ptrx ):transaction_receipt_header(executed),trx(ptrx){}
+
+      fc::static_variant<transaction_id_type, pruned_transaction> trx;
+
+      std::size_t maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const;
+
+      digest_type digest()const {
+         digest_type::encoder enc;
+         fc::raw::pack( enc, status );
+         fc::raw::pack( enc, cpu_usage_us );
+         fc::raw::pack( enc, net_usage_words );
+         if( trx.contains<transaction_id_type>() )
+            fc::raw::pack( enc, trx.get<transaction_id_type>() );
+         else
+            fc::raw::pack( enc, trx.get<pruned_transaction>().packed_digest() );
+         return enc.result();
+      }
+   };
+
+   struct pruned_block : public signed_block_header{
+   private:
+      pruned_block( const pruned_block& ) = default;
+   public:
+      enum class prune_state_type : uint8_t { incomplete, complete, complete_legacy };
+
+      pruned_block() = default;
+      explicit pruned_block( const signed_block_header& h ):signed_block_header(h){}
+      pruned_block( const signed_block&, bool legacy );
+      pruned_block( pruned_block&& ) = default;
+      pruned_block& operator=(const pruned_block&) = delete;
+      pruned_block clone() const { return *this; }
+
+      fc::enum_type<uint8_t,prune_state_type> prune_state{prune_state_type::complete_legacy};
+      deque<pruned_transaction_receipt>       transactions; /// new or generated transactions
+      extensions_type                         block_extensions;
+
+      std::size_t maximum_pruned_pack_size( pruned_transaction::cf_compression_type segment_compression ) const;
+
+      // Returns the maximum_pruned_padded_size.  It is the caller's responsibility to
+      // reserve enough space after the end if in-place pruning is desired.
+      template<typename Stream>
+      std::size_t pack(Stream& stream, pruned_transaction::cf_compression_type segment_compression) {
+         std::size_t padded_size = maximum_pruned_pack_size( segment_compression );
+         // TODO: This only handles legacy transactions.
+         fc::raw::pack(stream, *this);
+         return padded_size;
+      }
+      template<typename Stream>
+      void unpack(Stream& stream, pruned_transaction::cf_compression_type segment_compression) {
+         fc::raw::unpack(stream, *this);
+      }
+
+      flat_multimap<uint16_t, block_extension> validate_and_extract_extensions()const;
+   };
+   using pruned_block_ptr = std::shared_ptr<pruned_block>;
+
    struct producer_confirmation {
       block_id_type   block_id;
       digest_type     block_digest;
@@ -119,3 +180,6 @@ FC_REFLECT(eosio::chain::transaction_receipt_header, (status)(cpu_usage_us)(net_
 FC_REFLECT_DERIVED(eosio::chain::transaction_receipt, (eosio::chain::transaction_receipt_header), (trx) )
 FC_REFLECT(eosio::chain::additional_block_signatures_extension, (signatures));
 FC_REFLECT_DERIVED(eosio::chain::signed_block, (eosio::chain::signed_block_header), (transactions)(block_extensions) )
+
+FC_REFLECT_DERIVED(eosio::chain::pruned_transaction_receipt, (eosio::chain::transaction_receipt_header), (trx) )
+FC_REFLECT_DERIVED(eosio::chain::pruned_block, (eosio::chain::signed_block_header), (prune_state)(transactions)(block_extensions) )

libraries/chain/include/eosio/chain/exceptions.hpp

@@ -236,6 +236,8 @@ namespace eosio { namespace chain {
                                     3030012, "Invalid block extension" )
       FC_DECLARE_DERIVED_EXCEPTION( ill_formed_additional_block_signatures_extension, block_validate_exception,
                                     3030013, "Block includes an ill-formed additional block signature extension" )
+      FC_DECLARE_DERIVED_EXCEPTION( block_padding_exception, block_validate_exception,
+                                    3030014, "Invalid block padding" )
 
 
    FC_DECLARE_DERIVED_EXCEPTION( transaction_exception,             chain_exception,
@@ -277,6 +279,8 @@ namespace eosio { namespace chain {
                                     3040017, "Transaction includes disallowed extensions (invalid block)" )
       FC_DECLARE_DERIVED_EXCEPTION( tx_resource_exhaustion, transaction_exception,
                                     3040018, "Transaction exceeded transient resource limit" )
+      FC_DECLARE_DERIVED_EXCEPTION( tx_prune_exception, transaction_exception,
+                                    3040019, "Prunable data not found" )
 
 
    FC_DECLARE_DERIVED_EXCEPTION( action_validate_exception, chain_exception,

libraries/chain/include/eosio/chain/transaction.hpp

@@ -203,6 +203,115 @@ namespace eosio { namespace chain {
       transaction_id_type                     trx_id;
    };
 
+   struct prunable_transaction_data {
+      enum class compression_type {
+         none = 0,
+         zlib = 1,
+      };
+
+      struct none {
+         digest_type                     prunable_digest;
+      };
+
+      struct signatures_only {
+         std::vector<signature_type>     signatures;
+         digest_type                     context_free_mroot;
+      };
+
+      using segment_type = fc::static_variant<digest_type, std::vector<char>>;
+
+      struct partial {
+         std::vector<signature_type>     signatures;
+         std::vector<segment_type>       context_free_segments;
+      };
+
+      struct full {
+         std::vector<signature_type>     signatures;
+         std::vector<std::vector<char>>  context_free_segments;
+      };
+
+      struct full_legacy {
+         std::vector<signature_type>     signatures;
+         std::vector<char>               packed_context_free_data;
+      };
+
+      using prunable_data_type = fc::static_variant< full_legacy,
+                                                     none,
+                                                     signatures_only,
+                                                     partial,
+                                                     full >;
+
+      prunable_transaction_data prune_all() const;
+      digest_type digest() const;
+
+      // Returns the maximum pack size of any prunable_transaction_data that is reachable
+      // by pruning this object.
+      std::size_t maximum_pruned_pack_size( compression_type segment_compression ) const;
+
+      prunable_data_type  prunable_data;
+   };
+
+   struct pruned_transaction : fc::reflect_init {
+      using compression_type = packed_transaction::compression_type;
+      using cf_compression_type = prunable_transaction_data::compression_type;
+
+      pruned_transaction() = default;
+      pruned_transaction(pruned_transaction&&) = default;
+      explicit pruned_transaction(const pruned_transaction&) = default;
+      pruned_transaction& operator=(const pruned_transaction&) = delete;
+      pruned_transaction& operator=(pruned_transaction&&) = default;
+
+      pruned_transaction(const packed_transaction& other, bool legacy) : pruned_transaction(other.get_signed_transaction(), legacy, other.get_compression()) {}
+      explicit pruned_transaction(const signed_transaction& t, bool legacy, compression_type _compression = compression_type::none);
+      explicit pruned_transaction(signed_transaction&& t, bool legacy, compression_type _compression = compression_type::none);
+
+#if 0
+      // used by abi_serializer
+      packed_transaction( bytes&& packed_txn, vector<signature_type>&& sigs, bytes&& packed_cfd, compression_type _compression );
+      packed_transaction( bytes&& packed_txn, vector<signature_type>&& sigs, vector<bytes>&& cfd, compression_type _compression );
+      packed_transaction( transaction&& t, vector<signature_type>&& sigs, bytes&& packed_cfd, compression_type _compression );
+#endif
+
+      uint32_t get_unprunable_size()const;
+      uint32_t get_prunable_size()const;
+
+      digest_type packed_digest()const;
+
+      const transaction_id_type& id()const { return trx_id; }
+
+      time_point_sec                expiration()const { return unpacked_trx.expiration; }
+      const transaction&            get_transaction()const { return unpacked_trx; }
+      const signed_transaction&     get_signed_transaction()const { return unpacked_trx; }
+      // Returns nullptr if the signatures were pruned
+      const vector<signature_type>* get_signatures()const;
+      // Returns nullptr if any context_free_data segment was pruned
+      const vector<bytes>*          get_context_free_data()const;
+      // Returns nullptr if the context_free_data segment was pruned or segment_ordinal is out of range.
+      const bytes*                  get_context_free_data(std::size_t segment_ordinal);
+      const fc::enum_type<uint8_t,compression_type>& get_compression()const { return compression; }
+      const bytes&                  get_packed_transaction()const { return packed_trx; }
+
+      void prune_all();
+
+      std::size_t maximum_pruned_pack_size( cf_compression_type segment_compression ) const;
+
+   private:
+
+      friend struct fc::reflector<pruned_transaction>;
+      friend struct fc::reflector_init_visitor<pruned_transaction>;
+      friend struct fc::has_reflector_init<pruned_transaction>;
+      void reflector_init();
+   private:
+      fc::enum_type<uint8_t,compression_type> compression;
+      prunable_transaction_data               prunable_data;
+      bytes                                   packed_trx;
+
+   private:
+      // cache unpacked trx, for thread safety do not modify after construction
+      signed_transaction                      unpacked_trx;
+      transaction_id_type                     trx_id;
+   };
+
    using packed_transaction_ptr = std::shared_ptr<packed_transaction>;
 
    uint128_t transaction_id_to_sender_id( const transaction_id_type& tid );
@@ -217,3 +326,10 @@ FC_REFLECT_DERIVED( eosio::chain::signed_transaction, (eosio::chain::transaction
 FC_REFLECT_ENUM( eosio::chain::packed_transaction::compression_type, (none)(zlib))
 // @ignore unpacked_trx
 FC_REFLECT( eosio::chain::packed_transaction, (signatures)(compression)(packed_context_free_data)(packed_trx) )
+FC_REFLECT( eosio::chain::pruned_transaction, (compression)(prunable_data)(packed_trx) )
+FC_REFLECT( eosio::chain::prunable_transaction_data, (prunable_data));
+FC_REFLECT( eosio::chain::prunable_transaction_data::none, (prunable_digest))
+FC_REFLECT( eosio::chain::prunable_transaction_data::signatures_only, (signatures)(context_free_mroot))
+FC_REFLECT( eosio::chain::prunable_transaction_data::partial, (signatures)(context_free_segments))
+FC_REFLECT( eosio::chain::prunable_transaction_data::full, (signatures)(context_free_segments))
+FC_REFLECT( eosio::chain::prunable_transaction_data::full_legacy, (signatures)(packed_context_free_data))