Merge pull request #4 from eosnetworkfoundation/backport-chainbase

backport 2.1 chainbase

libraries/chain/controller.cpp

@@ -302,10 +302,10 @@ struct controller_impl {
     self(s),
     db( cfg.state_dir,
         cfg.read_only ? database::read_only : database::read_write,
-        cfg.state_size, false, cfg.db_map_mode, cfg.db_hugepage_paths ),
+        cfg.state_size, false, cfg.db_map_mode ),
     reversible_blocks( cfg.blocks_dir/config::reversible_blocks_dir_name,
         cfg.read_only ? database::read_only : database::read_write,
-        cfg.reversible_cache_size, false, cfg.db_map_mode, cfg.db_hugepage_paths ),
+        cfg.reversible_cache_size, false, cfg.db_map_mode ),
     blog( cfg.blocks_dir ),
     fork_db( cfg.state_dir ),
     wasmif( cfg.wasm_runtime, cfg.eosvmoc_tierup, db, cfg.state_dir, cfg.eosvmoc_config ),
@@ -1005,8 +1005,7 @@ struct controller_impl {
          if( name == config::system_account_name ) {
             // The initial eosio ABI value affects consensus; see  https://github.com/EOSIO/eos/issues/7794
             // TODO: This doesn't charge RAM; a fix requires a consensus upgrade.
-            a.abi.resize(sizeof(eosio_abi_bin));
-            memcpy(a.abi.data(), eosio_abi_bin, sizeof(eosio_abi_bin));
+            a.abi.assign(eosio_abi_bin, sizeof(eosio_abi_bin));
          }
       });
       db.create<account_metadata_object>([&](auto & a) {

libraries/chain/eosio_contract.cpp

@@ -209,11 +209,7 @@ void apply_eosio_setabi(apply_context& context) {
    int64_t new_size = abi_size;
 
    db.modify( account, [&]( auto& a ) {
-      if (abi_size > 0) {
-         a.abi.assign(act.abi.data(), abi_size);
-      } else {
-         a.abi.resize(0);
-      }
+      a.abi.assign(act.abi.data(), abi_size);
    });
 
    const auto& account_metadata = db.get<account_metadata_object, by_name>(act.account);

libraries/chain/include/eosio/chain/account_object.hpp

@@ -18,9 +18,10 @@ namespace eosio { namespace chain {
       shared_blob          abi;
 
       void set_abi( const eosio::chain::abi_def& a ) {
-         abi.resize( fc::raw::pack_size( a ) );
-         fc::datastream<char*> ds( abi.data(), abi.size() );
-         fc::raw::pack( ds, a );
+         abi.resize_and_fill( fc::raw::pack_size( a ), [&a](char* data, std::size_t size) {
+            fc::datastream<char*> ds( data, size );
+            fc::raw::pack( ds, a );
+         });
       }
 
       eosio::chain::abi_def get_abi()const {
@@ -103,7 +104,6 @@ CHAINBASE_SET_INDEX_TYPE(eosio::chain::account_object, eosio::chain::account_ind
 CHAINBASE_SET_INDEX_TYPE(eosio::chain::account_metadata_object, eosio::chain::account_metadata_index)
 CHAINBASE_SET_INDEX_TYPE(eosio::chain::account_ram_correction_object, eosio::chain::account_ram_correction_index)
 
-
 FC_REFLECT(eosio::chain::account_object, (name)(creation_date)(abi))
 FC_REFLECT(eosio::chain::account_metadata_object, (name)(recv_sequence)(auth_sequence)(code_sequence)(abi_sequence)
                                                   (code_hash)(last_code_update)(flags)(vm_type)(vm_version))

libraries/chain/include/eosio/chain/authority.hpp

@@ -111,9 +111,11 @@ struct shared_key_weight {
          },
          [&](const fc::crypto::webauthn::public_key& wa) {
             size_t psz = fc::raw::pack_size(wa);
-            shared_string wa_ss(psz, boost::container::default_init, std::move(allocator));
-            fc::datastream<char*> ds(wa_ss.data(), wa_ss.size());
-            fc::raw::pack(ds, wa);
+            shared_string wa_ss(std::move(allocator));
+            wa_ss.resize_and_fill( psz, [&wa]( char* data, std::size_t sz ) {
+               fc::datastream<char*> ds(data, sz);
+               fc::raw::pack(ds, wa);
+            });
 
             return shared_key_weight(std::move(wa_ss), k.weight);
          }
@@ -306,4 +308,4 @@ FC_REFLECT(eosio::chain::wait_weight, (wait_sec)(weight) )
 FC_REFLECT(eosio::chain::authority, (threshold)(keys)(accounts)(waits) )
 FC_REFLECT(eosio::chain::shared_key_weight, (key)(weight) )
 FC_REFLECT(eosio::chain::shared_authority, (threshold)(keys)(accounts)(waits) )
-FC_REFLECT(eosio::chain::shared_public_key, (pubkey))
+FC_REFLECT(eosio::chain::shared_public_key, (pubkey))

libraries/chain/include/eosio/chain/controller.hpp

@@ -92,7 +92,6 @@ namespace eosio { namespace chain {
             validation_mode          block_validation_mode  = validation_mode::FULL;
 
             pinnable_mapped_file::map_mode db_map_mode      = pinnable_mapped_file::map_mode::mapped;
-            vector<string>           db_hugepage_paths;
 
             flat_set<account_name>   resource_greylist;
             flat_set<account_name>   trusted_producers;

libraries/chain/include/eosio/chain/generated_transaction_object.hpp

@@ -35,9 +35,10 @@ namespace eosio { namespace chain {
 
          uint32_t set( const transaction& trx ) {
             auto trxsize = fc::raw::pack_size( trx );
-            packed_trx.resize( trxsize );
-            fc::datastream<char*> ds( packed_trx.data(), trxsize );
-            fc::raw::pack( ds, trx );
+            packed_trx.resize_and_fill( trxsize, [&trx](char* data, std::size_t size) {
+               fc::datastream<char*> ds( data, size );
+               fc::raw::pack( ds, trx );
+            });
             return trxsize;
          }
    };

libraries/chain/include/eosio/chain/reversible_block_object.hpp

@@ -16,9 +16,10 @@ namespace eosio { namespace chain {
       shared_string  packedblock;
 
       void set_block( const signed_block_ptr& b ) {
-         packedblock.resize( fc::raw::pack_size( *b ) );
-         fc::datastream<char*> ds( packedblock.data(), packedblock.size() );
-         fc::raw::pack( ds, *b );
+         packedblock.resize_and_fill( fc::raw::pack_size( *b ), [&b](char* data, std::size_t size) {
+            fc::datastream<char*> ds( data, size );
+            fc::raw::pack( ds, *b );
+         });
       }
 
       signed_block_ptr get_block()const {

libraries/chain/include/eosio/chain/types.hpp

@@ -85,7 +85,7 @@ namespace eosio { namespace chain {
    struct void_t{};
 
    using chainbase::allocator;
-   using shared_string = boost::interprocess::basic_string<char, std::char_traits<char>, allocator<char>>;
+   using shared_string = chainbase::shared_string;
    template<typename T>
    using shared_vector = boost::interprocess::vector<T, allocator<T>>;
    template<typename T>
@@ -103,17 +103,10 @@ namespace eosio { namespace chain {
          shared_blob() = delete;
          shared_blob(shared_blob&&) = default;
 
-         shared_blob(const shared_blob& s)
-         :shared_string(s.get_allocator())
-         {
-            assign(s.c_str(), s.size());
-         }
+         shared_blob(const shared_blob& s) = default;
 
 
-         shared_blob& operator=(const shared_blob& s) {
-            assign(s.c_str(), s.size());
-            return *this;
-         }
+         shared_blob& operator=(const shared_blob& s) = default;
 
          shared_blob& operator=(shared_blob&& ) = default;
 
@@ -393,4 +386,22 @@ namespace eosio { namespace chain {
 
 } }  // eosio::chain
 
+namespace chainbase {
+   // chainbase::shared_cow_string
+   template<typename DataStream> inline DataStream& operator<<( DataStream& s, const chainbase::shared_cow_string& v )  {
+      FC_ASSERT( v.size() <= MAX_SIZE_OF_BYTE_ARRAYS );
+      fc::raw::pack( s, fc::unsigned_int((uint32_t)v.size()));
+      if( v.size() ) s.write( v.data(), v.size() );
+      return s;
+   }
+
+   template<typename DataStream> inline DataStream& operator>>( DataStream& s, chainbase::shared_cow_string& v )  {
+      fc::unsigned_int size; fc::raw::unpack( s, size );
+      FC_ASSERT( size.value <= MAX_SIZE_OF_BYTE_ARRAYS );
+      FC_ASSERT( v.size() == 0 );
+      v.resize_and_fill(size.value, [&s](char* buf, std::size_t sz) { s.read(buf, sz); });
+      return s;
+   }
+}
+
 FC_REFLECT_EMPTY( eosio::chain::void_t )

libraries/state_history/create_deltas.cpp

@@ -59,8 +59,8 @@ std::vector<table_delta> create_deltas(const chainbase::database& db, bool full_
    std::vector<table_delta>                          deltas;
    const auto&                                       table_id_index = db.get_index<chain::table_id_multi_index>();
    std::map<uint64_t, const chain::table_id_object*> removed_table_id;
-   for (auto& rem : table_id_index.stack().back().removed_values)
-      removed_table_id[rem.first._id] = &rem.second;
+   for (auto& rem : table_id_index.last_undo_session().removed_values)
+      removed_table_id[rem.id._id] = &rem;
 
    auto get_table_id = [&](uint64_t tid) -> const chain::table_id_object& {
       auto obj = table_id_index.find(tid);
@@ -86,23 +86,20 @@ std::vector<table_delta> create_deltas(const chainbase::database& db, bool full_
          for (auto& row : index.indices())
             delta.rows.obj.emplace_back(true, pack_row(row));
       } else {
-         if (index.stack().empty())
-            return;
-         auto& undo = index.stack().back();
-         if (undo.old_values.empty() && undo.new_ids.empty() && undo.removed_values.empty())
+         auto undo = index.last_undo_session();
+         if (undo.old_values.empty() && undo.new_values.empty() && undo.removed_values.empty())
             return;
          deltas.push_back({});
          auto& delta = deltas.back();
          delta.name  = name;
          for (auto& old : undo.old_values) {
-            auto& row = index.get(old.first);
-            if (include_delta(old.second, row))
+            auto& row = index.get(old.id);
+            if (include_delta(old, row))
                delta.rows.obj.emplace_back(true, pack_row(row));
          }
          for (auto& old : undo.removed_values)
-            delta.rows.obj.emplace_back(false, pack_row(old.second));
-         for (auto id : undo.new_ids) {
-            auto& row = index.get(id);
+            delta.rows.obj.emplace_back(false, pack_row(old));
+         for (auto& row : undo.new_values) {
             delta.rows.obj.emplace_back(true, pack_row(row));
          }
       }

libraries/state_history/include/eosio/state_history/serialization.hpp

@@ -401,11 +401,12 @@ datastream<ST>& operator<<(datastream<ST>& ds, const history_serial_wrapper<eosi
       auto&       index  = obj.db.get_index<eosio::chain::permission_index>();
       const auto* parent = index.find(obj.obj.parent);
       if (!parent) {
-         auto& undo = index.stack().back();
-         auto  it   = undo.removed_values.find(obj.obj.parent);
+         auto undo = index.last_undo_session();
+         auto it   = std::find_if(undo.removed_values.begin(), undo.removed_values.end(),
+                                [&](auto& x) { return x.id._id == obj.obj.parent; });
          EOS_ASSERT(it != undo.removed_values.end(), eosio::chain::plugin_exception,
                     "can not find parent of permission_object");
-         parent = &it->second;
+         parent = &*it;
       }
       fc::raw::pack(ds, as_type<uint64_t>(parent->name.to_uint64_t()));
    } else {

plugins/chain_plugin/chain_plugin.cpp

@@ -287,11 +287,9 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
          ("database-map-mode", bpo::value<chainbase::pinnable_mapped_file::map_mode>()->default_value(chainbase::pinnable_mapped_file::map_mode::mapped),
           "Database map mode (\"mapped\", \"heap\", or \"locked\").\n"
           "In \"mapped\" mode database is memory mapped as a file.\n"
-          "In \"heap\" mode database is preloaded in to swappable memory.\n"
-#ifdef __linux__
-          "In \"locked\" mode database is preloaded, locked in to memory, and optionally can use huge pages.\n"
-#else
-          "In \"locked\" mode database is preloaded and locked in to memory.\n"
+#ifndef _WIN32
+          "In \"heap\" mode database is preloaded in to swappable memory and will use huge pages if available.\n"
+          "In \"locked\" mode database is preloaded, locked in to memory, and will use huge pages if available.\n"
 #endif
          )
 #ifdef __linux__
@@ -1052,10 +1050,6 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
       }
 
       my->chain_config->db_map_mode = options.at("database-map-mode").as<pinnable_mapped_file::map_mode>();
-#ifdef __linux__
-      if( options.count("database-hugepage-path") )
-         my->chain_config->db_hugepage_paths = options.at("database-hugepage-path").as<std::vector<std::string>>();
-#endif
 
 #ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED
       if( options.count("eos-vm-oc-cache-size-mb") )

plugins/history_plugin/history_plugin.cpp

@@ -258,9 +258,10 @@ namespace eosio {
 
                db.create<action_history_object>( [&]( auto& aho ) {
                   auto ps = fc::raw::pack_size( at );
-                  aho.packed_action_trace.resize(ps);
-                  datastream<char*> ds( aho.packed_action_trace.data(), ps );
-                  fc::raw::pack( ds, at );
+                  aho.packed_action_trace.resize_and_fill(ps, [&at](char* data, std::size_t size) {
+                     fc::datastream<char*> ds( data, size );
+                     fc::raw::pack( ds, at );
+                  });
                   aho.action_sequence_num = at.receipt->global_sequence;
                   aho.block_num = chain.head_block_num() + 1;
                   aho.block_time = chain.pending_block_time();

programs/eosio-blocklog/main.cpp

@@ -93,7 +93,7 @@ void blocklog::read_log() {
          reversible_blocks.reset();
       }
    } catch( const std::runtime_error& e ) {
-      if( std::string(e.what()).find("database dirty flag set") != std::string::npos ) {
+      if( std::string(e.what()).find("atabase dirty flag set") != std::string::npos ) {
          elog( "database dirty flag set (likely due to unclean shutdown): only block_log blocks are available" );
       } else {
          throw;

programs/nodeos/main.cpp

@@ -117,7 +117,7 @@ int main(int argc, char** argv)
       return NODE_MANAGEMENT_SUCCESS;
    } catch( const fc::exception& e ) {
       if( e.code() == fc::std_exception_code ) {
-         if( e.top_message().find( "database dirty flag set" ) != std::string::npos ) {
+         if( e.top_message().find( "atabase dirty flag set" ) != std::string::npos ) {
             elog( "database dirty flag set (likely due to unclean shutdown): replay required" );
             return DATABASE_DIRTY;
          }
@@ -131,7 +131,7 @@ int main(int argc, char** argv)
       elog("${e}", ("e",boost::diagnostic_information(e)));
       return OTHER_FAIL;
    } catch( const std::runtime_error& e ) {
-      if( std::string(e.what()).find("database dirty flag set") != std::string::npos ) {
+      if( std::string(e.what()).find("atabase dirty flag set") != std::string::npos ) {
          elog( "database dirty flag set (likely due to unclean shutdown): replay required" );
          return DATABASE_DIRTY;
       } else {

tests/Cluster.py

@@ -576,20 +576,14 @@ def doNodesHaveBlockNum(nodes, targetBlockNum, blockType, printCount):
     @staticmethod
     def getClientVersion(verbose=False):
         """Returns client version (string)"""
-        p = re.compile(r'^Build version:\s(\w+)\n$')
         try:
             cmd="%s version client" % (Utils.EosClientPath)
             if verbose: Utils.Print("cmd: %s" % (cmd))
             response=Utils.checkOutput(cmd.split())
             assert(response)
             assert(isinstance(response, str))
             if verbose: Utils.Print("response: <%s>" % (response))
-            m=p.match(response)
-            if m is None:
-                Utils.Print("ERROR: client version regex mismatch")
-                return None
-
-            verStr=m.group(1)
+            verStr=response.strip()
             return verStr
         except subprocess.CalledProcessError as ex:
             msg=ex.output.decode("utf-8")

tests/validate-dirty-db.py

@@ -102,8 +102,8 @@ def runNodeosAndGetOutput(myTimeout=3):
         retCode=ret[1]["returncode"]
         expectedRetCode=2
         if retCode != expectedRetCode:
-            errorExit("Expected return code to be %d, but instead received %d." % (expectedRetCode, retCode))
-        db_dirty_msg="database dirty flag set"
+            errorExit("Expected return code to be %d, but instead received %d. output={\n%s\n}" % (expectedRetCode, retCode, ret))
+        db_dirty_msg="atabase dirty flag set"
         if db_dirty_msg not in stderr:
             errorExit("stderr should have contained \"%s\" but it did not. stderr=\n%s" % (db_dirty_msg, stderr))