backport 29233b8 ship

.gitignore

@@ -87,3 +87,6 @@ var/lib/node_*
 .DS_Store
 
 !*.swagger.*
+
+node_modules
+package-lock.json

README.md

@@ -30,6 +30,7 @@ apt-get update && apt-get install   \
         libusb-1.0-0-dev            \
         libzstd-dev                 \
         llvm-11-dev                 \
+        npm                         \
         ninja-build                 \
         pkg-config                  \
         time
@@ -66,6 +67,7 @@ I highly recommend the ccache options. They don't speed up the first clean build
 
 ```
 cd build
+npm install
 
 # Runs parallelizable tests in parallel. This runs much faster when
 # -DDISABLE_WASM_SPEC_TESTS=yes is used.

libraries/CMakeLists.txt

@@ -7,6 +7,7 @@ add_subdirectory( appbase )
 add_subdirectory( chain )
 add_subdirectory( testing )
 add_subdirectory( version )
+add_subdirectory( state_history )
 
 set(USE_EXISTING_SOFTFLOAT ON CACHE BOOL "use pre-exisiting softfloat lib")
 set(ENABLE_TOOLS OFF CACHE BOOL "Build tools")

libraries/state_history/.clang-format

@@ -0,0 +1,8 @@
+BasedOnStyle: LLVM
+IndentWidth: 3
+ColumnLimit: 120
+PointerAlignment: Left
+AlwaysBreakTemplateDeclarations: true
+AlignConsecutiveAssignments: true
+AlignConsecutiveDeclarations: true
+BreakConstructorInitializers: BeforeComma

libraries/state_history/CMakeLists.txt

@@ -0,0 +1,17 @@
+file(GLOB HEADERS "include/eosio/state-history/*.hpp")
+
+add_library( state_history
+             abi.cpp
+             compression.cpp
+             create_deltas.cpp
+             trace_converter.cpp
+             ${HEADERS}
+           )
+
+target_link_libraries( state_history 
+                       PUBLIC eosio_chain fc chainbase softfloat
+                     )
+
+target_include_directories( state_history
+                            PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../wasm-jit/Include"
+                          )

plugins/state_history_plugin/state_history_plugin_abi.cpp → libraries/state_history/abi.cpp

@@ -17,7 +17,8 @@ extern const char* const state_history_plugin_abi = R"({
                 { "name": "trace_begin_block", "type": "uint32" },
                 { "name": "trace_end_block", "type": "uint32" },
                 { "name": "chain_state_begin_block", "type": "uint32" },
-                { "name": "chain_state_end_block", "type": "uint32" }
+                { "name": "chain_state_end_block", "type": "uint32" },
+                { "name": "chain_id", "type": "checksum256$" }
             ]
         },
         {

libraries/state_history/compression.cpp

@@ -0,0 +1,32 @@
+#include <eosio/state_history/compression.hpp>
+
+#include <boost/iostreams/device/back_inserter.hpp>
+#include <boost/iostreams/filter/zlib.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+
+namespace eosio {
+namespace state_history {
+
+namespace bio = boost::iostreams;
+bytes zlib_compress_bytes(const bytes& in) {
+   bytes                  out;
+   bio::filtering_ostream comp;
+   comp.push(bio::zlib_compressor(bio::zlib::default_compression));
+   comp.push(bio::back_inserter(out));
+   bio::write(comp, in.data(), in.size());
+   bio::close(comp);
+   return out;
+}
+
+bytes zlib_decompress(const bytes& in) {
+   bytes                  out;
+   bio::filtering_ostream decomp;
+   decomp.push(bio::zlib_decompressor());
+   decomp.push(bio::back_inserter(out));
+   bio::write(decomp, in.data(), in.size());
+   bio::close(decomp);
+   return out;
+}
+
+} // namespace state_history
+} // namespace eosio

libraries/state_history/create_deltas.cpp

@@ -0,0 +1,141 @@
+#include <eosio/state_history/create_deltas.hpp>
+#include <eosio/state_history/serialization.hpp>
+
+namespace eosio {
+namespace state_history {
+
+template <typename T>
+bool include_delta(const T& old, const T& curr) {
+   return true;
+}
+
+bool include_delta(const chain::table_id_object& old, const chain::table_id_object& curr) {
+   return old.payer != curr.payer;
+}
+
+bool include_delta(const chain::resource_limits::resource_limits_object& old,
+                   const chain::resource_limits::resource_limits_object& curr) {
+   return                                   //
+       old.net_weight != curr.net_weight || //
+       old.cpu_weight != curr.cpu_weight || //
+       old.ram_bytes != curr.ram_bytes;
+}
+
+bool include_delta(const chain::resource_limits::resource_limits_state_object& old,
+                   const chain::resource_limits::resource_limits_state_object& curr) {
+   return                                                                                       //
+       old.average_block_net_usage.last_ordinal != curr.average_block_net_usage.last_ordinal || //
+       old.average_block_net_usage.value_ex != curr.average_block_net_usage.value_ex ||         //
+       old.average_block_net_usage.consumed != curr.average_block_net_usage.consumed ||         //
+       old.average_block_cpu_usage.last_ordinal != curr.average_block_cpu_usage.last_ordinal || //
+       old.average_block_cpu_usage.value_ex != curr.average_block_cpu_usage.value_ex ||         //
+       old.average_block_cpu_usage.consumed != curr.average_block_cpu_usage.consumed ||         //
+       old.total_net_weight != curr.total_net_weight ||                                         //
+       old.total_cpu_weight != curr.total_cpu_weight ||                                         //
+       old.total_ram_bytes != curr.total_ram_bytes ||                                           //
+       old.virtual_net_limit != curr.virtual_net_limit ||                                       //
+       old.virtual_cpu_limit != curr.virtual_cpu_limit;
+}
+
+bool include_delta(const chain::account_metadata_object& old, const chain::account_metadata_object& curr) {
+   return                                               //
+       old.name != curr.name ||                         //
+       old.is_privileged() != curr.is_privileged() ||   //
+       old.last_code_update != curr.last_code_update || //
+       old.vm_type != curr.vm_type ||                   //
+       old.vm_version != curr.vm_version ||             //
+       old.code_hash != curr.code_hash;
+}
+
+bool include_delta(const chain::code_object& old, const chain::code_object& curr) { //
+   return false;
+}
+
+bool include_delta(const chain::protocol_state_object& old, const chain::protocol_state_object& curr) {
+   return old.activated_protocol_features != curr.activated_protocol_features;
+}
+
+std::vector<table_delta> create_deltas(const chainbase::database& db, bool full_snapshot) {
+   std::vector<table_delta>                          deltas;
+   const auto&                                       table_id_index = db.get_index<chain::table_id_multi_index>();
+   std::map<uint64_t, const chain::table_id_object*> removed_table_id;
+   for (auto& rem : table_id_index.stack().back().removed_values)
+      removed_table_id[rem.first._id] = &rem.second;
+
+   auto get_table_id = [&](uint64_t tid) -> const chain::table_id_object& {
+      auto obj = table_id_index.find(tid);
+      if (obj)
+         return *obj;
+      auto it = removed_table_id.find(tid);
+      EOS_ASSERT(it != removed_table_id.end(), chain::plugin_exception, "can not found table id ${tid}", ("tid", tid));
+      return *it->second;
+   };
+
+   auto pack_row          = [&](auto& row) { return fc::raw::pack(make_history_serial_wrapper(db, row)); };
+   auto pack_contract_row = [&](auto& row) {
+      return fc::raw::pack(make_history_context_wrapper(db, get_table_id(row.t_id._id), row));
+   };
+
+   auto process_table = [&](auto* name, auto& index, auto& pack_row) {
+      if (full_snapshot) {
+         if (index.indices().empty())
+            return;
+         deltas.push_back({});
+         auto& delta = deltas.back();
+         delta.name  = name;
+         for (auto& row : index.indices())
+            delta.rows.obj.emplace_back(true, pack_row(row));
+      } else {
+         if (index.stack().empty())
+            return;
+         auto& undo = index.stack().back();
+         if (undo.old_values.empty() && undo.new_ids.empty() && undo.removed_values.empty())
+            return;
+         deltas.push_back({});
+         auto& delta = deltas.back();
+         delta.name  = name;
+         for (auto& old : undo.old_values) {
+            auto& row = index.get(old.first);
+            if (include_delta(old.second, row))
+               delta.rows.obj.emplace_back(true, pack_row(row));
+         }
+         for (auto& old : undo.removed_values)
+            delta.rows.obj.emplace_back(false, pack_row(old.second));
+         for (auto id : undo.new_ids) {
+            auto& row = index.get(id);
+            delta.rows.obj.emplace_back(true, pack_row(row));
+         }
+      }
+   };
+
+   process_table("account", db.get_index<chain::account_index>(), pack_row);
+   process_table("account_metadata", db.get_index<chain::account_metadata_index>(), pack_row);
+   process_table("code", db.get_index<chain::code_index>(), pack_row);
+
+   process_table("contract_table", db.get_index<chain::table_id_multi_index>(), pack_row);
+   process_table("contract_row", db.get_index<chain::key_value_index>(), pack_contract_row);
+   process_table("contract_index64", db.get_index<chain::index64_index>(), pack_contract_row);
+   process_table("contract_index128", db.get_index<chain::index128_index>(), pack_contract_row);
+   process_table("contract_index256", db.get_index<chain::index256_index>(), pack_contract_row);
+   process_table("contract_index_double", db.get_index<chain::index_double_index>(), pack_contract_row);
+   process_table("contract_index_long_double", db.get_index<chain::index_long_double_index>(), pack_contract_row);
+
+   process_table("global_property", db.get_index<chain::global_property_multi_index>(), pack_row);
+   process_table("generated_transaction", db.get_index<chain::generated_transaction_multi_index>(), pack_row);
+   process_table("protocol_state", db.get_index<chain::protocol_state_multi_index>(), pack_row);
+
+   process_table("permission", db.get_index<chain::permission_index>(), pack_row);
+   process_table("permission_link", db.get_index<chain::permission_link_index>(), pack_row);
+
+   process_table("resource_limits", db.get_index<chain::resource_limits::resource_limits_index>(), pack_row);
+   process_table("resource_usage", db.get_index<chain::resource_limits::resource_usage_index>(), pack_row);
+   process_table("resource_limits_state", db.get_index<chain::resource_limits::resource_limits_state_index>(),
+                 pack_row);
+   process_table("resource_limits_config", db.get_index<chain::resource_limits::resource_limits_config_index>(),
+                 pack_row);
+
+   return deltas;
+}
+
+} // namespace state_history
+} // namespace eosio

libraries/state_history/include/eosio/state_history/compression.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <eosio/chain/types.hpp>
+
+namespace eosio {
+namespace state_history {
+
+using chain::bytes;
+
+bytes zlib_compress_bytes(const bytes& in);
+bytes zlib_decompress(const bytes& in);
+
+} // namespace state_history
+} // namespace eosio

libraries/state_history/include/eosio/state_history/create_deltas.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <eosio/state_history/types.hpp>
+
+namespace eosio {
+namespace state_history {
+
+std::vector<table_delta> create_deltas(const chainbase::database& db, bool full_snapshot);
+
+} // namespace state_history
+} // namespace eosio

plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_log.hpp → libraries/state_history/include/eosio/state_history/log.hpp

@@ -7,9 +7,9 @@
 #include <eosio/chain/block_header.hpp>
 #include <eosio/chain/exceptions.hpp>
 #include <eosio/chain/types.hpp>
-#include <fc/log/logger.hpp>
 #include <fc/io/cfile.hpp>
-#include <appbase/application.hpp>
+#include <fc/log/logger.hpp>
+
 namespace eosio {
 
 /*
@@ -87,52 +87,39 @@ class state_history_log {
 
    template <typename F>
    void write_entry(const state_history_log_header& header, const chain::block_id_type& prev_id, F write_payload) {
-      try {
-         auto block_num = chain::block_header::num_from_id(header.block_id);
-         EOS_ASSERT(_begin_block == _end_block || block_num <= _end_block, chain::plugin_exception,
-                  "missed a block in ${name}.log", ("name", name));
+      auto block_num = chain::block_header::num_from_id(header.block_id);
+      EOS_ASSERT(_begin_block == _end_block || block_num <= _end_block, chain::plugin_exception,
+                 "missed a block in ${name}.log", ("name", name));
 
-         if (_begin_block != _end_block && block_num > _begin_block) {
-            if (block_num == _end_block) {
-               EOS_ASSERT(prev_id == last_block_id, chain::plugin_exception, "missed a fork change in ${name}.log",
-                        ("name", name));
-            } else {
-               state_history_log_header prev;
-               get_entry(block_num - 1, prev);
-               EOS_ASSERT(prev_id == prev.block_id, chain::plugin_exception, "missed a fork change in ${name}.log",
-                        ("name", name));
-            }
+      if (_begin_block != _end_block && block_num > _begin_block) {
+         if (block_num == _end_block) {
+            EOS_ASSERT(prev_id == last_block_id, chain::plugin_exception, "missed a fork change in ${name}.log",
+                       ("name", name));
+         } else {
+            state_history_log_header prev;
+            get_entry(block_num - 1, prev);
+            EOS_ASSERT(prev_id == prev.block_id, chain::plugin_exception, "missed a fork change in ${name}.log",
+                       ("name", name));
          }
+      }
 
-         if (block_num < _end_block)
-            truncate(block_num);
-         log.seek_end(0);
-         uint64_t pos = log.tellp();
-         write_header(header);
-         write_payload(log);
-         uint64_t end = log.tellp();
-         EOS_ASSERT(end == pos + state_history_log_header_serial_size + header.payload_size, chain::plugin_exception,
-                  "wrote payload with incorrect size to ${name}.log", ("name", name));
-         log.write((char*)&pos, sizeof(pos));
+      if (block_num < _end_block)
+         truncate(block_num);
+      log.seek_end(0);
+      uint64_t pos = log.tellp();
+      write_header(header);
+      write_payload(log);
+      uint64_t end = log.tellp();
+      EOS_ASSERT(end == pos + state_history_log_header_serial_size + header.payload_size, chain::plugin_exception,
+                 "wrote payload with incorrect size to ${name}.log", ("name", name));
+      log.write((char*)&pos, sizeof(pos));
 
-         index.seek_end(0);
-         index.write((char*)&pos, sizeof(pos));
-         if (_begin_block == _end_block)
-            _begin_block = block_num;
-         _end_block    = block_num + 1;
-         last_block_id = header.block_id;
-      }
-      catch(const chain::plugin_exception& e) {
-         elog( "chain::plugin_exception: ${details}", ("details", e.to_detail_string()) );
-         // Both app().quit() and exception throwing are required. Without app().quit(),
-         // the exception would be caught and drop before reaching main(). The exception is
-         // to ensure the block won't be commited. 
-         appbase::app().quit();
-         EOS_THROW(
-             chain::state_history_write_exception,
-             "State history encountered an Error which it cannot recover from.  Please resolve the error and relaunch "
-             "the process");
-      }
+      index.seek_end(0);
+      index.write((char*)&pos, sizeof(pos));
+      if (_begin_block == _end_block)
+         _begin_block = block_num;
+      _end_block    = block_num + 1;
+      last_block_id = header.block_id;
    }
 
    // returns cfile positioned at payload
@@ -210,8 +197,8 @@ class state_history_log {
    }
 
    void open_log() {
-      log.set_file_path( log_filename );
-      log.open( "a+b" ); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app
+      log.set_file_path(log_filename);
+      log.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app
       log.seek_end(0);
       uint64_t size = log.tellp();
       if (size >= state_history_log_header_serial_size) {
@@ -233,14 +220,14 @@ class state_history_log {
    }
 
    void open_index() {
-      index.set_file_path( index_filename );
-      index.open( "a+b" ); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app
+      index.set_file_path(index_filename);
+      index.open("a+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::app
       index.seek_end(0);
       if (index.tellp() == (static_cast<int>(_end_block) - _begin_block) * sizeof(uint64_t))
          return;
       ilog("Regenerate ${name}.index", ("name", name));
       index.close();
-      index.open( "w+b" ); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc
+      index.open("w+b"); // std::ios_base::binary | std::ios_base::in | std::ios_base::out | std::ios_base::trunc
 
       log.seek_end(0);
       uint64_t size      = log.tellp();