Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Add support for block log splitting #9184

Merged
merged 22 commits into from
Jun 23, 2020
Merged

Add support for block log splitting #9184

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
e9cfda6
Add support for block log splitting
huangminghuang May 29, 2020
3ac568f
Merge branch 'develop' into block-log-split
huangminghuang Jun 10, 2020
efec3c7
bug fixes and PR comments
huangminghuang Jun 10, 2020
34c81d3
rename blocks-split-factor to blocks-log-stride
huangminghuang Jun 11, 2020
f9937b6
allows log file version upgrade after spliting
huangminghuang Jun 11, 2020
dce3cc3
more stride rename
huangminghuang Jun 11, 2020
0745026
add support to recover from incomplete block head automatically
huangminghuang Jun 11, 2020
dd10241
fix bug and modify restart chain test with block num with 3 digits.
huangminghuang Jun 15, 2020
68ec5ae
Merge branch 'develop' into block-log-split
huangminghuang Jun 16, 2020
ebf48b0
add test to split from v1 log
huangminghuang Jun 16, 2020
00eb6c9
use RAII to set blocklog version in unittests
huangminghuang Jun 16, 2020
d15e3b5
remove unneeded code
huangminghuang Jun 17, 2020
d2f8e5b
add comment to address PR concern
huangminghuang Jun 17, 2020
0ebb3fd
added tests for different max_retained_block_files configuration
huangminghuang Jun 18, 2020
5fb2af4
handle retained block log with overlapping ranges
huangminghuang Jun 18, 2020
b069f54
add assertion to protect against bad memory access
huangminghuang Jun 18, 2020
3063759
Address more PR comments
huangminghuang Jun 19, 2020
e2dd458
extend block-log-auto-fix to recovery from corrupted index file
huangminghuang Jun 22, 2020
a674825
change command line option description
huangminghuang Jun 22, 2020
ee3a733
more command line description change
huangminghuang Jun 22, 2020
c158c7b
Some more PR comments fix
huangminghuang Jun 23, 2020
3207d7d
rename allow_block_log_auto_fix to fix_irreversible_blocks
huangminghuang Jun 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
474 changes: 361 additions & 113 deletions libraries/chain/block_log.cpp
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion libraries/chain/controller.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ struct controller_impl {
reversible_blocks( cfg.blocks_dir/config::reversible_blocks_dir_name,
cfg.read_only ? database::read_only : database::read_write,
cfg.reversible_cache_size, false, cfg.db_map_mode, cfg.db_hugepage_paths ),
blog( cfg.blocks_dir ),
blog( cfg.blocks_dir, cfg.blocks_archive_dir, cfg.blocks_log_stride, cfg.max_retained_block_files),
fork_db( cfg.state_dir ),
wasmif( cfg.wasm_runtime, cfg.eosvmoc_tierup, db, cfg.state_dir, cfg.eosvmoc_config ),
resource_limits( db, [&s]() { return s.get_deep_mind_logger(); }),
Expand Down
3 changes: 2 additions & 1 deletion libraries/chain/include/eosio/chain/block_log.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ namespace eosio { namespace chain {

class block_log {
public:
block_log(const fc::path& data_dir);
block_log(const fc::path& data_dir, fc::path backup_dir = fc::path(), uint64_t stride=1000, uint16_t max_retained_files=10);
block_log(block_log&& other) = default;
~block_log();

Expand Down Expand Up @@ -83,6 +83,7 @@ namespace eosio { namespace chain {

// used for unit test to generate older version blocklog
static void set_version(uint32_t);
uint32_t version() const;

/**
* @param n Only test 1 block out of every n blocks. If n is 0, it is maximum between 1 and the ceiling of the total number blocks divided by 8.
Expand Down
3 changes: 3 additions & 0 deletions libraries/chain/include/eosio/chain/chain_id_type.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ namespace chain {

void reflector_init()const;

bool empty() const { return *this == chain_id_type{};}

private:
chain_id_type() = default;

Expand All @@ -51,6 +53,7 @@ namespace chain {
friend struct eosio::handshake_message;
friend class block_log;
friend struct block_log_preamble;
friend struct block_log_catalog;
friend class controller;
friend struct controller_impl;
friend class global_property_object;
Expand Down
5 changes: 4 additions & 1 deletion libraries/chain/include/eosio/chain/config.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,10 @@ namespace eosio { namespace chain { namespace config {
typedef __uint128_t uint128_t;

const static auto default_blocks_dir_name = "blocks";
const static auto reversible_blocks_dir_name = "reversible";
const static auto default_blocks_archive_dir_name = "archive";
const static auto default_blocks_log_stride = UINT32_MAX;
const static auto default_max_retained_block_files = 10;
const static auto reversible_blocks_dir_name = "reversible";
const static auto default_reversible_cache_size = 340*1024*1024ll;/// 1MB * 340 blocks based on 21 producer BFT delay
const static auto default_reversible_guard_size = 2*1024*1024ll;/// 1MB * 340 blocks based on 21 producer BFT delay

Expand Down
3 changes: 3 additions & 0 deletions libraries/chain/include/eosio/chain/controller.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,16 @@ namespace eosio { namespace chain {
flat_set< pair<account_name, action_name> > action_blacklist;
flat_set<public_key_type> key_blacklist;
path blocks_dir = chain::config::default_blocks_dir_name;
path blocks_archive_dir = chain::config::default_blocks_archive_dir_name;
path state_dir = chain::config::default_state_dir_name;
uint64_t state_size = chain::config::default_state_size;
uint64_t state_guard_size = chain::config::default_state_guard_size;
uint64_t reversible_cache_size = chain::config::default_reversible_cache_size;
uint64_t reversible_guard_size = chain::config::default_reversible_guard_size;
uint32_t sig_cpu_bill_pct = chain::config::default_sig_cpu_bill_pct;
uint16_t thread_pool_size = chain::config::default_controller_thread_pool_size;
uint16_t max_retained_block_files = chain::config::default_max_retained_block_files;
uint64_t blocks_log_stride = chain::config::default_blocks_log_stride;
fc::microseconds abi_serializer_max_time_us = fc::microseconds(chain::config::default_abi_serializer_max_time_us);
bool read_only = false;
bool force_all_checks = false;
Expand Down
1 change: 1 addition & 0 deletions libraries/fc/include/fc/filesystem.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ namespace fc {

bool is_relative()const;
bool is_absolute()const;
bool empty() const;

static char separator_char;

Expand Down
2 changes: 2 additions & 0 deletions libraries/fc/src/filesystem.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,8 @@ namespace fc {
bool path::is_relative()const { return _p->is_relative(); }
bool path::is_absolute()const { return _p->is_absolute(); }

bool path::empty() const { return _p->empty(); }

directory_iterator::directory_iterator( const fc::path& p )
:_p(p){}

Expand Down
12 changes: 11 additions & 1 deletion plugins/chain_plugin/chain_plugin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,6 @@ class chain_plugin_impl {


fc::optional<fork_database> fork_db;
fc::optional<block_log> block_logger;
fc::optional<controller::config> chain_config;
fc::optional<controller> chain;
fc::optional<genesis_state> genesis;
Expand Down Expand Up @@ -228,6 +227,14 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
cfg.add_options()
("blocks-dir", bpo::value<bfs::path>()->default_value("blocks"),
"the location of the blocks directory (absolute path or relative to application data dir)")
("blocks-log-stride", bpo::value<uint32_t>()->default_value(config::default_blocks_log_stride),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should add here (and also in PR API documentation) something like "When the stride is breached, the current blog log and index will be renamed 'blocks--.log/index' and a new current block log and index will be created with the most recent block. All files following this format will be used to construct an extended block log."

"split the block log file when the head block number is the multiple of the split factor")
("max-retained-block-files", bpo::value<uint16_t>()->default_value(config::default_max_retained_block_files),
"the maximum number of blocks files to retain so that the blocks in those files can be queried.\n"
"When the number is reached, the oldest block file would be move to archive dir or deleted if the archive dir is empty." )
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Make this match the documentation in the PR.

("blocks-archive-dir", bpo::value<bfs::path>()->default_value(config::default_blocks_archive_dir_name),
"the location of the blocks archive directory (absolute path or relative to blocks dir).\n"
"If the value is empty, blocks files beyond the retained limit will be deleted.")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Make this match the documentation in the PR.

("protocol-features-dir", bpo::value<bfs::path>()->default_value("protocol_features"),
"the location of the protocol_features directory (absolute path or relative to application config dir)")
("checkpoint", bpo::value<vector<string>>()->composing(), "Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints.")
Expand Down Expand Up @@ -742,6 +749,9 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
my->chain_config->blocks_dir = my->blocks_dir;
my->chain_config->state_dir = app().data_dir() / config::default_state_dir_name;
my->chain_config->read_only = my->readonly;
my->chain_config->blocks_archive_dir = options.at("blocks-archive-dir").as<bfs::path>();
my->chain_config->blocks_log_stride = options.at("blocks-log-stride").as<uint32_t>();
my->chain_config->max_retained_block_files = options.at("max-retained-block-files").as<uint16_t>();

if (auto resmon_plugin = app().find_plugin<resource_monitor_plugin>()) {
resmon_plugin->monitor_directory(my->chain_config->blocks_dir);
Expand Down
174 changes: 166 additions & 8 deletions unittests/restart_chain_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include <contracts.hpp>
#include <snapshots.hpp>
#include <boost/iostreams/device/mapped_file.hpp>
#include <fc/io/cfile.hpp>
#include "test_cfd_transaction.hpp"

using namespace eosio;
Expand Down Expand Up @@ -235,8 +236,162 @@ BOOST_FIXTURE_TEST_CASE(test_light_validation_restart_from_block_log_with_pruned
BOOST_REQUIRE_NO_THROW(block_log::repair_log(blocks_dir));
}

BOOST_AUTO_TEST_CASE(test_split_log) {
namespace bfs = boost::filesystem;
fc::temp_directory temp_dir;

tester chain(
temp_dir,
[](controller::config& config) {
config.blocks_log_stride = 20;
config.max_retained_block_files = 5;
},
true);
chain.produce_blocks(150);

auto blocks_dir = chain.get_config().blocks_dir;
auto blocks_archive_dir = chain.get_config().blocks_dir / chain.get_config().blocks_archive_dir;

BOOST_CHECK(bfs::exists( blocks_archive_dir / "blocks-1-20.log" ));
BOOST_CHECK(bfs::exists( blocks_archive_dir / "blocks-1-20.index" ));
BOOST_CHECK(bfs::exists( blocks_archive_dir / "blocks-21-40.log" ));
BOOST_CHECK(bfs::exists( blocks_archive_dir / "blocks-21-40.index" ));

BOOST_CHECK(bfs::exists( blocks_dir / "blocks-41-60.log" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-41-60.index" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-61-80.log" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-61-80.index" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-81-100.log" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-81-100.index" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-101-120.log" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-101-120.index" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-121-140.log" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-121-140.index" ));

BOOST_CHECK( ! chain.control->fetch_block_by_number(40) );

BOOST_CHECK( chain.control->fetch_block_by_number(81)->block_num() == 81 );
BOOST_CHECK( chain.control->fetch_block_by_number(90)->block_num() == 90 );
BOOST_CHECK( chain.control->fetch_block_by_number(100)->block_num() == 100 );

BOOST_CHECK( chain.control->fetch_block_by_number(41)->block_num() == 41 );
BOOST_CHECK( chain.control->fetch_block_by_number(50)->block_num() == 50 );
BOOST_CHECK( chain.control->fetch_block_by_number(60)->block_num() == 60 );

BOOST_CHECK( chain.control->fetch_block_by_number(121)->block_num() == 121 );
BOOST_CHECK( chain.control->fetch_block_by_number(130)->block_num() == 130 );
BOOST_CHECK( chain.control->fetch_block_by_number(140)->block_num() == 140 );

BOOST_CHECK( chain.control->fetch_block_by_number(145)->block_num() == 145);

BOOST_CHECK( ! chain.control->fetch_block_by_number(160));
}

BOOST_AUTO_TEST_CASE(test_split_log_no_archive) {

namespace bfs = boost::filesystem;
fc::temp_directory temp_dir;

tester chain(
temp_dir,
[](controller::config& config) {
config.blocks_archive_dir = "";
config.blocks_log_stride = 10;
config.max_retained_block_files = 5;
},
true);
chain.produce_blocks(75);

auto blocks_dir = chain.get_config().blocks_dir;
auto blocks_archive_dir = chain.get_config().blocks_dir / chain.get_config().blocks_archive_dir;

BOOST_CHECK(!bfs::exists( blocks_archive_dir / "blocks-1-10.log" ));
BOOST_CHECK(!bfs::exists( blocks_archive_dir / "blocks-1-10.index" ));
BOOST_CHECK(!bfs::exists( blocks_archive_dir / "blocks-11-20.log" ));
BOOST_CHECK(!bfs::exists( blocks_archive_dir / "blocks-11-20.index" ));
BOOST_CHECK(!bfs::exists( blocks_dir / "blocks-1-10.log" ));
BOOST_CHECK(!bfs::exists( blocks_dir / "blocks-1-10.index" ));
BOOST_CHECK(!bfs::exists( blocks_dir / "blocks-11-20.log" ));
BOOST_CHECK(!bfs::exists( blocks_dir / "blocks-11-20.index" ));

b1bart marked this conversation as resolved.
Show resolved Hide resolved
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-21-30.log" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-21-30.index" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-31-40.log" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-31-40.index" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-41-50.log" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-41-50.index" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-51-60.log" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-51-60.index" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-61-70.log" ));
BOOST_CHECK(bfs::exists( blocks_dir / "blocks-61-70.index" ));

BOOST_CHECK( ! chain.control->fetch_block_by_number(10));
BOOST_CHECK( chain.control->fetch_block_by_number(70));
BOOST_CHECK( ! chain.control->fetch_block_by_number(80));
}

BOOST_AUTO_TEST_CASE(test_split_log_replay) {

namespace bfs = boost::filesystem;
fc::temp_directory temp_dir;

tester chain(
temp_dir,
[](controller::config& config) {
config.blocks_log_stride = 20;
config.max_retained_block_files = 10;
},
true);
chain.produce_blocks(150);

controller::config copied_config = chain.get_config();
auto genesis = chain::block_log::extract_genesis_state(chain.get_config().blocks_dir);
BOOST_REQUIRE(genesis);

chain.close();

void trim_blocklog_front() {
// remove the state files to make sure we are starting from block log
remove_existing_states(copied_config);
tester from_block_log_chain(copied_config, *genesis);
BOOST_CHECK( from_block_log_chain.control->fetch_block_by_number(1)->block_num() == 1);
BOOST_CHECK( from_block_log_chain.control->fetch_block_by_number(75)->block_num() == 75);
BOOST_CHECK( from_block_log_chain.control->fetch_block_by_number(100)->block_num() == 100);
BOOST_CHECK( from_block_log_chain.control->fetch_block_by_number(150)->block_num() == 150);
}

BOOST_FIXTURE_TEST_CASE(restart_from_block_log_with_incomplete_head,restart_from_block_log_test_fixture) {
auto& config = chain.get_config();
auto blocks_path = config.blocks_dir;
// write a few random bytes to block log indicating the last block entry is incomplete
fc::cfile logfile;
logfile.set_file_path(config.blocks_dir / "blocks.log");
logfile.open("ab");
const char random_data[] = "12345678901231876983271649837";
logfile.write(random_data, sizeof(random_data));
}

BOOST_AUTO_TEST_CASE(test_split_from_v1_log) {
namespace bfs = boost::filesystem;
fc::temp_directory temp_dir;
block_log::set_version(1);
tester chain(
temp_dir,
[](controller::config& config) {
config.blocks_log_stride = 20;
config.max_retained_block_files = 5;
},
true);
chain.produce_blocks(75);

BOOST_CHECK( chain.control->fetch_block_by_number(1)->block_num() == 1 );
BOOST_CHECK( chain.control->fetch_block_by_number(21)->block_num() == 21 );
BOOST_CHECK( chain.control->fetch_block_by_number(41)->block_num() == 41 );
BOOST_CHECK( chain.control->fetch_block_by_number(75)->block_num() == 75 );
block_log::set_version(block_log::max_supported_version);
}

void trim_blocklog_front(uint32_t version) {
block_log::set_version(version);
tester chain;
chain.produce_blocks(10);
chain.produce_blocks(20);
Expand All @@ -256,28 +411,31 @@ void trim_blocklog_front() {

block_log old_log(blocks_dir);
block_log new_log(temp1.path);
// double check if the version has been set to the desired version
BOOST_CHECK(old_log.version() == version);
BOOST_CHECK(new_log.first_block_num() == 10);
BOOST_CHECK(new_log.head()->block_num() == old_log.head()->block_num());

int num_blocks_trimmed = 10 - 1;
BOOST_CHECK(fc::file_size(temp1.path / "blocks.index") == old_index_size - sizeof(uint64_t) * num_blocks_trimmed);
block_log::set_version(block_log::max_supported_version);
}

BOOST_AUTO_TEST_CASE(test_trim_blocklog_front) { trim_blocklog_front(); }
BOOST_AUTO_TEST_CASE(test_trim_blocklog_front) {
trim_blocklog_front(block_log::max_supported_version);
}

BOOST_AUTO_TEST_CASE(test_trim_blocklog_front_v1) {
block_log::set_version(1);
trim_blocklog_front();
block_log::set_version(1);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't need this anymore.

trim_blocklog_front(1);
}

BOOST_AUTO_TEST_CASE(test_trim_blocklog_front_v2) {
block_log::set_version(2);
trim_blocklog_front();
trim_blocklog_front(2);
}

BOOST_AUTO_TEST_CASE(test_trim_blocklog_front_v3) {
block_log::set_version(3);
trim_blocklog_front();
trim_blocklog_front(3);
}

BOOST_AUTO_TEST_SUITE_END()