From 2353b34f2707760a25f8974f1bdd7d7d82fe1673 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 18 Dec 2023 17:40:47 +0300 Subject: [PATCH] Improve CellDb migration * Fix deserializing cells * Use proxy actor * Add delays * Print stats every minute --- crypto/vm/boc.cpp | 4 +- crypto/vm/boc.h | 2 +- crypto/vm/db/CellStorage.cpp | 2 +- validator-engine/validator-engine.cpp | 14 +++--- validator/db/celldb.cpp | 61 ++++++++++++++++++++------- validator/db/celldb.hpp | 24 +++++++++++ 6 files changed, 82 insertions(+), 25 deletions(-) diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index 11583ede6..bd334cbfc 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -930,7 +930,7 @@ unsigned long long BagOfCells::get_idx_entry_raw(int index) { * */ -td::Result> std_boc_deserialize(td::Slice data, bool can_be_empty) { +td::Result> std_boc_deserialize(td::Slice data, bool can_be_empty, bool allow_nonzero_level) { if (data.empty() && can_be_empty) { return Ref(); } @@ -946,7 +946,7 @@ td::Result> std_boc_deserialize(td::Slice data, bool can_be_empty) { if (root.is_null()) { return td::Status::Error("bag of cells has null root cell (?)"); } - if (root->get_level() != 0) { + if (!allow_nonzero_level && root->get_level() != 0) { return td::Status::Error("bag of cells has a root with non-zero level"); } return std::move(root); diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index c7a1810d7..1bff5b257 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -323,7 +323,7 @@ class BagOfCells { std::vector* cell_should_cache); }; -td::Result> std_boc_deserialize(td::Slice data, bool can_be_empty = false); +td::Result> std_boc_deserialize(td::Slice data, bool can_be_empty = false, bool allow_nonzero_level = false); td::Result std_boc_serialize(Ref root, int mode = 0); td::Result>> std_boc_deserialize_multi(td::Slice data, diff --git a/crypto/vm/db/CellStorage.cpp b/crypto/vm/db/CellStorage.cpp index acc55898a..303d46503 100644 --- a/crypto/vm/db/CellStorage.cpp +++ b/crypto/vm/db/CellStorage.cpp @@ -98,7 +98,7 @@ class RefcntCellParser { auto size = parser.get_left_len(); td::Slice data = parser.template fetch_string_raw(size); if (stored_boc_) { - TRY_RESULT(boc, vm::std_boc_deserialize(data)); + TRY_RESULT(boc, vm::std_boc_deserialize(data, false, true)); TRY_RESULT(loaded_cell, boc->load_cell()); cell = std::move(loaded_cell.data_cell); return td::Status::OK(); diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index d98c296c9..1d9223f2b 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3775,11 +3775,15 @@ int main(int argc, char *argv[]) { acts.push_back([&x, at]() { td::actor::send_closure(x, &ValidatorEngine::schedule_shutdown, (double)at); }); return td::Status::OK(); }); - p.add_checked_option('\0', "celldb-compress-depth", "(default: 0)", [&](td::Slice arg) { - TRY_RESULT(value, td::to_integer_safe(arg)); - acts.push_back([&x, value]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_compress_depth, value); }); - return td::Status::OK(); - }); + p.add_checked_option('\0', "celldb-compress-depth", + "optimize celldb by storing cells of depth X with whole subtrees (experimental, default: 0)", + [&](td::Slice arg) { + TRY_RESULT(value, td::to_integer_safe(arg)); + acts.push_back([&x, value]() { + td::actor::send_closure(x, &ValidatorEngine::set_celldb_compress_depth, value); + }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 6a2b46992..d29126cea 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -23,6 +23,7 @@ #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" +#include "common/delay.h" namespace ton { @@ -68,14 +69,16 @@ CellDbIn::CellDbIn(td::actor::ActorId root_db, td::actor::ActorId>( + td::actor::create_actor("celldbmigration", actor_id(this))), compress_depth = opts_->get_celldb_compress_depth()](const vm::CellLoader::LoadResult& res) { if (res.cell_.is_null()) { return; } bool expected_stored_boc = res.cell_->get_depth() == compress_depth && compress_depth != 0; if (expected_stored_boc != res.stored_boc_) { - td::actor::send_closure(db, &CellDbIn::migrate_cell, td::Bits256{res.cell_->get_hash().bits()}); + td::actor::send_closure(*actor, &CellDbIn::MigrationProxy::migrate_cell, + td::Bits256{res.cell_->get_hash().bits()}); } }; @@ -156,6 +159,13 @@ void CellDbIn::alarm() { if (migrate_after_ && migrate_after_.is_in_past()) { migrate_cells(); } + if (migration_stats_ && migration_stats_->end_at_.is_in_past()) { + LOG(INFO) << "CellDb migration, " << migration_stats_->start_.elapsed() + << "s stats: batches=" << migration_stats_->batches_ << " migrated=" << migration_stats_->migrated_cells_ + << " checked=" << migration_stats_->checked_cells_ << " time=" << migration_stats_->total_time_ + << " queue_size=" << cells_to_migrate_.size(); + migration_stats_ = {}; + } auto E = get_block(get_empty_key_hash()).move_as_ok(); auto N = get_block(E.next).move_as_ok(); if (N.is_empty()) { @@ -291,23 +301,31 @@ void CellDbIn::set_block(KeyHash key_hash, DbEntry e) { void CellDbIn::migrate_cell(td::Bits256 hash) { cells_to_migrate_.insert(hash); - if (cells_to_migrate_.size() >= 32) { - migrate_cells(); - } else if (!migrate_after_) { - migrate_after_ = td::Timestamp::in(1.0); + if (!migration_active_) { + migration_active_ = true; + migrate_after_ = td::Timestamp::in(10.0); } } void CellDbIn::migrate_cells() { + migrate_after_ = td::Timestamp::never(); if (cells_to_migrate_.empty()) { + migration_active_ = false; return; } + td::Timer timer; + if (!migration_stats_) { + migration_stats_ = std::make_unique(); + } vm::CellStorer stor{*cell_db_}; auto loader = std::make_unique(cell_db_->snapshot()); boc_->set_loader(std::make_unique(*loader)).ensure(); cell_db_->begin_write_batch().ensure(); - td::uint32 cnt = 0; - for (const auto& hash : cells_to_migrate_) { + td::uint32 checked = 0, migrated = 0; + for (auto it = cells_to_migrate_.begin(); it != cells_to_migrate_.end() && checked < 128; ) { + ++checked; + td::Bits256 hash = *it; + it = cells_to_migrate_.erase(it); auto R = loader->load(hash.as_slice(), true, boc_->as_ext_cell_creator()); if (R.is_error()) { continue; @@ -318,18 +336,27 @@ void CellDbIn::migrate_cells() { bool expected_stored_boc = R.ok().cell_->get_depth() == opts_->get_celldb_compress_depth() && opts_->get_celldb_compress_depth() != 0; if (expected_stored_boc != R.ok().stored_boc_) { - ++cnt; + ++migrated; stor.set(R.ok().refcnt(), R.ok().cell_, expected_stored_boc).ensure(); } } - cells_to_migrate_.clear(); - if (cnt > 0) { - LOG(DEBUG) << "Migrated " << cnt << " cells"; - } cell_db_->commit_write_batch().ensure(); boc_->set_loader(std::make_unique(cell_db_->snapshot(), on_load_callback_)).ensure(); td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); - migrate_after_ = td::Timestamp::never(); + + double time = timer.elapsed(); + LOG(DEBUG) << "CellDb migration: migrated=" << migrated << " checked=" << checked << " time=" << time; + ++migration_stats_->batches_; + migration_stats_->migrated_cells_ += migrated; + migration_stats_->checked_cells_ += checked; + migration_stats_->total_time_ += time; + + if (cells_to_migrate_.empty()) { + migration_active_ = false; + } else { + delay_action([SelfId = actor_id(this)] { td::actor::send_closure(SelfId, &CellDbIn::migrate_cells); }, + td::Timestamp::in(time * 2)); + } } void CellDb::load_cell(RootHash hash, td::Promise> promise) { @@ -361,14 +388,16 @@ void CellDb::start_up() { boc_ = vm::DynamicBagOfCellsDb::create(); boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); cell_db_ = td::actor::create_actor("celldbin", root_db_, actor_id(this), path_, opts_); - on_load_callback_ = [db = cell_db_.get(), + on_load_callback_ = [actor = std::make_shared>( + td::actor::create_actor("celldbmigration", cell_db_.get())), compress_depth = opts_->get_celldb_compress_depth()](const vm::CellLoader::LoadResult& res) { if (res.cell_.is_null()) { return; } bool expected_stored_boc = res.cell_->get_depth() == compress_depth && compress_depth != 0; if (expected_stored_boc != res.stored_boc_) { - td::actor::send_closure(db, &CellDbIn::migrate_cell, td::Bits256{res.cell_->get_hash().bits()}); + td::actor::send_closure(*actor, &CellDbIn::MigrationProxy::migrate_cell, + td::Bits256{res.cell_->get_hash().bits()}); } }; } diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 6545d5970..a2a84ab4a 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -107,6 +107,30 @@ class CellDbIn : public CellDbBase { std::function on_load_callback_; std::set cells_to_migrate_; td::Timestamp migrate_after_ = td::Timestamp::never(); + bool migration_active_ = false; + + struct MigrationStats { + td::Timer start_; + td::Timestamp end_at_ = td::Timestamp::in(60.0); + size_t batches_ = 0; + size_t migrated_cells_ = 0; + size_t checked_cells_ = 0; + double total_time_ = 0.0; + }; + std::unique_ptr migration_stats_; + + public: + class MigrationProxy : public td::actor::Actor { + public: + explicit MigrationProxy(td::actor::ActorId cell_db) : cell_db_(cell_db) { + } + void migrate_cell(td::Bits256 hash) { + td::actor::send_closure(cell_db_, &CellDbIn::migrate_cell, hash); + } + + private: + td::actor::ActorId cell_db_; + }; }; class CellDb : public CellDbBase {