From 2b530f8d98115de06f4afbbdbbd812b82f113ec0 Mon Sep 17 00:00:00 2001 From: Harrm Date: Mon, 2 Sep 2024 15:34:48 +0300 Subject: [PATCH] Add precompilation for Glutton parachain (#2198) * Add precompilation for Glutton parachain used in 0013 zombienet test, also minor code cleanups --- cmake/functions.cmake | 2 ++ core/application/modes/precompile_wasm.cpp | 40 ++++++++++++++++------ core/log/profiling_logger.hpp | 11 +++--- core/parachain/CMakeLists.txt | 2 -- core/parachain/pvf/kagome_pvf_worker.cpp | 21 +++--------- core/parachain/pvf/pvf_impl.cpp | 2 +- core/parachain/pvf/pvf_worker_types.hpp | 5 +-- core/parachain/pvf/workers.cpp | 13 +++---- core/parachain/pvf/workers.hpp | 4 +-- core/runtime/wabt/stack_limiter.cpp | 6 ++-- core/utils/thread_pool.hpp | 2 +- zombienet/precompile.sh | 12 +++++++ 12 files changed, 69 insertions(+), 51 deletions(-) diff --git a/cmake/functions.cmake b/cmake/functions.cmake index 841308905e..8ec02e606f 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -40,6 +40,8 @@ function(add_flag flag) check_cxx_compiler_flag(${flag} FLAG${flag_var}) if(FLAG${flag_var} EQUAL 1) add_compile_options(${flag}) + else() + message(AUTHOR_WARNING "Compiler flag ${flag} requested, but not supported by ${CMAKE_CXX_COMPILER}") endif() endfunction() diff --git a/core/application/modes/precompile_wasm.cpp b/core/application/modes/precompile_wasm.cpp index c2dc7455a2..fb6f563c4f 100644 --- a/core/application/modes/precompile_wasm.cpp +++ b/core/application/modes/precompile_wasm.cpp @@ -5,9 +5,11 @@ */ #include "application/modes/precompile_wasm.hpp" +#include #include "blockchain/block_tree.hpp" #include "common/bytestr.hpp" +#include "log/logger.hpp" #include "parachain/pvf/pool.hpp" #include "parachain/pvf/session_params.hpp" #include "runtime/common/runtime_instances_pool.hpp" @@ -36,6 +38,30 @@ namespace kagome::application::mode { return EXIT_SUCCESS; } + std::optional readRuntimeFile(const std::filesystem::path &path, + const log::Logger &log) { + Buffer bytes; + auto res = readFile(bytes, path); + if (!res) { + SL_ERROR(log, "file {} read error {}", path.string(), res.error()); + return std::nullopt; + } + auto text = byte2str(bytes); + if (text.starts_with("{")) { + SL_ERROR(log, "expected WASM, got JSON, file {}", path.string()); + return std::nullopt; + } + if (text.starts_with("0x")) { + auto bytes_res = common::unhexWith0x(text); + if (!bytes_res) { + SL_ERROR(log, "failed to unhex a seemingly hex file {}", path.string()); + return std::nullopt; + } + bytes = std::move(bytes_res.value()); + } + return bytes; + } + outcome::result PrecompileWasmMode::runOutcome() const { auto block = block_tree_->bestBlock(); @@ -43,19 +69,11 @@ namespace kagome::application::mode { for (auto &path : config_.parachains) { SL_INFO(log_, "precompile parachain {}", path.string()); - Buffer bytes; - if (not readFile(bytes, path)) { - SL_WARN(log_, "read error, file {}", path.string()); - continue; - } - auto text = byte2str(bytes); - if (text.starts_with("{")) { - SL_WARN(log_, "expected WASM, got JSON, file {}", path.string()); + auto bytes_opt = readRuntimeFile(path, log_); + if (!bytes_opt) { continue; } - if (text.starts_with("0x")) { - BOOST_OUTCOME_TRY(bytes, common::unhexWith0x(text)); - } + auto &bytes = bytes_opt.value(); // https://github.com/paritytech/polkadot-sdk/blob/b4ae5b01da280f754ccc00b94314a30b658182a1/polkadot/parachain/src/primitives.rs#L74-L81 auto code_hash = hasher_->blake2b_256(bytes); OUTCOME_TRY(config, diff --git a/core/log/profiling_logger.hpp b/core/log/profiling_logger.hpp index 4254f88480..935bb6019c 100644 --- a/core/log/profiling_logger.hpp +++ b/core/log/profiling_logger.hpp @@ -17,8 +17,9 @@ namespace kagome::log { struct ProfileScope { using Clock = ::kagome::clock::SteadyClockImpl; - ProfileScope(std::string_view scope, log::Logger logger) - : scope{scope}, logger{logger} { + explicit ProfileScope(std::string_view scope, + log::Logger logger = profiling_logger) + : scope{scope}, logger{std::move(logger)} { BOOST_ASSERT(logger != nullptr); start = Clock{}.now(); } @@ -58,12 +59,9 @@ namespace kagome::log { #define KAGOME_PROFILE_START_L(logger, scope) \ auto _profiling_scope_##scope = ::kagome::log::ProfileScope{#scope, logger}; -#define KAGOME_PROFILE_END_L(logger, scope) _profiling_scope_##scope.end(); - #define KAGOME_PROFILE_START(scope) \ KAGOME_PROFILE_START_L(::kagome::log::profiling_logger, scope) -#define KAGOME_PROFILE_END(scope) \ - KAGOME_PROFILE_END_L(::kagome::log::profiling_logger, scope) +#define KAGOME_PROFILE_END(scope) _profiling_scope_##scope.end(); #else @@ -71,6 +69,5 @@ namespace kagome::log { #define KAGOME_PROFILE_END(scope) #define KAGOME_PROFILE_START_L(logger, scope) -#define KAGOME_PROFILE_END_L(logger, scope) #endif diff --git a/core/parachain/CMakeLists.txt b/core/parachain/CMakeLists.txt index 38ef7d48eb..667c101c65 100644 --- a/core/parachain/CMakeLists.txt +++ b/core/parachain/CMakeLists.txt @@ -66,7 +66,6 @@ target_link_libraries(kagome_pvf_worker soralog::soralog logger log_configurator - binaryen_wasm_memory binaryen_wasm_memory_factory binaryen_runtime_external_interface @@ -75,7 +74,6 @@ target_link_libraries(kagome_pvf_worker binaryen_instance_environment_factory binaryen_module_factory wasm_compiler - runtime_properties_cache sr25519_provider ed25519_provider diff --git a/core/parachain/pvf/kagome_pvf_worker.cpp b/core/parachain/pvf/kagome_pvf_worker.cpp index 9f6286e2c6..e4a931331f 100644 --- a/core/parachain/pvf/kagome_pvf_worker.cpp +++ b/core/parachain/pvf/kagome_pvf_worker.cpp @@ -12,7 +12,6 @@ #include #include #include -#include "runtime/wabt/instrument.hpp" #ifdef __linux__ #include @@ -45,6 +44,7 @@ #include "runtime/binaryen/module/module_factory_impl.hpp" #include "runtime/module_instance.hpp" #include "runtime/runtime_context.hpp" +#include "runtime/wabt/instrument.hpp" #include "utils/mkdirs.hpp" // rust reference: polkadot-sdk/polkadot/node/core/pvf/execute-worker/src/lib.rs @@ -237,18 +237,6 @@ namespace kagome::parachain { outcome::result> createModuleFactory( const auto &injector, RuntimeEngine engine) { - // class Foo { - // public:Foo( - // crypto::EcdsaProvider&, - // crypto::Secp256k1Provider&) {} - // }; - // - // std::ignore = injector.template create>(); - // std::ignore = injector.template create< - // std::shared_ptr>(); - // std::ignore = injector.template create< - // std::shared_ptr>(); - switch (engine) { case RuntimeEngine::kBinaryen: return injector.template create< @@ -343,8 +331,9 @@ namespace kagome::parachain { std::shared_ptr module; while (true) { OUTCOME_TRY(input, decodeInput()); - if (auto *code = std::get_if(&input)) { - OUTCOME_TRY(path, chroot_path(*code)); + + if (auto *code_path = std::get_if(&input)) { + OUTCOME_TRY(path, chroot_path(*code_path)); BOOST_OUTCOME_TRY(module, factory->loadCompiled(path)); continue; } @@ -390,7 +379,7 @@ namespace kagome::parachain { } if (auto r = pvf_worker_main_outcome(); not r) { - SL_ERROR(logger, "{}", r.error()); + SL_ERROR(logger, "PVF worker process failed: {}", r.error()); return EXIT_FAILURE; } return EXIT_SUCCESS; diff --git a/core/parachain/pvf/pvf_impl.cpp b/core/parachain/pvf/pvf_impl.cpp index 9702396834..072bcc6124 100644 --- a/core/parachain/pvf/pvf_impl.cpp +++ b/core/parachain/pvf/pvf_impl.cpp @@ -337,7 +337,7 @@ namespace kagome::parachain { auto &wasm_module = module_opt.value(); CB_TRY(auto instance, wasm_module->instantiate()); CB_TRY(auto ctx, runtime::RuntimeContextFactory::stateless(instance)); - KAGOME_PROFILE_END_L(log_, single_process_runtime_instantitation); + KAGOME_PROFILE_END(single_process_runtime_instantitation); KAGOME_PROFILE_START_L(log_, single_process_runtime_call); return cb(executor_->call(ctx, name, params)); } diff --git a/core/parachain/pvf/pvf_worker_types.hpp b/core/parachain/pvf/pvf_worker_types.hpp index 43daca7e83..7756e365ce 100644 --- a/core/parachain/pvf/pvf_worker_types.hpp +++ b/core/parachain/pvf/pvf_worker_types.hpp @@ -39,9 +39,10 @@ namespace kagome::parachain { bool force_disable_secure_mode; }; - using PvfWorkerInputCode = std::string; + using PvfWorkerInputCodePath = std::string; using PvfWorkerInputArgs = Buffer; - using PvfWorkerInput = std::variant; + using PvfWorkerInput = + std::variant; } // namespace kagome::parachain diff --git a/core/parachain/pvf/workers.cpp b/core/parachain/pvf/workers.cpp index 27e55e64e3..3efe41ad28 100644 --- a/core/parachain/pvf/workers.cpp +++ b/core/parachain/pvf/workers.cpp @@ -146,7 +146,7 @@ namespace kagome::parachain { } self->writeCode( std::move(job), - {.process = std::move(process), .code = std::nullopt}, + {.process = std::move(process), .code_path = std::nullopt}, std::move(used)); }); return; @@ -157,7 +157,7 @@ namespace kagome::parachain { void PvfWorkers::findFree(Job &&job) { auto it = std::find_if(free_.begin(), free_.end(), [&](const Worker &worker) { - return worker.code == job.code; + return worker.code_path == job.code_path; }); if (it == free_.end()) { it = free_.begin(); @@ -180,14 +180,15 @@ namespace kagome::parachain { void PvfWorkers::writeCode(Job &&job, Worker &&worker, std::shared_ptr &&used) { - if (worker.code == job.code) { + if (worker.code_path == job.code_path) { call(std::move(job), std::move(worker), std::move(used)); return; } - worker.code = job.code; - auto code = PvfWorkerInput{job.code}; + worker.code_path = job.code_path; + auto code_path = PvfWorkerInput{job.code_path}; + worker.process->writeScale( - code, + code_path, [WEAK_SELF, job{std::move(job)}, worker, used{std::move(used)}]( outcome::result r) mutable { WEAK_LOCK(self); diff --git a/core/parachain/pvf/workers.hpp b/core/parachain/pvf/workers.hpp index 9c8e4abb5b..6d4a8b8106 100644 --- a/core/parachain/pvf/workers.hpp +++ b/core/parachain/pvf/workers.hpp @@ -43,7 +43,7 @@ namespace kagome::parachain { using Cb = std::function)>; struct Job { - PvfWorkerInputCode code; + PvfWorkerInputCodePath code_path; Buffer args; Cb cb; }; @@ -52,7 +52,7 @@ namespace kagome::parachain { private: struct Worker { std::shared_ptr process; - std::optional code; + std::optional code_path; }; struct Used { Used(PvfWorkers &self); diff --git a/core/runtime/wabt/stack_limiter.cpp b/core/runtime/wabt/stack_limiter.cpp index f4accd99c0..68d036638d 100644 --- a/core/runtime/wabt/stack_limiter.cpp +++ b/core/runtime/wabt/stack_limiter.cpp @@ -754,7 +754,7 @@ namespace kagome::runtime { func_costs[i] = cost; SL_TRACE(logger, "cost {} = {}", func->name, cost); } - KAGOME_PROFILE_END_L(logger, count_costs); + KAGOME_PROFILE_END(count_costs); wabt::Global stack_height_global{""}; stack_height_global.type = wabt::Type::I32; @@ -778,7 +778,7 @@ namespace kagome::runtime { OUTCOME_TRY(detail::generate_thunks( logger, module, stack_height_var, stack_limit, func_costs)); - KAGOME_PROFILE_END_L(logger, instrument_wasm); + KAGOME_PROFILE_END(instrument_wasm); OUTCOME_TRY(wabtValidate(module)); return outcome::success(); @@ -789,7 +789,7 @@ namespace kagome::runtime { auto logger = stackLimiterLog(); KAGOME_PROFILE_START_L(logger, read_ir); OUTCOME_TRY(module, wabtDecode(uncompressed_wasm)); - KAGOME_PROFILE_END_L(logger, read_ir); + KAGOME_PROFILE_END(read_ir); OUTCOME_TRY(instrumentWithStackLimiter(module, stack_limit)); diff --git a/core/utils/thread_pool.hpp b/core/utils/thread_pool.hpp index 06d9bf1f8d..5698936d7e 100644 --- a/core/utils/thread_pool.hpp +++ b/core/utils/thread_pool.hpp @@ -36,7 +36,7 @@ namespace kagome { ThreadPool(std::shared_ptr watchdog, std::string_view pool_tag, size_t thread_count, - std::optional> ioc) + std::optional> ioc = {}) : log_(log::createLogger(fmt::format("ThreadPool:{}", pool_tag), "threads")), ioc_{ioc.has_value() ? std::move(ioc.value()) diff --git a/zombienet/precompile.sh b/zombienet/precompile.sh index 3496fcc4ed..5b1091c23f 100755 --- a/zombienet/precompile.sh +++ b/zombienet/precompile.sh @@ -15,6 +15,12 @@ undying-collator export-genesis-wasm > $DIR/undying-collator.json polkadot-parachain export-genesis-wasm > $DIR/polkadot-parachain.json polkadot build-spec --chain westend-local --raw > $DIR/westend-local.json +for id in {2000..2002} +do + polkadot-parachain export-genesis-wasm --chain glutton-westend-local-${id} \ + > $DIR/glutton-parachain-${id}.json +done + kagome --tmp --chain $DIR/rococo-local.json \ --precompile-para $DIR/adder-collator.json \ --precompile-para $DIR/undying-collator.json \ @@ -25,3 +31,9 @@ kagome --tmp --chain $DIR/westend-local.json \ kagome --tmp --chain ./old/0009-basic-warp-sync/gen-db-raw.json \ --precompile-relay + +for id in {2000..2002} +do + kagome --tmp --chain $DIR/rococo-local.json \ + --precompile-para $DIR/glutton-parachain-${id}.json +done