Add precompilation for Glutton parachain (#2198)

* Add precompilation for Glutton parachain used in 0013 zombienet test, also minor code cleanups

cmake/functions.cmake

@@ -40,6 +40,8 @@ function(add_flag flag)
   check_cxx_compiler_flag(${flag} FLAG${flag_var})
   if(FLAG${flag_var} EQUAL 1)
     add_compile_options(${flag})
+  else()
+    message(AUTHOR_WARNING "Compiler flag ${flag} requested, but not supported by ${CMAKE_CXX_COMPILER}")
   endif()
 endfunction()
 

core/application/modes/precompile_wasm.cpp

@@ -5,9 +5,11 @@
  */
 
 #include "application/modes/precompile_wasm.hpp"
+#include <qtils/error.hpp>
 
 #include "blockchain/block_tree.hpp"
 #include "common/bytestr.hpp"
+#include "log/logger.hpp"
 #include "parachain/pvf/pool.hpp"
 #include "parachain/pvf/session_params.hpp"
 #include "runtime/common/runtime_instances_pool.hpp"
@@ -36,26 +38,42 @@ namespace kagome::application::mode {
     return EXIT_SUCCESS;
   }
 
+  std::optional<Buffer> readRuntimeFile(const std::filesystem::path &path,
+                                        const log::Logger &log) {
+    Buffer bytes;
+    auto res = readFile(bytes, path);
+    if (!res) {
+      SL_ERROR(log, "file {} read error {}", path.string(), res.error());
+      return std::nullopt;
+    }
+    auto text = byte2str(bytes);
+    if (text.starts_with("{")) {
+      SL_ERROR(log, "expected WASM, got JSON, file {}", path.string());
+      return std::nullopt;
+    }
+    if (text.starts_with("0x")) {
+      auto bytes_res = common::unhexWith0x(text);
+      if (!bytes_res) {
+        SL_ERROR(log, "failed to unhex a seemingly hex file {}", path.string());
+        return std::nullopt;
+      }
+      bytes = std::move(bytes_res.value());
+    }
+    return bytes;
+  }
+
   outcome::result<void> PrecompileWasmMode::runOutcome() const {
     auto block = block_tree_->bestBlock();
 
     // relay already precompiled, when getting version for genesis state
 
     for (auto &path : config_.parachains) {
       SL_INFO(log_, "precompile parachain {}", path.string());
-      Buffer bytes;
-      if (not readFile(bytes, path)) {
-        SL_WARN(log_, "read error, file {}", path.string());
-        continue;
-      }
-      auto text = byte2str(bytes);
-      if (text.starts_with("{")) {
-        SL_WARN(log_, "expected WASM, got JSON, file {}", path.string());
+      auto bytes_opt = readRuntimeFile(path, log_);
+      if (!bytes_opt) {
         continue;
       }
-      if (text.starts_with("0x")) {
-        BOOST_OUTCOME_TRY(bytes, common::unhexWith0x(text));
-      }
+      auto &bytes = bytes_opt.value();
       // https://github.com/paritytech/polkadot-sdk/blob/b4ae5b01da280f754ccc00b94314a30b658182a1/polkadot/parachain/src/primitives.rs#L74-L81
       auto code_hash = hasher_->blake2b_256(bytes);
       OUTCOME_TRY(config,

core/log/profiling_logger.hpp

@@ -17,8 +17,9 @@ namespace kagome::log {
   struct ProfileScope {
     using Clock = ::kagome::clock::SteadyClockImpl;
 
-    ProfileScope(std::string_view scope, log::Logger logger)
-        : scope{scope}, logger{logger} {
+    explicit ProfileScope(std::string_view scope,
+                          log::Logger logger = profiling_logger)
+        : scope{scope}, logger{std::move(logger)} {
       BOOST_ASSERT(logger != nullptr);
       start = Clock{}.now();
     }
@@ -58,19 +59,15 @@ namespace kagome::log {
 #define KAGOME_PROFILE_START_L(logger, scope) \
   auto _profiling_scope_##scope = ::kagome::log::ProfileScope{#scope, logger};
 
-#define KAGOME_PROFILE_END_L(logger, scope) _profiling_scope_##scope.end();
-
 #define KAGOME_PROFILE_START(scope) \
   KAGOME_PROFILE_START_L(::kagome::log::profiling_logger, scope)
-#define KAGOME_PROFILE_END(scope) \
-  KAGOME_PROFILE_END_L(::kagome::log::profiling_logger, scope)
+#define KAGOME_PROFILE_END(scope) _profiling_scope_##scope.end();
 
 #else
 
 #define KAGOME_PROFILE_START(scope)
 #define KAGOME_PROFILE_END(scope)
 
 #define KAGOME_PROFILE_START_L(logger, scope)
-#define KAGOME_PROFILE_END_L(logger, scope)
 
 #endif

core/parachain/CMakeLists.txt

@@ -66,7 +66,6 @@ target_link_libraries(kagome_pvf_worker
     soralog::soralog
     logger
     log_configurator
-
     binaryen_wasm_memory
     binaryen_wasm_memory_factory
     binaryen_runtime_external_interface
@@ -75,7 +74,6 @@ target_link_libraries(kagome_pvf_worker
     binaryen_instance_environment_factory
     binaryen_module_factory
     wasm_compiler
-
     runtime_properties_cache
     sr25519_provider
     ed25519_provider

core/parachain/pvf/kagome_pvf_worker.cpp

@@ -12,7 +12,6 @@
 #include <string>
 #include <system_error>
 #include <vector>
-#include "runtime/wabt/instrument.hpp"
 
 #ifdef __linux__
 #include <linux/landlock.h>
@@ -45,6 +44,7 @@
 #include "runtime/binaryen/module/module_factory_impl.hpp"
 #include "runtime/module_instance.hpp"
 #include "runtime/runtime_context.hpp"
+#include "runtime/wabt/instrument.hpp"
 #include "utils/mkdirs.hpp"
 
 // rust reference: polkadot-sdk/polkadot/node/core/pvf/execute-worker/src/lib.rs
@@ -237,18 +237,6 @@ namespace kagome::parachain {
 
   outcome::result<std::shared_ptr<runtime::ModuleFactory>> createModuleFactory(
       const auto &injector, RuntimeEngine engine) {
-    // class Foo {
-    //  public:Foo(
-    //       crypto::EcdsaProvider&,
-    //       crypto::Secp256k1Provider&) {}
-    // };
-    //
-    // std::ignore = injector.template create<std::shared_ptr<Foo>>();
-    // std::ignore = injector.template create<
-    //     std::shared_ptr<crypto::EcdsaProvider>>();
-    // std::ignore = injector.template create<
-    //     std::shared_ptr<crypto::Secp256k1Provider>>();
-
     switch (engine) {
       case RuntimeEngine::kBinaryen:
         return injector.template create<
@@ -343,8 +331,9 @@ namespace kagome::parachain {
     std::shared_ptr<runtime::Module> module;
     while (true) {
       OUTCOME_TRY(input, decodeInput<PvfWorkerInput>());
-      if (auto *code = std::get_if<PvfWorkerInputCode>(&input)) {
-        OUTCOME_TRY(path, chroot_path(*code));
+
+      if (auto *code_path = std::get_if<PvfWorkerInputCodePath>(&input)) {
+        OUTCOME_TRY(path, chroot_path(*code_path));
         BOOST_OUTCOME_TRY(module, factory->loadCompiled(path));
         continue;
       }
@@ -390,7 +379,7 @@ namespace kagome::parachain {
     }
 
     if (auto r = pvf_worker_main_outcome(); not r) {
-      SL_ERROR(logger, "{}", r.error());
+      SL_ERROR(logger, "PVF worker process failed: {}", r.error());
       return EXIT_FAILURE;
     }
     return EXIT_SUCCESS;

core/parachain/pvf/pvf_impl.cpp

@@ -337,7 +337,7 @@ namespace kagome::parachain {
       auto &wasm_module = module_opt.value();
       CB_TRY(auto instance, wasm_module->instantiate());
       CB_TRY(auto ctx, runtime::RuntimeContextFactory::stateless(instance));
-      KAGOME_PROFILE_END_L(log_, single_process_runtime_instantitation);
+      KAGOME_PROFILE_END(single_process_runtime_instantitation);
       KAGOME_PROFILE_START_L(log_, single_process_runtime_call);
       return cb(executor_->call<ValidationResult>(ctx, name, params));
     }

core/parachain/pvf/pvf_worker_types.hpp

@@ -39,9 +39,10 @@ namespace kagome::parachain {
     bool force_disable_secure_mode;
   };
 
-  using PvfWorkerInputCode = std::string;
+  using PvfWorkerInputCodePath = std::string;
 
   using PvfWorkerInputArgs = Buffer;
 
-  using PvfWorkerInput = std::variant<PvfWorkerInputCode, PvfWorkerInputArgs>;
+  using PvfWorkerInput =
+      std::variant<PvfWorkerInputCodePath, PvfWorkerInputArgs>;
 }  // namespace kagome::parachain

core/parachain/pvf/workers.cpp

@@ -146,7 +146,7 @@ namespace kagome::parachain {
             }
             self->writeCode(
                 std::move(job),
-                {.process = std::move(process), .code = std::nullopt},
+                {.process = std::move(process), .code_path = std::nullopt},
                 std::move(used));
           });
       return;
@@ -157,7 +157,7 @@ namespace kagome::parachain {
   void PvfWorkers::findFree(Job &&job) {
     auto it =
         std::find_if(free_.begin(), free_.end(), [&](const Worker &worker) {
-          return worker.code == job.code;
+          return worker.code_path == job.code_path;
         });
     if (it == free_.end()) {
       it = free_.begin();
@@ -180,14 +180,15 @@ namespace kagome::parachain {
   void PvfWorkers::writeCode(Job &&job,
                              Worker &&worker,
                              std::shared_ptr<Used> &&used) {
-    if (worker.code == job.code) {
+    if (worker.code_path == job.code_path) {
       call(std::move(job), std::move(worker), std::move(used));
       return;
     }
-    worker.code = job.code;
-    auto code = PvfWorkerInput{job.code};
+    worker.code_path = job.code_path;
+    auto code_path = PvfWorkerInput{job.code_path};
+
     worker.process->writeScale(
-        code,
+        code_path,
         [WEAK_SELF, job{std::move(job)}, worker, used{std::move(used)}](
             outcome::result<void> r) mutable {
           WEAK_LOCK(self);

core/parachain/pvf/workers.hpp

@@ -43,7 +43,7 @@ namespace kagome::parachain {
 
     using Cb = std::function<void(outcome::result<Buffer>)>;
     struct Job {
-      PvfWorkerInputCode code;
+      PvfWorkerInputCodePath code_path;
       Buffer args;
       Cb cb;
     };
@@ -52,7 +52,7 @@ namespace kagome::parachain {
    private:
     struct Worker {
       std::shared_ptr<ProcessAndPipes> process;
-      std::optional<PvfWorkerInputCode> code;
+      std::optional<PvfWorkerInputCodePath> code_path;
     };
     struct Used {
       Used(PvfWorkers &self);

core/runtime/wabt/stack_limiter.cpp

@@ -754,7 +754,7 @@ namespace kagome::runtime {
       func_costs[i] = cost;
       SL_TRACE(logger, "cost {} = {}", func->name, cost);
     }
-    KAGOME_PROFILE_END_L(logger, count_costs);
+    KAGOME_PROFILE_END(count_costs);
 
     wabt::Global stack_height_global{""};
     stack_height_global.type = wabt::Type::I32;
@@ -778,7 +778,7 @@ namespace kagome::runtime {
     OUTCOME_TRY(detail::generate_thunks(
         logger, module, stack_height_var, stack_limit, func_costs));
 
-    KAGOME_PROFILE_END_L(logger, instrument_wasm);
+    KAGOME_PROFILE_END(instrument_wasm);
 
     OUTCOME_TRY(wabtValidate(module));
     return outcome::success();
@@ -789,7 +789,7 @@ namespace kagome::runtime {
     auto logger = stackLimiterLog();
     KAGOME_PROFILE_START_L(logger, read_ir);
     OUTCOME_TRY(module, wabtDecode(uncompressed_wasm));
-    KAGOME_PROFILE_END_L(logger, read_ir);
+    KAGOME_PROFILE_END(read_ir);
 
     OUTCOME_TRY(instrumentWithStackLimiter(module, stack_limit));
 

core/utils/thread_pool.hpp

@@ -36,7 +36,7 @@ namespace kagome {
     ThreadPool(std::shared_ptr<Watchdog> watchdog,
                std::string_view pool_tag,
                size_t thread_count,
-               std::optional<std::shared_ptr<boost::asio::io_context>> ioc)
+               std::optional<std::shared_ptr<boost::asio::io_context>> ioc = {})
         : log_(log::createLogger(fmt::format("ThreadPool:{}", pool_tag),
                                  "threads")),
           ioc_{ioc.has_value() ? std::move(ioc.value())

zombienet/precompile.sh

@@ -15,6 +15,12 @@ undying-collator export-genesis-wasm > $DIR/undying-collator.json
 polkadot-parachain export-genesis-wasm > $DIR/polkadot-parachain.json
 polkadot build-spec --chain westend-local --raw > $DIR/westend-local.json
 
+for id in {2000..2002}
+do
+  polkadot-parachain export-genesis-wasm --chain glutton-westend-local-${id} \
+    > $DIR/glutton-parachain-${id}.json
+done
+
 kagome --tmp --chain $DIR/rococo-local.json \
   --precompile-para $DIR/adder-collator.json \
   --precompile-para $DIR/undying-collator.json \
@@ -25,3 +31,9 @@ kagome --tmp --chain $DIR/westend-local.json \
 
 kagome --tmp --chain ./old/0009-basic-warp-sync/gen-db-raw.json \
   --precompile-relay
+
+for id in {2000..2002}
+do
+  kagome --tmp --chain $DIR/rococo-local.json \
+    --precompile-para $DIR/glutton-parachain-${id}.json
+done