Merge branch 'master' into ci/debian-trixie

.github/workflows/test.yml

@@ -131,7 +131,8 @@ jobs:
           - name: "Self-hosted: Linux: clang-16 TSAN WAVM"
             run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM
           - name: "Self-hosted: Linux: clang-16 UBSAN"
-            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DUBSAN=ON
+            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DUBSAN=ON -DUBSAN_TRAP=OFF -DUBSAN_ABORT=ON
+            env: UBSAN_OPTIONS=print_stacktrace=1
         # Need to fix
         #  - name: "Self-hosted: Linux: clang-16 External Project"
         #    run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake

.github/workflows/zombie-builder.yml

@@ -26,13 +26,18 @@ on:
         required: false
         default: 'false'
         type: 'boolean'
+      polkadot_sdk_tag:
+        description: 'Custom Polkadot SDK tag'
+        required: false
+        type: 'string'
 
 env:
   DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/
   GITHUB_HUNTER_USERNAME: ${{ secrets.HUNTER_USERNAME }}
   GITHUB_HUNTER_TOKEN: ${{ secrets.HUNTER_TOKEN }}
   CACHE_VERSION: v001
   CACHE_PATHS: ./zombienet/docker/cargo
+  POLKADOT_SDK_TAG: ${{ github.event.inputs.polkadot_sdk_tag }}
 
 jobs:
   build_polkadot_builder:

CMakeLists.txt

@@ -98,6 +98,12 @@ option(TSAN "Enable thread sanitizer" OFF)
 option(UBSAN "Enable UB sanitizer" OFF)
 # cmake-format: on
 
+# useful for tests, an error message will be printed
+option(UBSAN_ABORT "Make UB sanitizer abort on error" OFF)
+
+# useful to catch the exact wrong place with a debugger
+option(UBSAN_TRAP "Make UB sanitizer execute a trap instruction on error" OFF)
+
 set(RECOMMENDED_CLANG_FORMAT_VERSION 16)
 
 include(CheckCXXCompilerFlag)

cmake/Hunter/config.cmake

@@ -116,6 +116,12 @@ hunter_config(
     SHA1 884932112bc75996eeecd4a7bbcb932565fe9859
 )
 
+hunter_config(
+    scale
+    URL  https://github.com/qdrvm/scale-codec-cpp/archive/e1a3c7afafc2eeda0c8e2daed08da6b7789f44b3.zip
+    SHA1 b56bcda34fb0d293c88d8b642b2f3fdc2d16a3e5
+)
+
 hunter_config(
     erasure_coding_crust
 #    VERSION 0.0.8
@@ -128,7 +134,7 @@ hunter_config(
     soralog
 #    VERSION 0.2.4
     URL  https://github.com/qdrvm/soralog/archive/refs/tags/v0.2.4.tar.gz
-    SHA1 832a32cf134f8caab4a79c03024ac8408f848dae
+    SHA1 1de495d8a3a73c1e940be3fdddf263a2d673aec1
     KEEP_PACKAGE_SOURCES
 )
 

cmake/functions.cmake

@@ -15,7 +15,7 @@ function(addtest test_name)
   set(xml_output "--gtest_output=xml:${CMAKE_BINARY_DIR}/xunit/xunit-${test_name}.xml")
   add_test(
       NAME ${test_name}
-      COMMAND $<TARGET_FILE:${test_name}> ${xml_output}
+      COMMAND $<TARGET_FILE:${test_name}> ${xml_output} "--output-on-failure"
   )
   set_target_properties(${test_name} PROPERTIES
       RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test_bin

cmake/toolchain/flags/sanitize_undefined.cmake

@@ -10,7 +10,16 @@ set(FLAGS
     -fsanitize=undefined
     -fno-omit-frame-pointer
     -g
+    -O0
+    -fsanitize-ignorelist="${CMAKE_CURRENT_LIST_DIR}/ubsan_ignore.txt"
     )
+if (UBSAN_ABORT) 
+  list(APPEND FLAGS -fno-sanitize-recover=undefined)
+endif()
+if (UBSAN_TRAP)
+  list(APPEND FLAGS -fsanitize-trap=undefined)
+endif()
+
 foreach(FLAG IN LISTS FLAGS)
   add_cache_flag(CMAKE_CXX_FLAGS ${FLAG})
   add_cache_flag(CMAKE_C_FLAGS ${FLAG})

cmake/toolchain/flags/ubsan_ignore.txt

@@ -0,0 +1,2 @@
+# false-positive on applying non-zero offset to null pointer: the pointer is never dereferenced
+fun:_ZN9rapidjson8internal5StackINS_12CrtAllocatorEE7Reserve*

core/application/impl/app_state_manager_impl.cpp

@@ -10,9 +10,11 @@
 #include <functional>
 
 namespace kagome::application {
-  std::atomic_bool AppStateManagerImpl::signals_enabled{false};
+  std::weak_ptr<AppStateManagerImpl> AppStateManagerImpl::wp_to_myself;
+
+  std::atomic_bool AppStateManagerImpl::shutting_down_signals_enabled{false};
 
-  void AppStateManagerImpl::signalsEnable() {
+  void AppStateManagerImpl::shuttingDownSignalsEnable() {
     // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
     struct sigaction act;
     memset(&act, 0, sizeof(act));
@@ -25,13 +27,14 @@ namespace kagome::application {
     sigaction(SIGINT, &act, nullptr);
     sigaction(SIGTERM, &act, nullptr);
     sigaction(SIGQUIT, &act, nullptr);
-    signals_enabled.store(true);
+    shutting_down_signals_enabled.store(true);
     sigprocmask(SIG_UNBLOCK, &act.sa_mask, nullptr);
   }
 
-  void AppStateManagerImpl::signalsDisable() {
+  void AppStateManagerImpl::shuttingDownSignalsDisable() {
     auto expected = true;
-    if (not signals_enabled.compare_exchange_strong(expected, false)) {
+    if (not shutting_down_signals_enabled.compare_exchange_strong(expected,
+                                                                  false)) {
       return;
     }
     // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
@@ -43,24 +46,59 @@ namespace kagome::application {
     sigaction(SIGQUIT, &act, nullptr);
   }
 
-  std::weak_ptr<AppStateManagerImpl> AppStateManagerImpl::wp_to_myself;
-
   void AppStateManagerImpl::shuttingDownSignalsHandler(int signal) {
-    signalsDisable();
+    shuttingDownSignalsDisable();
     if (auto self = wp_to_myself.lock()) {
       SL_TRACE(self->logger_, "Shutdown signal {} received", signal);
       self->shutdown();
     }
   }
 
+  std::atomic_bool AppStateManagerImpl::log_rotate_signals_enabled{false};
+
+  void AppStateManagerImpl::logRotateSignalsEnable() {
+    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+    struct sigaction act;
+    memset(&act, 0, sizeof(act));
+    act.sa_handler = logRotateSignalsHandler;
+    sigemptyset(&act.sa_mask);
+    sigaddset(&act.sa_mask, SIGHUP);
+    sigprocmask(SIG_BLOCK, &act.sa_mask, nullptr);
+    sigaction(SIGHUP, &act, nullptr);
+    log_rotate_signals_enabled.store(true);
+    sigprocmask(SIG_UNBLOCK, &act.sa_mask, nullptr);
+  }
+
+  void AppStateManagerImpl::logRotateSignalsDisable() {
+    auto expected = true;
+    if (not log_rotate_signals_enabled.compare_exchange_strong(expected,
+                                                               false)) {
+      return;
+    }
+    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+    struct sigaction act;
+    memset(&act, 0, sizeof(act));
+    act.sa_handler = SIG_DFL;
+    sigaction(SIGHUP, &act, nullptr);
+  }
+
+  void AppStateManagerImpl::logRotateSignalsHandler(int signal) {
+    if (auto self = wp_to_myself.lock()) {
+      SL_TRACE(self->logger_, "Log rotate signal {} received", signal);
+      log::doLogRotate();
+    }
+  }
+
   AppStateManagerImpl::AppStateManagerImpl()
       : logger_(log::createLogger("AppStateManager", "application")) {
-    signalsEnable();
-    SL_TRACE(logger_, "Signal handler set up");
+    shuttingDownSignalsEnable();
+    logRotateSignalsEnable();
+    SL_TRACE(logger_, "Signal handlers set up");
   }
 
   AppStateManagerImpl::~AppStateManagerImpl() {
-    signalsDisable();
+    shuttingDownSignalsDisable();
+    logRotateSignalsDisable();
     wp_to_myself.reset();
   }
 
@@ -223,7 +261,7 @@ namespace kagome::application {
   }
 
   void AppStateManagerImpl::shutdown() {
-    signalsDisable();
+    shuttingDownSignalsDisable();
     if (state_.load() == State::ReadyToStop) {
       SL_TRACE(logger_, "Shutting down requested, but app is ready to stop");
       return;

core/application/impl/app_state_manager_impl.hpp

@@ -49,12 +49,18 @@ namespace kagome::application {
     void doShutdown() override;
 
    private:
-    static std::atomic_bool signals_enabled;
-    static void signalsEnable();
-    static void signalsDisable();
     static std::weak_ptr<AppStateManagerImpl> wp_to_myself;
+
+    static std::atomic_bool shutting_down_signals_enabled;
+    static void shuttingDownSignalsEnable();
+    static void shuttingDownSignalsDisable();
     static void shuttingDownSignalsHandler(int);
 
+    static std::atomic_bool log_rotate_signals_enabled;
+    static void logRotateSignalsEnable();
+    static void logRotateSignalsDisable();
+    static void logRotateSignalsHandler(int);
+
     void shutdownRequestWaiting();
 
     log::Logger logger_;

core/common/blob.hpp

@@ -89,8 +89,8 @@
   struct fmt::formatter<space_name::class_name>                                \
       : fmt::formatter<space_name::class_name::Base> {                         \
     template <typename FormatCtx>                                              \
-    auto format(const space_name::class_name &blob, FormatCtx &ctx) const      \
-        -> decltype(ctx.out()) {                                               \
+    auto format(const space_name::class_name &blob,                            \
+                FormatCtx &ctx) const -> decltype(ctx.out()) {                 \
       return fmt::formatter<space_name::class_name::Base>::format(blob, ctx);  \
     }                                                                          \
   };
@@ -269,19 +269,17 @@ struct fmt::formatter<kagome::common::Blob<N>> {
   // Formats the Blob using the parsed format specification (presentation)
   // stored in this formatter.
   template <typename FormatContext>
-  auto format(const kagome::common::Blob<N> &blob, FormatContext &ctx) const
-      -> decltype(ctx.out()) {
-    // ctx.out() is an output iterator to write to.
-
+  auto format(const kagome::common::Blob<N> &blob,
+              FormatContext &ctx) const -> decltype(ctx.out()) {
     if (presentation == 's') {
-      return fmt::format_to(
-          ctx.out(),
-          "0x{:04x}…{:04x}",
-          // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
-          htobe16(*reinterpret_cast<const uint16_t *>(blob.data())),
-          // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
-          htobe16(*reinterpret_cast<const uint16_t *>(blob.data() + blob.size()
-                                                      - sizeof(uint16_t))));
+      if constexpr (N > 4) {
+        uint16_t head = static_cast<uint16_t>(blob[1])
+                      | (static_cast<uint16_t>(blob[0]) << 8);
+        uint16_t tail = static_cast<uint16_t>(blob[blob.size() - 1])
+                      | (static_cast<uint16_t>(blob[blob.size() - 2]) << 8);
+        return fmt::format_to(ctx.out(), "0x{:04x}…{:04x}", head, tail);
+      }
+      // else fallback to normal print
     }
 
     return fmt::format_to(ctx.out(), "0x{}", blob.toHex());

core/dispute_coordinator/participation/impl/participation_impl.cpp

@@ -252,6 +252,7 @@ namespace kagome::dispute {
                       ctx->available_data->pov,
                       ctx->request.candidate_receipt,
                       ctx->validation_code.value(),
+                      runtime::PvfExecTimeoutKind::Approval,
                       [cb{std::move(cb)}](
                           const outcome::result<parachain::Pvf::Result> &res) {
                         // we cast votes (either positive or negative)

core/injector/application_injector.cpp

@@ -272,11 +272,14 @@ namespace {
     // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions)
     options.max_open_files = soft_limit.value() / 2;
 
+    const std::unordered_map<std::string, int32_t> column_ttl = {
+        {"avaliability_storage", 25 * 60 * 60}};  // 25 hours
     auto db_res =
         storage::RocksDb::create(app_config.databasePath(chain_spec->id()),
                                  options,
                                  app_config.dbCacheSize(),
-                                 prevent_destruction);
+                                 prevent_destruction,
+                                 column_ttl);
     if (!db_res) {
       auto log = log::createLogger("Injector", "injector");
       log->critical(
@@ -335,6 +338,13 @@ namespace {
         std::move(kademlia_config));
   }
 
+  sptr<libp2p::protocol::IdentifyConfig> get_identify_config() {
+    libp2p::protocol::IdentifyConfig identify_config;
+    identify_config.protocols = {"/ipfs/id/1.0.0", "/substrate/1.0"};
+    return std::make_shared<libp2p::protocol::IdentifyConfig>(
+        std::move(identify_config));
+  }
+
   template <typename Injector>
   sptr<blockchain::BlockTree> get_block_tree(const Injector &injector) {
     auto chain_events_engine =
@@ -890,6 +900,10 @@ namespace {
             di::bind<network::FetchAvailableDataProtocol>.template to<network::FetchAvailableDataProtocolImpl>(),
             di::bind<network::WarpProtocol>.template to<network::WarpProtocolImpl>(),
             di::bind<network::SendDisputeProtocol>.template to<network::SendDisputeProtocolImpl>(),
+            bind_by_lambda<libp2p::protocol::IdentifyConfig>(
+                [](const auto &injector) {
+                  return get_identify_config();
+                }),
 
             // user-defined overrides...
             std::forward<decltype(args)>(args)...);
@@ -923,7 +937,7 @@ namespace kagome::injector {
   KagomeNodeInjector::KagomeNodeInjector(
       sptr<application::AppConfiguration> app_config)
       : pimpl_{std::make_unique<KagomeNodeInjectorImpl>(
-          makeKagomeNodeInjector(std::move(app_config)))} {}
+            makeKagomeNodeInjector(std::move(app_config)))} {}
 
   sptr<application::AppConfiguration> KagomeNodeInjector::injectAppConfig() {
     return pimpl_->injector_

core/log/logger.cpp

@@ -120,6 +120,11 @@ namespace kagome::log {
     }
   }
 
+  void doLogRotate() {
+    auto logging_system = ensure_logger_system_is_initialized();
+    logging_system->callRotateForAllSinks();
+  }
+
   Logger createLogger(const std::string &tag) {
     auto logging_system = ensure_logger_system_is_initialized();
     return std::static_pointer_cast<soralog::LoggerFactory>(logging_system)

core/log/logger.hpp

@@ -31,6 +31,8 @@ namespace kagome::log {
 
   void tuneLoggingSystem(const std::vector<std::string> &cfg);
 
+  void doLogRotate();
+
   static const std::string defaultGroupName("kagome");
 
   [[nodiscard]] Logger createLogger(const std::string &tag);

core/parachain/approval/approval_distribution.cpp

@@ -1758,6 +1758,7 @@ namespace kagome::parachain {
                                   available_data.pov,
                                   candidate_receipt,
                                   validation_code,
+                                  runtime::PvfExecTimeoutKind::Approval,
                                   std::move(cb));
         };