backport: bitcoin#19160, #21663, #21669, #21732, #21738, #21750, #21775, #21812

.cirrus.yml

@@ -87,7 +87,7 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
 
 task:
-  name: '[previous releases, uses qt5 dev package and some depends packages] [unsigned char] [focal]'
+  name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [focal]'
   # For faster CI feedback, immediately schedule a task that compiles most modules
   << : *CREDITS_TEMPLATE
   << : *GLOBAL_TASK_TEMPLATE
@@ -116,10 +116,10 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
 
 task:
-  name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [focal]'
+  name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [hirsute]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: ubuntu:focal
+    image: ubuntu:hirsute
   env:
     FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
 
@@ -132,7 +132,7 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
 
 task:
-  name: '[multiprocess] [focal]'
+  name: '[multiprocess, DEBUG] [focal]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
     image: ubuntu:focal
@@ -157,8 +157,9 @@ task:
 
 task:
   name: 'macOS 11 native [gui] [no depends]'
-  macos_brew_addon_script:
-    - brew install boost libevent berkeley-db4 qt miniupnpc ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt
+  brew_install_script:
+    - brew update
+    - brew install boost libevent berkeley-db4 qt@5 miniupnpc ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt
   << : *GLOBAL_TASK_TEMPLATE
   macos_instance:
     # Use latest image, but hardcode version to avoid silent upgrades (and breaks)

.gitlab-ci.yml

@@ -203,16 +203,13 @@ x86_64-pc-linux-gnu-nowallet:
     HOST: x86_64-pc-linux-gnu
     DEP_OPTS: "NO_WALLET=1"
 
-## TODO: # Skip on non-depends, non-multiprocess PRs
-## if: type != pull_request OR commit_message =~ /depends:|multiprocess:/
-#x86_64-pc-linux-gnu-multiprocess:
-#  extends:
-#    - .build-depends-template
-#    - .skip-in-fast-mode-template
-#  variables:
-#    HOST: x86_64-pc-linux-gnu
-#    DEP_OPTS: "MULTIPROCESS=1"
-#
+x86_64-pc-linux-gnu-multiprocess:
+  extends:
+    - .build-depends-template
+    - .skip-in-fast-mode-template
+  variables:
+    HOST: x86_64-pc-linux-gnu
+    DEP_OPTS: "MULTIPROCESS=1"
 
 x86_64-apple-darwin:
   extends:
@@ -309,14 +306,14 @@ linux64_nowallet-build:
   variables:
     BUILD_TARGET: linux64_nowallet
 
-#linux64_multiprocess-build:
-#  extends:
-#    - .build-template
-#    - .skip-in-fast-mode-template
-#  needs:
-#    - x86_64-pc-linux-gnu-multiprocess
-#  variables:
-#    BUILD_TARGET: linux64_multiprocess
+linux64_multiprocess-build:
+  extends:
+    - .build-template
+    - .skip-in-fast-mode-template
+  needs:
+    - x86_64-pc-linux-gnu-multiprocess
+  variables:
+    BUILD_TARGET: linux64_multiprocess
 
 #linux64_valgrind-build:
 #  extends:
@@ -381,16 +378,14 @@ linux64_ubsan-test:
   variables:
     BUILD_TARGET: linux64_ubsan
 
-# TODO: enable multiprocess back in CI once it has any value
-# or in case if any new backports to test
-#linux64_multiprocess-test:
-#  extends:
-#    - .test-template
-#    - .skip-in-fast-mode-template
-#  needs:
-#    - linux64_multiprocess-build
-#  variables:
-#    BUILD_TARGET: linux64_multiprocess
+linux64_multiprocess-test:
+  extends:
+    - .test-template
+    - .skip-in-fast-mode-template
+  needs:
+    - linux64_multiprocess-build
+  variables:
+    BUILD_TARGET: linux64_multiprocess
 
 #linux64_valgrind-test:
 #  extends:

ci/test/00_setup_env_native_multiprocess.sh

@@ -7,8 +7,8 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_multiprocess
-export PACKAGES="cmake python3"
-export DEP_OPTS="MULTIPROCESS=1"
+export PACKAGES="cmake python3 llvm clang"
+export DEP_OPTS="DEBUG=1 MULTIPROCESS=1"
 export GOAL="install"
-export BITCOIN_CONFIG="--with-boost-process"
+export BITCOIN_CONFIG="--with-boost-process --enable-debug CC=clang CXX=clang++" # Use clang to avoid OOM
 export TEST_RUNNER_ENV="BITCOIND=dash-node"

depends/packages/native_libmultiprocess.mk

@@ -1,8 +1,8 @@
 package=native_libmultiprocess
-$(package)_version=5741d750a04e644a03336090d8979c6d033e32c0
+$(package)_version=d576d975debdc9090bd2582f83f49c76c0061698
 $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive
 $(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=ac848db49a6ed53e423c62d54bd87f1f08cbb0326254a8667e10bbfe5bf032a4
+$(package)_sha256_hash=9f8b055c8bba755dc32fe799b67c20b91e7b13e67cadafbc54c0f1def057a370
 $(package)_dependencies=native_capnp
 
 define $(package)_config_cmds

doc/multiprocess.md

@@ -15,7 +15,7 @@ Specific next steps after backporting [bitcoin#10102](https://github.com/bitcoin
 
 ## Debugging
 
-After backporting [bitcoin#10102](https://github.com/bitcoin/bitcoin/pull/10102), the `-debug=ipc` command line option can be used to see requests and responses between processes.
+The `-debug=ipc` command line option can be used to see requests and responses between processes.
 
 ## Installation
 
@@ -33,3 +33,40 @@ DASHD=dash-node test/functional/test_runner.py
 The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option).
 
 Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general.
+
+## IPC implementation details
+
+Cross process Node, Wallet, and Chain interfaces are defined in
+[`src/interfaces/`](../src/interfaces/). These are C++ classes which follow
+[conventions](developer-notes.md#internal-interface-guidelines), like passing
+serializable arguments so they can be called from different processes, and
+making methods pure virtual so they can have proxy implementations that forward
+calls between processes.
+
+When Wallet, Node, and Chain code is running in the same process, calling any
+interface method invokes the implementation directly. When code is running in
+different processes, calling an interface method invokes a proxy interface
+implementation that communicates with a remote process and invokes the real
+implementation in the remote process. The
+[libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) code
+generation tool internally generates proxy client classes and proxy server
+classes for this purpose that are thin wrappers around Cap'n Proto
+[client](https://capnproto.org/cxxrpc.html#clients) and
+[server](https://capnproto.org/cxxrpc.html#servers) classes, which handle the
+actual serialization and socket communication.
+
+As much as possible, calls between processes are meant to work the same as
+calls within a single process without adding limitations or requiring extra
+implementation effort. Processes communicate with each other by calling regular
+[C++ interface methods](../src/interfaces/README.md). Method arguments and
+return values are automatically serialized and sent between processes. Object
+references and `std::function` arguments are automatically tracked and mapped
+to allow invoked code to call back into invoking code at any time, and there is
+a 1:1 threading model where any thread invoking a method in another process has
+a corresponding thread in the invoked process responsible for executing all
+method calls from the source thread, without blocking I/O or holding up another
+call, and using the same thread local variables, locks, and callbacks between
+calls. The forwarding, tracking, and threading is implemented inside the
+[libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) library
+which has the design goal of making calls between processes look like calls in
+the same process to the extent possible.

src/Makefile.am

@@ -94,6 +94,7 @@ EXTRA_LIBRARIES += \
   $(LIBBITCOIN_CONSENSUS) \
   $(LIBBITCOIN_SERVER) \
   $(LIBBITCOIN_CLI) \
+  $(LIBBITCOIN_IPC) \
   $(LIBBITCOIN_WALLET) \
   $(LIBBITCOIN_WALLET_TOOL) \
   $(LIBBITCOIN_ZMQ)
@@ -219,7 +220,10 @@ BITCOIN_CORE_H = \
   init/common.h \
   interfaces/chain.h \
   interfaces/coinjoin.h \
+  interfaces/echo.h \
   interfaces/handler.h \
+  interfaces/init.h \
+  interfaces/ipc.h \
   interfaces/node.h \
   interfaces/wallet.h \
   key.h \
@@ -407,6 +411,8 @@ obj/build.h: FORCE
 	  "$(abs_top_srcdir)"
 libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h
 
+ipc/capnp/libbitcoin_ipc_a-ipc.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h)
+
 # server: shared between dashd and dash-qt
 # Contains code accessing mempool and chain state that is meant to be separated
 # from wallet and gui code (see node/README.md). Shared code should go in
@@ -772,7 +778,9 @@ libbitcoin_util_a_SOURCES = \
   clientversion.cpp \
   compat/strnlen.cpp \
   fs.cpp \
+  interfaces/echo.cpp \
   interfaces/handler.cpp \
+  interfaces/init.cpp \
   logging.cpp \
   messagesigner.cpp \
   random.cpp \
@@ -851,17 +859,17 @@ bitcoin_bin_ldadd = \
 
 bitcoin_bin_ldadd += $(BACKTRACE_LIB) $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(GMP_LIBS)
 
-dashd_SOURCES = $(bitcoin_daemon_sources)
+dashd_SOURCES = $(bitcoin_daemon_sources) init/bitcoind.cpp
 dashd_CPPFLAGS = $(bitcoin_bin_cppflags)
 dashd_CXXFLAGS = $(bitcoin_bin_cxxflags)
 dashd_LDFLAGS = $(bitcoin_bin_ldflags)
 dashd_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd)
 
-dash_node_SOURCES = $(bitcoin_daemon_sources)
+dash_node_SOURCES = $(bitcoin_daemon_sources) init/bitcoin-node.cpp
 dash_node_CPPFLAGS = $(bitcoin_bin_cppflags)
 dash_node_CXXFLAGS = $(bitcoin_bin_cxxflags)
 dash_node_LDFLAGS = $(bitcoin_bin_ldflags)
-dash_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd)
+dash_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) $(LIBBITCOIN_IPC) $(LIBMULTIPROCESS_LIBS)
 
 # dash-cli binary #
 dash_cli_SOURCES = bitcoin-cli.cpp
@@ -1001,6 +1009,39 @@ endif
 osx_debug: $(bin_PROGRAMS)
 	for i in $(bin_PROGRAMS); do mkdir -p $$i.dSYM/Contents/Resources/DWARF && $(DSYMUTIL_FLAT) -o $$i.dSYM/Contents/Resources/DWARF/$$(basename $$i) $$i &> /dev/null ; done
 
+libbitcoin_ipc_mpgen_input = \
+  ipc/capnp/echo.capnp \
+  ipc/capnp/init.capnp
+EXTRA_DIST += $(libbitcoin_ipc_mpgen_input)
+%.capnp:
+
+if BUILD_MULTIPROCESS
+LIBBITCOIN_IPC=libbitcoin_ipc.a
+libbitcoin_ipc_a_SOURCES = \
+  ipc/capnp/init-types.h \
+  ipc/capnp/protocol.cpp \
+  ipc/capnp/protocol.h \
+  ipc/exception.h \
+  ipc/interfaces.cpp \
+  ipc/process.cpp \
+  ipc/process.h \
+  ipc/protocol.h
+libbitcoin_ipc_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)
+libbitcoin_ipc_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(LIBMULTIPROCESS_CFLAGS)
+
+include $(MPGEN_PREFIX)/include/mpgen.mk
+libbitcoin_ipc_mpgen_output = \
+  $(libbitcoin_ipc_mpgen_input:=.c++) \
+  $(libbitcoin_ipc_mpgen_input:=.h) \
+  $(libbitcoin_ipc_mpgen_input:=.proxy-client.c++) \
+  $(libbitcoin_ipc_mpgen_input:=.proxy-server.c++) \
+  $(libbitcoin_ipc_mpgen_input:=.proxy-types.c++) \
+  $(libbitcoin_ipc_mpgen_input:=.proxy-types.h) \
+  $(libbitcoin_ipc_mpgen_input:=.proxy.h)
+nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output)
+CLEANFILES += $(libbitcoin_ipc_mpgen_output)
+endif
+
 include Makefile.crc32c.include
 include Makefile.leveldb.include
 include Makefile.test_util.include

src/bitcoind.cpp

@@ -13,6 +13,7 @@
 #include <compat.h>
 #include <init.h>
 #include <interfaces/chain.h>
+#include <interfaces/init.h>
 #include <node/context.h>
 #include <node/ui_interface.h>
 #include <noui.h>
@@ -109,10 +110,8 @@ int fork_daemon(bool nochdir, bool noclose, TokenPipeEnd& endpoint)
 
 #endif
 
-static bool AppInit(int argc, char* argv[])
+static bool AppInit(NodeContext& node, int argc, char* argv[])
 {
-    NodeContext node;
-
     bool fRet = false;
 
     util::ThreadSetInternalName("init");
@@ -264,10 +263,18 @@ MAIN_FUNCTION
     util::WinCmdLineArgs winArgs;
     std::tie(argc, argv) = winArgs.get();
 #endif
+
+    NodeContext node;
+    int exit_status;
+    std::unique_ptr<interfaces::Init> init = interfaces::MakeNodeInit(node, argc, argv, exit_status);
+    if (!init) {
+        return exit_status;
+    }
+
     SetupEnvironment();
 
     // Connect dashd signal handlers
     noui_connect();
 
-    return (AppInit(argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE);
+    return (AppInit(node, argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE);
 }

src/init.cpp

@@ -718,7 +718,7 @@ void SetupServerArgs(NodeContext& node)
     argsman.AddArg("-disablegovernance", strprintf("Disable governance validation (0-1, default: %u)", 0), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
-    argsman.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + "(default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
+    argsman.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-minsporkkeys=<n>", "Overrides minimum spork signers to change spork value. Only useful for regtest and devnet. Using this on mainnet or testnet will ban you.", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-printpriority", strprintf("Log transaction fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-pushversion", "Protocol version to report to other nodes", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);

src/init/bitcoin-node.cpp

@@ -0,0 +1,45 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <interfaces/echo.h>
+#include <interfaces/init.h>
+#include <interfaces/ipc.h>
+#include <node/context.h>
+
+#include <memory>
+
+namespace init {
+namespace {
+const char* EXE_NAME = "dash-node";
+
+class BitcoinNodeInit : public interfaces::Init
+{
+public:
+    BitcoinNodeInit(NodeContext& node, const char* arg0)
+        : m_node(node),
+          m_ipc(interfaces::MakeIpc(EXE_NAME, arg0, *this))
+    {
+        m_node.init = this;
+    }
+    std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); }
+    interfaces::Ipc* ipc() override { return m_ipc.get(); }
+    NodeContext& m_node;
+    std::unique_ptr<interfaces::Ipc> m_ipc;
+};
+} // namespace
+} // namespace init
+
+namespace interfaces {
+std::unique_ptr<Init> MakeNodeInit(NodeContext& node, int argc, char* argv[], int& exit_status)
+{
+    auto init = std::make_unique<init::BitcoinNodeInit>(node, argc > 0 ? argv[0] : "");
+    // Check if bitcoin-node is being invoked as an IPC server. If so, then
+    // bypass normal execution and just respond to requests over the IPC
+    // channel and return null.
+    if (init->m_ipc->startSpawnedProcess(argc, argv, exit_status)) {
+        return nullptr;
+    }
+    return init;
+}
+} // namespace interfaces