From ab65d84e0855c9364f73b8e3bd0fb00a5a6c4f2d Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Wed, 28 Dec 2022 15:47:09 +0200 Subject: [PATCH] Handle `SIGTERM` for the docker containers + relay (#1735) * Handle SIGTERM for some docker containers * Implement SIGTERM handling for the relay --- Cargo.lock | 18 ++++++++-- .../bridges/common/generate_messages.sh | 4 ++- ...rialto-parachain-resubmitter-entrypoint.sh | 2 +- ...elay-millau-rialto-parachain-entrypoint.sh | 2 +- deployments/monitoring/docker-compose.yml | 2 ++ .../rialto-chainspec-exporter-entrypoint.sh | 4 ++- .../rialto-parachain-registrar-entrypoint.sh | 2 +- deployments/ui/docker-compose.yml | 1 + relays/bin-substrate/Cargo.toml | 2 ++ relays/bin-substrate/src/cli/mod.rs | 36 +++++++++++++++++-- relays/bin-substrate/src/main.rs | 5 +-- 11 files changed, 65 insertions(+), 13 deletions(-) mode change 100644 => 100755 deployments/bridges/common/generate_messages.sh diff --git a/Cargo.lock b/Cargo.lock index fbaa5dfc21116..fcbc05b648749 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10472,14 +10472,26 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signal-hook" -version = "0.3.10" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c98891d737e271a2954825ef19e46bd16bdb98e2746f2eec4f7a4ef7946efd1" +checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" dependencies = [ "libc", "signal-hook-registry", ] +[[package]] +name = "signal-hook-async-std" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4aa94397e2023af5b7cff5b8d4785e935cfb77f0e4aab0cae3b26258ace556" +dependencies = [ + "async-io", + "futures-lite", + "libc", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -11653,6 +11665,8 @@ dependencies = [ "relay-wococo-client", "rialto-parachain-runtime", "rialto-runtime", + "signal-hook", + "signal-hook-async-std", "sp-core", "sp-keyring", "sp-runtime", diff --git a/deployments/bridges/common/generate_messages.sh b/deployments/bridges/common/generate_messages.sh old mode 100644 new mode 100755 index 565b82bc9170f..156fb5b5d2b16 --- a/deployments/bridges/common/generate_messages.sh +++ b/deployments/bridges/common/generate_messages.sh @@ -17,11 +17,13 @@ SECONDARY_EXTRA_ARGS=${SECONDARY_EXTRA_ARGS:-""} +trap "echo Exiting... TERM; exit $?" TERM + # Sleep a bit between messages rand_sleep() { SUBMIT_DELAY_S=`shuf -i 0-$MAX_SUBMIT_DELAY_S -n 1` echo "Sleeping $SUBMIT_DELAY_S seconds..." - sleep $SUBMIT_DELAY_S + sleep $SUBMIT_DELAY_S & wait $! NOW=`date "+%Y-%m-%d %H:%M:%S"` echo "Woke up at $NOW" } diff --git a/deployments/bridges/rialto-parachain-millau/entrypoints/relay-messages-to-rialto-parachain-resubmitter-entrypoint.sh b/deployments/bridges/rialto-parachain-millau/entrypoints/relay-messages-to-rialto-parachain-resubmitter-entrypoint.sh index cf4c4612d6938..4653ae396c6d4 100755 --- a/deployments/bridges/rialto-parachain-millau/entrypoints/relay-messages-to-rialto-parachain-resubmitter-entrypoint.sh +++ b/deployments/bridges/rialto-parachain-millau/entrypoints/relay-messages-to-rialto-parachain-resubmitter-entrypoint.sh @@ -15,7 +15,7 @@ sleep 15 # we need tip around `526186677695 - 17800827994 = 508_385_849_701`. Let's round it # up to `1_000_000_000_000`. -/home/user/substrate-relay resubmit-transactions millau \ +exec /home/user/substrate-relay resubmit-transactions millau \ --target-host millau-node-alice \ --target-port 9944 \ --target-signer //RialtoParachain.MessagesSender \ diff --git a/deployments/bridges/rialto-parachain-millau/entrypoints/relay-millau-rialto-parachain-entrypoint.sh b/deployments/bridges/rialto-parachain-millau/entrypoints/relay-millau-rialto-parachain-entrypoint.sh index 39c28fb4cf654..7685f5f5ec2b9 100755 --- a/deployments/bridges/rialto-parachain-millau/entrypoints/relay-millau-rialto-parachain-entrypoint.sh +++ b/deployments/bridges/rialto-parachain-millau/entrypoints/relay-millau-rialto-parachain-entrypoint.sh @@ -24,7 +24,7 @@ RIALTO_PARACHAIN_RELAY_ACCOUNT=${EXT_RIALTO_PARACHAIN_RELAY_ACCOUNT:-//Millau.He # Give chain a little bit of time to process initialization transaction sleep 6 -/home/user/substrate-relay relay-headers-and-messages millau-rialto-parachain \ +exec /home/user/substrate-relay relay-headers-and-messages millau-rialto-parachain \ --millau-host millau-node-alice \ --millau-port 9944 \ --millau-signer $MILLAU_RELAY_ACCOUNT \ diff --git a/deployments/monitoring/docker-compose.yml b/deployments/monitoring/docker-compose.yml index 1fa50c68d0e4b..05209041d13b8 100644 --- a/deployments/monitoring/docker-compose.yml +++ b/deployments/monitoring/docker-compose.yml @@ -20,6 +20,8 @@ services: - "3000:3000" depends_on: - prometheus-metrics + # SIGTERM won't work because of our custom entrypoint. Should be ok to use SIGKILL. + stop_signal: SIGKILL entrypoint: sh -c "echo 'sleeping for 10m' && sleep 600 && /run.sh" grafana-matrix-notifier: diff --git a/deployments/networks/entrypoints/rialto-chainspec-exporter-entrypoint.sh b/deployments/networks/entrypoints/rialto-chainspec-exporter-entrypoint.sh index 0898978096d33..eac2a80de87bd 100755 --- a/deployments/networks/entrypoints/rialto-chainspec-exporter-entrypoint.sh +++ b/deployments/networks/entrypoints/rialto-chainspec-exporter-entrypoint.sh @@ -1,6 +1,8 @@ #!/bin/bash set -xeu +trap "echo Exiting... TERM; exit $?" TERM + /home/user/rialto-bridge-node build-spec \ --chain local \ --raw \ @@ -11,4 +13,4 @@ set -xeu # by the container running this script. If this script ends, the volume will be detached # and our chain spec will be lost when it'll go online again. Hence the never-ending # script which keeps volume online until container is stopped. -tail -f /dev/null +tail -f /dev/null & wait $! diff --git a/deployments/networks/entrypoints/rialto-parachain-registrar-entrypoint.sh b/deployments/networks/entrypoints/rialto-parachain-registrar-entrypoint.sh index bb201ac5dad60..1c33dd0084160 100755 --- a/deployments/networks/entrypoints/rialto-parachain-registrar-entrypoint.sh +++ b/deployments/networks/entrypoints/rialto-parachain-registrar-entrypoint.sh @@ -3,7 +3,7 @@ set -xeu sleep 15 -/home/user/substrate-relay register-parachain rialto-parachain \ +exec /home/user/substrate-relay register-parachain rialto-parachain \ --parachain-host rialto-parachain-collator-alice \ --parachain-port 9944 \ --relaychain-host rialto-node-alice \ diff --git a/deployments/ui/docker-compose.yml b/deployments/ui/docker-compose.yml index 8b3f8178c36e5..2d400609c4428 100644 --- a/deployments/ui/docker-compose.yml +++ b/deployments/ui/docker-compose.yml @@ -9,5 +9,6 @@ services: LETSENCRYPT_EMAIL: admin@parity.io CHAIN_1_SUBSTRATE_PROVIDER: ${UI_CHAIN_1:-ws://localhost:9944} CHAIN_2_SUBSTRATE_PROVIDER: ${UI_CHAIN_2:-ws://localhost:19944} + stop_signal: SIGKILL ports: - "8080:80" diff --git a/relays/bin-substrate/Cargo.toml b/relays/bin-substrate/Cargo.toml index 9ae6a98868e39..de1807cdc9708 100644 --- a/relays/bin-substrate/Cargo.toml +++ b/relays/bin-substrate/Cargo.toml @@ -16,6 +16,8 @@ log = "0.4.17" num-format = "0.4" num-traits = "0.2" structopt = "0.3" +signal-hook = "0.3.14" +signal-hook-async-std = "0.2.2" strum = { version = "0.21.0", features = ["derive"] } # Bridge dependencies diff --git a/relays/bin-substrate/src/cli/mod.rs b/relays/bin-substrate/src/cli/mod.rs index 5733aa411d8e7..4bd735c6334ad 100644 --- a/relays/bin-substrate/src/cli/mod.rs +++ b/relays/bin-substrate/src/cli/mod.rs @@ -18,7 +18,11 @@ use std::convert::TryInto; +use async_std::prelude::*; use codec::{Decode, Encode}; +use futures::{select, FutureExt}; +use signal_hook::consts::*; +use signal_hook_async_std::Signals; use structopt::{clap::arg_enum, StructOpt}; use strum::{EnumString, EnumVariantNames}; @@ -37,6 +41,9 @@ mod relay_messages; mod relay_parachains; mod resubmit_transactions; +/// The target that will be used when publishing logs related to this pallet. +pub const LOG_TARGET: &str = "bridge"; + /// Parse relay CLI args. pub fn parse_args() -> Command { Command::from_args() @@ -100,8 +107,7 @@ impl Command { } /// Run the command. - pub async fn run(self) -> anyhow::Result<()> { - self.init_logger(); + async fn do_run(self) -> anyhow::Result<()> { match self { Self::RelayHeaders(arg) => arg.run().await?, Self::RelayMessages(arg) => arg.run().await?, @@ -114,6 +120,32 @@ impl Command { } Ok(()) } + + /// Run the command. + pub async fn run(self) { + self.init_logger(); + + let exit_signals = match Signals::new([SIGINT, SIGTERM]) { + Ok(signals) => signals, + Err(e) => { + log::error!(target: LOG_TARGET, "Could not register exit signals: {}", e); + return + }, + }; + let run = self.do_run().fuse(); + futures::pin_mut!(exit_signals, run); + + select! { + signal = exit_signals.next().fuse() => { + log::info!(target: LOG_TARGET, "Received exit signal {:?}", signal); + }, + result = run => { + if let Err(e) = result { + log::error!(target: LOG_TARGET, "substrate-relay: {}", e); + } + }, + } + } } arg_enum! { diff --git a/relays/bin-substrate/src/main.rs b/relays/bin-substrate/src/main.rs index bc84786ee27f2..8bd89a4193627 100644 --- a/relays/bin-substrate/src/main.rs +++ b/relays/bin-substrate/src/main.rs @@ -24,8 +24,5 @@ mod cli; fn main() { let command = cli::parse_args(); let run = command.run(); - let result = async_std::task::block_on(run); - if let Err(error) = result { - log::error!(target: "bridge", "substrate-relay: {}", error); - } + async_std::task::block_on(run); }