From e41aaa77fcf2470a9ad71caf5a29cd1e82e0f4f4 Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Fri, 18 Nov 2016 11:45:13 +0100 Subject: [PATCH 1/5] Waiter: add USER_SIGNALS USR1, USR2 --- lib/hutch/waiter.rb | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/hutch/waiter.rb b/lib/hutch/waiter.rb index 2cfb89c9..e2e52e03 100644 --- a/lib/hutch/waiter.rb +++ b/lib/hutch/waiter.rb @@ -3,8 +3,12 @@ module Hutch class Waiter include Logging - + + class SkipHere < Exception + end + SHUTDOWN_SIGNALS = %w(QUIT TERM INT).keep_if { |s| Signal.list.keys.include? s }.freeze + USER_SIGNALS = %w(USR1 USR2).keep_if { |s| Signal.list.keys.include?(s) }.freeze def self.wait_until_signaled new.wait_until_signaled @@ -14,10 +18,20 @@ def wait_until_signaled self.sig_read, self.sig_write = IO.pipe register_signal_handlers - wait_for_signal - - sig = sig_read.gets.strip.downcase - logger.info "caught sig#{sig}, stopping hutch..." + + begin + wait_for_signal + + sig = sig_read.gets.strip.downcase + if USER_SIGNALS.include?(sig.upcase) + logger.info "FUN TIME! #{sig}" + raise SkipHere + else + logger.info "caught sig#{sig}, stopping hutch..." + end + rescue SkipHere + retry + end end private @@ -29,7 +43,7 @@ def wait_for_signal end def register_signal_handlers - SHUTDOWN_SIGNALS.each do |sig| + (SHUTDOWN_SIGNALS + USER_SIGNALS).each do |sig| # This needs to be reentrant, so we queue up signals to be handled # in the run loop, rather than acting on signals here trap(sig) do From 2a2878ab269d3576b92a568a69afc91c95a8c497 Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Sat, 19 Nov 2016 11:56:33 +0100 Subject: [PATCH 2/5] Waiter: TTIN signal as Thread backtrace outputter --- lib/hutch/waiter.rb | 74 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/lib/hutch/waiter.rb b/lib/hutch/waiter.rb index e2e52e03..c3ff7d29 100644 --- a/lib/hutch/waiter.rb +++ b/lib/hutch/waiter.rb @@ -1,14 +1,20 @@ require 'hutch/logging' module Hutch + # Signal-handling class. class Waiter include Logging - - class SkipHere < Exception + + class ContinueProcessingSignals < RuntimeError + end + + def self.supported_signals_of(list) + list.keep_if { |s| Signal.list.keys.include? s } end - - SHUTDOWN_SIGNALS = %w(QUIT TERM INT).keep_if { |s| Signal.list.keys.include? s }.freeze - USER_SIGNALS = %w(USR1 USR2).keep_if { |s| Signal.list.keys.include?(s) }.freeze + + SHUTDOWN_SIGNALS = supported_signals_of(%w(QUIT TERM INT)).freeze + USER_SIGNALS = supported_signals_of(%w(TTIN USR1 USR2)).freeze + REGISTERED_SIGNALS = (SHUTDOWN_SIGNALS + USER_SIGNALS).freeze def self.wait_until_signaled new.wait_until_signaled @@ -18,24 +24,58 @@ def wait_until_signaled self.sig_read, self.sig_write = IO.pipe register_signal_handlers - + begin wait_for_signal - sig = sig_read.gets.strip.downcase - if USER_SIGNALS.include?(sig.upcase) - logger.info "FUN TIME! #{sig}" - raise SkipHere - else - logger.info "caught sig#{sig}, stopping hutch..." - end - rescue SkipHere + sig = sig_read.gets.strip + handle_signal(sig) + rescue ContinueProcessingSignals retry end end + def handle_signal(sig) + raise ContinueProcessingSignals unless REGISTERED_SIGNALS.include?(sig) + if user_signal?(sig) + handle_user_signal(sig) + else + handle_shutdown_signal(sig) + end + end + + def handle_shutdown_signal(sig) + logger.info "caught SIG#{sig}, stopping hutch..." + end + + # @raises ContinueProcessingSignals + def handle_user_signal(sig) + case sig + when 'TTIN' then handle_ttin + when 'USR1', 'USR2' then handle_user_defined_signals + else + raise "Assertion failed - unhandled signal: #{sig.inspect}" + end + raise ContinueProcessingSignals + end + private + def handle_ttin + Thread.list.each do |thread| + logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" + if thread.backtrace + logger.warn thread.backtrace.join("\n") + else + logger.warn '' + end + end + end + + def handle_user_defined_signals + logger.warn "SIG#{sig} noted. Continuing." # TODO: Find useful behavior + end + attr_accessor :sig_read, :sig_write def wait_for_signal @@ -43,7 +83,7 @@ def wait_for_signal end def register_signal_handlers - (SHUTDOWN_SIGNALS + USER_SIGNALS).each do |sig| + REGISTERED_SIGNALS.each do |sig| # This needs to be reentrant, so we queue up signals to be handled # in the run loop, rather than acting on signals here trap(sig) do @@ -51,5 +91,9 @@ def register_signal_handlers end end end + + def user_signal?(sig) + USER_SIGNALS.include?(sig) + end end end From 1eeed8f2f8d76f9546baf5253319fdc1d75632fb Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Sat, 19 Nov 2016 12:11:30 +0100 Subject: [PATCH 3/5] Spec: upcase signal name in message --- spec/hutch/waiter_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/hutch/waiter_spec.rb b/spec/hutch/waiter_spec.rb index b5c8cd5c..38d1dfab 100644 --- a/spec/hutch/waiter_spec.rb +++ b/spec/hutch/waiter_spec.rb @@ -20,7 +20,7 @@ def start_kill_thread(signal) context "a #{signal} signal is received" do it "logs that hutch is stopping" do expect(Hutch::Logging.logger).to receive(:info) - .with("caught sig#{signal.downcase}, stopping hutch...") + .with("caught SIG#{signal}, stopping hutch...") start_kill_thread(signal) described_class.wait_until_signaled From 461161517fffa890aad063bbfff2a3a3aabf824f Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Sat, 19 Nov 2016 13:05:47 +0100 Subject: [PATCH 4/5] Waiter: USR2 is JRuby-supported - to have a user-facing feature signal, we pick one that can be used on JVM --- lib/hutch/waiter.rb | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/lib/hutch/waiter.rb b/lib/hutch/waiter.rb index c3ff7d29..93ada63a 100644 --- a/lib/hutch/waiter.rb +++ b/lib/hutch/waiter.rb @@ -13,7 +13,8 @@ def self.supported_signals_of(list) end SHUTDOWN_SIGNALS = supported_signals_of(%w(QUIT TERM INT)).freeze - USER_SIGNALS = supported_signals_of(%w(TTIN USR1 USR2)).freeze + # We have chosen a JRuby-supported signal + USER_SIGNALS = supported_signals_of(%w(USR2)).freeze REGISTERED_SIGNALS = (SHUTDOWN_SIGNALS + USER_SIGNALS).freeze def self.wait_until_signaled @@ -51,17 +52,15 @@ def handle_shutdown_signal(sig) # @raises ContinueProcessingSignals def handle_user_signal(sig) case sig - when 'TTIN' then handle_ttin - when 'USR1', 'USR2' then handle_user_defined_signals - else - raise "Assertion failed - unhandled signal: #{sig.inspect}" + when 'USR2' then handle_usr2 + else raise "Assertion failed - unhandled signal: #{sig.inspect}" end raise ContinueProcessingSignals end private - def handle_ttin + def handle_usr2 Thread.list.each do |thread| logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" if thread.backtrace @@ -72,10 +71,6 @@ def handle_ttin end end - def handle_user_defined_signals - logger.warn "SIG#{sig} noted. Continuing." # TODO: Find useful behavior - end - attr_accessor :sig_read, :sig_write def wait_for_signal From 77c2a20b6b0910abcd1afaa480a780a0cd691169 Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Sun, 20 Nov 2016 19:40:14 +0100 Subject: [PATCH 5/5] Waiter: clarify thread dump logging --- lib/hutch/waiter.rb | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/hutch/waiter.rb b/lib/hutch/waiter.rb index 93ada63a..bb5bcb8d 100644 --- a/lib/hutch/waiter.rb +++ b/lib/hutch/waiter.rb @@ -2,6 +2,9 @@ module Hutch # Signal-handling class. + # + # Currently, the signal USR2 performs a thread dump, + # while QUIT, TERM and INT all perform a graceful shutdown. class Waiter include Logging @@ -45,29 +48,34 @@ def handle_signal(sig) end end - def handle_shutdown_signal(sig) - logger.info "caught SIG#{sig}, stopping hutch..." - end - # @raises ContinueProcessingSignals def handle_user_signal(sig) case sig - when 'USR2' then handle_usr2 + when 'USR2' then log_thread_backtraces else raise "Assertion failed - unhandled signal: #{sig.inspect}" end raise ContinueProcessingSignals end + def handle_shutdown_signal(sig) + logger.info "caught SIG#{sig}, stopping hutch..." + end + private - def handle_usr2 + def log_thread_backtraces + logger.info 'Requested a VM-wide thread stack trace dump...' Thread.list.each do |thread| - logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" - if thread.backtrace - logger.warn thread.backtrace.join("\n") - else - logger.warn '' - end + logger.info "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" + logger.info backtrace_for(thread) + end + end + + def backtrace_for(thread) + if thread.backtrace + thread.backtrace.join("\n") + else + '' end end