From 112b704194ebeb05b8715aa2cf9ebbc28099b8e7 Mon Sep 17 00:00:00 2001 From: Michael Nikitochkin Date: Fri, 9 Sep 2022 23:12:36 +0200 Subject: [PATCH] Add gracefull shutdown to HTTP server. Make sure toxiproxy HTTP server close gracefully connection. Extract endpoints initialization in separate method Routes. Update notify signal to handle SIGTERM and SIGINT. --- CHANGELOG.md | 2 + api.go | 77 ++++++++++++++++++++++++++------------ api_test.go | 1 + cmd/server/server.go | 45 +++++++++++++++------- scripts/test-e2e-hazelcast | 4 +- 5 files changed, 90 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65ec9a421..4dbdab936 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # [Unreleased] +* Gracefull shutdown of HTTP server. (@miry) + # [2.5.0] - 2022-09-10 * Update Release steps. (#369, @neufeldtech) diff --git a/api.go b/api.go index c9c4cbedf..90efd1b29 100644 --- a/api.go +++ b/api.go @@ -1,6 +1,7 @@ package toxiproxy import ( + "context" "encoding/json" "fmt" "net" @@ -34,8 +35,14 @@ type ApiServer struct { Collection *ProxyCollection Metrics *metricsContainer Logger *zerolog.Logger + http *http.Server } +const ( + wait_timeout = 30 * time.Second + read_timeout = 15 * time.Second +) + func NewServer(m *metricsContainer, logger zerolog.Logger) *ApiServer { return &ApiServer{ Collection: NewProxyCollection(), @@ -44,23 +51,46 @@ func NewServer(m *metricsContainer, logger zerolog.Logger) *ApiServer { } } -func (server *ApiServer) PopulateConfig(filename string) { - file, err := os.Open(filename) - logger := server.Logger - if err != nil { - logger.Err(err).Str("config", filename).Msg("Error reading config file") - return +func (server *ApiServer) Listen(host string, port string) error { + addr := net.JoinHostPort(host, port) + server.Logger. + Info(). + Str("address", addr). + Msg("Starting Toxiproxy HTTP server") + + server.http = &http.Server{ + Addr: addr, + Handler: server.Routes(), + WriteTimeout: wait_timeout, + ReadTimeout: read_timeout, + IdleTimeout: 60 * time.Second, } - proxies, err := server.Collection.PopulateJson(server, file) + err := server.http.ListenAndServe() + if err == http.ErrServerClosed { + err = nil + } + + return err +} + +func (server *ApiServer) Shutdown() error { + if server.http == nil { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), wait_timeout) + defer cancel() + + err := server.http.Shutdown(ctx) if err != nil { - logger.Err(err).Msg("Failed to populate proxies from file") - } else { - logger.Info().Int("proxies", len(proxies)).Msg("Populated proxies from file") + return err } + + return nil } -func (server *ApiServer) Listen(host string, port string) { +func (server *ApiServer) Routes() *mux.Router { r := mux.NewRouter() r.Use(hlog.NewHandler(*server.Logger)) r.Use(hlog.RequestIDHandler("request_id", "X-Toxiproxy-Request-Id")) @@ -111,23 +141,22 @@ func (server *ApiServer) Listen(host string, port string) { r.Handle("/metrics", server.Metrics.handler()).Name("Metrics") } - server.Logger. - Info(). - Str("host", host). - Str("port", port). - Str("version", Version). - Msgf("Starting HTTP server on endpoint %s:%s", host, port) + return r +} - srv := &http.Server{ - Handler: r, - Addr: net.JoinHostPort(host, port), - WriteTimeout: 30 * time.Second, - ReadTimeout: 10 * time.Second, +func (server *ApiServer) PopulateConfig(filename string) { + file, err := os.Open(filename) + logger := server.Logger + if err != nil { + logger.Err(err).Str("config", filename).Msg("Error reading config file") + return } - err := srv.ListenAndServe() + proxies, err := server.Collection.PopulateJson(server, file) if err != nil { - server.Logger.Fatal().Err(err).Msg("ListenAndServe finished with error") + logger.Err(err).Msg("Failed to populate proxies from file") + } else { + logger.Info().Int("proxies", len(proxies)).Msg("Populated proxies from file") } } diff --git a/api_test.go b/api_test.go index eec6c06da..a0138c3c6 100644 --- a/api_test.go +++ b/api_test.go @@ -42,6 +42,7 @@ func WithServer(t *testing.T, f func(string)) { f("http://localhost:8475") } + func TestRequestId(t *testing.T) { WithServer(t, func(addr string) { client := http.Client{} diff --git a/cmd/server/server.go b/cmd/server/server.go index e414a6958..465fb7f7a 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -50,28 +50,31 @@ func parseArguments() cliArguments { } func main() { - // Handle SIGTERM to exit cleanly - signals := make(chan os.Signal, 1) - signal.Notify(signals, syscall.SIGTERM) - go func() { - <-signals - os.Exit(0) - }() + err := run() + if err != nil { + fmt.Printf("error: %v", err) + os.Exit(1) + } + os.Exit(0) +} +func run() error { cli := parseArguments() - run(cli) -} -func run(cli cliArguments) { if cli.printVersion { fmt.Printf("toxiproxy-server version %s\n", toxiproxy.Version) - return + return nil } + rand.Seed(cli.seed) + logger := setupLogger() log.Logger = logger - rand.Seed(cli.seed) + logger. + Info(). + Str("version", toxiproxy.Version). + Msg("Starting Toxiproxy") metrics := toxiproxy.NewMetricsContainer(prometheus.NewRegistry()) server := toxiproxy.NewServer(metrics, logger) @@ -81,11 +84,27 @@ func run(cli cliArguments) { if cli.runtimeMetrics { server.Metrics.RuntimeMetrics = collectors.NewRuntimeMetricCollectors() } + if len(cli.config) > 0 { server.PopulateConfig(cli.config) } - server.Listen(cli.host, cli.port) + go func(server *toxiproxy.ApiServer, host, port string) { + err := server.Listen(host, port) + if err != nil { + server.Logger.Err(err).Msg("Server finished with error") + } + }(server, cli.host, cli.port) + + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + <-signals + server.Logger.Info().Msg("Shutdown started") + err := server.Shutdown() + if err != nil { + logger.Err(err).Msg("Shutdown finished with error") + } + return nil } func setupLogger() zerolog.Logger { diff --git a/scripts/test-e2e-hazelcast b/scripts/test-e2e-hazelcast index 27f17726c..2f675fe01 100755 --- a/scripts/test-e2e-hazelcast +++ b/scripts/test-e2e-hazelcast @@ -26,8 +26,8 @@ function cleanup() { docker kill -s SIGQUIT member-proxy docker logs -t member-proxy fi - docker stop member-proxy member0 member1 member2 &> /dev/null || true - docker network rm toxiproxy-e2e &> /dev/null || true + docker stop member-proxy member0 member1 member2 &>/dev/null || true + docker network rm toxiproxy-e2e &>/dev/null || true } trap "cleanup" EXIT SIGINT SIGTERM