Add gracefull shutdown to HTTP server.

Make sure toxiproxy HTTP server close gracefully connection.
Extract endpoints initialization in separate method Routes.
Update notify signal to handle SIGTERM and SIGINT.

CHANGELOG.md

@@ -1,5 +1,7 @@
 # [Unreleased]
 
+* Gracefull shutdown of HTTP server. (#439, @miry)
+
 # [2.5.0] - 2022-09-10
 
 * Update Release steps. (#369, @neufeldtech)

api.go

@@ -1,6 +1,7 @@
 package toxiproxy
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net"
@@ -34,8 +35,14 @@ type ApiServer struct {
 	Collection *ProxyCollection
 	Metrics    *metricsContainer
 	Logger     *zerolog.Logger
+	http       *http.Server
 }
 
+const (
+	wait_timeout = 30 * time.Second
+	read_timeout = 15 * time.Second
+)
+
 func NewServer(m *metricsContainer, logger zerolog.Logger) *ApiServer {
 	return &ApiServer{
 		Collection: NewProxyCollection(),
@@ -44,23 +51,46 @@ func NewServer(m *metricsContainer, logger zerolog.Logger) *ApiServer {
 	}
 }
 
-func (server *ApiServer) PopulateConfig(filename string) {
-	file, err := os.Open(filename)
-	logger := server.Logger
-	if err != nil {
-		logger.Err(err).Str("config", filename).Msg("Error reading config file")
-		return
+func (server *ApiServer) Listen(host string, port string) error {
+	addr := net.JoinHostPort(host, port)
+	server.Logger.
+		Info().
+		Str("address", addr).
+		Msg("Starting Toxiproxy HTTP server")
+
+	server.http = &http.Server{
+		Addr:         addr,
+		Handler:      server.Routes(),
+		WriteTimeout: wait_timeout,
+		ReadTimeout:  read_timeout,
+		IdleTimeout:  60 * time.Second,
 	}
 
-	proxies, err := server.Collection.PopulateJson(server, file)
+	err := server.http.ListenAndServe()
+	if err == http.ErrServerClosed {
+		err = nil
+	}
+
+	return err
+}
+
+func (server *ApiServer) Shutdown() error {
+	if server.http == nil {
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), wait_timeout)
+	defer cancel()
+
+	err := server.http.Shutdown(ctx)
 	if err != nil {
-		logger.Err(err).Msg("Failed to populate proxies from file")
-	} else {
-		logger.Info().Int("proxies", len(proxies)).Msg("Populated proxies from file")
+		return err
 	}
+
+	return nil
 }
 
-func (server *ApiServer) Listen(host string, port string) {
+func (server *ApiServer) Routes() *mux.Router {
 	r := mux.NewRouter()
 	r.Use(hlog.NewHandler(*server.Logger))
 	r.Use(hlog.RequestIDHandler("request_id", "X-Toxiproxy-Request-Id"))
@@ -111,23 +141,22 @@ func (server *ApiServer) Listen(host string, port string) {
 		r.Handle("/metrics", server.Metrics.handler()).Name("Metrics")
 	}
 
-	server.Logger.
-		Info().
-		Str("host", host).
-		Str("port", port).
-		Str("version", Version).
-		Msgf("Starting HTTP server on endpoint %s:%s", host, port)
+	return r
+}
 
-	srv := &http.Server{
-		Handler:      r,
-		Addr:         net.JoinHostPort(host, port),
-		WriteTimeout: 30 * time.Second,
-		ReadTimeout:  10 * time.Second,
+func (server *ApiServer) PopulateConfig(filename string) {
+	file, err := os.Open(filename)
+	logger := server.Logger
+	if err != nil {
+		logger.Err(err).Str("config", filename).Msg("Error reading config file")
+		return
 	}
 
-	err := srv.ListenAndServe()
+	proxies, err := server.Collection.PopulateJson(server, file)
 	if err != nil {
-		server.Logger.Fatal().Err(err).Msg("ListenAndServe finished with error")
+		logger.Err(err).Msg("Failed to populate proxies from file")
+	} else {
+		logger.Info().Int("proxies", len(proxies)).Msg("Populated proxies from file")
 	}
 }
 

api_test.go

@@ -42,6 +42,7 @@ func WithServer(t *testing.T, f func(string)) {
 
 	f("http://localhost:8475")
 }
+
 func TestRequestId(t *testing.T) {
 	WithServer(t, func(addr string) {
 		client := http.Client{}

cmd/server/server.go

@@ -50,28 +50,31 @@ func parseArguments() cliArguments {
 }
 
 func main() {
-	// Handle SIGTERM to exit cleanly
-	signals := make(chan os.Signal, 1)
-	signal.Notify(signals, syscall.SIGTERM)
-	go func() {
-		<-signals
-		os.Exit(0)
-	}()
+	err := run()
+	if err != nil {
+		fmt.Printf("error: %v", err)
+		os.Exit(1)
+	}
+	os.Exit(0)
+}
 
+func run() error {
 	cli := parseArguments()
-	run(cli)
-}
 
-func run(cli cliArguments) {
 	if cli.printVersion {
 		fmt.Printf("toxiproxy-server version %s\n", toxiproxy.Version)
-		return
+		return nil
 	}
 
+	rand.Seed(cli.seed)
+
 	logger := setupLogger()
 	log.Logger = logger
 
-	rand.Seed(cli.seed)
+	logger.
+		Info().
+		Str("version", toxiproxy.Version).
+		Msg("Starting Toxiproxy")
 
 	metrics := toxiproxy.NewMetricsContainer(prometheus.NewRegistry())
 	server := toxiproxy.NewServer(metrics, logger)
@@ -81,11 +84,27 @@ func run(cli cliArguments) {
 	if cli.runtimeMetrics {
 		server.Metrics.RuntimeMetrics = collectors.NewRuntimeMetricCollectors()
 	}
+
 	if len(cli.config) > 0 {
 		server.PopulateConfig(cli.config)
 	}
 
-	server.Listen(cli.host, cli.port)
+	go func(server *toxiproxy.ApiServer, host, port string) {
+		err := server.Listen(host, port)
+		if err != nil {
+			server.Logger.Err(err).Msg("Server finished with error")
+		}
+	}(server, cli.host, cli.port)
+
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
+	<-signals
+	server.Logger.Info().Msg("Shutdown started")
+	err := server.Shutdown()
+	if err != nil {
+		logger.Err(err).Msg("Shutdown finished with error")
+	}
+	return nil
 }
 
 func setupLogger() zerolog.Logger {

scripts/test-e2e-hazelcast

@@ -26,8 +26,8 @@ function cleanup() {
     docker kill -s SIGQUIT member-proxy
     docker logs -t member-proxy
   fi
-  docker stop member-proxy member0 member1 member2 &> /dev/null || true
-  docker network rm toxiproxy-e2e &> /dev/null || true
+  docker stop member-proxy member0 member1 member2 &>/dev/null || true
+  docker network rm toxiproxy-e2e &>/dev/null || true
 }
 trap "cleanup" EXIT SIGINT SIGTERM