UPSTREAM: <carry>: server: add support for log rotation (etcd-io#12774)

bill-of-materials.json

@@ -431,6 +431,15 @@
 			}
 		]
 	},
+	{
+		"project": "gopkg.in/natefinch/lumberjack.v2",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 1
+			}
+		]
+	},
 	{
 		"project": "gopkg.in/yaml.v2",
 		"licenses": [

embed/config.go

@@ -67,6 +67,15 @@ const (
 	StdErrLogOutput  = "stderr"
 	StdOutLogOutput  = "stdout"
 
+	// DefaultLogRotationConfig is the default configuration used for log rotation.
+	// Log rotation is disabled by default.
+	// MaxSize    = 100 // MB
+	// MaxAge     = 0 // days (no limit)
+	// MaxBackups = 0 // no limit
+	// LocalTime  = false // use computers local time, UTC by default
+	// Compress   = false // compress the rotated log in gzip format
+	DefaultLogRotationConfig = `{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}`
+
 	// DefaultStrictReconfigCheck is the default value for "--strict-reconfig-check" flag.
 	// It's enabled by default.
 	DefaultStrictReconfigCheck = true
@@ -85,6 +94,7 @@ var (
 	ErrConflictBootstrapFlags = fmt.Errorf("multiple discovery or bootstrap flags are set. " +
 		"Choose one of \"initial-cluster\", \"discovery\" or \"discovery-srv\"")
 	ErrUnsetAdvertiseClientURLsFlag = fmt.Errorf("--advertise-client-urls is required when --listen-client-urls is set explicitly")
+	ErrLogRotationInvalidLogOutput  = fmt.Errorf("--log-outputs requires a single file path when --log-rotate-config-json is defined")
 
 	DefaultInitialAdvertisePeerURLs = "http://localhost:2380"
 	DefaultAdvertiseClientURLs      = "http://localhost:2379"
@@ -313,7 +323,10 @@ type Config struct {
 	//  - file path to append server logs to.
 	// It can be multiple when "Logger" is zap.
 	LogOutputs []string `json:"log-outputs"`
-
+	// EnableLogRotation enables log rotation of a single LogOutputs file target.
+	EnableLogRotation bool `json:"enable-log-rotation"`
+	// LogRotationConfigJSON is a passthrough allowing a log rotation JSON config to be passed directly.
+	LogRotationConfigJSON string `json:"log-rotation-config-json"`
 	// ZapLoggerBuilder is used to build the zap logger.
 	ZapLoggerBuilder func(*Config) error
 
@@ -430,14 +443,16 @@ func NewConfig() *Config {
 
 		PreVote: false, // TODO: enable by default in v3.5
 
-		loggerMu:            new(sync.RWMutex),
-		logger:              nil,
-		Logger:              "capnslog",
-		DeprecatedLogOutput: []string{DefaultLogOutput},
-		LogOutputs:          []string{DefaultLogOutput},
-		Debug:               false,
-		LogLevel:            logutil.DefaultLogLevel,
-		LogPkgLevels:        "",
+		loggerMu:              new(sync.RWMutex),
+		logger:                nil,
+		Logger:                "capnslog",
+		DeprecatedLogOutput:   []string{DefaultLogOutput},
+		LogOutputs:            []string{DefaultLogOutput},
+		Debug:                 false,
+		LogLevel:              logutil.DefaultLogLevel,
+		LogPkgLevels:          "",
+		EnableLogRotation:     false,
+		LogRotationConfigJSON: DefaultLogRotationConfig,
 	}
 	cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
 	return cfg

embed/config_logging.go

@@ -16,9 +16,11 @@ package embed
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"os"
 	"reflect"
 	"sync"
@@ -30,6 +32,7 @@ import (
 	"go.uber.org/zap/zapcore"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/grpclog"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
 // GetLogger returns the logger.
@@ -143,6 +146,11 @@ func (cfg *Config) setupLogging() error {
 				}
 			}
 		}
+		if cfg.EnableLogRotation {
+			if err := setupLogRotation(cfg.LogOutputs, cfg.LogRotationConfigJSON); err != nil {
+				return err
+			}
+		}
 
 		outputPaths, errOutputPaths := make([]string, 0), make([]string, 0)
 		isJournal := false
@@ -164,8 +172,15 @@ func (cfg *Config) setupLogging() error {
 				errOutputPaths = append(errOutputPaths, StdOutLogOutput)
 
 			default:
-				outputPaths = append(outputPaths, v)
-				errOutputPaths = append(errOutputPaths, v)
+				var path string
+				if cfg.EnableLogRotation {
+					// append rotate scheme to logs managed by lumberjack log rotation
+					path = fmt.Sprintf("rotate:%s", v)
+				} else {
+					path = v
+				}
+				outputPaths = append(outputPaths, path)
+				errOutputPaths = append(errOutputPaths, path)
 			}
 		}
 
@@ -310,3 +325,46 @@ func NewZapCoreLoggerBuilder(lg *zap.Logger, cr zapcore.Core, syncer zapcore.Wri
 		return nil
 	}
 }
+
+type logRotationConfig struct {
+	*lumberjack.Logger
+}
+
+// Sync implements zap.Sink
+func (logRotationConfig) Sync() error { return nil }
+
+// setupLogRotation initializes log rotation for a single file path target.
+func setupLogRotation(logOutputs []string, logRotateConfigJSON string) error {
+	var logRotationConfig logRotationConfig
+	outputFilePaths := 0
+	for _, v := range logOutputs {
+		switch v {
+		case DefaultLogOutput, StdErrLogOutput, StdOutLogOutput:
+			continue
+		default:
+			outputFilePaths++
+		}
+	}
+	// log rotation requires file target
+	if len(logOutputs) == 1 && outputFilePaths == 0 {
+		return ErrLogRotationInvalidLogOutput
+	}
+	// support max 1 file target for log rotation
+	if outputFilePaths > 1 {
+		return ErrLogRotationInvalidLogOutput
+	}
+
+	if err := json.Unmarshal([]byte(logRotateConfigJSON), &logRotationConfig); err != nil {
+		switch err.(type) {
+		case *json.SyntaxError:
+			return fmt.Errorf("improperly formatted log rotation config: %w", err)
+		case *json.UnmarshalTypeError:
+			return fmt.Errorf("invalid log rotation config: %w", err)
+		}
+	}
+	zap.RegisterSink("rotate", func(u *url.URL) (zap.Sink, error) {
+		logRotationConfig.Filename = u.Path
+		return &logRotationConfig, nil
+	})
+	return nil
+}

embed/config_test.go

@@ -15,6 +15,7 @@
 package embed
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/url"
@@ -202,3 +203,77 @@ func TestAutoCompactionModeParse(t *testing.T) {
 		}
 	}
 }
+
+func TestLogRotation(t *testing.T) {
+	tests := []struct {
+		name              string
+		logOutputs        []string
+		logRotationConfig string
+		wantErr           bool
+		wantErrMsg        error
+	}{
+		{
+			name:              "mixed log output targets",
+			logOutputs:        []string{"stderr", "/tmp/path"},
+			logRotationConfig: `{"maxsize": 1}`,
+		},
+		{
+			name:              "no file targets",
+			logOutputs:        []string{"stderr"},
+			logRotationConfig: `{"maxsize": 1}`,
+			wantErr:           true,
+			wantErrMsg:        ErrLogRotationInvalidLogOutput,
+		},
+		{
+			name:              "multiple file targets",
+			logOutputs:        []string{"/tmp/path1", "/tmp/path2"},
+			logRotationConfig: DefaultLogRotationConfig,
+			wantErr:           true,
+			wantErrMsg:        ErrLogRotationInvalidLogOutput,
+		},
+		{
+			name:              "default output",
+			logRotationConfig: `{"maxsize": 1}`,
+			wantErr:           true,
+			wantErrMsg:        ErrLogRotationInvalidLogOutput,
+		},
+		{
+			name:              "default log rotation config",
+			logOutputs:        []string{"/tmp/path"},
+			logRotationConfig: DefaultLogRotationConfig,
+		},
+		{
+			name:              "invalid logger config",
+			logOutputs:        []string{"/tmp/path"},
+			logRotationConfig: `{"maxsize": true}`,
+			wantErr:           true,
+			wantErrMsg:        errors.New("invalid log rotation config: json: cannot unmarshal bool into Go struct field logRotationConfig.maxsize of type int"),
+		},
+		{
+			name:              "improperly formatted logger config",
+			logOutputs:        []string{"/tmp/path"},
+			logRotationConfig: `{"maxsize": true`,
+			wantErr:           true,
+			wantErrMsg:        errors.New("improperly formatted log rotation config: unexpected end of JSON input"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := NewConfig()
+			cfg.Logger = "zap"
+			cfg.LogOutputs = tt.logOutputs
+			cfg.EnableLogRotation = true
+			cfg.LogRotationConfigJSON = tt.logRotationConfig
+			err := cfg.Validate()
+			if err != nil && !tt.wantErr {
+				t.Errorf("test %q, unexpected error %v", tt.name, err)
+			}
+			if err != nil && tt.wantErr && tt.wantErrMsg.Error() != err.Error() {
+				t.Errorf("test %q, expected error: %+v, got: %+v", tt.name, tt.wantErrMsg, err)
+			}
+			if err == nil && tt.wantErr {
+				t.Errorf("test %q, expected error, got nil", tt.name)
+			}
+		})
+	}
+}

etcdmain/config.go

@@ -164,6 +164,8 @@ func newConfig() *config {
 	fs.DurationVar(&cfg.ec.GRPCKeepAliveTimeout, "grpc-keepalive-timeout", cfg.ec.GRPCKeepAliveTimeout, "Additional duration of wait before closing a non-responsive connection (0 to disable).")
 	fs.BoolVar(&cfg.ec.SocketOpts.ReusePort, "socket-reuse-port", cfg.ec.SocketOpts.ReusePort, "Enable to set socket option SO_REUSEPORT on listeners allowing rebinding of a port already in use.")
 	fs.BoolVar(&cfg.ec.SocketOpts.ReuseAddress, "socket-reuse-address", cfg.ec.SocketOpts.ReuseAddress, "Enable to set socket option SO_REUSEADDR on listeners allowing binding to an address in `TIME_WAIT` state.")
+	fs.BoolVar(&cfg.ec.EnableLogRotation, "enable-log-rotation", false, "Enable log rotation of a single log-outputs file target.")
+	fs.StringVar(&cfg.ec.LogRotationConfigJSON, "log-rotation-config-json", embed.DefaultLogRotationConfig, "Configures log rotation if enabled with a JSON logger config. Default: MaxSize=100(MB), MaxAge=0(days,no limit), MaxBackups=0(no limit), LocalTime=false(UTC), Compress=false(gzip)")
 
 	// clustering
 	fs.Var(

etcdmain/help.go

@@ -184,6 +184,10 @@ Logging:
     Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd, or list of comma separated output targets.
   --log-level 'info'
     Configures log level. Only supports debug, info, warn, error, panic, or fatal.
+  --enable-log-rotation 'false'
+    Enable log rotation of a single log-outputs file target.
+  --log-rotation-config-json '{"maxsize": 100, "maxage": 0, "maxbackups": 0, "localtime": false, "compress": false}'
+    Configures log rotation if enabled with a JSON logger config. MaxSize(MB), MaxAge(days,0=no limit), MaxBackups(0=no limit), LocalTime(use computers local time), Compress(gzip)".
 
 v2 Proxy (to be deprecated in v4):
   --proxy 'off'

go.mod

@@ -5,6 +5,7 @@ require (
 	github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa
 	github.com/coreos/go-semver v0.2.0
 	github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7
+	github.com/coreos/license-bill-of-materials v0.0.0-20190913234955-13baff47494e // indirect
 	github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf
 	github.com/creack/pty v1.1.11
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
@@ -46,10 +47,9 @@ require (
 	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
 	golang.org/x/text v0.3.3 // indirect
 	golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2
-	golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 // indirect
 	google.golang.org/grpc v1.26.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.25
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v2 v2.2.2
-	honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect
 	sigs.k8s.io/yaml v1.1.0
 )

go.sum

@@ -1,4 +1,5 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -15,6 +16,8 @@ github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazu
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/license-bill-of-materials v0.0.0-20190913234955-13baff47494e h1:vHRufSa2k8tfkcDdia1vJFa+oiBvvPxW94mg76PPAoA=
+github.com/coreos/license-bill-of-materials v0.0.0-20190913234955-13baff47494e/go.mod h1:4xMOusJ7xxc84WclVxKT8+lNfGYDwojOUC2OQNCwcj4=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBtyYFaUT/WmOqsJjgtihT0vMI=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
@@ -201,6 +204,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=