config: Add log configuration items to the config file (#2182)

cmd/server.go

@@ -96,8 +96,11 @@ func runEServer(cmd *cobra.Command, args []string) error {
 	}
 
 	cancel := initCmd(cmd, &logutil.Config{
-		File:  conf.LogFile,
-		Level: conf.LogLevel,
+		File:           conf.LogFile,
+		Level:          conf.LogLevel,
+		FileMaxSize:    conf.Log.File.MaxSize,
+		FileMaxDays:    conf.Log.File.MaxDays,
+		FileMaxBackups: conf.Log.File.MaxBackups,
 	})
 	defer cancel()
 	tz, err := util.GetTimezone(conf.TZ)

cmd/server_test.go

@@ -138,10 +138,17 @@ func (s *serverSuite) TestLoadAndVerifyServerConfig(c *check.C) {
 	cfg, err = loadAndVerifyServerConfig(cmd)
 	c.Assert(err, check.IsNil)
 	c.Assert(cfg, check.DeepEquals, &config.ServerConfig{
-		Addr:                   "127.5.5.1:8833",
-		AdvertiseAddr:          "127.5.5.1:7777",
-		LogFile:                "/root/cdc.log",
-		LogLevel:               "debug",
+		Addr:          "127.5.5.1:8833",
+		AdvertiseAddr: "127.5.5.1:7777",
+		LogFile:       "/root/cdc.log",
+		LogLevel:      "debug",
+		Log: &config.LogConfig{
+			File: &config.LogFileConfig{
+				MaxSize:    300,
+				MaxDays:    0,
+				MaxBackups: 0,
+			},
+		},
 		DataDir:                dataDir,
 		GcTTL:                  10,
 		TZ:                     "UTC",
@@ -179,6 +186,7 @@ advertise-addr = "127.0.0.1:1111"
 
 log-file = "/root/cdc1.log"
 log-level = "warn"
+
 data-dir = "%+v"
 gc-ttl = 500
 tz = "US"
@@ -187,6 +195,11 @@ capture-session-ttl = 10
 owner-flush-interval = "600ms"
 processor-flush-interval = "600ms"
 
+[log.file]
+max-size = 200
+max-days = 1
+max-backups = 1
+
 [sorter]
 chunk-size-limit = 10000000
 max-memory-consumption = 2000000
@@ -203,10 +216,17 @@ sort-dir = "/tmp/just_a_test"
 	cfg, err = loadAndVerifyServerConfig(cmd)
 	c.Assert(err, check.IsNil)
 	c.Assert(cfg, check.DeepEquals, &config.ServerConfig{
-		Addr:                   "128.0.0.1:1234",
-		AdvertiseAddr:          "127.0.0.1:1111",
-		LogFile:                "/root/cdc1.log",
-		LogLevel:               "warn",
+		Addr:          "128.0.0.1:1234",
+		AdvertiseAddr: "127.0.0.1:1111",
+		LogFile:       "/root/cdc1.log",
+		LogLevel:      "warn",
+		Log: &config.LogConfig{
+			File: &config.LogFileConfig{
+				MaxSize:    200,
+				MaxDays:    1,
+				MaxBackups: 1,
+			},
+		},
 		DataDir:                dataDir,
 		GcTTL:                  500,
 		TZ:                     "US",
@@ -260,10 +280,17 @@ cert-allowed-cn = ["dd","ee"]
 	cfg, err = loadAndVerifyServerConfig(cmd)
 	c.Assert(err, check.IsNil)
 	c.Assert(cfg, check.DeepEquals, &config.ServerConfig{
-		Addr:                   "127.5.5.1:8833",
-		AdvertiseAddr:          "127.0.0.1:1111",
-		LogFile:                "/root/cdc.log",
-		LogLevel:               "debug",
+		Addr:          "127.5.5.1:8833",
+		AdvertiseAddr: "127.0.0.1:1111",
+		LogFile:       "/root/cdc.log",
+		LogLevel:      "debug",
+		Log: &config.LogConfig{
+			File: &config.LogFileConfig{
+				MaxSize:    200,
+				MaxDays:    1,
+				MaxBackups: 1,
+			},
+		},
 		DataDir:                dataDir,
 		GcTTL:                  10,
 		TZ:                     "UTC",

cmd/ticdc.toml

@@ -23,6 +23,16 @@ log-level = "info"
 # the time zone of TiCDC cluster, default: "System"
 # tz = "System"
 
+[log.file]
+# Max log file size in MB (upper limit to 4096MB).
+max-size = 300
+
+# Max log file keep days. No clean up by default.
+max-days = 0
+
+# Maximum number of old log files to retain. No clean up by default.
+max-backups = 0
+
 [security]
 # ca-path = ""
 # cert-path = ""

pkg/config/config.go

@@ -146,14 +146,33 @@ func GetDefaultReplicaConfig() *ReplicaConfig {
 // SecurityConfig represents security config for server
 type SecurityConfig = security.Credential
 
+// LogFileConfig represents log file config for server
+type LogFileConfig struct {
+	MaxSize    int `toml:"max-size" json:"max-size"`
+	MaxDays    int `toml:"max-days" json:"max-days"`
+	MaxBackups int `toml:"max-backups" json:"max-backups"`
+}
+
+// LogConfig represents log config for server
+type LogConfig struct {
+	File *LogFileConfig `toml:"file" json:"file"`
+}
+
 var defaultServerConfig = &ServerConfig{
 	Addr:          "127.0.0.1:8300",
 	AdvertiseAddr: "",
 	LogFile:       "",
 	LogLevel:      "info",
-	DataDir:       "",
-	GcTTL:         24 * 60 * 60, // 24H
-	TZ:            "System",
+	Log: &LogConfig{
+		File: &LogFileConfig{
+			MaxSize:    300,
+			MaxDays:    0,
+			MaxBackups: 0,
+		},
+	},
+	DataDir: "",
+	GcTTL:   24 * 60 * 60, // 24H
+	TZ:      "System",
 	// The default election-timeout in PD is 3s and minimum session TTL is 5s,
 	// which is calculated by `math.Ceil(3 * election-timeout / 2)`, we choose
 	// default capture session ttl to 10s to increase robust to PD jitter,
@@ -183,9 +202,11 @@ type ServerConfig struct {
 	Addr          string `toml:"addr" json:"addr"`
 	AdvertiseAddr string `toml:"advertise-addr" json:"advertise-addr"`
 
-	LogFile  string `toml:"log-file" json:"log-file"`
-	LogLevel string `toml:"log-level" json:"log-level"`
-	DataDir  string `toml:"data-dir" json:"data-dir"`
+	LogFile  string     `toml:"log-file" json:"log-file"`
+	LogLevel string     `toml:"log-level" json:"log-level"`
+	Log      *LogConfig `toml:"log" json:"log"`
+
+	DataDir string `toml:"data-dir" json:"data-dir"`
 
 	GcTTL int64  `toml:"gc-ttl" json:"gc-ttl"`
 	TZ    string `toml:"tz" json:"tz"`

pkg/config/config_test.go

@@ -80,7 +80,7 @@ var _ = check.Suite(&serverConfigSuite{})
 
 func (s *serverConfigSuite) TestMarshal(c *check.C) {
 	defer testleak.AfterTest(c)()
-	rawConfig := `{"addr":"192.155.22.33:8887","advertise-addr":"","log-file":"","log-level":"info","data-dir":"","gc-ttl":86400,"tz":"System","capture-session-ttl":10,"owner-flush-interval":200000000,"processor-flush-interval":100000000,"sorter":{"num-concurrent-worker":4,"chunk-size-limit":999,"max-memory-percentage":30,"max-memory-consumption":17179869184,"num-workerpool-goroutine":16,"sort-dir":"/tmp/sorter"},"security":{"ca-path":"","cert-path":"","key-path":"","cert-allowed-cn":null},"per-table-memory-quota":20971520,"kv-client":{"worker-concurrent":8,"worker-pool-size":0,"region-scan-limit":40}}`
+	rawConfig := `{"addr":"192.155.22.33:8887","advertise-addr":"","log-file":"","log-level":"info","log":{"file":{"max-size":300,"max-days":0,"max-backups":0}},"data-dir":"","gc-ttl":86400,"tz":"System","capture-session-ttl":10,"owner-flush-interval":200000000,"processor-flush-interval":100000000,"sorter":{"num-concurrent-worker":4,"chunk-size-limit":999,"max-memory-percentage":30,"max-memory-consumption":17179869184,"num-workerpool-goroutine":16,"sort-dir":"/tmp/sorter"},"security":{"ca-path":"","cert-path":"","key-path":"","cert-allowed-cn":null},"per-table-memory-quota":20971520,"kv-client":{"worker-concurrent":8,"worker-pool-size":0,"region-scan-limit":40}}`
 
 	conf := GetDefaultServerConfig()
 	conf.Addr = "192.155.22.33:8887"