[libbeat] Add flag for rotating log files on startup

CHANGELOG.next.asciidoc

@@ -120,6 +120,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- Add an option to append to existing logs rather than always rotate on start. {pull}11953[11953]
 - Add `network` condition to processors for matching IP addresses against CIDRs. {pull}10743[10743]
 - Add if/then/else support to processors. {pull}10744[10744]
 - Add `community_id` processor for computing network flow hashes. {pull}10745[10745]

auditbeat/auditbeat.reference.yml

@@ -1173,6 +1173,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

filebeat/filebeat.reference.yml

@@ -1900,6 +1900,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

heartbeat/heartbeat.reference.yml

@@ -1317,6 +1317,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

journalbeat/journalbeat.reference.yml

@@ -1113,6 +1113,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

libbeat/_meta/config.reference.yml

@@ -1061,6 +1061,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

libbeat/common/file/rotator.go

@@ -67,6 +67,7 @@ type Rotator struct {
 	permissions     os.FileMode
 	log             Logger // Optional Logger (may be nil).
 	interval        time.Duration
+	rotateOnStartup bool
 	intervalRotator *intervalRotator // Optional, may be nil
 	redirectStderr  bool
 
@@ -124,6 +125,14 @@ func Interval(d time.Duration) RotatorOption {
 	}
 }
 
+// RotateOnStartup immediately rotates files on startup rather than appending to
+// the existing file. The default is true.
+func RotateOnStartup(b bool) RotatorOption {
+	return func(r *Rotator) {
+		r.rotateOnStartup = b
+	}
+}
+
 // RedirectStderr causes all writes to standard error to be redirected
 // to this rotator.
 func RedirectStderr(redirect bool) RotatorOption {
@@ -135,11 +144,12 @@ func RedirectStderr(redirect bool) RotatorOption {
 // NewFileRotator returns a new Rotator.
 func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) {
 	r := &Rotator{
-		filename:     filename,
-		maxSizeBytes: 10 * 1024 * 1024, // 10 MiB
-		maxBackups:   7,
-		permissions:  0600,
-		interval:     0,
+		filename:        filename,
+		maxSizeBytes:    10 * 1024 * 1024, // 10 MiB
+		maxBackups:      7,
+		permissions:     0600,
+		interval:        0,
+		rotateOnStartup: true,
 	}
 
 	for _, opt := range options {
@@ -260,6 +270,8 @@ func (r *Rotator) dirMode() os.FileMode {
 	return os.FileMode(mode)
 }
 
+// openNew opens r's log file for the first time, creating it if it doesn't
+// exist, and performing an initial rotation if r.rotateOnStartup is set.
 func (r *Rotator) openNew() error {
 	err := os.MkdirAll(r.dir(), r.dirMode())
 	if err != nil {
@@ -268,6 +280,11 @@ func (r *Rotator) openNew() error {
 
 	_, err = os.Stat(r.filename)
 	if err == nil {
+		if !r.rotateOnStartup {
+			// If the file exists and rotateOnStartup is false, then we just want to
+			// append to the existing file.
+			return r.appendToFile()
+		}
 		if err = r.rotate(rotateReasonInitializing); err != nil {
 			return err
 		}
@@ -292,6 +309,21 @@ func (r *Rotator) openFile() error {
 	return nil
 }
 
+// appendToFile opens an existing log file for appending. Unlike openFile it
+// does not call MkdirAll because it is an error for the file to not already
+// exist.
+func (r *Rotator) appendToFile() error {
+	var err error
+	r.file, err = os.OpenFile(r.filename, os.O_WRONLY|os.O_APPEND, r.permissions)
+	if err != nil {
+		return errors.Wrap(err, "failed to append to existing file")
+	}
+	if r.redirectStderr {
+		RedirectStandardError(r.file)
+	}
+	return nil
+}
+
 func (r *Rotator) closeFile() error {
 	if r.file == nil {
 		return nil
@@ -366,6 +398,9 @@ func (r *Rotator) rotate(reason rotateReason) error {
 
 	var err error
 	if r.intervalRotator != nil {
+		// Interval and size rotation use different filename patterns, so we use
+		// rotateByInterval if interval rotation is enabled, even if this specific
+		// rotation is triggered by size.
 		err = r.rotateByInterval(reason)
 	} else {
 		err = r.rotateBySize(reason)

libbeat/common/file/rotator_test.go

@@ -145,6 +145,9 @@ func TestDailyRotation(t *testing.T) {
 	}
 	defer r.Close()
 
+	// The backups exceeding the max of 2 aren't deleted until the first rotation.
+	AssertDirContents(t, dir, files...)
+
 	Rotate(t, r)
 
 	AssertDirContents(t, dir, logname+"-"+yesterday+"-12", logname+"-"+yesterday+"-13")
@@ -168,6 +171,49 @@ func TestDailyRotation(t *testing.T) {
 	AssertDirContents(t, dir, logname+"-"+today+"-1", logname+"-"+today+"-2", logname)
 }
 
+// Tests the FileConfig.RotateOnStartup parameter
+func TestRotateOnStartup(t *testing.T) {
+	dir, err := ioutil.TempDir("", "rotate_on_open")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	logname := "rotate_on_open"
+	filename := filepath.Join(dir, logname)
+
+	// Create an existing log file with this name.
+	CreateFile(t, filename)
+	AssertDirContents(t, dir, logname)
+
+	r, err := file.NewFileRotator(filename, file.RotateOnStartup(false))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+	WriteMsg(t, r)
+
+	// The line should have been appended to the existing file without rotation.
+	AssertDirContents(t, dir, logname)
+
+	// Close the first rotator early (the deferred close will be a no-op if
+	// we haven't hit an error by now), so it can't interfere with the second one.
+	r.Close()
+
+	// Create a second rotator with the default setting of rotateOnStartup=true
+	r, err = file.NewFileRotator(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer r.Close()
+
+	// The directory contents shouldn't change until the first Write.
+	AssertDirContents(t, dir, logname)
+
+	WriteMsg(t, r)
+	AssertDirContents(t, dir, logname, logname+".1")
+}
+
 func CreateFile(t *testing.T, filename string) {
 	t.Helper()
 	f, err := os.Create(filename)

libbeat/docs/loggingconfig.asciidoc

@@ -206,6 +206,13 @@ reported by the local system clock. All other intervals are calculated from the
 unix epoch. Defaults to disabled.
 endif::serverless[]
 
+[float]
+==== `logging.files.rotateonstartup`
+
+If the log file already exists on startup, immediately rotate it and start
+writing to a new file instead of appending to the existing one. Defaults to
+true.
+
 [float]
 ==== `logging.json`
 

libbeat/logp/config.go

@@ -42,23 +42,25 @@ type Config struct {
 
 // FileConfig contains the configuration options for the file output.
 type FileConfig struct {
-	Path           string        `config:"path"`
-	Name           string        `config:"name"`
-	MaxSize        uint          `config:"rotateeverybytes" validate:"min=1"`
-	MaxBackups     uint          `config:"keepfiles" validate:"max=1024"`
-	Permissions    uint32        `config:"permissions"`
-	Interval       time.Duration `config:"interval"`
-	RedirectStderr bool          `config:"redirect_stderr"`
+	Path            string        `config:"path"`
+	Name            string        `config:"name"`
+	MaxSize         uint          `config:"rotateeverybytes" validate:"min=1"`
+	MaxBackups      uint          `config:"keepfiles" validate:"max=1024"`
+	Permissions     uint32        `config:"permissions"`
+	Interval        time.Duration `config:"interval"`
+	RotateOnStartup bool          `config:"rotateonstartup"`
+	RedirectStderr  bool          `config:"redirect_stderr"`
 }
 
 var defaultConfig = Config{
 	Level:   InfoLevel,
 	ToFiles: true,
 	Files: FileConfig{
-		MaxSize:     10 * 1024 * 1024,
-		MaxBackups:  7,
-		Permissions: 0600,
-		Interval:    0,
+		MaxSize:         10 * 1024 * 1024,
+		MaxBackups:      7,
+		Permissions:     0600,
+		Interval:        0,
+		RotateOnStartup: true,
 	},
 	addCaller: true,
 }

libbeat/logp/core.go

@@ -197,6 +197,7 @@ func makeFileOutput(cfg Config) (zapcore.Core, error) {
 		file.MaxBackups(cfg.Files.MaxBackups),
 		file.Permissions(os.FileMode(cfg.Files.Permissions)),
 		file.Interval(cfg.Files.Interval),
+		file.RotateOnStartup(cfg.Files.RotateOnStartup),
 		file.RedirectStderr(cfg.Files.RedirectStderr),
 	)
 	if err != nil {

metricbeat/metricbeat.reference.yml

@@ -1796,6 +1796,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

packetbeat/packetbeat.reference.yml

@@ -1541,6 +1541,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

winlogbeat/winlogbeat.reference.yml

@@ -1094,6 +1094,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

x-pack/auditbeat/auditbeat.reference.yml

@@ -1212,6 +1212,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

x-pack/filebeat/filebeat.reference.yml

@@ -2016,6 +2016,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

x-pack/functionbeat/functionbeat.reference.yml

@@ -1230,6 +1230,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

x-pack/metricbeat/metricbeat.reference.yml

@@ -1854,6 +1854,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false
 

x-pack/winlogbeat/winlogbeat.reference.yml

@@ -1093,6 +1093,10 @@ logging.files:
   # Unix epoch. Defaults to disabled.
   #interval: 0
 
+  # Rotate existing logs on startup rather than appending to the existing
+  # file. Defaults to true.
+  # rotateonstartup: true
+
 # Set to true to log messages in JSON format.
 #logging.json: false