Update filebeat registry configuration (#10504)

Change registry config and directory layout, preparing for future
changes.

- remove settings `filebeat.registry_file` and replace with
  `filebeat.registry.path`. Path will be a directory, and the
  actual contents will be stored under
  `${filebeat.registry.path}/filebeat/data.json`.
- introduce `<registry path>/filebeat/meta.json` with version number of
  the current directories layout.
- move `filebeat.registry_flush` to `filebeat.registry.flush`
- move `filebeat.registry_file_permission` to
  `filebeat.registry.file_permission`
- update tests

CHANGELOG.next.asciidoc

@@ -85,6 +85,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
   still used for matcher. {issue}10505[10505] {pull}10506[10506]
 - Change type of haproxy.source from text to keyword. {pull}10506[10506]
 - Rename `event.type` to `suricata.eve.event_type` in Suricata module because event.type is reserved for future use by ECS. {pull}10575[10575]
+- Populate more ECS fields in the Suricata module. {pull}10006[10006]
+- Rename setting `filebeat.registry_flush` to `filebeat.registry.flush`. {pull}10504[10504]
+- Rename setting `filebeat.registry_file_permission` to `filebeat.registry.file_permission`. {pull}10504[10504]
+- Remove setting `filebeat.registry_file` in favor of `filebeat.registry.path`. The registry file will be stored in a sub-directory by now. {pull}10504[10504]
 
 *Heartbeat*
 

filebeat/_meta/common.reference.p2.yml

@@ -17,20 +17,20 @@
 
 #========================= Filebeat global options ============================
 
-# Name of the registry file. If a relative path is used, it is considered relative to the
+# Registry data path. If a relative path is used, it is considered relative to the
 # data path.
-#filebeat.registry_file: ${path.data}/registry
+#filebeat.registry.path: ${path.data}/registry
 
-# The permissions mask to apply on registry file. The default value is 0600.
-# Must be a valid Unix-style file permissions mask expressed in octal notation.
-# This option is not supported on Windows.
-#filebeat.registry_file_permissions: 0600
+# The permissions mask to apply on registry data, and meta files. The default
+# value is 0600.  Must be a valid Unix-style file permissions mask expressed in
+# octal notation.  This option is not supported on Windows.
+#filebeat.registry.file_permissions: 0600
 
 # The timeout value that controls when registry entries are written to disk
-# (flushed). When an unwritten update exceeds this value, it triggers a write to
-# disk. When registry_flush is set to 0s, the registry is written to disk after
-# each batch of events has been published successfully. The default value is 0s.
-#filebeat.registry_flush: 0s
+# (flushed). When an unwritten update exceeds this value, it triggers a write
+# to disk. When flush is set to 0s, the registry is written to disk after each
+# batch of events has been published successfully. The default value is 0s.
+#filebeat.registry.flush: 0s
 
 # By default Ingest pipelines are not updated if a pipeline with the same ID
 # already exists. If this option is enabled Filebeat overwrites pipelines

filebeat/beater/filebeat.go

@@ -76,6 +76,9 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 		rawConfig,
 		"prospectors",
 		"config.prospectors",
+		"registry_file",
+		"registry_file_permissions",
+		"registry_flush",
 	); err != nil {
 		return nil, err
 	}
@@ -293,7 +296,7 @@ func (fb *Filebeat) Run(b *beat.Beat) error {
 	finishedLogger := newFinishedLogger(wgEvents)
 
 	// Setup registrar to persist state
-	registrar, err := registrar.New(config.RegistryFile, config.RegistryFilePermissions, config.RegistryFlush, finishedLogger)
+	registrar, err := registrar.New(config.Registry, finishedLogger)
 	if err != nil {
 		logp.Err("Could not init registrar: %v", err)
 		return err

filebeat/config/config.go

@@ -39,25 +39,33 @@ const (
 )
 
 type Config struct {
-	Inputs                  []*common.Config     `config:"inputs"`
-	RegistryFile            string               `config:"registry_file"`
-	RegistryFilePermissions os.FileMode          `config:"registry_file_permissions"`
-	RegistryFlush           time.Duration        `config:"registry_flush"`
-	ConfigDir               string               `config:"config_dir"`
-	ShutdownTimeout         time.Duration        `config:"shutdown_timeout"`
-	Modules                 []*common.Config     `config:"modules"`
-	ConfigInput             *common.Config       `config:"config.inputs"`
-	ConfigModules           *common.Config       `config:"config.modules"`
-	Autodiscover            *autodiscover.Config `config:"autodiscover"`
-	OverwritePipelines      bool                 `config:"overwrite_pipelines"`
+	Inputs             []*common.Config     `config:"inputs"`
+	Registry           Registry             `config:"registry"`
+	ConfigDir          string               `config:"config_dir"`
+	ShutdownTimeout    time.Duration        `config:"shutdown_timeout"`
+	Modules            []*common.Config     `config:"modules"`
+	ConfigInput        *common.Config       `config:"config.inputs"`
+	ConfigModules      *common.Config       `config:"config.modules"`
+	Autodiscover       *autodiscover.Config `config:"autodiscover"`
+	OverwritePipelines bool                 `config:"overwrite_pipelines"`
+}
+
+type Registry struct {
+	Path         string        `config:"path"`
+	Permissions  os.FileMode   `config:"file_permissions"`
+	FlushTimeout time.Duration `config:"flush"`
+	MigrateFile  string        `config:"migrate_file"`
 }
 
 var (
 	DefaultConfig = Config{
-		RegistryFile:            "registry",
-		RegistryFilePermissions: 0600,
-		ShutdownTimeout:         0,
-		OverwritePipelines:      false,
+		Registry: Registry{
+			Path:        "registry",
+			Permissions: 0600,
+			MigrateFile: "",
+		},
+		ShutdownTimeout:    0,
+		OverwritePipelines: false,
 	}
 )
 

filebeat/docs/faq.asciidoc

@@ -53,7 +53,7 @@ Make sure that you read the documentation for these configuration options before
 [[reduce-registry-size]]
 === Registry file is too large?
 
-{beatname_uc} keeps the state of each file and persists the state to disk in the `registry_file`. The file state is used to continue file reading at a previous position when {beatname_uc} is restarted. If a large number of new files are produced every day, the registry file might grow to be too large. To reduce the size of the registry file, there are two configuration options available: <<{beatname_lc}-input-log-clean-removed,`clean_removed`>> and <<{beatname_lc}-input-log-clean-inactive,`clean_inactive`>>.
+{beatname_uc} keeps the state of each file and persists the state to disk in the registry file. The file state is used to continue file reading at a previous position when {beatname_uc} is restarted. If a large number of new files are produced every day, the registry file might grow to be too large. To reduce the size of the registry file, there are two configuration options available: <<{beatname_lc}-input-log-clean-removed,`clean_removed`>> and <<{beatname_lc}-input-log-clean-inactive,`clean_inactive`>>.
 
 For old files that you no longer touch and are ignored (see <<{beatname_lc}-input-log-ignore-older,`ignore_older`>>), we recommended that you use `clean_inactive`. If old files get removed from disk, then use the `clean_removed` option.
 

filebeat/docs/filebeat-general-options.asciidoc

@@ -17,25 +17,30 @@ Beats.
 These options are in the `filebeat` namespace.
 
 [float]
-==== `registry_file`
+==== `registry.path`
 
-The name of the registry file. If a relative path is used, it is considered relative to the
-data path. See the <<directory-layout>> section for details. The default is `${path.data}/registry`.
+The root path of the registry.  If a relative path is used, it is considered
+relative to the data path. See the <<directory-layout>> section for details.
+The default is `${path.data}/registry`.
 
 [source,yaml]
 -------------------------------------------------------------------------------------
-filebeat.registry_file: registry
+filebeat.registry.path: registry
 -------------------------------------------------------------------------------------
 
-It is not possible to use a symlink as registry file.
-
-NOTE: The registry file is only updated when new events are flushed and not on a predefined period.
+NOTE: The registry is only updated when new events are flushed and not on a predefined period.
 That means in case there are some states where the TTL expired, these are only removed when new event are processed.
 
+NOTE: The registry stores it's data in the subdirectory filebeat/data.json. It
+also contains a meta data file named filebeat/meta.json. The meta file contains
+the file format version number.
+
+NOTE: The content stored in filebeat/data.json is compatible to the old registry file data format.
+
 [float]
-==== `registry_file_permissions`
+==== `registry.file_permissions`
 
-The permissions mask to apply on registry file. The default value is 0600. The permissions option must be a valid Unix-style file permissions mask expressed in octal notation. In Go, numbers in octal notation must start with 0.
+The permissions mask to apply on registry data file. The default value is 0600. The permissions option must be a valid Unix-style file permissions mask expressed in octal notation. In Go, numbers in octal notation must start with 0.
 
 This option is not supported on Windows.
 
@@ -47,25 +52,25 @@ Examples:
 
 [source,yaml]
 -------------------------------------------------------------------------------------
-filebeat.registry_file_permissions: 0600
+filebeat.registry.file_permissions: 0600
 -------------------------------------------------------------------------------------
 
 [float]
-==== `registry_flush`
+==== `registry.flush`
 
 
 The timeout value that controls when registry entries are written to disk
 (flushed). When an unwritten update exceeds this value, it triggers a write to
-disk. When registry_flush is set to 0s, the registry is written to disk after
+disk. When `registry.flush` is set to 0s, the registry is written to disk after
 each batch of events has been published successfully. The default value is 0s.
 
 NOTE: The registry is always updated when Filebeat shuts down normally. After an
-abnormal shutdown, the registry will not be up-to-date if the registry_flush
+abnormal shutdown, the registry will not be up-to-date if the `registry.flush`
 value is >0s. Filebeat will send published events again (depending on values in
 the last updated registry file).
 
 NOTE: Filtering out a huge number of logs can cause many registry updates, slowing
-down processing. Setting registry_flush to a value >0s reduces write operations,
+down processing. Setting `registry.flush` to a value >0s reduces write operations,
 helping Filebeat process more events.
 
 

filebeat/filebeat.reference.yml

@@ -775,20 +775,20 @@ filebeat.inputs:
 
 #========================= Filebeat global options ============================
 
-# Name of the registry file. If a relative path is used, it is considered relative to the
+# Registry data path. If a relative path is used, it is considered relative to the
 # data path.
-#filebeat.registry_file: ${path.data}/registry
+#filebeat.registry.path: ${path.data}/registry
 
-# The permissions mask to apply on registry file. The default value is 0600.
-# Must be a valid Unix-style file permissions mask expressed in octal notation.
-# This option is not supported on Windows.
-#filebeat.registry_file_permissions: 0600
+# The permissions mask to apply on registry data, and meta files. The default
+# value is 0600.  Must be a valid Unix-style file permissions mask expressed in
+# octal notation.  This option is not supported on Windows.
+#filebeat.registry.file_permissions: 0600
 
 # The timeout value that controls when registry entries are written to disk
-# (flushed). When an unwritten update exceeds this value, it triggers a write to
-# disk. When registry_flush is set to 0s, the registry is written to disk after
-# each batch of events has been published successfully. The default value is 0s.
-#filebeat.registry_flush: 0s
+# (flushed). When an unwritten update exceeds this value, it triggers a write
+# to disk. When flush is set to 0s, the registry is written to disk after each
+# batch of events has been published successfully. The default value is 0s.
+#filebeat.registry.flush: 0s
 
 # By default Ingest pipelines are not updated if a pipeline with the same ID
 # already exists. If this option is enabled Filebeat overwrites pipelines