Allow rollover alias to use reference like agent.version or agent.name

CHANGELOG.asciidoc

@@ -104,6 +104,7 @@ https://github.com/elastic/beats/compare/v7.0.0-rc1...v7.0.0-rc2[Check the HEAD
 *Affecting all Beats*
 
 - Fixed OS family classification in `add_host_metadata` for Amazon Linux, Raspbian, and RedHat Linux. {issue}9134[9134] {pull}11494[11494]
+- Allow 'ilm.rollover_alias' to expand global fields like `agent.version`. {issue}12233[12233]
 
 *Auditbeat*
 

libbeat/common/fmtstr/formatevents.go

@@ -43,9 +43,10 @@ import (
 // Default values are given defined by the colon operator. For example:
 // `%{[field.name]:default value}`.
 type EventFormatString struct {
-	formatter StringFormatter
-	fields    []fieldInfo
-	timestamp bool
+	expression string
+	formatter  StringFormatter
+	fields     []fieldInfo
+	timestamp  bool
 }
 
 type eventFieldEvaler struct {
@@ -142,9 +143,10 @@ func CompileEvent(in string) (*EventFormatString, error) {
 
 	ctx.keys = make([]string, len(keys))
 	efs := &EventFormatString{
-		formatter: sf,
-		fields:    keys,
-		timestamp: efComp.timestamp,
+		expression: in,
+		formatter:  sf,
+		fields:     keys,
+		timestamp:  efComp.timestamp,
 	}
 	return efs, nil
 }
@@ -266,6 +268,12 @@ func (fs *EventFormatString) collectFields(
 	return nil
 }
 
+// IsEmpty returns true if the format string expression is an empty string, can be used
+// when validating to make defining a string mandatory.
+func (fs *EventFormatString) IsEmpty() bool {
+	return len(fs.expression) == 0
+}
+
 func (e *eventFieldCompiler) compileExpression(
 	s string,
 	opts []VariableOp,

libbeat/common/fmtstr/formatevents_test.go

@@ -35,6 +35,13 @@ func TestEventFormatString(t *testing.T) {
 		expected string
 		fields   []string
 	}{
+		{
+			"empty string",
+			"",
+			beat.Event{},
+			"",
+			nil,
+		},
 		{
 			"no fields configured",
 			"format string",
@@ -261,3 +268,15 @@ func TestEventFormatStringFromConfig(t *testing.T) {
 		assert.Equal(t, test.expected, actual)
 	}
 }
+
+func TestIsEmpty(t *testing.T) {
+	t.Run("when string is Empty", func(t *testing.T) {
+		fs := MustCompileEvent("")
+		assert.True(t, fs.IsEmpty())
+	})
+	t.Run("when string is not Empty", func(t *testing.T) {
+		fs := MustCompileEvent("hello")
+		assert.False(t, fs.IsEmpty())
+	})
+
+}

libbeat/docs/shared-ilm.asciidoc

@@ -51,8 +51,7 @@ required license; otherwise, {beatname_uc} creates daily indices.
 ==== `setup.ilm.rollover_alias`
 
 The index lifecycle write alias name. The default is
-+{beatname_lc}-\{{beat_version_key}\}+. Setting this option changes the prefix
-in the alias name. It doesn't remove +{beat_version_key}+ from the alias name. 
++{beatname_lc}-\{{beat_version_key}\}+. Setting this option changes the alias name.
 
 NOTE: If you modify this setting after loading the index template, you must
 overwrite the template to apply the changes.

libbeat/idxmgmt/ilm/config.go

@@ -32,7 +32,7 @@ type Config struct {
 	Mode          Mode                     `config:"enabled"`
 	PolicyName    fmtstr.EventFormatString `config:"policy_name"`
 	PolicyFile    string                   `config:"policy_file"`
-	RolloverAlias string                   `config:"rollover_alias"`
+	RolloverAlias fmtstr.EventFormatString `config:"rollover_alias"`
 	Pattern       string                   `config:"pattern"`
 
 	// CheckExists can disable the check for an existing policy. Check required
@@ -103,20 +103,20 @@ func (m *Mode) Unpack(in string) error {
 
 //Validate verifies that expected config options are given and valid
 func (cfg *Config) Validate() error {
-	if cfg.RolloverAlias == "" && cfg.Mode != ModeDisabled {
+	if cfg.RolloverAlias.IsEmpty() && cfg.Mode != ModeDisabled {
 		return fmt.Errorf("rollover_alias must be set when ILM is not disabled")
 	}
 	return nil
 }
 
 func defaultConfig(info beat.Info) Config {
-	name := fmt.Sprintf("%s-%s", info.Beat, info.Version)
+	name := info.Beat + "-%{[agent.version]}"
 	nameFmt := fmtstr.MustCompileEvent(name)
 
 	return Config{
 		Mode:          ModeAuto,
 		PolicyName:    *nameFmt,
-		RolloverAlias: name,
+		RolloverAlias: *nameFmt,
 		Pattern:       ilmDefaultPattern,
 		PolicyFile:    "",
 		CheckExists:   true,

libbeat/idxmgmt/ilm/ilm.go

@@ -104,8 +104,13 @@ func StdSupport(log *logp.Logger, info beat.Info, config *common.Config) (Suppor
 		return nil, errors.Wrap(err, "failed to read ilm policy name")
 	}
 
+	rolloverAlias, err := applyStaticFmtstr(info, &cfg.RolloverAlias)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read the ilm rollover alias")
+	}
+
 	alias := Alias{
-		Name:    cfg.RolloverAlias,
+		Name:    rolloverAlias,
 		Pattern: cfg.Pattern,
 	}
 

libbeat/idxmgmt/ilm/ilm_test.go

@@ -52,6 +52,19 @@ func TestDefaultSupport_Init(t *testing.T) {
 		}
 	})
 
+	t.Run("with an empty rollover_alias", func(t *testing.T) {
+		_, err := DefaultSupport(nil, info, common.MustNewConfigFrom(
+			map[string]interface{}{
+				"enabled":        true,
+				"rollover_alias": "",
+				"pattern":        "01",
+				"check_exists":   false,
+				"overwrite":      true,
+			},
+		))
+		require.Error(t, err)
+	})
+
 	t.Run("with custom config", func(t *testing.T) {
 		tmp, err := DefaultSupport(nil, info, common.MustNewConfigFrom(
 			map[string]interface{}{
@@ -74,6 +87,47 @@ func TestDefaultSupport_Init(t *testing.T) {
 		assert.Equal(Alias{Name: "alias", Pattern: "01"}, s.Alias())
 	})
 
+	t.Run("with custom alias config with fieldref", func(t *testing.T) {
+		tmp, err := DefaultSupport(nil, info, common.MustNewConfigFrom(
+			map[string]interface{}{
+				"enabled":        true,
+				"rollover_alias": "alias-%{[agent.version]}",
+				"pattern":        "01",
+				"check_exists":   false,
+				"overwrite":      true,
+			},
+		))
+		require.NoError(t, err)
+
+		s := tmp.(*stdSupport)
+		assert := assert.New(t)
+		assert.Equal(true, s.overwrite)
+		assert.Equal(false, s.checkExists)
+		assert.Equal(ModeEnabled, s.Mode())
+		assert.Equal(DefaultPolicy, common.MapStr(s.Policy().Body))
+		assert.Equal(Alias{Name: "alias-9.9.9", Pattern: "01"}, s.Alias())
+	})
+
+	t.Run("with default alias", func(t *testing.T) {
+		tmp, err := DefaultSupport(nil, info, common.MustNewConfigFrom(
+			map[string]interface{}{
+				"enabled":      true,
+				"pattern":      "01",
+				"check_exists": false,
+				"overwrite":    true,
+			},
+		))
+		require.NoError(t, err)
+
+		s := tmp.(*stdSupport)
+		assert := assert.New(t)
+		assert.Equal(true, s.overwrite)
+		assert.Equal(false, s.checkExists)
+		assert.Equal(ModeEnabled, s.Mode())
+		assert.Equal(DefaultPolicy, common.MapStr(s.Policy().Body))
+		assert.Equal(Alias{Name: "test-9.9.9", Pattern: "01"}, s.Alias())
+	})
+
 	t.Run("load external policy", func(t *testing.T) {
 		s, err := DefaultSupport(nil, info, common.MustNewConfigFrom(
 			common.MapStr{"policy_file": "testfiles/custom.json"},

libbeat/tests/system/test_cmd_setup_index_management.py

@@ -144,6 +144,26 @@ def test_setup_rollover_alias(self):
         self.idxmgmt.assert_docs_written_to_alias(self.custom_alias)
         self.idxmgmt.assert_alias_created(self.custom_alias)
 
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    def test_setup_rollover_alias_with_fieldref(self):
+        """
+        Test setup --index-management when ilm.rollover_alias is configured and using field reference.
+        """
+        aliasFieldRef = "%{[agent.name]}-myalias"
+        self.render_config()
+        exit_code = self.run_beat(logging_args=["-v", "-d", "*"],
+                                  extra_args=["setup", self.cmd,
+                                              "-E", "setup.ilm.rollover_alias=" + aliasFieldRef])
+
+        self.custom_alias = self.beat_name + "-myalias"
+
+        assert exit_code == 0
+        self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias)
+        self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"])
+        self.idxmgmt.assert_docs_written_to_alias(self.custom_alias)
+        self.idxmgmt.assert_alias_created(self.custom_alias)
+
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
     @attr('integration')
     def test_setup_template_name_and_pattern(self):