Add migration.enabled config option

CHANGELOG.next.asciidoc

@@ -129,6 +129,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `name` config option to `add_host_metadata` processor. {pull}9943[9943]
 - Add `add_labels` and `add_tags` processors. {pull}9973[9973]
 - Add missing file encoding to readers. {pull}10080[10080]
+- Introduce `migration.enabled` configuration. {pull}9805[9805]
 - Add alias field support in Kibana index pattern. {pull}10075[10075]
 
 *Auditbeat*

auditbeat/auditbeat.reference.yml

@@ -1246,3 +1246,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

auditbeat/auditbeat.yml

@@ -177,3 +177,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

filebeat/filebeat.reference.yml

@@ -1941,3 +1941,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

filebeat/filebeat.yml

@@ -205,3 +205,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

heartbeat/heartbeat.reference.yml

@@ -1390,3 +1390,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

heartbeat/heartbeat.yml

@@ -162,3 +162,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

journalbeat/journalbeat.reference.yml

@@ -1180,3 +1180,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

journalbeat/journalbeat.yml

@@ -168,3 +168,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

libbeat/_meta/config.reference.yml

@@ -1134,3 +1134,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

libbeat/_meta/config.yml

@@ -122,3 +122,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

libbeat/cmd/export/template.go

@@ -66,7 +66,7 @@ func GenTemplateConfigCmd(settings instance.Settings, name, idxPrefix, beatVersi
 				fmt.Fprintf(os.Stderr, "Invalid Elasticsearch version: %s\n", err)
 			}
 
-			tmpl, err := template.New(b.Info.Version, index, *esVersion, cfg)
+			tmpl, err := template.New(b.Info.Version, index, *esVersion, cfg, b.Config.Migration.Enabled())
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Error generating template: %+v", err)
 				os.Exit(1)

libbeat/cmd/instance/beat.go

@@ -63,7 +63,7 @@ import (
 	svc "github.com/elastic/beats/libbeat/service"
 	"github.com/elastic/beats/libbeat/template"
 	"github.com/elastic/beats/libbeat/version"
-	"github.com/elastic/go-sysinfo"
+	sysinfo "github.com/elastic/go-sysinfo"
 	"github.com/elastic/go-sysinfo/types"
 	ucfg "github.com/elastic/go-ucfg"
 )
@@ -105,6 +105,7 @@ type beatConfig struct {
 	Dashboards *common.Config `config:"setup.dashboards"`
 	Template   *common.Config `config:"setup.template"`
 	Kibana     *common.Config `config:"setup.kibana"`
+	Migration  *common.Config `config:"migration"`
 
 	// ILM Config options
 	ILM *common.Config `config:"output.elasticsearch.ilm"`
@@ -877,7 +878,7 @@ func (b *Beat) templateLoadingCallback() (func(esClient *elasticsearch.Client) e
 			b.Config.Template = common.NewConfig()
 		}
 
-		loader, err := template.NewLoader(b.Config.Template, esClient, b.Info, b.Fields)
+		loader, err := template.NewLoader(b.Config.Template, esClient, b.Info, b.Fields, b.Config.Migration.Enabled())
 		if err != nil {
 			return fmt.Errorf("Error creating Elasticsearch template loader: %v", err)
 		}

libbeat/common/field.go

@@ -52,6 +52,7 @@ type Field struct {
 	CopyTo         string      `config:"copy_to"`
 	IgnoreAbove    int         `config:"ignore_above"`
 	AliasPath      string      `config:"path"`
+	MigrationAlias bool        `config:"migration"`
 
 	ObjectType            string          `config:"object_type"`
 	ObjectTypeMappingType string          `config:"object_type_mapping_type"`
@@ -64,6 +65,7 @@ type Field struct {
 	Searchable   *bool  `config:"searchable"`
 	Aggregatable *bool  `config:"aggregatable"`
 	Script       string `config:"script"`
+
 	// Kibana params
 	Pattern              string              `config:"pattern"`
 	InputFormat          string              `config:"input_format"`

libbeat/template/load.go

@@ -38,14 +38,15 @@ type ESClient interface {
 }
 
 type Loader struct {
-	config   TemplateConfig
-	client   ESClient
-	beatInfo beat.Info
-	fields   []byte
+	config    TemplateConfig
+	client    ESClient
+	beatInfo  beat.Info
+	fields    []byte
+	migration bool
 }
 
 // NewLoader creates a new template loader
-func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields []byte) (*Loader, error) {
+func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields []byte, migration bool) (*Loader, error) {
 	config := DefaultConfig
 
 	err := cfg.Unpack(&config)
@@ -54,10 +55,11 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields [
 	}
 
 	return &Loader{
-		config:   config,
-		client:   client,
-		beatInfo: beatInfo,
-		fields:   fields,
+		config:    config,
+		client:    client,
+		beatInfo:  beatInfo,
+		fields:    fields,
+		migration: migration,
 	}, nil
 }
 
@@ -66,7 +68,7 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields [
 // template is written to index
 func (l *Loader) Load() error {
 
-	tmpl, err := New(l.beatInfo.Version, l.beatInfo.IndexPrefix, l.client.GetVersion(), l.config)
+	tmpl, err := New(l.beatInfo.Version, l.beatInfo.IndexPrefix, l.client.GetVersion(), l.config, l.migration)
 	if err != nil {
 		return fmt.Errorf("error creating template instance: %v", err)
 	}

libbeat/template/load_integration_test.go

@@ -70,7 +70,7 @@ func TestLoadTemplate(t *testing.T) {
 	fieldsPath := absPath + "/fields.yml"
 	index := "testbeat"
 
-	tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{})
+	tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}, false)
 	assert.NoError(t, err)
 	content, err := tmpl.LoadFile(fieldsPath)
 	assert.NoError(t, err)
@@ -140,7 +140,7 @@ func TestLoadBeatsTemplate(t *testing.T) {
 		fieldsPath := absPath + "/fields.yml"
 		index := beat
 
-		tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{})
+		tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}, false)
 		assert.NoError(t, err)
 		content, err := tmpl.LoadFile(fieldsPath)
 		assert.NoError(t, err)
@@ -189,7 +189,7 @@ func TestTemplateSettings(t *testing.T) {
 	config := TemplateConfig{
 		Settings: settings,
 	}
-	tmpl, err := New(version.GetDefaultVersion(), "testbeat", client.GetVersion(), config)
+	tmpl, err := New(version.GetDefaultVersion(), "testbeat", client.GetVersion(), config, false)
 	assert.NoError(t, err)
 	content, err := tmpl.LoadFile(fieldsPath)
 	assert.NoError(t, err)
@@ -240,7 +240,7 @@ func TestOverwrite(t *testing.T) {
 		Enabled: true,
 		Fields:  absPath + "/fields.yml",
 	})
-	loader, err := NewLoader(config, client, beatInfo, nil)
+	loader, err := NewLoader(config, client, beatInfo, nil, false)
 	assert.NoError(t, err)
 	err = loader.Load()
 	assert.NoError(t, err)
@@ -255,7 +255,7 @@ func TestOverwrite(t *testing.T) {
 			},
 		},
 	})
-	loader, err = NewLoader(config, client, beatInfo, nil)
+	loader, err = NewLoader(config, client, beatInfo, nil, false)
 	assert.NoError(t, err)
 	err = loader.Load()
 	assert.NoError(t, err)
@@ -275,7 +275,7 @@ func TestOverwrite(t *testing.T) {
 			},
 		},
 	})
-	loader, err = NewLoader(config, client, beatInfo, nil)
+	loader, err = NewLoader(config, client, beatInfo, nil, false)
 	assert.NoError(t, err)
 	err = loader.Load()
 	assert.NoError(t, err)
@@ -339,7 +339,7 @@ func TestTemplateWithData(t *testing.T) {
 	// Setup ES
 	client := estest.GetTestingElasticsearch(t)
 
-	tmpl, err := New(version.GetDefaultVersion(), "testindex", client.GetVersion(), TemplateConfig{})
+	tmpl, err := New(version.GetDefaultVersion(), "testindex", client.GetVersion(), TemplateConfig{}, false)
 	assert.NoError(t, err)
 	content, err := tmpl.LoadFile(fieldsPath)
 	assert.NoError(t, err)

libbeat/template/processor.go

@@ -27,6 +27,7 @@ import (
 // Processor struct to process fields to template
 type Processor struct {
 	EsVersion common.Version
+	Migration bool
 }
 
 var (
@@ -264,9 +265,14 @@ func (p *Processor) alias(f *common.Field) common.MapStr {
 		return nil
 	}
 
+	// In case migration is disabled and it's a migration alias, field is not created
+	if !p.Migration && f.MigrationAlias {
+		return nil
+	}
 	properties := getDefaultProperties(f)
 	properties["type"] = "alias"
 	properties["path"] = f.AliasPath
+
 	return properties
 }
 

libbeat/template/processor_test.go

@@ -28,7 +28,8 @@ import (
 func TestProcessor(t *testing.T) {
 	falseVar := false
 	trueVar := true
-	p := &Processor{}
+	p := &Processor{EsVersion: *common.MustNewVersion("7.0.0")}
+	migrationP := &Processor{EsVersion: *common.MustNewVersion("7.0.0"), Migration: true}
 	pEsVersion2 := &Processor{EsVersion: *common.MustNewVersion("2.0.0")}
 	pEsVersion64 := &Processor{EsVersion: *common.MustNewVersion("6.4.0")}
 	pEsVersion63 := &Processor{EsVersion: *common.MustNewVersion("6.3.6")}
@@ -282,6 +283,22 @@ func TestProcessor(t *testing.T) {
 				"type": "text", "doc_values": true,
 			},
 		},
+		{
+			output:   p.alias(&common.Field{Type: "alias", AliasPath: "a.c", MigrationAlias: false}),
+			expected: common.MapStr{"path": "a.c", "type": "alias"},
+		},
+		{
+			output:   p.alias(&common.Field{Type: "alias", AliasPath: "a.d", MigrationAlias: true}),
+			expected: nil,
+		},
+		{
+			output:   migrationP.alias(&common.Field{Type: "alias", AliasPath: "a.e", MigrationAlias: false}),
+			expected: common.MapStr{"path": "a.e", "type": "alias"},
+		},
+		{
+			output:   migrationP.alias(&common.Field{Type: "alias", AliasPath: "a.f", MigrationAlias: true}),
+			expected: common.MapStr{"path": "a.f", "type": "alias"},
+		},
 	}
 
 	for _, test := range tests {

libbeat/template/template.go

@@ -48,10 +48,11 @@ type Template struct {
 	beatVersion common.Version
 	esVersion   common.Version
 	config      TemplateConfig
+	migration   bool
 }
 
 // New creates a new template instance
-func New(beatVersion string, beatName string, esVersion common.Version, config TemplateConfig) (*Template, error) {
+func New(beatVersion string, beatName string, esVersion common.Version, config TemplateConfig, migration bool) (*Template, error) {
 	bV, err := common.NewVersion(beatVersion)
 	if err != nil {
 		return nil, err
@@ -116,6 +117,7 @@ func New(beatVersion string, beatName string, esVersion common.Version, config T
 		beatVersion: *bV,
 		esVersion:   esVersion,
 		config:      config,
+		migration:   migration,
 	}, nil
 }
 
@@ -139,7 +141,7 @@ func (t *Template) load(fields common.Fields) (common.MapStr, error) {
 
 	// Start processing at the root
 	properties := common.MapStr{}
-	processor := Processor{EsVersion: t.esVersion}
+	processor := Processor{EsVersion: t.esVersion, Migration: t.migration}
 	if err := processor.Process(fields, "", properties); err != nil {
 		return nil, err
 	}

libbeat/template/template_test.go

@@ -35,7 +35,7 @@ func TestNumberOfRoutingShards(t *testing.T) {
 
 	// Test it exists in 6.1
 	ver := common.MustNewVersion("6.1.0")
-	template, err := New(beatVersion, beatName, *ver, config)
+	template, err := New(beatVersion, beatName, *ver, config, false)
 	assert.NoError(t, err)
 
 	data := template.Generate(nil, nil)
@@ -46,7 +46,7 @@ func TestNumberOfRoutingShards(t *testing.T) {
 
 	// Test it does not exist in 6.0
 	ver = common.MustNewVersion("6.0.0")
-	template, err = New(beatVersion, beatName, *ver, config)
+	template, err = New(beatVersion, beatName, *ver, config, false)
 	assert.NoError(t, err)
 
 	data = template.Generate(nil, nil)
@@ -67,7 +67,7 @@ func TestNumberOfRoutingShardsOverwrite(t *testing.T) {
 
 	// Test it exists in 6.1
 	ver := common.MustNewVersion("6.1.0")
-	template, err := New(beatVersion, beatName, *ver, config)
+	template, err := New(beatVersion, beatName, *ver, config, false)
 	assert.NoError(t, err)
 
 	data := template.Generate(nil, nil)

libbeat/template/testdata/fields.yml

@@ -24,3 +24,13 @@
     - name: alias
       type: alias
       path: keyword
+
+    - name: migration_alias_true
+      type: alias
+      path: keyword
+      migration: true
+
+    - name: migration_alias_false
+      type: alias
+      path: keyword
+      migration_alias: false