From a3b2d5304e4d6f08d97f2393bddcd7beeccfe906 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 5 Aug 2020 11:57:01 +0200 Subject: [PATCH] [Ingest Manager] Send datastreams fields (#20402) (#20442) [Ingest Manager] Send datastreams fields (#20402) --- .../elastic-agent/pkg/agent/operation/monitoring.go | 12 ++++++------ .../program/testdata/constraints_config-filebeat.yml | 4 ++-- .../testdata/enabled_output_true-filebeat.yml | 4 ++-- .../agent/program/testdata/enabled_true-filebeat.yml | 4 ++-- .../program/testdata/single_config-filebeat.yml | 8 ++++---- .../program/testdata/single_config-metricbeat.yml | 12 ++++++------ x-pack/elastic-agent/pkg/agent/transpiler/rules.go | 6 ++++-- 7 files changed, 26 insertions(+), 24 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index 62372cc3f543..cd9d3d95a2cc 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -190,10 +190,10 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ - "target": "dataset", + "target": "data_stream", "fields": map[string]interface{}{ "type": "logs", - "name": "elastic.agent", + "dataset": "elastic.agent", "namespace": "default", }, }, @@ -224,10 +224,10 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ - "target": "dataset", + "target": "data_stream", "fields": map[string]interface{}{ "type": "logs", - "name": fmt.Sprintf("elastic.agent.%s", name), + "dataset": fmt.Sprintf("elastic.agent.%s", name), "namespace": "default", }, }, @@ -274,10 +274,10 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ - "target": "dataset", + "target": "data_stream", "fields": map[string]interface{}{ "type": "metrics", - "name": fmt.Sprintf("elastic.agent.%s", name), + "dataset": fmt.Sprintf("elastic.agent.%s", name), "namespace": "default", }, }, diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml index 15f6b71a9531..15013da3377b 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml @@ -7,10 +7,10 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml index c2e8c0d26ec0..8edc27061b00 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml @@ -7,10 +7,10 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml index 1da1c701d81c..8bd5d93a3b97 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml @@ -8,10 +8,10 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml index 0fb1a4356b55..b996e13b5318 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml @@ -9,10 +9,10 @@ filebeat: var: value processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" @@ -27,10 +27,10 @@ filebeat: var: value processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: testtype - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml index 67a3815e4a76..c62882ff6da6 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml @@ -6,10 +6,10 @@ metricbeat: hosts: ["http://127.0.0.1:8080"] processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: metrics - name: docker.status + dataset: docker.status namespace: default - add_fields: target: "event" @@ -21,10 +21,10 @@ metricbeat: hosts: ["http://127.0.0.1:8080"] processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: metrics - name: generic + dataset: generic namespace: default - add_fields: target: "event" @@ -39,10 +39,10 @@ metricbeat: fields: should_be: first - add_fields: - target: "dataset" + target: "data_stream" fields: type: metrics - name: generic + dataset: generic namespace: testing - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go index fe98386a150c..a9fc8f4497da 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go @@ -632,16 +632,18 @@ func (r *InjectStreamProcessorRule) Apply(ast *AST) error { return errors.New("InjectStreamProcessorRule: processors is not a list") } + // datastream processorMap := &Dict{value: make([]Node, 0)} - processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "dataset"}}) + processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "data_stream"}}) processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ &Key{name: "type", value: &StrVal{value: datasetType}}, &Key{name: "namespace", value: &StrVal{value: namespace}}, - &Key{name: "name", value: &StrVal{value: dataset}}, + &Key{name: "dataset", value: &StrVal{value: dataset}}, }}}) addFieldsMap := &Dict{value: []Node{&Key{"add_fields", processorMap}}} processorsList.value = mergeStrategy(r.OnConflict).InjectItem(processorsList.value, addFieldsMap) + // event processorMap = &Dict{value: make([]Node, 0)} processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "event"}}) processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{