From 314921bba0963ed3aea607f97488754bbc817d9b Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 23 Jun 2021 13:15:26 +0200 Subject: [PATCH 1/4] change index on change --- .../emitter/modifiers/monitoring_decorator.go | 1 + .../pkg/agent/operation/monitoring.go | 38 +++++++++++-------- .../pkg/core/monitoring/config/config.go | 3 ++ 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go index 5c1d2d037fd..8006b601502 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go @@ -49,6 +49,7 @@ func InjectMonitoring(agentInfo *info.AgentInfo, outputGroup string, rootAst *tr transpiler.NewKey("logs", transpiler.NewBoolVal(true)), transpiler.NewKey("metrics", transpiler.NewBoolVal(true)), transpiler.NewKey("use_output", transpiler.NewStrVal("default")), + transpiler.NewKey("namespace", transpiler.NewStrVal("default")), }) transpiler.Insert(rootAst, transpiler.NewKey("monitoring", monitoringNode), "settings") diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index d7c81d9a3a9..58f3b2bfae8 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -26,6 +26,7 @@ const ( metricsProcessName = "metricbeat" artifactPrefix = "beats" agentName = "elastic-agent" + defaultNamespace = "default" ) func (o *Operator) handleStartSidecar(s configrequest.Step) (result error) { @@ -129,10 +130,15 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [ watchLogs := o.monitor.WatchLogs() watchMetrics := o.monitor.WatchMetrics() + monitoringNamespace := defaultNamespace + if o.config.MonitoringConfig.Namespace != "" { + monitoringNamespace = o.config.MonitoringConfig.Namespace + } + // generate only when monitoring is running (for config refresh) or // state changes (turning on/off) if watchLogs != o.isMonitoringLogs() || watchLogs { - fbConfig, any := o.getMonitoringFilebeatConfig(output) + fbConfig, any := o.getMonitoringFilebeatConfig(output, monitoringNamespace) stepID := configrequest.StepRun if !watchLogs || !any { stepID = configrequest.StepRemove @@ -149,7 +155,7 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [ steps = append(steps, filebeatStep) } if watchMetrics != o.isMonitoringMetrics() || watchMetrics { - mbConfig, any := o.getMonitoringMetricbeatConfig(output) + mbConfig, any := o.getMonitoringMetricbeatConfig(output, monitoringNamespace) stepID := configrequest.StepRun if !watchMetrics || !any { stepID = configrequest.StepRemove @@ -182,12 +188,12 @@ func loadSpecFromSupported(processName string) program.Spec { } } -func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]interface{}, bool) { +func (o *Operator) getMonitoringFilebeatConfig(output interface{}, monitoringNamespace string) (map[string]interface{}, bool) { inputs := []interface{}{ map[string]interface{}{ "type": "filestream", "parsers": []map[string]interface{}{ - map[string]interface{}{ + { "ndjson": map[string]interface{}{ "overwrite_keys": true, "message_key": "message", @@ -200,7 +206,7 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log"), filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log*"), }, - "index": "logs-elastic_agent-default", + "index": fmt.Sprintf("logs-elastic_agent-%s", monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -208,7 +214,7 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i "fields": map[string]interface{}{ "type": "logs", "dataset": "elastic_agent", - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -247,7 +253,7 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i inputs = append(inputs, map[string]interface{}{ "type": "filestream", "parsers": []map[string]interface{}{ - map[string]interface{}{ + { "ndjson": map[string]interface{}{ "overwrite_keys": true, "message_key": "message", @@ -255,7 +261,7 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i }, }, "paths": paths, - "index": fmt.Sprintf("logs-elastic_agent.%s-default", name), + "index": fmt.Sprintf("logs-elastic_agent.%s-%s", name, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -263,7 +269,7 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i "fields": map[string]interface{}{ "type": "logs", "dataset": fmt.Sprintf("elastic_agent.%s", name), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -311,7 +317,7 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i return result, true } -func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string]interface{}, bool) { +func (o *Operator) getMonitoringMetricbeatConfig(output interface{}, monitoringNamespace string) (map[string]interface{}, bool) { hosts := o.getMetricbeatEndpoints() if len(hosts) == 0 { return nil, false @@ -325,7 +331,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string "metricsets": []string{"stats", "state"}, "period": "10s", "hosts": endpoints, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", name), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", name, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -333,7 +339,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", name), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -363,7 +369,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string "period": "10s", "path": "/stats", "hosts": endpoints, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -371,7 +377,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", fixedAgentName), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -446,7 +452,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string "period": "10s", "path": "/stats", "hosts": []string{beats.AgentPrefixedMonitoringEndpoint(o.config.DownloadConfig.OS(), o.config.MonitoringConfig.HTTP)}, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -454,7 +460,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", fixedAgentName), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, diff --git a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go index 2ce067d4e19..fe18b0fb73e 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go @@ -5,6 +5,7 @@ package config const defaultPort = 6791 +const defaultNamespace = "default" // MonitoringConfig describes a configuration of a monitoring type MonitoringConfig struct { @@ -12,6 +13,7 @@ type MonitoringConfig struct { MonitorLogs bool `yaml:"logs" config:"logs"` MonitorMetrics bool `yaml:"metrics" config:"metrics"` HTTP *MonitoringHTTPConfig `yaml:"http" config:"http"` + Namespace string `yaml:"namespace" config:"namespace"` } // MonitoringHTTPConfig is a config defining HTTP endpoint published by agent @@ -33,5 +35,6 @@ func DefaultConfig() *MonitoringConfig { Enabled: false, Port: defaultPort, }, + Namespace: defaultNamespace, } } From 6f4da855a674d5ee90532e39bfaa7aa4e1bff999 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 23 Jun 2021 13:33:46 +0200 Subject: [PATCH 2/4] make it reloadable --- x-pack/elastic-agent/pkg/agent/operation/monitoring.go | 7 +------ .../elastic-agent/pkg/agent/operation/monitoring_test.go | 3 +++ .../pkg/core/monitoring/beats/beats_monitor.go | 9 +++++++++ x-pack/elastic-agent/pkg/core/monitoring/monitor.go | 1 + .../pkg/core/monitoring/noop/noop_monitor.go | 3 +++ 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index 58f3b2bfae8..c478700aea0 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -26,7 +26,6 @@ const ( metricsProcessName = "metricbeat" artifactPrefix = "beats" agentName = "elastic-agent" - defaultNamespace = "default" ) func (o *Operator) handleStartSidecar(s configrequest.Step) (result error) { @@ -129,11 +128,7 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [ var steps []configrequest.Step watchLogs := o.monitor.WatchLogs() watchMetrics := o.monitor.WatchMetrics() - - monitoringNamespace := defaultNamespace - if o.config.MonitoringConfig.Namespace != "" { - monitoringNamespace = o.config.MonitoringConfig.Namespace - } + monitoringNamespace := o.monitor.MonitoringNamespace() // generate only when monitoring is running (for config refresh) or // state changes (turning on/off) diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go index cbf9edf3266..a36d45c2f48 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go @@ -214,6 +214,9 @@ func (b *testMonitor) Reload(cfg *config.Config) error { return nil } // IsMonitoringEnabled returns true if monitoring is configured. func (b *testMonitor) IsMonitoringEnabled() bool { return b.monitorLogs || b.monitorMetrics } +// MonitoringNamespace returns monitoring namespace configured. +func (b *testMonitor) MonitoringNamespace() string { return "default" } + // WatchLogs return true if monitoring is configured and monitoring logs is enabled. func (b *testMonitor) WatchLogs() bool { return b.monitorLogs } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go index 743d44118d6..1c0c4ba61ad 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go @@ -21,6 +21,7 @@ import ( ) const httpPlusPrefix = "http+" +const defaultMonitoringNamespace = "default" // Monitor is a monitoring interface providing information about the way // how beat is monitored @@ -69,6 +70,14 @@ func (b *Monitor) Close() { // IsMonitoringEnabled returns true if monitoring is enabled. func (b *Monitor) IsMonitoringEnabled() bool { return b.config.Enabled } +// MonitoringNamespace returns monitoring namespace configured. +func (b *Monitor) MonitoringNamespace() string { + if b.config.Namespace == "" { + return defaultMonitoringNamespace + } + return b.config.Namespace +} + // WatchLogs returns true if monitoring is enabled and monitor should watch logs. func (b *Monitor) WatchLogs() bool { return b.config.Enabled && b.config.MonitorLogs } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/monitor.go index 00c7a50003a..6c71f4f65fc 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/monitor.go @@ -23,6 +23,7 @@ type Monitor interface { Cleanup(spec program.Spec, pipelineID string) error Reload(cfg *config.Config) error IsMonitoringEnabled() bool + MonitoringNamespace() string WatchLogs() bool WatchMetrics() bool Close() diff --git a/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go index 9ea8f08a788..d98deb90888 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go @@ -66,3 +66,6 @@ func (b *Monitor) WatchLogs() bool { return false } // WatchMetrics return true if monitoring is configured and monitoring metrics is enabled. func (b *Monitor) WatchMetrics() bool { return false } + +// MonitoringNamespace returns monitoring namespace configured. +func (b *Monitor) MonitoringNamespace() string { return "default" } From 80be0eeacbc2564e3b7206e038b7921129726fa5 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 23 Jun 2021 13:38:22 +0200 Subject: [PATCH 3/4] changelog --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index e1ccd6c4c62..919360e5bff 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -114,3 +114,4 @@ - Use `filestream` input for internal log collection. {pull}25660[25660] - Enable agent to send custom headers to kibana/ES {pull}26275[26275] - Set `agent.id` to the Fleet Agent ID in events published from inputs backed by Beats. {issue}21121[21121] {pull}26394[26394] +- Enable configuring monitoring namespace {issue}26439[26439] From 11ebaab55a3ea6f812a12f0ec1abd3049d395d96 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 25 Jun 2021 14:37:39 +0200 Subject: [PATCH 4/4] unit --- .../pkg/agent/program/program_test.go | 4 + .../testdata/namespace-endpoint-security.yml | 114 ++++++++++ .../program/testdata/namespace-filebeat.yml | 68 ++++++ .../testdata/namespace-fleet-server.yml | 16 ++ .../program/testdata/namespace-heartbeat.yml | 30 +++ .../program/testdata/namespace-metricbeat.yml | 88 ++++++++ .../program/testdata/namespace-packetbeat.yml | 35 +++ .../pkg/agent/program/testdata/namespace.yml | 201 ++++++++++++++++++ 8 files changed, 556 insertions(+) create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go index 4498f7e5236..5ca35de0136 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/program_test.go @@ -383,6 +383,10 @@ func TestConfiguration(t *testing.T) { empty bool err bool }{ + "namespace": { + programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"}, + expected: 6, + }, "single_config": { programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"}, expected: 6, diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml new file mode 100644 index 00000000000..7e9f04dc411 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml @@ -0,0 +1,114 @@ +fleet: + enabled: true + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + +output: + elasticsearch: + hosts: + - "127.0.0.1:9200" + - "127.0.0.1:9300" + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= + +inputs: +- id: endpoint-id + type: endpoint + name: endpoint-1 + enabled: true + package: + name: endpoint + version: 0.3.0 + data_stream: + namespace: default + artifact_manifest: + schema_version: v22 + manifest_version: v21 + artifacts: + - endpoint-allowlist-windows: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-windows + - endpoint-allowlist-macos: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-macos + - endpoint-allowlist-linux: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-linux + policy: + linux: + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + windows: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + registry: logs-endpoint.events.registry-default + process: logs-endpoint.events.process-default + driver: logs-endpoint.events.driver-default + library: logs-endpoint.events.library-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + registry: true + process: true + security: true + file: true + dns: false + dll_and_driver_load: false + network: true + mac: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml new file mode 100644 index 00000000000..83df83e56e0 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml @@ -0,0 +1,68 @@ +filebeat: + inputs: + - type: log + paths: + - /var/log/hello1.log + - /var/log/hello2.log + index: logs-generic-default + vars: + var: value + processors: + - add_fields: + target: "data_stream" + fields: + type: logs + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - type: log + paths: + - /var/log/hello3.log + - /var/log/hello4.log + index: testtype-generic-default + vars: + var: value + processors: + - add_fields: + target: "data_stream" + fields: + type: testtype + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml new file mode 100644 index 00000000000..c03696aff1f --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml @@ -0,0 +1,16 @@ +fleet: + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + +output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword + +inputs: + - id: fleet-server-id + type: fleet-server diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml new file mode 100644 index 00000000000..f34b204f5fa --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml @@ -0,0 +1,30 @@ +inputs: +- type: synthetics/http + id: unique-http-id + name: my-http + schedule: '*/5 * * * * * *' + host: "http://localhost:80/service/status" + timeout: 16s + wait: 1s + data_stream.namespace: default + processors: + - add_fields: + target: 'elastic_agent' + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: 'agent' + fields: + id: agent-id +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml new file mode 100644 index 00000000000..3f16a9d9e21 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml @@ -0,0 +1,88 @@ +metricbeat: + modules: + - module: docker + metricsets: [status] + index: metrics-docker.status-default + hosts: ["http://127.0.0.1:8080"] + processors: + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: docker.status + namespace: default + - add_fields: + target: "event" + fields: + dataset: docker.status + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - module: docker + metricsets: [info] + index: metrics-generic-default + hosts: ["http://127.0.0.1:8080"] + processors: + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - module: apache + metricsets: [info] + index: metrics-generic-testing + hosts: ["http://apache.remote"] + processors: + - add_fields: + fields: + should_be: first + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: generic + namespace: testing + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id +output: + elasticsearch: + hosts: [127.0.0.1:9200, 127.0.0.1:9300] + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml new file mode 100644 index 00000000000..d71499bdef4 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml @@ -0,0 +1,35 @@ +inputs: +- type: packet + processors: + - add_fields: + target: 'elastic_agent' + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: 'agent' + fields: + id: agent-id + streams: + - type: flow + timeout: 10s + period: 10s + keep_null: false + data_stream: + dataset: packet.flow + type: logs + - type: icmp + data_stream: + dataset: packet.icmp + type: logs +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml new file mode 100644 index 00000000000..c2f83a9abf0 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml @@ -0,0 +1,201 @@ +name: Production Website DB Servers +fleet: + enabled: true + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + server: + output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword + +outputs: + default: + type: elasticsearch + namespace: test_namespace + hosts: [127.0.0.1:9200, 127.0.0.1:9300] + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= + + monitoring: + type: elasticsearch + api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + hosts: ["monitoring:9200"] + ca_sha256: "7lHLiyp4J8m9kw38SJ7SURJP4bXRZv/BNxyyXkCcE/M=" + +inputs: +- id: fleet-server-id + type: fleet-server + use_output: default + data_stream: + type: default +- type: docker/metrics + use_output: default + streams: + - metricset: status + processors: + - null + data_stream: + dataset: docker.status + - metricset: info + data_stream: + dataset: "" + hosts: ["http://127.0.0.1:8080"] +- type: logfile + use_output: default + streams: + - paths: + - /var/log/hello1.log + - /var/log/hello2.log + vars: + var: value +- type: logfile + data_stream: + type: testtype + use_output: default + streams: + - paths: + - /var/log/hello3.log + - /var/log/hello4.log + vars: + var: value +- id: apache-metrics-id + type: apache/metrics + data_stream: + namespace: testing + use_output: default + processors: + - add_fields: + fields: + should_be: first + streams: + - enabled: true + metricset: info + hosts: ["http://apache.remote"] + hosts: ["http://apache.local"] +- type: synthetics/http + id: unique-http-id + name: my-http + schedule: '*/5 * * * * * *' + host: "http://localhost:80/service/status" + timeout: 16s + wait: 1s +- type: packet + streams: + - type: flow + timeout: 10s + period: 10s + keep_null: false + data_stream: + dataset: packet.flow + type: logs + - type: icmp + data_stream: + dataset: packet.icmp + type: logs +- id: endpoint-id + type: endpoint + name: endpoint-1 + enabled: true + package: + name: endpoint + version: 0.3.0 + data_stream: + namespace: default + artifact_manifest: + schema_version: v22 + manifest_version: v21 + artifacts: + - endpoint-allowlist-windows: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-windows + - endpoint-allowlist-macos: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-macos + - endpoint-allowlist-linux: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-linux + policy: + linux: + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + windows: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + registry: logs-endpoint.events.registry-default + process: logs-endpoint.events.process-default + driver: logs-endpoint.events.driver-default + library: logs-endpoint.events.library-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + registry: true + process: true + security: true + file: true + dns: false + dll_and_driver_load: false + network: true + mac: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + +agent.monitoring: + use_output: monitoring + +agent: + reload: 123