diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 5926e86cc71..12d76adabc1 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -22,6 +22,8 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Move Fields from package libbeat/common to libbeat/mapping. {pull}11198[11198] - For "metricbeat style" generated custom beats, the mage target `GoTestIntegration` has changed to `GoIntegTest` and `GoTestUnit` has changed to `GoUnitTest`. {pull}13341[13341] +- Build docker and kubernetes features only on supported platforms. {pull}13509[13509] +- Need to register new processors to be used in the JS processor in their `init` functions. {pull}13509[13509] ==== Bugfixes diff --git a/filebeat/tests/system/test_processors.py b/filebeat/tests/system/test_processors.py index c6514986120..95673cfdb5e 100644 --- a/filebeat/tests/system/test_processors.py +++ b/filebeat/tests/system/test_processors.py @@ -2,6 +2,8 @@ from filebeat import BaseTest import io import os +import unittest +import sys """ Contains tests for filtering. @@ -300,6 +302,42 @@ def test_decode_csv_fields_all_options(self): ["42", "hello world", "string\twith tabs and \"broken\" quotes"], ]) + def test_javascript_processor_add_host_metadata(self): + """ + Check JS processor with add_host_metadata + """ + + self._test_javascript_processor_with_source("""\'var processor = require("processor"); +var addHostMetadata = new processor.AddHostMetadata({"netinfo.enabled": true}); + +function process(evt) { + addHostMetadata.Run(evt); +}\' +""") + + output = self.read_output() + for evt in output: + assert "host.hostname" in evt + + def _test_javascript_processor_with_source(self, script_source): + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/test.log", + processors=[ + { + "script": { + "lang": "javascript", + "source": script_source, + }, + }, + ] + ) + + self._init_and_read_test_input([ + u"test line 1\n", + u"test line 2\n", + u"test line 3\n", + ]) + def _init_and_read_test_input(self, input_lines): with io.open(self.working_dir + "/test.log", "w", encoding="utf-8") as f: for line in input_lines: diff --git a/heartbeat/scripts/generate_imports_helper.py b/heartbeat/scripts/generate_imports_helper.py index 9a7e7ecd5cd..be85b8a2318 100644 --- a/heartbeat/scripts/generate_imports_helper.py +++ b/heartbeat/scripts/generate_imports_helper.py @@ -23,7 +23,11 @@ def imports(mode): package = "monitors/{}".format(mode) return [format(package, m) for m in collect_monitors(package)] - return sorted(imports("active") + imports("passive")) + return [{ + "file_suffix": "", + "build_tags": "", + "imported_lines": sorted(imports("active") + imports("passive")) + }] def collect_monitors(package): diff --git a/libbeat/autodiscover/providers/docker/config.go b/libbeat/autodiscover/providers/docker/config.go index 6e103b208c6..cba67993677 100644 --- a/libbeat/autodiscover/providers/docker/config.go +++ b/libbeat/autodiscover/providers/docker/config.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package docker import ( diff --git a/libbeat/autodiscover/providers/docker/docker.go b/libbeat/autodiscover/providers/docker/docker.go index 4e9b8aec631..a37ff756934 100644 --- a/libbeat/autodiscover/providers/docker/docker.go +++ b/libbeat/autodiscover/providers/docker/docker.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package docker import ( diff --git a/libbeat/autodiscover/providers/kubernetes/config.go b/libbeat/autodiscover/providers/kubernetes/config.go index 486ff3bc531..d7acea09034 100644 --- a/libbeat/autodiscover/providers/kubernetes/config.go +++ b/libbeat/autodiscover/providers/kubernetes/config.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package kubernetes import ( diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index 98c729c3329..d0de3f314f5 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package kubernetes import ( diff --git a/libbeat/cmd/instance/imports.go b/libbeat/cmd/instance/imports_common.go similarity index 86% rename from libbeat/cmd/instance/imports.go rename to libbeat/cmd/instance/imports_common.go index 240eca87e30..6b78ffc7d56 100644 --- a/libbeat/cmd/instance/imports.go +++ b/libbeat/cmd/instance/imports_common.go @@ -19,15 +19,11 @@ package instance import ( _ "github.com/elastic/beats/libbeat/autodiscover/appenders/config" // Register autodiscover appenders - _ "github.com/elastic/beats/libbeat/autodiscover/providers/docker" // Register autodiscover providers _ "github.com/elastic/beats/libbeat/autodiscover/providers/jolokia" - _ "github.com/elastic/beats/libbeat/autodiscover/providers/kubernetes" _ "github.com/elastic/beats/libbeat/monitoring/report/elasticsearch" // Register default monitoring reporting _ "github.com/elastic/beats/libbeat/processors/actions" // Register default processors. _ "github.com/elastic/beats/libbeat/processors/add_cloud_metadata" - _ "github.com/elastic/beats/libbeat/processors/add_docker_metadata" _ "github.com/elastic/beats/libbeat/processors/add_host_metadata" - _ "github.com/elastic/beats/libbeat/processors/add_kubernetes_metadata" _ "github.com/elastic/beats/libbeat/processors/add_locale" _ "github.com/elastic/beats/libbeat/processors/add_observer_metadata" _ "github.com/elastic/beats/libbeat/processors/add_process_metadata" diff --git a/libbeat/cmd/instance/imports_docker.go b/libbeat/cmd/instance/imports_docker.go new file mode 100644 index 00000000000..faca620d087 --- /dev/null +++ b/libbeat/cmd/instance/imports_docker.go @@ -0,0 +1,27 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux darwin windows + +package instance + +import ( + _ "github.com/elastic/beats/libbeat/autodiscover/providers/docker" // Register autodiscover providers + _ "github.com/elastic/beats/libbeat/autodiscover/providers/kubernetes" + _ "github.com/elastic/beats/libbeat/processors/add_docker_metadata" + _ "github.com/elastic/beats/libbeat/processors/add_kubernetes_metadata" +) diff --git a/libbeat/common/docker/client.go b/libbeat/common/docker/client.go index d7e38030781..12337910977 100644 --- a/libbeat/common/docker/client.go +++ b/libbeat/common/docker/client.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package docker import ( diff --git a/libbeat/common/docker/client_test.go b/libbeat/common/docker/client_test.go index a9a93a0f755..14268e78e76 100644 --- a/libbeat/common/docker/client_test.go +++ b/libbeat/common/docker/client_test.go @@ -16,6 +16,7 @@ // under the License. // +build integration +// +build linux darwin windows package docker diff --git a/libbeat/common/docker/watcher.go b/libbeat/common/docker/watcher.go index 61b69c37e21..efabbd1928d 100644 --- a/libbeat/common/docker/watcher.go +++ b/libbeat/common/docker/watcher.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package docker import ( diff --git a/libbeat/common/docker/watcher_test.go b/libbeat/common/docker/watcher_test.go index 98f13b2d107..faed944401d 100644 --- a/libbeat/common/docker/watcher_test.go +++ b/libbeat/common/docker/watcher_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package docker import ( diff --git a/libbeat/processors/actions/add_fields.go b/libbeat/processors/actions/add_fields.go index 22e2fd02fe4..39b41b0c905 100644 --- a/libbeat/processors/actions/add_fields.go +++ b/libbeat/processors/actions/add_fields.go @@ -25,6 +25,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) type addFields struct { @@ -40,6 +41,8 @@ func init() { checks.ConfigChecked(CreateAddFields, checks.RequireFields(FieldsKey), checks.AllowedFields(FieldsKey, "target", "when"))) + + jsprocessor.RegisterPlugin("AddFields", CreateAddFields) } // CreateAddFields constructs an add_fields processor from config. diff --git a/libbeat/processors/actions/copy_fields.go b/libbeat/processors/actions/copy_fields.go index af5709d54e1..c67dd996385 100644 --- a/libbeat/processors/actions/copy_fields.go +++ b/libbeat/processors/actions/copy_fields.go @@ -27,6 +27,7 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) type copyFields struct { @@ -45,6 +46,7 @@ func init() { checks.RequireFields("fields"), ), ) + jsprocessor.RegisterPlugin("CopyFields", NewCopyFields) } // NewCopyFields returns a new copy_fields processor. diff --git a/libbeat/processors/actions/decode_base64_field.go b/libbeat/processors/actions/decode_base64_field.go index c4e33d4a20e..bf2c92dc6e4 100644 --- a/libbeat/processors/actions/decode_base64_field.go +++ b/libbeat/processors/actions/decode_base64_field.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) const ( @@ -50,6 +51,7 @@ func init() { checks.ConfigChecked(NewDecodeBase64Field, checks.RequireFields("field"), checks.AllowedFields("field", "when", "ignore_missing", "fail_on_error"))) + jsprocessor.RegisterPlugin("DecodeBase64Field", NewDecodeBase64Field) } // NewDecodeBase64Field construct a new decode_base64_field processor. diff --git a/libbeat/processors/actions/decode_json_fields.go b/libbeat/processors/actions/decode_json_fields.go index a2666a8b8cc..ca85a0d4bfd 100644 --- a/libbeat/processors/actions/decode_json_fields.go +++ b/libbeat/processors/actions/decode_json_fields.go @@ -31,6 +31,7 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) type decodeJSONFields struct { @@ -66,6 +67,8 @@ func init() { checks.ConfigChecked(NewDecodeJSONFields, checks.RequireFields("fields"), checks.AllowedFields("fields", "max_depth", "overwrite_keys", "add_error_key", "process_array", "target", "when"))) + + jsprocessor.RegisterPlugin("DecodeJSONFields", NewDecodeJSONFields) } // NewDecodeJSONFields construct a new decode_json_fields processor. diff --git a/libbeat/processors/actions/rename.go b/libbeat/processors/actions/rename.go index c0c8f778745..89e92d2a3e6 100644 --- a/libbeat/processors/actions/rename.go +++ b/libbeat/processors/actions/rename.go @@ -27,6 +27,7 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) type renameFields struct { @@ -48,6 +49,8 @@ func init() { processors.RegisterPlugin("rename", checks.ConfigChecked(NewRenameFields, checks.RequireFields("fields"))) + + jsprocessor.RegisterPlugin("Rename", NewRenameFields) } // NewRenameFields returns a new rename processor. diff --git a/libbeat/processors/actions/truncate_fields.go b/libbeat/processors/actions/truncate_fields.go index ca87eb35c5e..1c1256f69bc 100644 --- a/libbeat/processors/actions/truncate_fields.go +++ b/libbeat/processors/actions/truncate_fields.go @@ -30,6 +30,7 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) type truncateFieldsConfig struct { @@ -54,6 +55,7 @@ func init() { checks.MutuallyExclusiveRequiredFields("max_bytes", "max_characters"), ), ) + jsprocessor.RegisterPlugin("TruncateFields", NewTruncateFields) } // NewTruncateFields returns a new truncate_fields processor. diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index d61992457e4..451763801d5 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -34,6 +34,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) const ( @@ -53,6 +54,7 @@ var debugf = logp.MakeDebug("filters") // init registers the add_cloud_metadata processor. func init() { processors.RegisterPlugin("add_cloud_metadata", New) + jsprocessor.RegisterPlugin("AddCloudMetadata", New) } type schemaConv func(m map[string]interface{}) common.MapStr diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index 70b2aaa5e50..6148a4be2eb 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package add_docker_metadata import ( @@ -35,6 +37,7 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/actions" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) const ( @@ -49,6 +52,7 @@ var processCgroupPaths = cgroup.ProcessCgroupPaths func init() { processors.RegisterPlugin(processorName, New) + jsprocessor.RegisterPlugin("AddDockerMetadata", New) } type addDockerMetadata struct { diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index 7fc1c00338e..9a99ac1cd02 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package add_docker_metadata import ( diff --git a/libbeat/processors/add_docker_metadata/config.go b/libbeat/processors/add_docker_metadata/config.go index aa78dd08ca0..930ca4ecdc6 100644 --- a/libbeat/processors/add_docker_metadata/config.go +++ b/libbeat/processors/add_docker_metadata/config.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package add_docker_metadata import ( diff --git a/libbeat/processors/add_host_metadata/add_host_metadata.go b/libbeat/processors/add_host_metadata/add_host_metadata.go index 1e1266deaf2..6cfc938e5f0 100644 --- a/libbeat/processors/add_host_metadata/add_host_metadata.go +++ b/libbeat/processors/add_host_metadata/add_host_metadata.go @@ -29,12 +29,14 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/metric/system/host" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" "github.com/elastic/beats/libbeat/processors/util" "github.com/elastic/go-sysinfo" ) func init() { processors.RegisterPlugin("add_host_metadata", New) + jsprocessor.RegisterPlugin("AddHostMetadata", New) } type addHostMetadata struct { diff --git a/libbeat/processors/add_kubernetes_metadata/cache.go b/libbeat/processors/add_kubernetes_metadata/cache.go index 156ae38e3bd..f5fb4ba52e6 100644 --- a/libbeat/processors/add_kubernetes_metadata/cache.go +++ b/libbeat/processors/add_kubernetes_metadata/cache.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package add_kubernetes_metadata import ( diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index ea635314b07..e16f1043fe4 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package add_kubernetes_metadata import ( @@ -26,6 +28,7 @@ import ( "github.com/elastic/beats/libbeat/common/kubernetes" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) const ( @@ -41,6 +44,7 @@ type kubernetesAnnotator struct { func init() { processors.RegisterPlugin("add_kubernetes_metadata", New) + jsprocessor.RegisterPlugin("AddKubernetesMetadata", New) // Register default indexers Indexing.AddIndexer(PodNameIndexerName, NewPodNameIndexer) diff --git a/libbeat/processors/add_locale/add_locale.go b/libbeat/processors/add_locale/add_locale.go index 2e0eb81c4a5..9d98c045df7 100644 --- a/libbeat/processors/add_locale/add_locale.go +++ b/libbeat/processors/add_locale/add_locale.go @@ -27,6 +27,7 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) type addLocale struct { @@ -53,6 +54,7 @@ func (t TimezoneFormat) String() string { func init() { processors.RegisterPlugin("add_locale", New) + jsprocessor.RegisterPlugin("AddLocale", New) } // New constructs a new add_locale processor. diff --git a/libbeat/processors/add_observer_metadata/add_observer_metadata.go b/libbeat/processors/add_observer_metadata/add_observer_metadata.go index 050e971dd97..b1740df0067 100644 --- a/libbeat/processors/add_observer_metadata/add_observer_metadata.go +++ b/libbeat/processors/add_observer_metadata/add_observer_metadata.go @@ -28,12 +28,14 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" "github.com/elastic/beats/libbeat/processors/util" "github.com/elastic/go-sysinfo" ) func init() { processors.RegisterPlugin("add_observer_metadata", New) + jsprocessor.RegisterPlugin("AddObserverMetadata", New) } type observerMetadata struct { diff --git a/libbeat/processors/add_process_metadata/add_process_metadata.go b/libbeat/processors/add_process_metadata/add_process_metadata.go index e600da7bb5d..2b78fcb817e 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata.go @@ -29,6 +29,7 @@ import ( "github.com/elastic/beats/libbeat/common/atomic" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) const ( @@ -72,6 +73,7 @@ type processMetadataProvider interface { func init() { processors.RegisterPlugin(processorName, New) + jsprocessor.RegisterPlugin("AddProcessMetadata", New) } // New constructs a new add_process_metadata processor. diff --git a/libbeat/processors/communityid/communityid.go b/libbeat/processors/communityid/communityid.go index cc13e73175d..f5c2812f6b1 100644 --- a/libbeat/processors/communityid/communityid.go +++ b/libbeat/processors/communityid/communityid.go @@ -31,12 +31,14 @@ import ( "github.com/elastic/beats/libbeat/common/flowhash" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) const logName = "processor.community_id" func init() { processors.RegisterPlugin("community_id", New) + jsprocessor.RegisterPlugin("CommunityID", New) } type processor struct { diff --git a/libbeat/processors/convert/convert.go b/libbeat/processors/convert/convert.go index 8e84d93de23..fbc6b5b3014 100644 --- a/libbeat/processors/convert/convert.go +++ b/libbeat/processors/convert/convert.go @@ -30,12 +30,14 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) const logName = "processor.convert" func init() { processors.RegisterPlugin("convert", New) + jsprocessor.RegisterPlugin("Convert", New) } type processor struct { diff --git a/libbeat/processors/decode_csv_fields/decode_csv_fields.go b/libbeat/processors/decode_csv_fields/decode_csv_fields.go index 2084cf6b50e..e65fe2231ac 100644 --- a/libbeat/processors/decode_csv_fields/decode_csv_fields.go +++ b/libbeat/processors/decode_csv_fields/decode_csv_fields.go @@ -29,6 +29,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) type decodeCSVFields struct { @@ -60,6 +61,8 @@ func init() { checks.ConfigChecked(NewDecodeCSVField, checks.RequireFields("fields"), checks.AllowedFields("fields", "ignore_missing", "overwrite_keys", "separator", "trim_leading_space", "overwrite_keys", "fail_on_error", "when"))) + + jsprocessor.RegisterPlugin("DecodeCSVField", NewDecodeCSVField) } // NewDecodeCSVField construct a new decode_csv_field processor. diff --git a/libbeat/processors/dissect/processor.go b/libbeat/processors/dissect/processor.go index d47b595e06c..ab5c1890309 100644 --- a/libbeat/processors/dissect/processor.go +++ b/libbeat/processors/dissect/processor.go @@ -25,6 +25,7 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) const flagParsingError = "dissect_parsing_error" @@ -35,6 +36,7 @@ type processor struct { func init() { processors.RegisterPlugin("dissect", NewProcessor) + jsprocessor.RegisterPlugin("Dissect", NewProcessor) } // NewProcessor constructs a new dissect processor. diff --git a/libbeat/processors/dns/dns.go b/libbeat/processors/dns/dns.go index d1c93a36f24..7dc447d44c6 100644 --- a/libbeat/processors/dns/dns.go +++ b/libbeat/processors/dns/dns.go @@ -31,6 +31,7 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/monitoring" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) const logName = "processor.dns" @@ -40,6 +41,7 @@ var instanceID = atomic.MakeUint32(0) func init() { processors.RegisterPlugin("dns", New) + jsprocessor.RegisterPlugin("DNS", New) } type processor struct { diff --git a/libbeat/processors/extract_array/extract_array.go b/libbeat/processors/extract_array/extract_array.go index 2bb473589cd..ce76b0962ae 100644 --- a/libbeat/processors/extract_array/extract_array.go +++ b/libbeat/processors/extract_array/extract_array.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/checks" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) type config struct { @@ -61,6 +62,8 @@ func init() { checks.ConfigChecked(New, checks.RequireFields("field", "mappings"), checks.AllowedFields("field", "mappings", "ignore_missing", "overwrite_keys", "fail_on_error", "when", "omit_empty"))) + + jsprocessor.RegisterPlugin("ExtractArray", New) } // Unpack unpacks the processor's configuration. diff --git a/libbeat/processors/namespace.go b/libbeat/processors/namespace.go index cea3e53ef23..1dd10f4b517 100644 --- a/libbeat/processors/namespace.go +++ b/libbeat/processors/namespace.go @@ -121,4 +121,13 @@ func (ns *Namespace) Plugin() Constructor { }) } +// Constructors returns all registered processor constructors and its names. +func (ns *Namespace) Constructors() map[string]Constructor { + c := make(map[string]Constructor, len(ns.reg)) + for name, p := range ns.reg { + c[name] = p.Plugin() + } + return c +} + func (p plugin) Plugin() Constructor { return p.c } diff --git a/libbeat/processors/registered_domain/registered_domain.go b/libbeat/processors/registered_domain/registered_domain.go index 0480c7c9186..b378da75233 100644 --- a/libbeat/processors/registered_domain/registered_domain.go +++ b/libbeat/processors/registered_domain/registered_domain.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) const ( @@ -37,6 +38,7 @@ const ( func init() { processors.RegisterPlugin(procName, New) + jsprocessor.RegisterPlugin("RegisteredDomain", New) } type processor struct { diff --git a/libbeat/processors/script/javascript/module/processor/chain.go b/libbeat/processors/script/javascript/module/processor/chain.go index cbf09d1ed30..947903c57ef 100644 --- a/libbeat/processors/script/javascript/module/processor/chain.go +++ b/libbeat/processors/script/javascript/module/processor/chain.go @@ -42,7 +42,7 @@ func newChainBuilder(runtime *goja.Runtime) func(call goja.ConstructorCall) *goj } c := &chainBuilder{runtime: runtime, this: call.This} - for name, fn := range constructors { + for name, fn := range registry.Constructors() { c.this.Set(name, c.makeBuilderFunc(fn)) } call.This.Set("Add", c.Add) diff --git a/libbeat/processors/script/javascript/module/processor/processor.go b/libbeat/processors/script/javascript/module/processor/processor.go index 758e7abc273..da5a914cd92 100644 --- a/libbeat/processors/script/javascript/module/processor/processor.go +++ b/libbeat/processors/script/javascript/module/processor/processor.go @@ -22,50 +22,23 @@ import ( "github.com/dop251/goja_nodejs/require" "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" - "github.com/elastic/beats/libbeat/processors/actions" - "github.com/elastic/beats/libbeat/processors/add_cloud_metadata" - "github.com/elastic/beats/libbeat/processors/add_docker_metadata" - "github.com/elastic/beats/libbeat/processors/add_host_metadata" - "github.com/elastic/beats/libbeat/processors/add_kubernetes_metadata" - "github.com/elastic/beats/libbeat/processors/add_locale" - "github.com/elastic/beats/libbeat/processors/add_observer_metadata" - "github.com/elastic/beats/libbeat/processors/add_process_metadata" - "github.com/elastic/beats/libbeat/processors/communityid" - "github.com/elastic/beats/libbeat/processors/convert" - "github.com/elastic/beats/libbeat/processors/decode_csv_fields" - "github.com/elastic/beats/libbeat/processors/dissect" - "github.com/elastic/beats/libbeat/processors/dns" - "github.com/elastic/beats/libbeat/processors/extract_array" - "github.com/elastic/beats/libbeat/processors/registered_domain" "github.com/elastic/beats/libbeat/processors/script/javascript" - "github.com/elastic/beats/libbeat/processors/timestamp" ) // Create constructors for most of the Beat processors. // Note that script is omitted to avoid nesting. -var constructors = map[string]processors.Constructor{ - "AddCloudMetadata": add_cloud_metadata.New, - "AddDockerMetadata": add_docker_metadata.New, - "AddFields": actions.CreateAddFields, - "AddHostMetadata": add_host_metadata.New, - "AddKubernetesMetadata": add_kubernetes_metadata.New, - "AddObserverMetadata": add_observer_metadata.New, - "AddLocale": add_locale.New, - "AddProcessMetadata": add_process_metadata.New, - "CommunityID": communityid.New, - "Convert": convert.New, - "CopyFields": actions.NewCopyFields, - "DecodeBase64Field": actions.NewDecodeBase64Field, - "DecodeCSVField": decode_csv_fields.NewDecodeCSVField, - "DecodeJSONFields": actions.NewDecodeJSONFields, - "Dissect": dissect.NewProcessor, - "DNS": dns.New, - "ExtractArray": extract_array.New, - "RegisteredDomain": registered_domain.New, - "Rename": actions.NewRenameFields, - "Timestamp": timestamp.New, - "TruncateFields": actions.NewTruncateFields, +var registry = processors.NewNamespace() + +// RegisterPlugin registeres processor plugins for the javascript processor. +func RegisterPlugin(name string, c processors.Constructor) { + logp.L().Named("javascript").Debugf("Register script processor %s", name) + + err := registry.Register(name, c) + if err != nil { + panic(err) + } } // beatProcessor wraps a processor for javascript. @@ -140,7 +113,7 @@ func newConstructor( func Require(runtime *goja.Runtime, module *goja.Object) { o := module.Get("exports").(*goja.Object) - for name, fn := range constructors { + for name, fn := range registry.Constructors() { o.Set(name, newConstructor(runtime, fn)) } diff --git a/libbeat/processors/script/javascript/module/processor/processor_test.go b/libbeat/processors/script/javascript/module/processor/processor_test.go index b78d0cfaf00..7faddd241c5 100644 --- a/libbeat/processors/script/javascript/module/processor/processor_test.go +++ b/libbeat/processors/script/javascript/module/processor/processor_test.go @@ -18,8 +18,8 @@ package processor import ( - "os" - "runtime" + "encoding/json" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -27,11 +27,16 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/processors" "github.com/elastic/beats/libbeat/processors/script/javascript" _ "github.com/elastic/beats/libbeat/processors/script/javascript/module/require" ) +func init() { + RegisterPlugin("Mock", newMock) +} + func testEvent() *beat.Event { return &beat.Event{ Fields: common.MapStr{ @@ -49,14 +54,14 @@ func testEvent() *beat.Event { } } -func TestNewProcessorAddHostMetadata(t *testing.T) { +func TestNewProcessorDummyProcessor(t *testing.T) { const script = ` var processor = require('processor'); -var addHostMetadata = new processor.AddHostMetadata({"netinfo.enabled": true}); +var mock = new processor.Mock({"fields": {"added": "new_value"}}); function process(evt) { - addHostMetadata.Run(evt); + mock.Run(evt); } ` @@ -71,75 +76,32 @@ function process(evt) { t.Fatal(err) } - _, err = evt.GetValue("host.hostname") - assert.NoError(t, err) -} - -func TestNewProcessorAddLocale(t *testing.T) { - const script = ` -var processor = require('processor'); - -var addLocale = new processor.AddLocale(); - -function process(evt) { - addLocale.Run(evt); -} -` - - logp.TestingSetup() - p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) - if err != nil { - t.Fatal(err) - } - - evt, err := p.Run(testEvent()) - if err != nil { - t.Fatal(err) - } - - _, err = evt.GetValue("event.timezone") - assert.NoError(t, err) + checkEvent(t, evt, "added", "new_value") } -func TestNewProcessorAddProcessMetadata(t *testing.T) { +func TestChainOfDummyProcessors(t *testing.T) { const script = ` var processor = require('processor'); -var addProcessMetadata = new processor.AddProcessMetadata({ - match_pids: "process.pid", - overwrite_keys: true, -}); +var hungarianHello = new processor.Mock({"fields": {"helló": "világ"}}); +var germanHello = new processor.Mock({"fields": {"hallo": "Welt"}}); -function process(evt) { - addProcessMetadata.Run(evt); -} -` - - logp.TestingSetup() - p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) - if err != nil { - t.Fatal(err) - } - - evt := &beat.Event{Fields: common.MapStr{"process": common.MapStr{"pid": os.Getppid()}}} - evt, err = p.Run(evt) - if err != nil { - t.Fatal(err) - } - - _, err = evt.GetValue("process.name") - assert.NoError(t, err) - t.Logf("%+v", evt.Fields) -} - -func TestNewProcessorCommunityID(t *testing.T) { - const script = ` -var processor = require('processor'); - -var communityID = new processor.CommunityID(); +var chain = new processor.Chain() + .Add(hungarianHello) + .Mock({ + fields: { "hola": "mundo" }, + }) + .Add(function(evt) { + evt.Put("hello", "world"); + }) + .Build(); +var chainOfChains = new processor.Chain() + .Add(chain) + .Add(germanHello) + .Build(); function process(evt) { - communityID.Run(evt); + chainOfChains.Run(evt); } ` @@ -154,242 +116,49 @@ function process(evt) { t.Fatal(err) } - id, _ := evt.GetValue("network.community_id") - assert.Equal(t, "1:15+Ly6HsDg0sJdTmNktf6rko+os=", id) + // checking if hello world is added to the event in different languages + checkEvent(t, evt, "helló", "világ") + checkEvent(t, evt, "hola", "mundo") + checkEvent(t, evt, "hello", "world") + checkEvent(t, evt, "hallo", "Welt") } -func TestNewCopyFields(t *testing.T) { - const script = ` -var processor = require('processor'); - -var copy = new processor.CopyFields({ - fields: [ - {from: "message", to: "log.original"}, - ], -}); - -function process(evt) { - copy.Run(evt); -} -` - - logp.TestingSetup() - p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) - if err != nil { - t.Fatal(err) - } - - evt, err := p.Run(testEvent()) - if err != nil { - t.Fatal(err) - } - - _, err = evt.GetValue("log.original") +func checkEvent(t *testing.T, evt *beat.Event, key, value string) { + s, err := evt.GetValue(key) assert.NoError(t, err) -} - -func TestNewProcessorDecodeJSONFields(t *testing.T) { - const script = ` -var processor = require('processor'); - -var decodeJSON = new processor.DecodeJSONFields({ - fields: ["message"], - target: "", -}); - -function process(evt) { - decodeJSON.Run(evt); -} -` - - logp.TestingSetup() - p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) - if err != nil { - t.Fatal(err) - } - - evt := testEvent() - evt.PutValue("message", `{"hello": "world"}`) - - _, err = p.Run(evt) - if err != nil { - t.Fatal(err) - } - - v, _ := evt.GetValue("hello") - assert.Equal(t, "world", v) -} - -func TestNewProcessorDissect(t *testing.T) { - const script = ` -var processor = require('processor'); - -var chopLog = new processor.Dissect({ - tokenizer: "key=%{key}", - field: "message", -}); -function process(evt) { - chopLog.Run(evt); -} -` - - logp.TestingSetup() - p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) - if err != nil { - t.Fatal(err) - } - - evt, err := p.Run(testEvent()) - if err != nil { - t.Fatal(err) - } - - key, _ := evt.GetValue("dissect.key") - assert.Equal(t, "hello", key) -} - -func TestNewProcessorDNS(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("Windows requires explicit DNS server configuration") - } - - const script = ` -var processor = require('processor'); - -var dns = new processor.DNS({ - type: "reverse", - fields: { - "source.ip": "source.domain", - "destination.ip": "destination.domain" - }, - tag_on_failure: ["_dns_reverse_lookup_failed"], -}); - -function process(evt) { - dns.Run(evt); - if (evt.Get().tags[0] !== "_dns_reverse_lookup_failed") { - throw "missing tag"; - } -} -` - - logp.TestingSetup() - p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) - if err != nil { - t.Fatal(err) - } - - _, err = p.Run(testEvent()) - if err != nil { - t.Fatal(err) - } -} - -func TestNewRename(t *testing.T) { - const script = ` -var processor = require('processor'); - -var rename = new processor.Rename({ - fields: [ - {from: "message", to: "log.original"}, - ], -}); - -function process(evt) { - rename.Run(evt); -} -` - - logp.TestingSetup() - p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) - if err != nil { - t.Fatal(err) + switch ss := s.(type) { + case string: + assert.Equal(t, ss, value) + default: + t.Fatal("unexpected type") } - - evt, err := p.Run(testEvent()) - if err != nil { - t.Fatal(err) - } - - _, err = evt.GetValue("log.original") - assert.NoError(t, err) } -func TestNewTruncateFields(t *testing.T) { - const script = ` -var processor = require('processor'); - -var truncate = new processor.TruncateFields({ - fields: [ - "message", - ], - max_characters: 4, -}); - -function process(evt) { - truncate.Run(evt); +type mockProcessor struct { + fields common.MapStr } -` - - logp.TestingSetup() - p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) - if err != nil { - t.Fatal(err) - } - evt, err := p.Run(testEvent()) +func newMock(c *common.Config) (processors.Processor, error) { + config := struct { + Fields common.MapStr `config:"fields" validate:"required"` + }{} + err := c.Unpack(&config) if err != nil { - t.Fatal(err) + return nil, fmt.Errorf("fail to unpack the mock processor configuration: %s", err) } - msg, _ := evt.GetValue("message") - assert.Equal(t, "key=", msg) + return &mockProcessor{ + fields: config.Fields, + }, nil } -func TestNewProcessorChain(t *testing.T) { - const script = ` -var processor = require('processor'); - -var localeProcessor = new processor.AddLocale(); - -var chain = new processor.Chain() - .Add(localeProcessor) - .Rename({ - fields: [ - {from: "event.timezone", to: "timezone"}, - ], - }) - .Add(function(evt) { - evt.Put("hello", "world"); - }) - .Build(); - -var chainOfChains = new processor.Chain() - .Add(chain) - .AddFields({fields: {foo: "bar"}}) - .Build(); - -function process(evt) { - chainOfChains.Run(evt); +func (m *mockProcessor) Run(event *beat.Event) (*beat.Event, error) { + event.Fields.DeepUpdate(m.fields) + return event, nil } -` - logp.TestingSetup() - p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) - if err != nil { - t.Fatal(err) - } - - evt, err := p.Run(testEvent()) - if err != nil { - t.Fatal(err) - } - - _, err = evt.GetValue("timezone") - assert.NoError(t, err) - v, _ := evt.GetValue("hello") - assert.Equal(t, "world", v) - v, _ = evt.GetValue("fields.foo") - assert.Equal(t, "bar", v) +func (m *mockProcessor) String() string { + s, _ := json.Marshal(m.fields) + return fmt.Sprintf("mock=%s", s) } diff --git a/libbeat/processors/timestamp/timestamp.go b/libbeat/processors/timestamp/timestamp.go index 975073b43f3..108ce7f50e1 100644 --- a/libbeat/processors/timestamp/timestamp.go +++ b/libbeat/processors/timestamp/timestamp.go @@ -28,12 +28,14 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/processors" + jsprocessor "github.com/elastic/beats/libbeat/processors/script/javascript/module/processor" ) const logName = "processor.timestamp" func init() { processors.RegisterPlugin("timestamp", New) + jsprocessor.RegisterPlugin("Timestamp", New) } type processor struct { diff --git a/metricbeat/include/list.go b/metricbeat/include/list_common.go similarity index 83% rename from metricbeat/include/list.go rename to metricbeat/include/list_common.go index 3f8a4af49db..989ceb91ca3 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list_common.go @@ -49,16 +49,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/couchbase/node" _ "github.com/elastic/beats/metricbeat/module/couchdb" _ "github.com/elastic/beats/metricbeat/module/couchdb/server" - _ "github.com/elastic/beats/metricbeat/module/docker" - _ "github.com/elastic/beats/metricbeat/module/docker/container" - _ "github.com/elastic/beats/metricbeat/module/docker/cpu" - _ "github.com/elastic/beats/metricbeat/module/docker/diskio" - _ "github.com/elastic/beats/metricbeat/module/docker/event" - _ "github.com/elastic/beats/metricbeat/module/docker/healthcheck" - _ "github.com/elastic/beats/metricbeat/module/docker/image" - _ "github.com/elastic/beats/metricbeat/module/docker/info" - _ "github.com/elastic/beats/metricbeat/module/docker/memory" - _ "github.com/elastic/beats/metricbeat/module/docker/network" _ "github.com/elastic/beats/metricbeat/module/dropwizard" _ "github.com/elastic/beats/metricbeat/module/dropwizard/collector" _ "github.com/elastic/beats/metricbeat/module/elasticsearch" @@ -98,25 +88,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/kibana" _ "github.com/elastic/beats/metricbeat/module/kibana/stats" _ "github.com/elastic/beats/metricbeat/module/kibana/status" - _ "github.com/elastic/beats/metricbeat/module/kubernetes" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/apiserver" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/container" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/controllermanager" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/event" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/node" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/pod" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/proxy" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/scheduler" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_container" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_cronjob" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_deployment" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_node" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_pod" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_replicaset" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_statefulset" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/system" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/util" - _ "github.com/elastic/beats/metricbeat/module/kubernetes/volume" _ "github.com/elastic/beats/metricbeat/module/kvm" _ "github.com/elastic/beats/metricbeat/module/kvm/dommemstat" _ "github.com/elastic/beats/metricbeat/module/logstash" diff --git a/metricbeat/include/list_docker.go b/metricbeat/include/list_docker.go new file mode 100644 index 00000000000..f23a50488e9 --- /dev/null +++ b/metricbeat/include/list_docker.go @@ -0,0 +1,60 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by 'make imports' - DO NOT EDIT. + +/* +Package include imports all Module and MetricSet packages so that they register +their factories with the global registry. This package can be imported in the +main package to automatically register all of the standard supported Metricbeat +modules. +*/ +// +build linux darwin windows + +package include + +import ( + _ "github.com/elastic/beats/metricbeat/module/docker" + _ "github.com/elastic/beats/metricbeat/module/docker/container" + _ "github.com/elastic/beats/metricbeat/module/docker/cpu" + _ "github.com/elastic/beats/metricbeat/module/docker/diskio" + _ "github.com/elastic/beats/metricbeat/module/docker/event" + _ "github.com/elastic/beats/metricbeat/module/docker/healthcheck" + _ "github.com/elastic/beats/metricbeat/module/docker/image" + _ "github.com/elastic/beats/metricbeat/module/docker/info" + _ "github.com/elastic/beats/metricbeat/module/docker/memory" + _ "github.com/elastic/beats/metricbeat/module/docker/network" + _ "github.com/elastic/beats/metricbeat/module/kubernetes" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/apiserver" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/container" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/controllermanager" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/event" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/node" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/pod" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/proxy" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/scheduler" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_container" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_cronjob" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_deployment" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_node" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_pod" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_replicaset" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_statefulset" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/system" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/util" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/volume" +) diff --git a/metricbeat/module/docker/container/container.go b/metricbeat/module/docker/container/container.go index 940d6e333f1..b3523464db3 100644 --- a/metricbeat/module/docker/container/container.go +++ b/metricbeat/module/docker/container/container.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package container import ( diff --git a/metricbeat/module/docker/cpu/cpu.go b/metricbeat/module/docker/cpu/cpu.go index a9fcbbc77b8..a07b4f8466f 100644 --- a/metricbeat/module/docker/cpu/cpu.go +++ b/metricbeat/module/docker/cpu/cpu.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package cpu import ( diff --git a/metricbeat/module/docker/diskio/diskio.go b/metricbeat/module/docker/diskio/diskio.go index fbfadc09052..dacf86374c3 100644 --- a/metricbeat/module/docker/diskio/diskio.go +++ b/metricbeat/module/docker/diskio/diskio.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package diskio import ( diff --git a/metricbeat/module/docker/docker.go b/metricbeat/module/docker/docker.go index adee725af69..623dc1699f4 100644 --- a/metricbeat/module/docker/docker.go +++ b/metricbeat/module/docker/docker.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package docker import ( diff --git a/metricbeat/module/docker/event/event.go b/metricbeat/module/docker/event/event.go index f81168b9de9..c8ab896de6b 100644 --- a/metricbeat/module/docker/event/event.go +++ b/metricbeat/module/docker/event/event.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package event import ( diff --git a/metricbeat/module/docker/healthcheck/healthcheck.go b/metricbeat/module/docker/healthcheck/healthcheck.go index c1af433a0dd..1b90ccd5471 100644 --- a/metricbeat/module/docker/healthcheck/healthcheck.go +++ b/metricbeat/module/docker/healthcheck/healthcheck.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package healthcheck import ( diff --git a/metricbeat/module/docker/memory/memory.go b/metricbeat/module/docker/memory/memory.go index 873f556268d..8108b30d46c 100644 --- a/metricbeat/module/docker/memory/memory.go +++ b/metricbeat/module/docker/memory/memory.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package memory import ( diff --git a/metricbeat/module/docker/network/network.go b/metricbeat/module/docker/network/network.go index a9b0b54c60d..84cdd4d1c59 100644 --- a/metricbeat/module/docker/network/network.go +++ b/metricbeat/module/docker/network/network.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build linux darwin windows + package network import ( diff --git a/metricbeat/scripts/generate_imports_helper.py b/metricbeat/scripts/generate_imports_helper.py index 8a4fe83e25e..3a1df4a35a0 100644 --- a/metricbeat/scripts/generate_imports_helper.py +++ b/metricbeat/scripts/generate_imports_helper.py @@ -6,22 +6,60 @@ main package to automatically register all of the standard supported Metricbeat modules.""" +modules_by_platform = [ + { + "file_suffix": "_docker", + "build_tags": "// +build linux darwin windows\n\n", + "modules": ["docker", "kubernetes"], + }, +] + def get_importable_lines(go_beat_path, import_line): path = abspath("module") - imported_lines = [] + imports_by_module = [] + common_lines = [] modules = [m for m in listdir(path) if isdir(join(path, m)) and m != "_meta"] - for module in modules: - module_import = import_line.format(beat_path=go_beat_path, module="module", name=module) - imported_lines.append(module_import) - - module_path = join(path, module) - ignore = ["_meta", "vendor", "mtest"] - metricsets = [m for m in listdir(module_path) if isdir(join(module_path, m)) and m not in ignore] - for metricset in metricsets: - metricset_name = "{}/{}".format(module, metricset) - metricset_import = import_line.format(beat_path=go_beat_path, module="module", name=metricset_name) - imported_lines.append(metricset_import) + not_common_modules = [] + for m in modules_by_platform: + not_common_modules.extend(m["modules"]) + + for platform_info in modules_by_platform: + lines = [] + for module in modules: + module_import = import_line.format(beat_path=go_beat_path, module="module", name=module) + if module in not_common_modules: + lines = _collect_imports_from_module(path, module, module_import, go_beat_path, import_line, lines) + else: + common_lines = _collect_imports_from_module( + path, module, module_import, go_beat_path, import_line, common_lines) + + if lines is not None: + imports_by_module.append({ + "file_suffix": platform_info["file_suffix"], + "build_tags": platform_info["build_tags"], + "imported_lines": lines, + }) + + imports_by_module.append({ + "file_suffix": "_common", + "build_tags": "", + "imported_lines": sorted(common_lines), + }) + + return imports_by_module + + +def _collect_imports_from_module(path, module, module_import, go_beat_path, import_line, imported_lines): + imported_lines.append(module_import) + + module_path = join(path, module) + ignore = ["_meta", "vendor", "mtest"] + metricsets = [m for m in listdir(module_path) if isdir(join(module_path, m)) and m not in ignore] + for metricset in metricsets: + metricset_name = "{}/{}".format(module, metricset) + metricset_import = import_line.format(beat_path=go_beat_path, module="module", name=metricset_name) + imported_lines.append(metricset_import) return sorted(imported_lines) diff --git a/script/generate_imports.py b/script/generate_imports.py index 56bf8644b1e..ef094a5efa8 100644 --- a/script/generate_imports.py +++ b/script/generate_imports.py @@ -30,7 +30,7 @@ /* {comment} */ -package {package} +{buildtags}package {package} import ( {imports} @@ -39,19 +39,22 @@ def generate_and_write_to_file(outfile, go_beat_path): - imported_beat_lines = get_importable_lines(go_beat_path, import_line_format) - imported_lines = "\n".join(imported_beat_lines) - package = basename(dirname(outfile)) - list_go = import_template.format(package=package, - comment=comment, - imports=imported_lines) - with open(outfile, "w") as output: - output.write(list_go) + outputs = get_importable_lines(go_beat_path, import_line_format) + + for output_data in outputs: + imported_lines = "\n".join(output_data["imported_lines"]) + package = basename(dirname(outfile)) + list_go = import_template.format(package=package, + comment=comment, + buildtags=output_data["build_tags"], + imports=imported_lines) + with open(outfile.format(suffix=output_data["file_suffix"]), "w") as output: + output.write(list_go) if __name__ == "__main__": parser = ArgumentParser(description="Generate imports for Beats packages") - parser.add_argument("--out", default="include/list.go") + parser.add_argument("--out", default="include/list{suffix}.go") parser.add_argument("beats_path") args = parser.parse_args() diff --git a/x-pack/filebeat/module/cisco/ios/pipeline_test.go b/x-pack/filebeat/module/cisco/ios/pipeline_test.go index ce9065d52b0..c3e92afc8a7 100644 --- a/x-pack/filebeat/module/cisco/ios/pipeline_test.go +++ b/x-pack/filebeat/module/cisco/ios/pipeline_test.go @@ -18,6 +18,9 @@ import ( // Register JS "require" modules. _ "github.com/elastic/beats/libbeat/processors/script/javascript/module" + // Register required processors. + _ "github.com/elastic/beats/libbeat/cmd/instance" + _ "github.com/elastic/beats/libbeat/processors/timestamp" ) var logInputHeaders = []string{ diff --git a/x-pack/winlogbeat/module/security/test/security_windows_test.go b/x-pack/winlogbeat/module/security/test/security_windows_test.go index 03198ce5279..84747627832 100644 --- a/x-pack/winlogbeat/module/security/test/security_windows_test.go +++ b/x-pack/winlogbeat/module/security/test/security_windows_test.go @@ -8,6 +8,10 @@ import ( "testing" "github.com/elastic/beats/x-pack/winlogbeat/module" + + // Register required processors. + _ "github.com/elastic/beats/libbeat/cmd/instance" + _ "github.com/elastic/beats/libbeat/processors/timestamp" ) func TestSecurity(t *testing.T) { diff --git a/x-pack/winlogbeat/module/sysmon/test/sysmon_windows_test.go b/x-pack/winlogbeat/module/sysmon/test/sysmon_windows_test.go index 2f1e367cc51..a82f5ba2e4c 100644 --- a/x-pack/winlogbeat/module/sysmon/test/sysmon_windows_test.go +++ b/x-pack/winlogbeat/module/sysmon/test/sysmon_windows_test.go @@ -8,6 +8,10 @@ import ( "testing" "github.com/elastic/beats/x-pack/winlogbeat/module" + + // Register required processors. + _ "github.com/elastic/beats/libbeat/cmd/instance" + _ "github.com/elastic/beats/libbeat/processors/timestamp" ) // Ignore these fields so that the tests will pass if Sysmon is not installed.