diff --git a/go.mod b/go.mod index e0db1f86d..f496b05b8 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.1 github.com/elastic/go-elasticsearch/v7 v7.9.0 github.com/elastic/go-ucfg v0.8.3 - github.com/elastic/package-spec/code/go v0.0.0-20201210164239-22bc835bcf04 + github.com/elastic/package-spec/code/go v0.0.0-20210111200056-23a4cfe4f518 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.1.0 github.com/go-openapi/strfmt v0.19.6 // indirect @@ -27,7 +27,6 @@ require ( github.com/stretchr/testify v1.6.1 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect - golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ) diff --git a/go.sum b/go.sum index 83e7992b5..c7e1c9cfa 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/elastic/go-elasticsearch/v7 v7.9.0 h1:UEau+a1MiiE/F+UrDj60kqIHFWdzU1M github.com/elastic/go-elasticsearch/v7 v7.9.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo= github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= -github.com/elastic/package-spec/code/go v0.0.0-20201210164239-22bc835bcf04 h1:DfZR0hCpA1yE/bG9q1FJoeajvbrjWn945bXyavC8knE= -github.com/elastic/package-spec/code/go v0.0.0-20201210164239-22bc835bcf04/go.mod h1:3W6uyBFCE4/NPcVPb+ZuoLJTMLu8BCTc+PRFDutSvfE= +github.com/elastic/package-spec/code/go v0.0.0-20210111200056-23a4cfe4f518 h1:zMaYyFfVNHmyjPE/iqjWr/EbfHPVxv+GCXAIEhkKKM4= +github.com/elastic/package-spec/code/go v0.0.0-20210111200056-23a4cfe4f518/go.mod h1:wjN5+18SNfDKwMmrT80XbcD6wpwig3XYYBUctkNmZjI= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -314,6 +314,7 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -361,6 +362,8 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -419,6 +422,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -507,6 +511,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d h1:W07d4xkoAUSNOkOzdzXCdFGxT7o2rW4q8M34tB2i//k= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e h1:Z2uDrs8MyXUWJbwGc4V+nGjV4Ygo+oubBbWSVQw21/I= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 8f607d2cc..f9edc43f7 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -35,6 +35,10 @@ const ( // Maximum number of events to query. elasticsearchQuerySize = 500 + + // Folder path where log files produced by the service + // are stored on the Agent container's filesystem. + serviceLogsAgentDir = "/tmp/service_logs" ) type runner struct { @@ -79,93 +83,199 @@ func (r *runner) TearDown() error { if err := r.resetAgentPolicyHandler(); err != nil { return err } + r.resetAgentPolicyHandler = nil } if r.deleteTestPolicyHandler != nil { if err := r.deleteTestPolicyHandler(); err != nil { return err } + r.deleteTestPolicyHandler = nil } if r.shutdownServiceHandler != nil { if err := r.shutdownServiceHandler(); err != nil { return err } + r.shutdownServiceHandler = nil } if r.wipeDataStreamHandler != nil { if err := r.wipeDataStreamHandler(); err != nil { return err } + r.wipeDataStreamHandler = nil } return nil } -func (r *runner) run() ([]testrunner.TestResult, error) { - result := testrunner.TestResult{ - TestType: TestType, - Package: r.options.TestFolder.Package, - DataStream: r.options.TestFolder.DataStream, +type resultComposer struct { + testrunner.TestResult + startTime time.Time +} + +func (r *runner) newResult(name string) resultComposer { + return resultComposer{ + TestResult: testrunner.TestResult{ + TestType: TestType, + Name: name, + Package: r.options.TestFolder.Package, + DataStream: r.options.TestFolder.DataStream, + }, + startTime: time.Now(), } - startTime := time.Now() - resultsWith := func(tr testrunner.TestResult, err error) ([]testrunner.TestResult, error) { - tr.TimeElapsed = time.Now().Sub(startTime) - if err == nil { - return []testrunner.TestResult{tr}, nil +} + +func (rc *resultComposer) withError(err error) ([]testrunner.TestResult, error) { + rc.TimeElapsed = time.Now().Sub(rc.startTime) + if err == nil { + return []testrunner.TestResult{rc.TestResult}, nil + } + + if tcf, ok := err.(testrunner.ErrTestCaseFailed); ok { + rc.FailureMsg += tcf.Reason + rc.FailureDetails += tcf.Details + return []testrunner.TestResult{rc.TestResult}, nil + } + + rc.ErrorMsg += err.Error() + return []testrunner.TestResult{rc.TestResult}, err +} + +func (rc *resultComposer) withSuccess() ([]testrunner.TestResult, error) { + return rc.withError(nil) +} + +func (r *runner) run() (results []testrunner.TestResult, err error) { + result := r.newResult("(init)") + serviceLogsDir, err := install.ServiceLogsDir() + if err != nil { + return result.withError(errors.Wrap(err, "reading service logs directory failed")) + } + + files, err := listConfigFiles(r.options.TestFolder.Path) + if err != nil { + return result.withError(errors.Wrap(err, "failed listing test case config files")) + } + for _, cfgFile := range files { + var ctxt servicedeployer.ServiceContext + ctxt.Name = r.options.TestFolder.Package + ctxt.Logs.Folder.Local = serviceLogsDir + ctxt.Logs.Folder.Agent = serviceLogsAgentDir + testConfig, err := newConfig(filepath.Join(r.options.TestFolder.Path, cfgFile), ctxt) + if err != nil { + return result.withError(errors.Wrapf(err, "unable to load system test case file '%s'", cfgFile)) + } + partial, err := r.runTest(testConfig, ctxt) + results = append(results, partial...) + if err != nil { + return results, err + } + if err = r.TearDown(); err != nil { + return results, errors.Wrap(err, "failed to teardown runner") } + } + return results, nil +} - if tcf, ok := err.(testrunner.ErrTestCaseFailed); ok { - tr.FailureMsg = tcf.Reason - tr.FailureDetails = tcf.Details - return []testrunner.TestResult{tr}, nil +func (r *runner) hasNumDocs( + dataStream string, + fieldsValidator *fields.Validator, + checker func(int) bool) func() (bool, error) { + return func() (bool, error) { + resp, err := r.options.ESClient.Search( + r.options.ESClient.Search.WithIndex(dataStream), + r.options.ESClient.Search.WithSort("@timestamp:asc"), + r.options.ESClient.Search.WithSize(elasticsearchQuerySize), + ) + if err != nil { + return false, errors.Wrap(err, "could not search data stream") } + defer resp.Body.Close() - tr.ErrorMsg = err.Error() - return []testrunner.TestResult{tr}, err + var results struct { + Hits struct { + Total struct { + Value int + } + Hits []struct { + Source common.MapStr `json:"_source"` + } + } + } + + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + return false, errors.Wrap(err, "could not decode search results response") + } + + numHits := results.Hits.Total.Value + logger.Debugf("found %d hits in %s data stream", numHits, dataStream) + if !checker(numHits) { + return false, nil + } + + var multiErr multierror.Error + for _, hit := range results.Hits.Hits { + if message, err := hit.Source.GetValue("error.message"); err != common.ErrKeyNotFound { + multiErr = append(multiErr, fmt.Errorf("found error.message in event: %v", message)) + continue + } + + errs := fieldsValidator.ValidateDocumentMap(hit.Source) + if errs != nil { + multiErr = append(multiErr, errs...) + continue + } + } + + if len(multiErr) > 0 { + multiErr = multiErr.Unique() + return false, testrunner.ErrTestCaseFailed{ + Reason: fmt.Sprintf("one or more errors found in documents stored in %s data stream", dataStream), + Details: multiErr.Error(), + } + } + return true, nil } +} + +func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext) ([]testrunner.TestResult, error) { + result := r.newResult(config.Name()) pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) if err != nil { - return resultsWith(result, errors.Wrap(err, "reading package manifest failed")) + return result.withError(errors.Wrap(err, "reading package manifest failed")) } dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.options.TestFolder.Path) if err != nil { - return resultsWith(result, errors.Wrap(err, "locating data stream root failed")) + return result.withError(errors.Wrap(err, "locating data stream root failed")) } if !found { - return resultsWith(result, errors.New("data stream root not found")) + return result.withError(errors.New("data stream root not found")) } dataStreamManifest, err := packages.ReadDataStreamManifest(filepath.Join(dataStreamPath, packages.DataStreamManifestFile)) if err != nil { - return resultsWith(result, errors.Wrap(err, "reading data stream manifest failed")) + return result.withError(errors.Wrap(err, "reading data stream manifest failed")) } - serviceLogsDir, err := install.ServiceLogsDir() - if err != nil { - return resultsWith(result, errors.Wrap(err, "reading service logs directory failed")) - } - - // Step 1. Setup service. - // Step 1a. (Deferred) Tear down service. + // Setup service. logger.Debug("setting up service...") serviceDeployer, err := servicedeployer.Factory(r.options.PackageRootPath) if err != nil { - return resultsWith(result, errors.Wrap(err, "could not create service runner")) + return result.withError(errors.Wrap(err, "could not create service runner")) } - var ctxt servicedeployer.ServiceContext - ctxt.Name = r.options.TestFolder.Package - ctxt.Logs.Folder.Local = serviceLogsDir - + if config.Service != "" { + ctxt.Name = config.Service + } service, err := serviceDeployer.SetUp(ctxt) if err != nil { - return resultsWith(result, errors.Wrap(err, "could not setup service")) + return result.withError(errors.Wrap(err, "could not setup service")) } ctxt = service.Context() - r.shutdownServiceHandler = func() error { logger.Debug("tearing down service...") if err := service.TearDown(); err != nil { @@ -175,10 +285,16 @@ func (r *runner) run() ([]testrunner.TestResult, error) { return nil } - // Step 2. Configure package (single data stream) via Ingest Manager APIs. + // Reload test config with ctx variable substitution. + config, err = newConfig(config.Path, ctxt) + if err != nil { + return result.withError(errors.Wrap(err, "unable to reload system test case configuration")) + } + + // Configure package (single data stream) via Ingest Manager APIs. kib, err := kibana.NewClient() if err != nil { - return resultsWith(result, errors.Wrap(err, "could not create ingest manager client")) + return result.withError(errors.Wrap(err, "could not create ingest manager client")) } logger.Debug("creating test policy...") @@ -190,7 +306,7 @@ func (r *runner) run() ([]testrunner.TestResult, error) { } policy, err := kib.CreatePolicy(p) if err != nil { - return resultsWith(result, errors.Wrap(err, "could not create test policy")) + return result.withError(errors.Wrap(err, "could not create test policy")) } r.deleteTestPolicyHandler = func() error { logger.Debug("deleting test policy...") @@ -200,31 +316,35 @@ func (r *runner) run() ([]testrunner.TestResult, error) { return nil } - testConfig, err := newConfig(r.options.TestFolder.Path, ctxt) - if err != nil { - return resultsWith(result, errors.Wrap(err, "unable to load system test configuration")) - } - logger.Debug("adding package data stream to test policy...") - ds := createPackageDatastream(*policy, *pkgManifest, *dataStreamManifest, *testConfig) + + ds := createPackageDatastream(*policy, *pkgManifest, *dataStreamManifest, *config) if err := kib.AddPackageDataStreamToPolicy(ds); err != nil { - return resultsWith(result, errors.Wrap(err, "could not add data stream config to policy")) + return result.withError(errors.Wrap(err, "could not add data stream config to policy")) } // Get enrolled agent ID agents, err := kib.ListAgents() if err != nil { - return resultsWith(result, errors.Wrap(err, "could not list agents")) + return result.withError(errors.Wrap(err, "could not list agents")) } if agents == nil || len(agents) == 0 { - return resultsWith(result, errors.New("no agents found")) + return result.withError(errors.New("no agents found")) } agent := agents[0] origPolicy := kibana.Policy{ ID: agent.PolicyID, } + // Create field validator + fieldsValidator, err := fields.CreateValidatorForDataStream(dataStreamPath, + fields.WithNumericKeywordFields(config.NumericKeywordFields)) + if err != nil { + return result.withError(errors.Wrapf(err, "creating fields validator for data stream failed (path: %s)", dataStreamPath)) + } + // Delete old data + logger.Debug("deleting old data in data stream...") dataStream := fmt.Sprintf( "%s-%s-%s", ds.Inputs[0].Streams[0].DataStream.Type, @@ -240,15 +360,24 @@ func (r *runner) run() ([]testrunner.TestResult, error) { return nil } - logger.Debug("deleting old data in data stream...") if err := deleteDataStreamDocs(r.options.ESClient, dataStream); err != nil { - return resultsWith(result, errors.Wrapf(err, "error deleting old data in data stream: %s", dataStream)) + return result.withError(errors.Wrapf(err, "error deleting old data in data stream: %s", dataStream)) + } + + cleared, err := waitUntilTrue(r.hasNumDocs(dataStream, fieldsValidator, func(n int) bool { + return n == 0 + }), 2*time.Minute) + if err != nil || !cleared { + if err == nil { + err = errors.New("unable to clear previous data") + } + return result.withError(err) } // Assign policy to agent logger.Debug("assigning package data stream to agent...") if err := kib.AssignPolicyToAgent(agent, *policy); err != nil { - return resultsWith(result, errors.Wrap(err, "could not assign policy to agent")) + return result.withError(errors.Wrap(err, "could not assign policy to agent")) } r.resetAgentPolicyHandler = func() error { logger.Debug("reassigning original policy back to agent...") @@ -258,77 +387,20 @@ func (r *runner) run() ([]testrunner.TestResult, error) { return nil } - fieldsValidator, err := fields.CreateValidatorForDataStream(dataStreamPath, - fields.WithNumericKeywordFields(testConfig.NumericKeywordFields)) - if err != nil { - return resultsWith(result, errors.Wrapf(err, "creating fields validator for data stream failed (path: %s)", dataStreamPath)) - } - - // Step 4. (TODO in future) Optionally exercise service to generate load. + // (TODO in future) Optionally exercise service to generate load. logger.Debug("checking for expected data in data stream...") - passed, err := waitUntilTrue(func() (bool, error) { - resp, err := r.options.ESClient.Search( - r.options.ESClient.Search.WithIndex(dataStream), - r.options.ESClient.Search.WithSort("@timestamp:asc"), - r.options.ESClient.Search.WithSize(elasticsearchQuerySize), - ) - if err != nil { - return false, errors.Wrap(err, "could not search data stream") - } - defer resp.Body.Close() + passed, err := waitUntilTrue(r.hasNumDocs(dataStream, fieldsValidator, func(n int) bool { + return n > 0 + }), 2*time.Minute) - var results struct { - Hits struct { - Total struct { - Value int - } - Hits []struct { - Source common.MapStr `json:"_source"` - } - } - } - - if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { - return false, errors.Wrap(err, "could not decode search results response") - } - - numHits := results.Hits.Total.Value - logger.Debugf("found %d hits in %s data stream", numHits, dataStream) - if numHits == 0 { - return false, nil - } - - var multiErr multierror.Error - for _, hit := range results.Hits.Hits { - if message, err := hit.Source.GetValue("error.message"); err != common.ErrKeyNotFound { - multiErr = append(multiErr, fmt.Errorf("found error.message in event: %v", message)) - continue - } - - errs := fieldsValidator.ValidateDocumentMap(hit.Source) - if errs != nil { - multiErr = append(multiErr, errs...) - continue - } - } - - if len(multiErr) > 0 { - multiErr = multiErr.Unique() - return false, testrunner.ErrTestCaseFailed{ - Reason: fmt.Sprintf("one or more errors found in documents stored in %s data stream", dataStream), - Details: multiErr.Error(), - } - } - return true, nil - }, 2*time.Minute) if err != nil { - return resultsWith(result, err) + return result.withError(err) } if !passed { result.FailureMsg = fmt.Sprintf("could not find hits in %s data stream", dataStream) } - return resultsWith(result, nil) + return result.withSuccess() } func createPackageDatastream( diff --git a/internal/testrunner/runners/system/servicedeployer/compose.go b/internal/testrunner/runners/system/servicedeployer/compose.go index 1307729ea..ed94c97d1 100644 --- a/internal/testrunner/runners/system/servicedeployer/compose.go +++ b/internal/testrunner/runners/system/servicedeployer/compose.go @@ -17,8 +17,6 @@ import ( "github.com/elastic/elastic-package/internal/stack" ) -const serviceLogsAgentDir = "/tmp/service_logs" - // DockerComposeServiceDeployer knows how to deploy a service defined via // a Docker Compose file. type DockerComposeServiceDeployer struct { @@ -58,19 +56,18 @@ func (r *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer if err != nil { return nil, errors.Wrap(err, "removing service logs failed") } - outCtxt.Logs.Folder.Agent = serviceLogsAgentDir // Boot up service + serviceName := inCtxt.Name opts := compose.CommandOptions{ Env: []string{fmt.Sprintf("%s=%s", serviceLogsDirEnv, outCtxt.Logs.Folder.Local)}, - ExtraArgs: []string{"--build", "-d"}, + ExtraArgs: []string{"--build", "-d", serviceName}, } if err := p.Up(opts); err != nil { return nil, errors.Wrap(err, "could not boot up service using docker compose") } // Build service container name - serviceName := inCtxt.Name serviceContainer := fmt.Sprintf("%s_%s_1", service.project, serviceName) outCtxt.Hostname = serviceContainer diff --git a/internal/testrunner/runners/system/test_config.go b/internal/testrunner/runners/system/test_config.go index e65a863c4..33f326153 100644 --- a/internal/testrunner/runners/system/test_config.go +++ b/internal/testrunner/runners/system/test_config.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "path/filepath" + "regexp" "github.com/aymerick/raymond" "github.com/pkg/errors" @@ -19,10 +20,11 @@ import ( "github.com/elastic/elastic-package/internal/testrunner/runners/system/servicedeployer" ) -const configFileName = "config.yml" +var systemTestConfigFilePattern = regexp.MustCompile(`^test-([a-z0-9_.-]+)-config.yml$`) type testConfig struct { Input string `config:"input"` + Service string `config:"service"` Vars map[string]packages.VarValue `config:"vars"` DataStream struct { Vars map[string]packages.VarValue `config:"vars"` @@ -31,10 +33,19 @@ type testConfig struct { // NumericKeywordFields holds a list of fields that have keyword // type but can be ingested as numeric type. NumericKeywordFields []string `config:"numeric_keyword_fields"` + + Path string +} + +func (t testConfig) Name() string { + name := filepath.Base(t.Path) + if matches := systemTestConfigFilePattern.FindStringSubmatch(name); len(matches) > 1 { + name = matches[1] + } + return name } -func newConfig(systemTestFolderPath string, ctxt servicedeployer.ServiceContext) (*testConfig, error) { - configFilePath := filepath.Join(systemTestFolderPath, configFileName) +func newConfig(configFilePath string, ctxt servicedeployer.ServiceContext) (*testConfig, error) { data, err := ioutil.ReadFile(configFilePath) if err != nil && os.IsNotExist(err) { return nil, errors.Wrapf(err, "unable to find system test configuration file: %s", configFilePath) @@ -54,13 +65,32 @@ func newConfig(systemTestFolderPath string, ctxt servicedeployer.ServiceContext) if err != nil { return nil, errors.Wrapf(err, "unable to load system test configuration file: %s", configFilePath) } - if err := cfg.Unpack(&c); err != nil { return nil, errors.Wrapf(err, "unable to unpack system test configuration file: %s", configFilePath) } + // Save path + c.Path = configFilePath return &c, nil } +func listConfigFiles(systemTestFolderPath string) (files []string, err error) { + fHandle, err := os.Open(systemTestFolderPath) + if err != nil { + return nil, err + } + defer fHandle.Close() + dirEntries, err := fHandle.Readdir(0) + if err != nil { + return nil, err + } + for _, entry := range dirEntries { + if !entry.IsDir() && systemTestConfigFilePattern.MatchString(entry.Name()) { + files = append(files, entry.Name()) + } + } + return files, nil +} + // applyContext takes the given system test configuration (data) and replaces any placeholder variables in // it with values from the given context (ctxt). The context may be populated from various sources but usually the // most interesting context values will be set by a ServiceDeployer in its SetUp method. diff --git a/test/packages/apache/data_stream/access/_dev/test/system/config.yml b/test/packages/apache/data_stream/access/_dev/test/system/test-default-config.yml similarity index 100% rename from test/packages/apache/data_stream/access/_dev/test/system/config.yml rename to test/packages/apache/data_stream/access/_dev/test/system/test-default-config.yml diff --git a/test/packages/apache/data_stream/error/_dev/test/system/config.yml b/test/packages/apache/data_stream/error/_dev/test/system/test-default-config.yml similarity index 100% rename from test/packages/apache/data_stream/error/_dev/test/system/config.yml rename to test/packages/apache/data_stream/error/_dev/test/system/test-default-config.yml diff --git a/test/packages/apache/data_stream/status/_dev/test/system/config.yml b/test/packages/apache/data_stream/status/_dev/test/system/test-default-config.yml similarity index 100% rename from test/packages/apache/data_stream/status/_dev/test/system/config.yml rename to test/packages/apache/data_stream/status/_dev/test/system/test-default-config.yml diff --git a/test/packages/multiinput/_dev/deploy/docker/Dockerfile b/test/packages/multiinput/_dev/deploy/docker/Dockerfile new file mode 100644 index 000000000..b52af4192 --- /dev/null +++ b/test/packages/multiinput/_dev/deploy/docker/Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.15 as builder + +ADD . /app + +WORKDIR /app + +RUN go mod download + +ENV CGO_ENABLED=0 +RUN go build + +# ------------------------------------------------------------------------------ +FROM scratch + +COPY --chown=0:0 --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --chown=0:0 --from=builder /app/stream /stream + +CMD ["/stream"] diff --git a/test/packages/multiinput/_dev/deploy/docker/docker-compose.yml b/test/packages/multiinput/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 000000000..d7389f444 --- /dev/null +++ b/test/packages/multiinput/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,14 @@ +version: '2.3' +services: + test-tcp: + tty: true + build: . + volumes: + - ./logs:/logs/:ro + command: /stream -proto tcp -dest elastic-agent:9999 /logs/generated.log + test-udp: + tty: true + build: . + volumes: + - ./logs:/logs/:ro + command: /stream -proto udp -delay 100ms -dest elastic-agent:9999 /logs/generated.log diff --git a/test/packages/multiinput/_dev/deploy/docker/go.mod b/test/packages/multiinput/_dev/deploy/docker/go.mod new file mode 100644 index 000000000..c3ede82b6 --- /dev/null +++ b/test/packages/multiinput/_dev/deploy/docker/go.mod @@ -0,0 +1,3 @@ +module github.com/elastic/elastic-package/packages/multiinput/_dev/deploy/docker/stream + +go 1.15 diff --git a/test/packages/multiinput/_dev/deploy/docker/logs/generated.log b/test/packages/multiinput/_dev/deploy/docker/logs/generated.log new file mode 100644 index 000000000..6bf53ab90 --- /dev/null +++ b/test/packages/multiinput/_dev/deploy/docker/logs/generated.log @@ -0,0 +1,100 @@ +ntpd[1001]: kernel time sync enabled utl +restorecond: : Reset file context quasiarc: liqua +auditd[5699]: Audit daemon rotating log files +anacron[5066]: Normal exit ehend +restorecond: : Reset file context vol: luptat +heartbeat: : < Processing command: accept +restorecond: : Reset file context nci: ofdeFin +auditd[6668]: Audit daemon rotating log files +anacron[1613]: Normal exit mvolu +ntpd[2959]: ntpd gelit-r tatno +anacron[654]: Updated timestamp for job rmagni to sit +dmd: : < Health state for metric"seq3874.mail.domain" "quid" changed to "fug", reason: "success" +auditd[2067]: Audit daemon rotating log files +pm[5969]: < check_license_validity(), tae +logrotate: : ALERT exited abnormally with temUten +sshd: : < error: Bind to port Duisau on psum failed: failure +configd: : < itaut@rveli: command: accept +authd: : < authd_signal_handler(), quam +xinetd[6547]: Started working: onproide available services +logrotate: : ALERT exited abnormally with tfug +heartbeat: : < Processing command: deny +rsyslogd: : Warning: rehe +sshd: : < error: Bind to port erc on amqu failed: unknown +ntpd[4515]: ntpd emp-r aperia +restorecond: : Reset file context run: vol +logrotate: : ALERT exited abnormally with mporain +heartbeat: : < connect: atu +cmd: : < cmd starting adeseru +cli[7108]: <<-uam.low> tmo@::fficiade:10.2.53.125 : CLI launched +pm[7061]: < ntpd will start in tlabo +poller[795]: < Querying content system for job results. +runner[6134]: < Processing command: allow +epmd: : epmd: epmd running orpor +runner[602]: < Failed to exec olup +shutdown[2807]: shutting down non +configd: : < sperna@sintocc: command: cancel +auditd[2986]: Audit daemon rotating log files +configd: : < CREATE onsequ +auditd[1243]: Audit daemon rotating log files +xinetd[6599]: Started working: naal available services +xinetd[5850]: Started working: rQu available services +heartbeat: : < queips: undefined symbol: ncidi +authd: : < authd_close(): npr +anacron[6373]: Anacron 1.3962 started on epre +cli[3979]: <<-iduntu.medium> temUt@avol752.www5.test : Processing command accept +cmd: : < cmd starting isiuta +sshd[5227]: dutp(psaquaea:taevita): pam_putenv: ameiusm +ccd: : < Device elitse6672.internal.localdomain: mquisno +runner[1859]: < Failed to exec umSe +shutdown[6110]: shutting down itau +sshd[2415]: PAM lorsita more authentication failure; dolore +rsyslogd: : Warning: tio +cli[802]: <<-gnaaliqu.very-high> velillu@::cteturad:10.18.204.87 : Processing a secure command... +heartbeat: : < connect: inimveni +authd: : < authd_close(): psumqu +runner[2558]: < Failed to exec edquiac +anacron[4538]: Updated timestamp for job remips to uisaute +auditd[6837]: Audit daemon rotating log files +pm[1493]: < print_msg(), dic +configd: : < Device "itation4168.api.domain" completed command(s) accept ;; CPL generated by Visual Policy Manager: isciv ;rroqu ; nofd ; dipisci +epmd: : epmd: invalid packet size (mquae) +runner[429]: < File reading failed +shutdown[7595]: shutting down emqu +heartbeat: : < The HB command is accept +authd: : < authd_signal_handler(), isetquas +authd: : < authd_signal_handler(), gnaal +logrotate: : ALERT exited abnormally with voluptas +ntpd[627]: ntpd exiting on signal orin +restorecond: : Reset file context ecillu: mmodoc +cli[1140]: <<-abore.high> modocon@ipsu3680.mail.test : Processing command: deny +sshd: : bad username mquisn +ntpd[1313]: ntpd derit-r orese +ccd: : < Device Communication Daemon online +rsyslogd: : Warning: moles +restorecond: : Reset file context olup: aco +shutdown[609]: shutting down ser +ntpd[2991]: ntpd orinrep-r quiavol +dmd: : < inserted device id = sBonor2001.www5.example and serial number = amc into DB +ccd: : < ccd_handle_read_failure(), uid +cmd: : < cmd starting lmolesti +dmd: : < inserted device id = ersp6625.internal.domain and serial number = seq into DB +cmd: : < cmd starting uipexe +heartbeat: : < The HB command is cancel +anacron[7360]: Normal exit tperspic +dmd: : < Filter on (tetura) things. riosamni +ccd: : < Device eleumiu2454.api.local: tat +schedulerd: : < System time changed, recomputing job run times. +xinetd[3450]: Started working: aconsequ available services +authd: : < handle_authd unknown message =utemvel +rsyslogd: : Warning: iusm +ntpd[16]: time reset stquido +ccd: : < Device olu5333.www.domain: orumSe +anacron[80]: Normal exit ici +ntpd[7612]: kernel time sync enabled nturmag +cli[7128]: eseruntm(lpaquiof:oloreeu): pam_putenv: olor +schedulerd: : < Executing Job "tquo" execution iatnu +logrotate: : ALERT exited abnormally with ntut +poller[7151]: < Querying content system for job results. +ntpd[2314]: ntpd litanim-r rQuisaut +heartbeat: : < Processing command: block diff --git a/test/packages/multiinput/_dev/deploy/docker/main.go b/test/packages/multiinput/_dev/deploy/docker/main.go new file mode 100644 index 000000000..7ec86354b --- /dev/null +++ b/test/packages/multiinput/_dev/deploy/docker/main.go @@ -0,0 +1,78 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "bufio" + "errors" + "flag" + "fmt" + "log" + "net" + "os" + "syscall" + "time" +) + +var ( + dest string + proto string + delay time.Duration +) + +func init() { + flag.StringVar(&dest, "dest", "localhost:514", "destination tcp address") + flag.StringVar(&proto, "proto", "tcp", "protocol to use (tcp or udp)") + flag.DurationVar(&delay, "delay", 0, "delay between messages") +} + +func main() { + log.SetFlags(0) + flag.Parse() + + var err error + fmt.Fprintf(os.Stderr, "Using proto=%s dest=%s\n", proto, dest) + var conn net.Conn + for { + conn, err = net.Dial(proto, dest) + if err != nil { + log.Println(err) + time.Sleep(time.Second) + continue + } + break + } + defer conn.Close() + + for _, input := range flag.Args() { + fmt.Fprintf(os.Stderr, "Delivering file %v\n", input) + f, err := os.Open(input) + if err != nil { + log.Fatal(err) + } + defer f.Close() + + scanner := bufio.NewScanner(f) + count := 0 + + for scanner.Scan() { + count += 1 + time.Sleep(delay) + data := append(scanner.Bytes(), '\n') + n, err := conn.Write(data) + if err != nil || n != len(data) { + if proto == "udp" && errors.Is(err, syscall.ECONNREFUSED) { + time.Sleep(time.Second) + log.Printf("Restarted count=%d", count) + f.Seek(0, 0) + scanner = bufio.NewScanner(f) + count = 0 + continue + } + log.Fatalf("Error sending message %d: %v", count, err) + } + } + } +} diff --git a/test/packages/multiinput/data_stream/test/_dev/test/system/test-tcp-config.yml b/test/packages/multiinput/data_stream/test/_dev/test/system/test-tcp-config.yml new file mode 100644 index 000000000..10fb7aa19 --- /dev/null +++ b/test/packages/multiinput/data_stream/test/_dev/test/system/test-tcp-config.yml @@ -0,0 +1,7 @@ +input: tcp +service: test-tcp +vars: ~ +data_stream: + vars: + tcp_host: 0.0.0.0 + tcp_port: 9999 diff --git a/test/packages/multiinput/data_stream/test/_dev/test/system/test-udp-config.yml b/test/packages/multiinput/data_stream/test/_dev/test/system/test-udp-config.yml new file mode 100644 index 000000000..2219cbfc6 --- /dev/null +++ b/test/packages/multiinput/data_stream/test/_dev/test/system/test-udp-config.yml @@ -0,0 +1,7 @@ +input: udp +service: test-udp +vars: ~ +data_stream: + vars: + udp_host: 0.0.0.0 + udp_port: 9999 diff --git a/test/packages/multiinput/data_stream/test/agent/stream/stream.yml.hbs b/test/packages/multiinput/data_stream/test/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..2cdbbeb73 --- /dev/null +++ b/test/packages/multiinput/data_stream/test/agent/stream/stream.yml.hbs @@ -0,0 +1,25 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +fields_under_root: true +fields: + observer: + vendor: Test + product: Test + type: test +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} + +processors: +- add_locale: ~ +- add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/test/packages/multiinput/data_stream/test/agent/stream/tcp.yml.hbs b/test/packages/multiinput/data_stream/test/agent/stream/tcp.yml.hbs new file mode 100644 index 000000000..2c57f3fa0 --- /dev/null +++ b/test/packages/multiinput/data_stream/test/agent/stream/tcp.yml.hbs @@ -0,0 +1,21 @@ +host: "{{tcp_host}}:{{tcp_port}}" +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +fields_under_root: true +fields: + observer: + vendor: Test + product: test + type: test +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} + +processors: +- add_locale: ~ +- add_fields: + target: '' + fields: + ecs.version: 1.7.0 diff --git a/test/packages/multiinput/data_stream/test/agent/stream/udp.yml.hbs b/test/packages/multiinput/data_stream/test/agent/stream/udp.yml.hbs new file mode 100644 index 000000000..bcceb4fea --- /dev/null +++ b/test/packages/multiinput/data_stream/test/agent/stream/udp.yml.hbs @@ -0,0 +1,21 @@ +host: "{{udp_host}}:{{udp_port}}" +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +fields_under_root: true +fields: + observer: + vendor: Test + product: test + type: test +{{#contains tags "forwarded"}} +publisher_pipeline.disable_host: true +{{/contains}} + +processors: +- add_locale: ~ +- add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/test/packages/multiinput/data_stream/test/elasticsearch/ingest_pipeline/default.yml b/test/packages/multiinput/data_stream/test/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..bef3b2c8b --- /dev/null +++ b/test/packages/multiinput/data_stream/test/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,12 @@ +--- +description: Test Pipeline + +processors: + # ECS event.ingested + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' +on_failure: + - append: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/test/packages/multiinput/data_stream/test/fields/base-fields.yml b/test/packages/multiinput/data_stream/test/fields/base-fields.yml new file mode 100644 index 000000000..0ec2cc7e0 --- /dev/null +++ b/test/packages/multiinput/data_stream/test/fields/base-fields.yml @@ -0,0 +1,38 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: container.id + description: Unique container id. + ignore_above: 1024 + type: keyword +- name: input.type + description: Type of Filebeat input. + type: keyword +- name: log.file.path + description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + type: keyword +- name: log.source.address + description: Source address from which the log event was read / sent from. + type: keyword +- name: log.flags + description: Flags for the log file. + type: keyword +- name: log.offset + description: Offset of the entry in the log file. + type: long +- name: tags + description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + type: keyword diff --git a/test/packages/multiinput/data_stream/test/fields/ecs.yml b/test/packages/multiinput/data_stream/test/fields/ecs.yml new file mode 100644 index 000000000..03e4a9fec --- /dev/null +++ b/test/packages/multiinput/data_stream/test/fields/ecs.yml @@ -0,0 +1,994 @@ +- name: log + type: group + fields: + - name: original + level: core + type: keyword + ignore_above: 1024 + description: 'This is the original log message and contains the full log message before splitting it up in multiple parts. + + In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. + + This field is not indexed and doc_values are disabled so it can''t be queried but the value can be retrieved from `_source`.' + example: Sep 19 08:26:10 localhost My log + index: false + - name: level + level: core + type: keyword + ignore_above: 1024 + description: 'Original log level of the log event. + + If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn''t specify one, you may put your event transport''s severity here (e.g. Syslog severity). + + Some examples are `warn`, `err`, `i`, `informational`.' + example: error + - name: syslog + type: group + fields: + - name: priority + level: extended + type: long + format: string + description: 'Syslog numeric priority of the event, if available. + + According to RFCs 5424 and 3164, the priority is 8 * facility + severity. This number is therefore expected to contain a value between 0 and 191.' + example: 135 + - name: facility + type: group + fields: + - name: code + level: extended + type: long + format: string + description: 'The Syslog numeric facility of the log event, if available. + + According to RFCs 5424 and 3164, this value should be an integer between 0 and 23.' + example: 23 + - name: severity + type: group + fields: + - name: code + level: extended + type: long + description: 'The Syslog numeric severity of the log event, if available. + + If the event source publishing via Syslog provides a different numeric severity value (e.g. firewall, IDS), your source''s numeric severity should go to `event.severity`. If the event source does not specify a distinct severity, you can optionally copy the Syslog severity to `event.severity`.' + example: 3 +- name: event + type: group + fields: + - name: code + level: extended + type: keyword + ignore_above: 1024 + description: 'Identification code for this event, if one exists. + + Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID.' + example: 4648 + - name: action + level: core + type: keyword + ignore_above: 1024 + description: 'The action captured by the event. + + This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer.' + example: user-password-change + - name: outcome + level: core + type: keyword + ignore_above: 1024 + description: 'This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. + + `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. + + Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. + + Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. + + Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense.' + example: success + - name: timezone + level: extended + type: keyword + ignore_above: 1024 + description: 'This field should be populated when the event''s timestamp does not include timezone information already (e.g. default Syslog timestamps). It''s optional otherwise. + + Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00").' + - name: ingested + level: core + type: date + description: 'Timestamp when an event arrived in the central data store. + + This is different from `@timestamp`, which is when the event originally occurred. It''s also different from `event.created`, which is meant to capture the first time an agent saw the event. + + In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` < `event.created` < `event.ingested`.' + example: '2016-05-23T08:05:35.101Z' + default_field: false + - name: original + level: core + type: keyword + ignore_above: 1024 + description: 'Raw text message of entire event. Used to demonstrate log integrity. + + This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`.' + example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 + index: false +- name: '@timestamp' + level: core + required: true + type: date + description: 'Date/time when the event originated. + + This is the date/time extracted from the event, typically representing when the event was generated by the source. + + If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. + + Required field for all events.' + example: '2016-05-23T08:05:34.853Z' +- name: related + type: group + fields: + - name: user + level: extended + type: keyword + ignore_above: 1024 + description: All the user names seen on your event. + default_field: false + - name: hosts + level: extended + type: keyword + ignore_above: 1024 + description: All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. + default_field: false + - name: ip + level: extended + type: ip + description: All of the IPs seen on your event. +- name: user + type: group + fields: + - name: name + level: core + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Short name or login of the user. + example: albert + - name: full_name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: User's full name, if available. + example: Albert Einstein + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the directory the user is a member of. + + For example, an LDAP or Active Directory domain name.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique identifier of the user. +- name: message + level: core + type: text + description: 'For log events the message field contains the log message, optimized for viewing in a log viewer. + + For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. + + If multiple messages exist, they can be combined into one message.' + example: Hello World +- name: source + type: group + fields: + - name: ip + level: core + type: ip + description: IP address of the source (IPv4 or IPv6). + - name: port + level: core + type: long + format: string + description: Port of the source. + - name: address + level: extended + type: keyword + ignore_above: 1024 + description: 'Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.' + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: MAC address of the source. + - name: bytes + level: core + type: long + format: bytes + description: Bytes sent from the source to the destination. + example: 184 + - name: nat + type: group + fields: + - name: ip + level: extended + type: ip + description: 'Translated ip of source based NAT sessions (e.g. internal client to internet) + + Typically connections traversing load balancers, firewalls, or routers.' + - name: port + level: extended + type: long + format: string + description: 'Translated port of source based NAT sessions. (e.g. internal client to internet) + + Typically used with load balancers, firewalls, or routers.' + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Source domain. + - name: registered_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The highest registered source domain, stripped of the subdomain. + + For example, the registered domain for "foo.example.com" is "example.com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk".' + example: example.com + - name: subdomain + level: extended + type: keyword + ignore_above: 1024 + description: 'The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. + + For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period.' + example: east + - name: top_level_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk".' + example: co.uk + - name: geo + type: group + fields: + - name: country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: location + type: group + fields: + - type: double + name: lat + - type: double + name: lon + - name: city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal + - name: as + type: group + fields: + - name: number + level: extended + type: long + description: Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. + example: 15169 + - name: organization + type: group + fields: + - name: name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Organization name. + example: Google LLC +- name: host + type: group + fields: + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. +- name: destination + type: group + fields: + - name: ip + level: core + type: ip + description: IP address of the destination (IPv4 or IPv6). + - name: port + level: core + type: long + format: string + description: Port of the destination. + - name: address + level: extended + type: keyword + ignore_above: 1024 + description: 'Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. + + Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.' + - name: bytes + level: core + type: long + format: bytes + description: Bytes sent from the destination to the source. + example: 184 + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: MAC address of the destination. + - name: nat + type: group + fields: + - name: ip + level: extended + type: ip + description: 'Translated ip of destination based NAT sessions (e.g. internet to private DMZ) + + Typically used with load balancers, firewalls, or routers.' + - name: port + level: extended + type: long + format: string + description: 'Port the source session is translated to by NAT Device. + + Typically used with load balancers, firewalls, or routers.' + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Destination domain. + - name: registered_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The highest registered source domain, stripped of the subdomain. + + For example, the registered domain for "foo.example.com" is "example.com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk".' + example: example.com + - name: subdomain + level: extended + type: keyword + ignore_above: 1024 + description: 'The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. + + For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period.' + example: east + - name: top_level_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk".' + example: co.uk + - name: geo + type: group + fields: + - name: country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: location + type: group + fields: + - type: double + name: lat + - type: double + name: lon + - name: city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal + - name: as + type: group + fields: + - name: number + level: extended + type: long + description: Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. + example: 15169 + - name: organization + type: group + fields: + - name: name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Organization name. + example: Google LLC +- name: network + type: group + fields: + - name: protocol + level: core + type: keyword + ignore_above: 1024 + description: 'L7 Network protocol name. ex. http, lumberjack, transport protocol. + + The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS".' + example: http + - name: application + level: extended + type: keyword + ignore_above: 1024 + description: 'A name given to an application level protocol. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. + + The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS".' + example: aim + - name: interface + type: group + fields: + - type: keyword + name: name + - name: bytes + level: core + type: long + format: bytes + description: 'Total bytes transferred in both directions. + + If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum.' + example: 368 + - name: direction + level: core + type: keyword + ignore_above: 1024 + description: "Direction of the network traffic.\nRecommended values are:\n * ingress\n * egress\n * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values \"ingress\" or \"egress\".\nWhen mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values \"inbound\", \"outbound\", \"internal\" or \"external\".\nNote that \"internal\" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that \"external\" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers." + example: inbound + - name: packets + level: core + type: long + description: 'Total packets transferred in both directions. + + If `source.packets` and `destination.packets` are known, `network.packets` is their sum.' + example: 24 + - name: forwarded_ip + level: core + type: ip + description: Host IP address when the source IP address is the proxy. + example: 192.1.1.2 +- name: observer + type: group + fields: + - name: version + level: core + type: keyword + ignore_above: 1024 + description: Observer version. + - name: product + level: extended + type: keyword + ignore_above: 1024 + description: The product name of the observer. + example: s200 + - name: ingress + type: group + fields: + - name: interface + type: group + fields: + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Interface name as reported by the system. + example: eth0 + default_field: false + - name: egress + type: group + fields: + - name: interface + type: group + fields: + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Interface name as reported by the system. + example: eth0 + default_field: false + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'The type of the observer the data is coming from. + + There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`.' + example: firewall + - name: vendor + level: core + type: keyword + ignore_above: 1024 + description: Vendor name of the observer. + example: Symantec +- name: file + type: group + fields: + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the file including the extension, without the directory. + example: example.png + - name: directory + level: extended + type: keyword + ignore_above: 1024 + description: Directory where the file is located. It should include the drive letter, when appropriate. + example: /home/alice + - name: size + level: extended + type: long + description: 'File size in bytes. + + Only relevant when `file.type` is "file".' + example: 16384 + - name: type + level: extended + type: keyword + ignore_above: 1024 + description: File type (file, dir, or symlink). + example: file + - name: extension + level: extended + type: keyword + ignore_above: 1024 + description: 'File extension, excluding the leading dot. + + Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz").' + example: png + - name: path + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Full path to the file, including the file name. It should include the drive letter, when appropriate. + example: /home/alice/example.png + - name: attributes + level: extended + type: keyword + ignore_above: 1024 + description: 'Array of file attributes. + + Attributes names will vary by platform. Here''s a non-exhaustive list of values that are expected in this field: archive, compressed, directory, encrypted, execute, hidden, read, readonly, system, write.' + example: '["readonly", "system"]' + default_field: false +- name: url + type: group + fields: + - name: original + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: 'Unmodified original url as seen in the event source. + + Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. + + This field is meant to represent the URL as it was observed, complete or not.' + example: https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch + - name: query + level: extended + type: keyword + ignore_above: 1024 + description: 'The query field describes the query string of the request, such as "q=elasticsearch". + + The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases.' + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Domain of the url, such as "www.elastic.co". + + In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. + + If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field.' + example: www.elastic.co + - name: path + level: extended + type: keyword + ignore_above: 1024 + description: Path of the request, such as "/search". + - name: top_level_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk".' + example: co.uk + - name: registered_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The highest registered url domain, stripped of the subdomain. + + For example, the registered domain for "foo.example.com" is "example.com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk".' + example: example.com +- name: service + type: group + fields: + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the service data is collected from. + + The name of the service is normally user given. This allows for distributed services that run on multiple hosts to correlate the related instances based on the name. + + In the case of Elasticsearch the `service.name` could contain the cluster name. For Beats the `service.name` is by default a copy of the `service.type` field if no name is specified.' + example: elasticsearch-metrics +- name: client + type: group + fields: + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Server domain. + - name: registered_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The highest registered source domain, stripped of the subdomain. + + For example, the registered domain for "foo.example.com" is "example.com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk".' + example: example.com + - name: subdomain + level: extended + type: keyword + ignore_above: 1024 + description: 'The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. + + For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period.' + example: east + - name: top_level_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk".' + example: co.uk +- name: server + type: group + fields: + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Server domain. + - name: registered_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The highest registered source domain, stripped of the subdomain. + + For example, the registered domain for "foo.example.com" is "example.com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk".' + example: example.com + - name: subdomain + level: extended + type: keyword + ignore_above: 1024 + description: 'The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. + + For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period.' + example: east + - name: top_level_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk".' + example: co.uk +- name: group + type: group + fields: + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the group. + - name: id + level: extended + type: keyword + ignore_above: 1024 + description: Unique identifier for the group on the system/platform. +- name: process + type: group + fields: + - name: pid + level: core + type: long + format: string + description: Process id. + example: 4242 + - name: name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: 'Process name. + + Sometimes called program name or similar.' + example: ssh + - name: ppid + level: extended + type: long + format: string + description: Parent process' pid. + example: 4241 + - name: parent + type: group + fields: + - name: name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + description: 'Process name. + + Sometimes called program name or similar.' + example: ssh + default_field: false + - name: title + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + description: 'Process title. + + The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened.' + default_field: false + - name: title + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: 'Process title. + + The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened.' +- name: rule + type: group + fields: + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: The name of the rule or signature generating the event. + example: BLOCK_DNS_over_TLS + default_field: false +- name: user_agent + type: group + fields: + - name: original + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + description: Unparsed user_agent string. + example: Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 +- name: http + type: group + fields: + - name: request + type: group + fields: + - name: referrer + level: extended + type: keyword + ignore_above: 1024 + description: Referrer for this HTTP request. + example: https://blog.example.com/ + - name: method + level: extended + type: keyword + ignore_above: 1024 + description: 'HTTP request method. + + Prior to ECS 1.6.0 the following guidance was provided: + + "The field value must be normalized to lowercase for querying." + + As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0' + example: GET, POST, PUT, PoST +- name: geo + type: group + fields: + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: 'User-defined description of a location, at the level of granularity they care about. + + Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. + + Not typically used in automated geolocation.' + example: boston-dc + - name: country_name + level: core + type: keyword + ignore_above: 1024 + description: Country name. + example: Canada + - name: city_name + level: core + type: keyword + ignore_above: 1024 + description: City name. + example: Montreal + - name: region_name + level: core + type: keyword + ignore_above: 1024 + description: Region name. + example: Quebec +- name: dns + type: group + fields: + - name: question + type: group + fields: + - name: type + level: extended + type: keyword + ignore_above: 1024 + description: The type of record being queried. + example: AAAA + - name: domain + level: core + type: keyword + ignore_above: 1024 + description: Server domain. + - name: registered_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The highest registered source domain, stripped of the subdomain. + + For example, the registered domain for "foo.example.com" is "example.com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk".' + example: example.com + - name: subdomain + level: extended + type: keyword + ignore_above: 1024 + description: 'The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. + + For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period.' + example: east + - name: top_level_domain + level: extended + type: keyword + ignore_above: 1024 + description: 'The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". + + This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk".' + example: co.uk + - name: answers + type: group + fields: + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: 'The domain name to which this resource record pertains. + + If a chain of CNAME is being resolved, each answer''s `name` should be the one that corresponds with the answer''s `data`. It should not simply be the original `question.name` repeated.' + example: www.example.com + - name: type + level: extended + type: keyword + ignore_above: 1024 + description: The type of data contained in this resource record. + example: CNAME +- name: error + type: group + fields: + - name: message + level: core + type: text + description: Error message. +- name: tags + level: core + type: keyword + ignore_above: 1024 + description: List of keywords used to tag each event. + example: '["production", "env2"]' +- name: ecs.version + type: keyword + description: ECS version this event conforms to. diff --git a/test/packages/multiinput/data_stream/test/manifest.yml b/test/packages/multiinput/data_stream/test/manifest.yml new file mode 100644 index 000000000..d92248116 --- /dev/null +++ b/test/packages/multiinput/data_stream/test/manifest.yml @@ -0,0 +1,79 @@ +title: Test +release: experimental +type: logs +streams: + - input: udp + title: UDP logs + description: Collect UDP logs + template_path: udp.yml.hbs + vars: + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - name: udp_host + type: text + title: UDP host to listen on + multi: false + required: true + show_user: true + default: localhost + - name: udp_port + type: integer + title: UDP port to listen on + multi: false + required: true + show_user: true + default: 9999 + - input: tcp + title: TCP logs + description: Collect TCP logs + template_path: tcp.yml.hbs + vars: + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - name: tcp_host + type: text + title: TCP host to listen on + multi: false + required: true + show_user: true + default: localhost + - name: tcp_port + type: integer + title: TCP port to listen on + multi: false + required: true + show_user: true + default: 9511 + - input: file + title: File logs + description: Collect logs from file + enabled: false + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/file.log + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded diff --git a/test/packages/multiinput/docs/README.md b/test/packages/multiinput/docs/README.md new file mode 100644 index 000000000..e0ef7b4a1 --- /dev/null +++ b/test/packages/multiinput/docs/README.md @@ -0,0 +1,2 @@ +# Test integration + diff --git a/test/packages/multiinput/manifest.yml b/test/packages/multiinput/manifest.yml new file mode 100644 index 000000000..4c293aef9 --- /dev/null +++ b/test/packages/multiinput/manifest.yml @@ -0,0 +1,27 @@ +format_version: 1.0.0 +name: multiinput +title: Multi-input test +version: 0.0.1 +description: Test for multiple input tests +categories: ["network"] +release: experimental +license: basic +type: integration +conditions: + kibana.version: '^7.10.0' +policy_templates: + - name: test + title: Test + description: Description + inputs: + - type: udp + title: Collect udp logs. + description: Collecting logs via UDP. + - type: tcp + title: Collect tcp logs. + description: Collecting logs via TCP. + - type: file + title: Collect logs via file. + description: Collecting logs via file. +owner: + github: elastic/security-external-integrations