Add support for multiple inputs per dataset

CHANGELOG.md

@@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add `all` query param to return all packages. By default is set false. [#301](https://github.com/elastic/integrations-registry/pull/301)
 * Add `multiple` config for datasource. By default true. [#361](https://github.com/elastic/integrations-registry/pull/361)
 * Add `removable` flag to package manifest. Default is true. [#359](https://github.com/elastic/integrations-registry/pull/359)
+* Add stream template to package json. [#335](https://github.com/elastic/integrations-registry/pull/335)
+* Add support for multiple inputs per dataset. [#346](https://github.com/elastic/integrations-registry/pull/346)
 
 ### Deprecated
 

dev/packages/example/nginx-1.2.0/dataset/error/manifest.yml

@@ -8,6 +8,8 @@ streams:
   - input: logs
     title: Nginx error logs
     description: Collecting the Nginx error logs from file.
+    # Name of the stream file
+    template_path: stream.yml
     vars:
       - name: paths
         required: true
@@ -23,6 +25,8 @@ streams:
   - input: syslog
     title: Nginx error logs
     description: Collecting the Nginx error logs from syslog.
+    # Name of the stream file
+    template_path: syslog.yml
     vars:
       # Are udp and tcp syslog input two different inputs?
       - name: protocol.udp.host

docs/api/package.json

@@ -103,6 +103,7 @@
                 }
               ],
               "dataset": "bar",
+              "template_path": "stream.yml",
               "template": "foo: bar\n"
             }
           ]

testdata/package/datasources-1.0.0/dataset/examplelog1/agent/stream/syslog.yml

@@ -0,0 +1 @@
+syslog: bar

testdata/package/datasources-1.0.0/dataset/examplelog1/manifest.yml

@@ -5,6 +5,7 @@ streams:
   - input: logs
     title: Title of the stream
     description: Description of the stream with more details.
+    template_path: logs.yml
     vars:
       - name: paths
         required: true
@@ -21,3 +22,8 @@ streams:
         os.windows:
           default:
             - c:/programdata/nginx/logs/*error.log*
+
+  - input: syslog
+    title: Title of the stream
+    description: Description of the stream with more details.
+    template_path: syslog.yml

testdata/public/package/datasources/1.0.0/dataset/examplelog1/agent/stream/syslog.yml

@@ -0,0 +1 @@
+syslog: bar

testdata/public/package/datasources/1.0.0/dataset/examplelog1/manifest.yml

@@ -5,6 +5,7 @@ streams:
   - input: logs
     title: Title of the stream
     description: Description of the stream with more details.
+    template_path: logs.yml
     vars:
       - name: paths
         required: true
@@ -21,3 +22,8 @@ streams:
         os.windows:
           default:
             - c:/programdata/nginx/logs/*error.log*
+
+  - input: syslog
+    title: Title of the stream
+    description: Description of the stream with more details.
+    template_path: syslog.yml

testdata/public/package/datasources/1.0.0/index.json

@@ -24,7 +24,8 @@
     "/package/datasources/1.0.0/dataset/examplelog1/fields/stream.yml",
     "/package/datasources/1.0.0/dataset/examplelog2/fields/stream.yml",
     "/package/datasources/1.0.0/dataset/examplemetric/fields/stream.yml",
-    "/package/datasources/1.0.0/dataset/examplelog1/agent/stream/stream.yml",
+    "/package/datasources/1.0.0/dataset/examplelog1/agent/stream/logs.yml",
+    "/package/datasources/1.0.0/dataset/examplelog1/agent/stream/syslog.yml",
     "/package/datasources/1.0.0/dataset/examplelog2/agent/stream/stream.yml",
     "/package/datasources/1.0.0/dataset/examplemetric/agent/stream/stream.yml"
   ],
@@ -62,6 +63,13 @@
               }
             }
           ],
+          "template_path": "logs.yml",
+          "title": "Title of the stream",
+          "description": "Description of the stream with more details."
+        },
+        {
+          "input": "syslog",
+          "template_path": "syslog.yml",
           "title": "Title of the stream",
           "description": "Description of the stream with more details."
         }
@@ -196,6 +204,7 @@
                 }
               ],
               "dataset": "datasources.examplemetric",
+              "template_path": "stream.yml",
               "template": "metric: foo\n",
               "title": "Title of the stream",
               "description": "Description of the stream with more details."
@@ -234,6 +243,7 @@
                 }
               ],
               "dataset": "datasources.examplelog1",
+              "template_path": "logs.yml",
               "template": "foo: bar\n",
               "title": "Title of the stream",
               "description": "Description of the stream with more details."
@@ -266,14 +276,25 @@
                 }
               ],
               "dataset": "datasources.examplelog2",
+              "template_path": "stream.yml",
               "template": "foo: bar\n",
               "title": "Title of the stream",
               "description": "Description of the stream with more details."
             }
           ]
         },
         {
-          "type": "syslog"
+          "type": "syslog",
+          "streams": [
+            {
+              "input": "syslog",
+              "dataset": "datasources.examplelog1",
+              "template_path": "syslog.yml",
+              "template": "syslog: bar\n",
+              "title": "Title of the stream",
+              "description": "Description of the stream with more details."
+            }
+          ]
         }
       ],
       "multiple": true

testdata/public/package/default-pipeline/0.0.2/index.json

@@ -71,6 +71,7 @@
                 }
               ],
               "dataset": "default-pipeline.foo",
+              "template_path": "stream.yml",
               "template": "foo: bar\n"
             }
           ]

testdata/public/package/example/1.0.0/index.json

@@ -103,6 +103,7 @@
                 }
               ],
               "dataset": "bar",
+              "template_path": "stream.yml",
               "template": "foo: bar\n"
             }
           ]

testdata/public/package/multiple-false/0.0.1/index.json

@@ -71,6 +71,7 @@
                 }
               ],
               "dataset": "multiple-false.foo",
+              "template_path": "stream.yml",
               "template": "foo: bar\n"
             }
           ]

util/dataset.go

@@ -45,12 +45,14 @@ type Input struct {
 }
 
 type Stream struct {
-	Input       string     `config:"input" json:"input" validate:"required"`
-	Vars        []Variable `config:"vars" json:"vars,omitempty" yaml:"vars,omitempty"`
-	Dataset     string     `config:"dataset" json:"dataset,omitempty" yaml:"dataset,omitempty"`
-	Template    string     `config:"template" json:"template,omitempty" yaml:"template,omitempty"`
-	Title       string     `config:"title" json:"title,omitempty" yaml:"title,omitempty"`
-	Description string     `config:"description" json:"description,omitempty" yaml:"description,omitempty"`
+	Input   string     `config:"input" json:"input" validate:"required"`
+	Vars    []Variable `config:"vars" json:"vars,omitempty" yaml:"vars,omitempty"`
+	Dataset string     `config:"dataset" json:"dataset,omitempty" yaml:"dataset,omitempty"`
+	// TODO: This might cause issues when consuming the json as the key contains . (had been an issue in the past if I remember correctly)
+	TemplatePath    string `config:"template_path" json:"template_path,omitempty" yaml:"template-path,omitempty"`
+	TemplateContent string `json:"template,omitempty"` // This is always generated in the json output
+	Title           string `config:"title" json:"title,omitempty" yaml:"title,omitempty"`
+	Description     string `config:"description" json:"description,omitempty" yaml:"description,omitempty"`
 }
 
 type Variable struct {

util/package.go

@@ -345,18 +345,20 @@ func (p *Package) LoadDataSets(packagePath string) error {
 			for iK, _ := range datasource.Inputs {
 				for _, stream := range d.Streams {
 					if stream.Input == p.Datasources[dK].Inputs[iK].Type {
+						if stream.TemplatePath == "" {
+							stream.TemplatePath = "stream.yml"
+						}
 						stream.Dataset = d.ID
-						streamTemplate := filepath.Join(datasetBasePath, "agent", "stream", "stream.yml")
+						streamTemplate := filepath.Join(datasetBasePath, "agent", "stream", stream.TemplatePath)
 
 						streamTemplateData, err := ioutil.ReadFile(streamTemplate)
 						if err != nil {
 							return err
 						}
 
-						stream.Template = string(streamTemplateData)
+						stream.TemplateContent = string(streamTemplateData)
 
 						// Add template to stream
-
 						p.Datasources[dK].Inputs[iK].Streams = append(p.Datasources[dK].Inputs[iK].Streams, stream)
 					}
 				}