Support zip files without directory entries

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Bugfixes
 
+* Data streams are properly read from Zip packages without entries for directories. [#817](https://github.com/elastic/package-registry/pull/817)
+
 ### Added
 
 * Prepare stub for Storage Indexer. Disable fetching packages from Package Storage v1. [#811](https://github.com/elastic/package-registry/pull/811)

main_test.go

@@ -289,6 +289,7 @@ func TestZippedArtifacts(t *testing.T) {
 		handler  func(w http.ResponseWriter, r *http.Request)
 	}{
 		{"/epr/example/example-1.0.1.zip", artifactsRouterPath, "example-1.0.1.zip-preview.txt", artifactsHandler},
+		{"/epr/example/nodirentries-1.0.0.zip", artifactsRouterPath, "nodirentries-1.0.0.zip-preview.txt", artifactsHandler},
 		{"/epr/example/example-999.0.2.zip", artifactsRouterPath, "artifact-package-version-not-found.txt", artifactsHandler},
 		{"/package/example/1.0.1/docs/README.md", staticRouterPath, "example-1.0.1-README.md", staticHandler},
 		{"/package/example/1.0.1/img/kibana-envoyproxy.jpg", staticRouterPath, "example-1.0.1-screenshot.jpg", staticHandler},
@@ -320,6 +321,7 @@ func TestPackageIndex(t *testing.T) {
 	}{
 		{"/package/example/1.0.0/", packageIndexRouterPath, "package.json", packageIndexHandler},
 		{"/package/example/1.0.1/", packageIndexRouterPath, "package-zip.json", packageIndexHandler},
+		{"/package/nodirentries/1.0.0/", packageIndexRouterPath, "package-zip-nodirentries.json", packageIndexHandler},
 		{"/package/missing/1.0.0/", packageIndexRouterPath, "index-package-not-found.txt", packageIndexHandler},
 		{"/package/example/999.0.0/", packageIndexRouterPath, "index-package-revision-not-found.txt", packageIndexHandler},
 		{"/package/example/a.b.c/", packageIndexRouterPath, "index-package-invalid-version.txt", packageIndexHandler},

packages/package.go

@@ -522,18 +522,9 @@ func (p *Package) GetDataStreamPaths() ([]string, error) {
 
 	dataStreamBasePath := "data_stream"
 
-	// Check if this package has dataStreams
-	_, err = fs.Stat(dataStreamBasePath)
-	// If no dataStreams exist, just return
-	if os.IsNotExist(err) {
-		return nil, nil
-	}
-	// An other error happened, report it
-	if err != nil {
-		return nil, err
-	}
-
-	paths, err := fs.Glob(filepath.Join(dataStreamBasePath, "*"))
+	// Look for a file here that a data_stream must have, some file systems as Zip files
+	// may not have entries for directories.
+	paths, err := fs.Glob(filepath.Join(dataStreamBasePath, "*", "manifest.yml"))
 	if err != nil {
 		return nil, err
 	}
@@ -543,7 +534,7 @@ func (p *Package) GetDataStreamPaths() ([]string, error) {
 		if err != nil {
 			return nil, fmt.Errorf("failed to get data stream path inside package (%s): %w", dataStreamBasePath, err)
 		}
-		paths[i] = relPath
+		paths[i] = filepath.Dir(relPath)
 	}
 
 	return paths, nil

testdata/generated/nodirentries-1.0.0.zip-preview.txt

@@ -0,0 +1,19 @@
+538 nodirentries-1.0.0/manifest.yml
+892 nodirentries-1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-entry.json
+3584 nodirentries-1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-plaintext.json
+887 nodirentries-1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-json.json
+2071 nodirentries-1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-http.json
+900 nodirentries-1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-tcp.json
+355 nodirentries-1.0.0/data_stream/foo/fields/base-fields.yml
+333 nodirentries-1.0.0/data_stream/foo/manifest.yml
+9 nodirentries-1.0.0/data_stream/foo/agent/stream/stream.yml.hbs
+19596 nodirentries-1.0.0/img/icon.png
+482070 nodirentries-1.0.0/img/kibana-envoyproxy.jpg
+36 nodirentries-1.0.0/docs/README.md
+2221 nodirentries-1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json
+1982 nodirentries-1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json
+1920 nodirentries-1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json
+1849 nodirentries-1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json
+1863 nodirentries-1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json
+1995 nodirentries-1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json
+2572 nodirentries-1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json

testdata/generated/package-zip-nodirentries.json

@@ -0,0 +1,96 @@
+{
+  "name": "nodirentries",
+  "title": "Example Integration",
+  "version": "1.0.0",
+  "release": "ga",
+  "description": "This is a zip package without directory entries.",
+  "type": "integration",
+  "download": "/epr/nodirentries/nodirentries-1.0.0.zip",
+  "path": "/package/nodirentries/1.0.0",
+  "conditions": {
+    "kibana": {
+      "version": "~7.x.x"
+    }
+  },
+  "owner": {
+    "github": "ruflin"
+  },
+  "categories": [
+    "crm",
+    "azure"
+  ],
+  "format_version": "1.0.0",
+  "readme": "/package/nodirentries/1.0.0/docs/README.md",
+  "license": "basic",
+  "screenshots": [
+    {
+      "src": "/img/kibana-envoyproxy.jpg",
+      "path": "/package/nodirentries/1.0.0/img/kibana-envoyproxy.jpg",
+      "title": "IP Tables Ubiquity Dashboard",
+      "size": "1492x1464",
+      "type": "image/png"
+    }
+  ],
+  "assets": [
+    "/package/nodirentries/1.0.0/manifest.yml",
+    "/package/nodirentries/1.0.0/img/icon.png",
+    "/package/nodirentries/1.0.0/img/kibana-envoyproxy.jpg",
+    "/package/nodirentries/1.0.0/docs/README.md",
+    "/package/nodirentries/1.0.0/data_stream/foo/manifest.yml",
+    "/package/nodirentries/1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json",
+    "/package/nodirentries/1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json",
+    "/package/nodirentries/1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json",
+    "/package/nodirentries/1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json",
+    "/package/nodirentries/1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json",
+    "/package/nodirentries/1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json",
+    "/package/nodirentries/1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json",
+    "/package/nodirentries/1.0.0/data_stream/foo/fields/base-fields.yml",
+    "/package/nodirentries/1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-entry.json",
+    "/package/nodirentries/1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-plaintext.json",
+    "/package/nodirentries/1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-json.json",
+    "/package/nodirentries/1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-http.json",
+    "/package/nodirentries/1.0.0/data_stream/foo/elasticsearch/ingest_pipeline/pipeline-tcp.json",
+    "/package/nodirentries/1.0.0/data_stream/foo/agent/stream/stream.yml.hbs"
+  ],
+  "policy_templates": [
+    {
+      "name": "logs",
+      "title": "Logs datasource",
+      "description": "Datasource for your log files.",
+      "inputs": [
+        {
+          "type": "foo"
+        }
+      ],
+      "multiple": true
+    }
+  ],
+  "data_streams": [
+    {
+      "type": "logs",
+      "dataset": "nodirentries.foo",
+      "title": "Foo",
+      "release": "ga",
+      "ingest_pipeline": "pipeline-entry",
+      "streams": [
+        {
+          "input": "foo",
+          "vars": [
+            {
+              "name": "paths",
+              "type": "text",
+              "description": "Path to log files to be collected",
+              "multi": true,
+              "required": true,
+              "show_user": false
+            }
+          ],
+          "template_path": "stream.yml.hbs",
+          "enabled": true
+        }
+      ],
+      "package": "nodirentries",
+      "path": "foo"
+    }
+  ]
+}