Ship packages as zip instead of tar.gz

This is a breaking change and first needs changes on the Kibana side. The new registry and the new version of Kibana should be either rolled out in sync or Kibana supports both for a bit. Closes elastic#474
ruflin · Aug 27, 2020 · fa20118 · fa20118
1 parent 8a8cf34
commit fa20118
Show file tree

Hide file tree

Showing 38 changed files with 154 additions and 150 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Breaking changes
 
+* Ship packages as zip instead of tar.gz [#628](https://github.com/elastic/package-registry/pull/628)
+
 ### Bugfixes
 
 ### Added

diff --git a/archiver/archive.go b/archiver/archive.go
@@ -5,8 +5,7 @@
 package archiver
 
 import (
-	"archive/tar"
-	"compress/gzip"
+	"archive/zip"
 	"fmt"
 	"io"
 	"os"
@@ -26,23 +25,17 @@ type PackageProperties struct {
 
 // ArchivePackage method builds and streams an archive with package content.
 func ArchivePackage(w io.Writer, properties PackageProperties) (err error) {
-	gzipWriter := gzip.NewWriter(w)
-	tarWriter := tar.NewWriter(gzipWriter)
+	zipWriter := zip.NewWriter(w)
 	defer func() {
 		var multiErr multierror.Errors
 
 		if err != nil {
 			multiErr = append(multiErr, err)
 		}
 
-		err = tarWriter.Close()
+		err = zipWriter.Close()
 		if err != nil {
-			multiErr = append(multiErr, errors.Wrapf(err, "closing tar writer failed"))
-		}
-
-		err = gzipWriter.Close()
-		if err != nil {
-			multiErr = append(multiErr, errors.Wrapf(err, "closing gzip writer failed"))
+			multiErr = append(multiErr, errors.Wrapf(err, "closing zip writer failed"))
 		}
 
 		if multiErr != nil {
@@ -70,13 +63,13 @@ func ArchivePackage(w io.Writer, properties PackageProperties) (err error) {
 			return errors.Wrapf(err, "building archive header failed (path: %s)", relativePath)
 		}
 
-		err = tarWriter.WriteHeader(header)
+		w, err = zipWriter.CreateHeader(header)
 		if err != nil {
 			return errors.Wrapf(err, "writing header failed (path: %s)", relativePath)
 		}
 
 		if !info.IsDir() {
-			err = writeFileContentToArchive(path, tarWriter)
+			err = writeFileContentToArchive(path, w)
 			if err != nil {
 				return errors.Wrapf(err, "archiving file content failed (path: %s)", path)
 			}
@@ -87,24 +80,20 @@ func ArchivePackage(w io.Writer, properties PackageProperties) (err error) {
 		return errors.Wrapf(err, "processing package path '%s' failed", properties.Path)
 	}
 
-	err = tarWriter.Flush()
-	if err != nil {
-		return errors.Wrap(err, "flushing tar writer failed")
-	}
-
-	err = gzipWriter.Flush()
+	err = zipWriter.Flush()
 	if err != nil {
 		return errors.Wrap(err, "flushing gzip writer failed")
 	}
 	return nil
 }
 
-func buildArchiveHeader(info os.FileInfo, relativePath string) (*tar.Header, error) {
-	header, err := tar.FileInfoHeader(info, "")
+func buildArchiveHeader(info os.FileInfo, relativePath string) (*zip.FileHeader, error) {
+	header, err := zip.FileInfoHeader(info)
 	if err != nil {
 		return nil, errors.Wrapf(err, "reading file info header failed (info: %s)", info.Name())
 	}
 
+	header.Method = zip.Deflate
 	header.Name = relativePath
 	if info.IsDir() && !strings.HasSuffix(header.Name, "/") {
 		header.Name = header.Name + "/"

diff --git a/artifacts.go b/artifacts.go
@@ -16,7 +16,7 @@ import (
 	"github.com/elastic/package-registry/archiver"
 )
 
-const artifactsRouterPath = "/epr/{packageName}/{packageName:[a-z0-9_]+}-{packageVersion}.tar.gz"
+const artifactsRouterPath = "/epr/{packageName}/{packageName:[a-z0-9_]+}-{packageVersion}.zip"
 
 var errArtifactNotFound = errors.New("artifact not found")
 

diff --git a/main_content_type_test.go b/main_content_type_test.go
@@ -19,7 +19,7 @@ func TestContentTypes(t *testing.T) {
 		expectedContentType string
 	}{
 		{"/index.json", "application/json"},
-		{"/activemq-0.0.1.tar.gz", "application/gzip"},
+		{"/activemq-0.0.1.zip", "application/zip"},
 		{"/favicon.ico", "image/x-icon"},
 		{"/metricbeat-mysql.png", "image/png"},
 		{"/kibana-coredns.jpg", "image/jpeg"},

diff --git a/main_test.go b/main_test.go
@@ -5,12 +5,10 @@
 package main
 
 import (
-	"archive/tar"
+	"archive/zip"
 	"bytes"
-	"compress/gzip"
 	"flag"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -84,10 +82,10 @@ func TestArtifacts(t *testing.T) {
 		file     string
 		handler  func(w http.ResponseWriter, r *http.Request)
 	}{
-		{"/epr/example/example-0.0.2.tar.gz", artifactsRouterPath, "example-0.0.2.tar.gz-preview.txt", artifactsHandler},
-		{"/epr/example/example-999.0.2.tar.gz", artifactsRouterPath, "artifact-package-version-not-found.txt", artifactsHandler},
-		{"/epr/example/missing-0.1.2.tar.gz", artifactsRouterPath, "artifact-package-not-found.txt", artifactsHandler},
-		{"/epr/example/example-a.b.c.tar.gz", artifactsRouterPath, "artifact-package-invalid-version.txt", artifactsHandler},
+		{"/epr/example/example-0.0.2.zip", artifactsRouterPath, "example-0.0.2.zip-preview.txt", artifactsHandler},
+		{"/epr/example/example-999.0.2.zip", artifactsRouterPath, "artifact-package-version-not-found.txt", artifactsHandler},
+		{"/epr/example/missing-0.1.2.zip", artifactsRouterPath, "artifact-package-not-found.txt", artifactsHandler},
+		{"/epr/example/example-a.b.c.zip", artifactsRouterPath, "artifact-package-invalid-version.txt", artifactsHandler},
 	}
 
 	for _, test := range tests {
@@ -206,21 +204,14 @@ func runEndpoint(t *testing.T, endpoint, path, file string, handler func(w http.
 }
 
 func listArchivedFiles(t *testing.T, body []byte) []byte {
-	gzippedReader, err := gzip.NewReader(bytes.NewReader(body))
+	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
 	require.NoError(t, err)
 
-	tarReader := tar.NewReader(gzippedReader)
-
 	var listing bytes.Buffer
 
-	for {
-		header, err := tarReader.Next()
-		if err == io.EOF {
-			break
-		}
-		require.NoError(t, err)
+	for _, f := range zipReader.File {
+		listing.WriteString(fmt.Sprintf("%d %s\n", f.UncompressedSize64, f.Name))
 
-		listing.WriteString(fmt.Sprintf("%d %s\n", header.Size, header.Name))
 	}
 	return listing.Bytes()
 }
diff --git a/mime_types.go b/mime_types.go
@@ -12,7 +12,7 @@ import (
 // init method defines MIME types important for the package content. Definitions ensure that the same Content-Type
 // will be returned if the /etc/mime.types is empty or tiny.
 func init() {
-	mustAddMimeExtensionType(".gz", "application/gzip")
+	mustAddMimeExtensionType(".zip", "application/zip")
 	mustAddMimeExtensionType(".ico", "image/x-icon")
 	mustAddMimeExtensionType(".md", "text/markdown; charset=utf-8")
 	mustAddMimeExtensionType(".yml", "text/yaml; charset=UTF-8")

diff --git a/testdata/content-types/activemq-0.0.1.tar.gz b/testdata/content-types/activemq-0.0.1.tar.gz
diff --git a/testdata/content-types/activemq-0.0.1.zip b/testdata/content-types/activemq-0.0.1.zip
diff --git a/testdata/generated/example-0.0.2.zip-preview.txt b/testdata/generated/example-0.0.2.zip-preview.txt
@@ -0,0 +1,22 @@
+0 example-0.0.2/docs/
+622 example-0.0.2/docs/README.md
+0 example-0.0.2/elasticsearch/
+0 example-0.0.2/elasticsearch/ingest_pipeline/
+892 example-0.0.2/elasticsearch/ingest_pipeline/pipeline-entry.json
+2071 example-0.0.2/elasticsearch/ingest_pipeline/pipeline-http.json
+887 example-0.0.2/elasticsearch/ingest_pipeline/pipeline-json.json
+3584 example-0.0.2/elasticsearch/ingest_pipeline/pipeline-plaintext.json
+900 example-0.0.2/elasticsearch/ingest_pipeline/pipeline-tcp.json
+0 example-0.0.2/img/
+482070 example-0.0.2/img/kibana-envoyproxy.jpg
+0 example-0.0.2/kibana/
+0 example-0.0.2/kibana/dashboard/
+2221 example-0.0.2/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json
+0 example-0.0.2/kibana/visualization/
+1863 example-0.0.2/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json
+1982 example-0.0.2/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json
+2572 example-0.0.2/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json
+1995 example-0.0.2/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json
+1849 example-0.0.2/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json
+1920 example-0.0.2/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json
+193 example-0.0.2/manifest.yml
diff --git a/testdata/generated/package.json b/testdata/generated/package.json
@@ -5,7 +5,7 @@
   "release": "ga",
   "description": "This is the example integration",
   "type": "integration",
-  "download": "/epr/example/example-1.0.0.tar.gz",
+  "download": "/epr/example/example-1.0.0.zip",
   "path": "/package/example/1.0.0",
   "format_version": "1.0.0",
   "readme": "/package/example/1.0.0/docs/README.md",

diff --git a/testdata/generated/package/datasources/1.0.0/index.json b/testdata/generated/package/datasources/1.0.0/index.json
@@ -5,7 +5,7 @@
   "release": "beta",
   "description": "Package with data sources",
   "type": "integration",
-  "download": "/epr/datasources/datasources-1.0.0.tar.gz",
+  "download": "/epr/datasources/datasources-1.0.0.zip",
   "path": "/package/datasources/1.0.0",
   "format_version": "1.0.0",
   "readme": "/package/datasources/1.0.0/docs/README.md",

diff --git a/testdata/generated/package/default_pipeline/0.0.2/index.json b/testdata/generated/package/default_pipeline/0.0.2/index.json
@@ -5,7 +5,7 @@
   "release": "beta",
   "description": "Tests if no pipeline is set, it defaults to the default one",
   "type": "integration",
-  "download": "/epr/default_pipeline/default_pipeline-0.0.2.tar.gz",
+  "download": "/epr/default_pipeline/default_pipeline-0.0.2.zip",
   "path": "/package/default_pipeline/0.0.2",
   "format_version": "1.0.0",
   "readme": "/package/default_pipeline/0.0.2/docs/README.md",

diff --git a/testdata/generated/package/ecs_style_dataset/0.0.1/index.json b/testdata/generated/package/ecs_style_dataset/0.0.1/index.json
@@ -5,7 +5,7 @@
   "release": "beta",
   "description": "Tests the registry validations works for dataset fields using the ecs style format",
   "type": "integration",
-  "download": "/epr/ecs_style_dataset/ecs_style_dataset-0.0.1.tar.gz",
+  "download": "/epr/ecs_style_dataset/ecs_style_dataset-0.0.1.zip",
   "path": "/package/ecs_style_dataset/0.0.1",
   "format_version": "1.0.0",
   "readme": "/package/ecs_style_dataset/0.0.1/docs/README.md",

diff --git a/testdata/generated/package/example/0.0.2/index.json b/testdata/generated/package/example/0.0.2/index.json
@@ -5,7 +5,7 @@
   "release": "beta",
   "description": "This is the example integration.",
   "type": "integration",
-  "download": "/epr/example/example-0.0.2.tar.gz",
+  "download": "/epr/example/example-0.0.2.zip",
   "path": "/package/example/0.0.2",
   "format_version": "1.0.0",
   "readme": "/package/example/0.0.2/docs/README.md",

diff --git a/testdata/generated/package/example/1.0.0/index.json b/testdata/generated/package/example/1.0.0/index.json
@@ -5,7 +5,7 @@
   "release": "ga",
   "description": "This is the example integration",
   "type": "integration",
-  "download": "/epr/example/example-1.0.0.tar.gz",
+  "download": "/epr/example/example-1.0.0.zip",
   "path": "/package/example/1.0.0",
   "format_version": "1.0.0",
   "readme": "/package/example/1.0.0/docs/README.md",

diff --git a/testdata/generated/package/experimental/0.0.1/index.json b/testdata/generated/package/experimental/0.0.1/index.json
@@ -5,7 +5,7 @@
   "release": "experimental",
   "description": "Experimental package, should be set by default",
   "type": "solution",
-  "download": "/epr/experimental/experimental-0.0.1.tar.gz",
+  "download": "/epr/experimental/experimental-0.0.1.zip",
   "path": "/package/experimental/0.0.1",
   "format_version": "1.0.0",
   "readme": "/package/experimental/0.0.1/docs/README.md",

diff --git a/testdata/generated/package/foo/1.0.0/index.json b/testdata/generated/package/foo/1.0.0/index.json
@@ -5,7 +5,7 @@
   "release": "beta",
   "description": "This is the foo integration",
   "type": "solution",
-  "download": "/epr/foo/foo-1.0.0.tar.gz",
+  "download": "/epr/foo/foo-1.0.0.zip",
   "path": "/package/foo/1.0.0",
   "format_version": "1.0.0",
   "readme": "/package/foo/1.0.0/docs/README.md",

diff --git a/testdata/generated/package/internal/1.2.0/index.json b/testdata/generated/package/internal/1.2.0/index.json
@@ -5,7 +5,7 @@
   "release": "beta",
   "description": "Internal package",
   "type": "integration",
-  "download": "/epr/internal/internal-1.2.0.tar.gz",
+  "download": "/epr/internal/internal-1.2.0.zip",
   "path": "/package/internal/1.2.0",
   "internal": true,
   "format_version": "1.0.0",

diff --git a/testdata/generated/package/longdocs/1.0.4/index.json b/testdata/generated/package/longdocs/1.0.4/index.json
@@ -5,7 +5,7 @@
   "release": "ga",
   "description": "This integration contains pretty long documentation.\nIt is used to show the different visualisations inside a documentation to test how we handle it.\nThe integration does not contain any assets except the documentation page.\n",
   "type": "integration",
-  "download": "/epr/longdocs/longdocs-1.0.4.tar.gz",
+  "download": "/epr/longdocs/longdocs-1.0.4.zip",
   "path": "/package/longdocs/1.0.4",
   "icons": [
     {

diff --git a/testdata/generated/package/metricsonly/2.0.1/index.json b/testdata/generated/package/metricsonly/2.0.1/index.json
@@ -5,7 +5,7 @@
   "release": "ga",
   "description": "This is an integration with only the metrics category.\n",
   "type": "integration",
-  "download": "/epr/metricsonly/metricsonly-2.0.1.tar.gz",
+  "download": "/epr/metricsonly/metricsonly-2.0.1.zip",
   "path": "/package/metricsonly/2.0.1",
   "icons": [
     {

diff --git a/testdata/generated/package/multiple_false/0.0.1/index.json b/testdata/generated/package/multiple_false/0.0.1/index.json
@@ -5,7 +5,7 @@
   "release": "beta",
   "description": "Tests that multiple can be set to false",
   "type": "integration",
-  "download": "/epr/multiple_false/multiple_false-0.0.1.tar.gz",
+  "download": "/epr/multiple_false/multiple_false-0.0.1.zip",
   "path": "/package/multiple_false/0.0.1",
   "format_version": "1.0.0",
   "readme": "/package/multiple_false/0.0.1/docs/README.md",

diff --git a/testdata/generated/package/multiversion/1.0.3/index.json b/testdata/generated/package/multiversion/1.0.3/index.json
@@ -5,7 +5,7 @@
   "release": "ga",
   "description": "Multiple versions of this integration exist.\n",
   "type": "integration",
-  "download": "/epr/multiversion/multiversion-1.0.3.tar.gz",
+  "download": "/epr/multiversion/multiversion-1.0.3.zip",
   "path": "/package/multiversion/1.0.3",
   "icons": [
     {

diff --git a/testdata/generated/package/multiversion/1.0.4/index.json b/testdata/generated/package/multiversion/1.0.4/index.json
@@ -5,7 +5,7 @@
   "release": "ga",
   "description": "Multiple versions of this integration exist.\n",
   "type": "integration",
-  "download": "/epr/multiversion/multiversion-1.0.4.tar.gz",
+  "download": "/epr/multiversion/multiversion-1.0.4.zip",
   "path": "/package/multiversion/1.0.4",
   "icons": [
     {

diff --git a/testdata/generated/package/multiversion/1.1.0/index.json b/testdata/generated/package/multiversion/1.1.0/index.json
@@ -5,7 +5,7 @@
   "release": "ga",
   "description": "Multiple versions of this integration exist.\n",
   "type": "integration",
-  "download": "/epr/multiversion/multiversion-1.1.0.tar.gz",
+  "download": "/epr/multiversion/multiversion-1.1.0.zip",
   "path": "/package/multiversion/1.1.0",
   "icons": [
     {

diff --git a/testdata/generated/package/no_stream_configs/1.0.0/index.json b/testdata/generated/package/no_stream_configs/1.0.0/index.json
@@ -5,7 +5,7 @@
   "release": "beta",
   "description": "This package does contain a dataset but not stream configs.\n",
   "type": "integration",
-  "download": "/epr/no_stream_configs/no_stream_configs-1.0.0.tar.gz",
+  "download": "/epr/no_stream_configs/no_stream_configs-1.0.0.zip",
   "path": "/package/no_stream_configs/1.0.0",
   "format_version": "1.0.0",
   "readme": "/package/no_stream_configs/1.0.0/docs/README.md",

diff --git a/testdata/generated/package/reference/1.0.0/index.json b/testdata/generated/package/reference/1.0.0/index.json
@@ -5,7 +5,7 @@
   "release": "ga",
   "description": "This package is used for defining all the properties of a package, the possible assets etc. It serves as a reference on all the config options which are possible.\n",
   "type": "integration",
-  "download": "/epr/reference/reference-1.0.0.tar.gz",
+  "download": "/epr/reference/reference-1.0.0.zip",
   "path": "/package/reference/1.0.0",
   "icons": [
     {

diff --git a/testdata/generated/package/yamlpipeline/1.0.0/index.json b/testdata/generated/package/yamlpipeline/1.0.0/index.json
@@ -5,7 +5,7 @@
   "release": "beta",
   "description": "This package contains a yaml pipeline.\n",
   "type": "integration",
-  "download": "/epr/yamlpipeline/yamlpipeline-1.0.0.tar.gz",
+  "download": "/epr/yamlpipeline/yamlpipeline-1.0.0.zip",
   "path": "/package/yamlpipeline/1.0.0",
   "format_version": "1.0.0",
   "readme": "/package/yamlpipeline/1.0.0/docs/README.md",