From 7da8f29ffb8a0756104a8581d231fc66469a380f Mon Sep 17 00:00:00 2001 From: utam0k Date: Fri, 14 Oct 2022 02:21:17 +0000 Subject: [PATCH] Support for zstd compression --- cmd/core/build.go | 18 ++++++++++- cmd/core/combine.go | 17 +++++++++- example.sh | 4 +-- pkg/dazzle/build.go | 28 ++++++++-------- pkg/dazzle/build_test.go | 66 ++++++++++++++++++++++++++------------ pkg/dazzle/combiner.go | 4 +-- pkg/dazzle/compression.go | 50 +++++++++++++++++++++++++++++ pkg/dazzle/project_test.go | 2 +- 8 files changed, 148 insertions(+), 41 deletions(-) create mode 100644 pkg/dazzle/compression.go diff --git a/cmd/core/build.go b/cmd/core/build.go index 55793bc..0f7cd83 100644 --- a/cmd/core/build.go +++ b/cmd/core/build.go @@ -22,6 +22,7 @@ package core import ( "context" + "fmt" "github.com/moby/buildkit/client" "github.com/spf13/cobra" @@ -60,7 +61,21 @@ var buildCmd = &cobra.Command{ return err } - err = prj.Build(context.Background(), session) + var compression dazzle.Compression + c, err := cmd.Flags().GetString("compression") + if err != nil { + return err + } + switch c { + case "gzip": + compression = dazzle.Gzip + case "zstd": + compression = dazzle.Zstd + default: + return fmt.Errorf("unknow a compression type: %s", c) + } + + err = prj.Build(context.Background(), session, compression) if err != nil { return err } @@ -77,4 +92,5 @@ func init() { buildCmd.Flags().Bool("no-cache", false, "disables the buildkit build cache") buildCmd.Flags().Bool("plain-output", false, "produce plain output") buildCmd.Flags().Bool("chunked-without-hash", false, "disable hash qualification for chunked image") + buildCmd.Flags().String("compression", "gzip", "compression type for layers") } diff --git a/cmd/core/combine.go b/cmd/core/combine.go index 690e65c..c3b8ba8 100644 --- a/cmd/core/combine.go +++ b/cmd/core/combine.go @@ -85,6 +85,20 @@ var combineCmd = &cobra.Command{ bldref = targetref.String() } + var compression dazzle.Compression + c, err := cmd.Flags().GetString("compression") + if err != nil { + return err + } + switch c { + case "gzip": + compression = dazzle.Gzip + case "zstd": + compression = dazzle.Zstd + default: + return fmt.Errorf("unknow a compression type: %s", c) + } + cl, err := client.New(context.Background(), rootCfg.BuildkitAddr, client.WithFailFast()) if err != nil { return err @@ -112,7 +126,7 @@ var combineCmd = &cobra.Command{ } log.WithField("combination", cmb.Name).WithField("chunks", cmb.Chunks).WithField("ref", destref.String()).Warn("producing chunk combination") - err = prj.Combine(context.Background(), cmb.Chunks, destref, sess, opts...) + err = prj.Combine(context.Background(), cmb.Chunks, destref, sess, compression, opts...) if err != nil { return err } @@ -130,4 +144,5 @@ func init() { combineCmd.Flags().String("combination", "", "build a specific combination") combineCmd.Flags().Bool("all", false, "build all combinations") combineCmd.Flags().String("build-ref", "", "use a different build-ref than the target-ref") + combineCmd.Flags().String("compression", "gzip", "compression type for layers") } diff --git a/example.sh b/example.sh index c6a1410..839cceb 100755 --- a/example.sh +++ b/example.sh @@ -2,5 +2,5 @@ # Build and run the example gp await-port 5000 -go run main.go build --context example localhost:5000/dazzle -go run main.go combine --context example localhost:5000/dazzle --all +go run main.go build --context example localhost:5000/dazzle --compression gzip +go run main.go combine --context example localhost:5000/dazzle --all --compression gzip diff --git a/pkg/dazzle/build.go b/pkg/dazzle/build.go index 0502476..e61ea71 100644 --- a/pkg/dazzle/build.go +++ b/pkg/dazzle/build.go @@ -126,7 +126,7 @@ func WithChunkedWithoutHash(enable bool) BuildOpt { } // Build builds all images in a project -func (p *Project) Build(ctx context.Context, session *BuildSession) error { +func (p *Project) Build(ctx context.Context, session *BuildSession, compression Compression) error { ctx = clog.WithLogger(ctx, log.NewEntry(log.New())) // Relying on the buildkit cache alone does not result in fixed content hashes. @@ -141,7 +141,7 @@ func (p *Project) Build(ctx context.Context, session *BuildSession) error { } log.WithField("ref", baseref.String()).Warn("building base image") - absbaseref, err := p.Base.buildAsBase(ctx, baseref, session) + absbaseref, err := p.Base.buildAsBase(ctx, baseref, session, compression) if err != nil { return fmt.Errorf("cannot build base image: %w", err) } @@ -169,12 +169,12 @@ func (p *Project) Build(ctx context.Context, session *BuildSession) error { session.baseBuildFinished(absbaseref, basemf, basecfg) for _, chk := range p.Chunks { - _, _, err := chk.test(ctx, session) + _, _, err := chk.test(ctx, session, compression) if err != nil { return fmt.Errorf("cannot test chunk %s: %w", chk.Name, err) } - _, _, err = chk.build(ctx, session) + _, _, err = chk.build(ctx, session, compression) if err != nil { return fmt.Errorf("cannot build chunk %s: %w", chk.Name, err) } @@ -281,7 +281,7 @@ func (s *BuildSession) baseBuildFinished(ref reference.Digested, mf *ociv1.Manif s.baseCfg = cfg } -func removeBaseLayer(ctx context.Context, opts removeBaseLayerOpts) (chkmf *ociv1.Manifest, didbuild bool, err error) { +func removeBaseLayer(ctx context.Context, opts removeBaseLayerOpts, compression Compression) (chkmf *ociv1.Manifest, didbuild bool, err error) { _, chkmf, chkcfg, err := getImageMetadata(ctx, opts.chunkref, opts.registry) if err != nil { return @@ -334,7 +334,7 @@ func removeBaseLayer(ctx context.Context, opts removeBaseLayerOpts) (chkmf *ociv } chkmf.Layers = chkmf.Layers[len(opts.basemf.Layers):] for i := range chkmf.Layers { - chkmf.Layers[i].MediaType = ociv1.MediaTypeImageLayerGzip + chkmf.Layers[i].MediaType = compression.Extension() } if chkmf.Annotations == nil { chkmf.Annotations = make(map[string]string) @@ -465,7 +465,7 @@ func (p *Project) BaseRef(build reference.Named) (reference.NamedTagged, error) return reference.WithTag(build, fmt.Sprintf("base--%s", hash)) } -func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, sess *BuildSession) (absref reference.Digested, err error) { +func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, sess *BuildSession, compression Compression) (absref reference.Digested, err error) { _, desc, err := sess.opts.Resolver.Resolve(ctx, dest.String()) if err == nil { // if err == nil the image exists already @@ -504,6 +504,7 @@ func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, se "name": dest.String(), "push": "true", "oci-mediatypes": "true", + "compression": compression.String(), }, }, }, @@ -552,7 +553,7 @@ func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, se return resref, nil } -func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession) (ok bool, didRun bool, err error) { +func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession, compression Compression) (ok bool, didRun bool, err error) { if sess == nil { return false, false, errors.New("cannot test without a session") } @@ -574,7 +575,7 @@ func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession) (ok bool, d } // build temp image for testing - testRef, _, err := p.buildImage(ctx, ImageTypeTest, sess) + testRef, _, err := p.buildImage(ctx, ImageTypeTest, sess, compression) if err != nil { return false, false, err } @@ -600,9 +601,9 @@ func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession) (ok bool, d return true, true, nil } -func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession) (chkRef reference.NamedTagged, didBuild bool, err error) { +func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession, compression Compression) (chkRef reference.NamedTagged, didBuild bool, err error) { // build actual full image - fullRef, didBuild, err := p.buildImage(ctx, ImageTypeFull, sess) + fullRef, didBuild, err := p.buildImage(ctx, ImageTypeFull, sess, compression) if err != nil { return } @@ -618,7 +619,7 @@ func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession) (chkRef re } log.WithField("chunk", p.Name).WithField("ref", chkRef).Warn("building chunked image") opts := removeBaseLayerOpts{sess.opts.Resolver, sess.opts.Registry, sess.baseRef, sess.baseMF, sess.baseCfg, fullRef, chkRef} - mf, didBuild, err := removeBaseLayer(ctx, opts) + mf, didBuild, err := removeBaseLayer(ctx, opts, compression) if err != nil { return } @@ -628,7 +629,7 @@ func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession) (chkRef re return } -func (p *ProjectChunk) buildImage(ctx context.Context, tpe ChunkImageType, sess *BuildSession) (tgt reference.Named, didBuild bool, err error) { +func (p *ProjectChunk) buildImage(ctx context.Context, tpe ChunkImageType, sess *BuildSession, compression Compression) (tgt reference.Named, didBuild bool, err error) { tgt, err = p.ImageName(tpe, sess) if err != nil { return @@ -690,6 +691,7 @@ func (p *ProjectChunk) buildImage(ctx context.Context, tpe ChunkImageType, sess "name": tgt.String(), "push": "true", "oci-mediatypes": "true", + "compression": compression.String(), }, }, }, diff --git a/pkg/dazzle/build_test.go b/pkg/dazzle/build_test.go index 1bab31f..8906746 100644 --- a/pkg/dazzle/build_test.go +++ b/pkg/dazzle/build_test.go @@ -51,12 +51,13 @@ func TestProjectChunk_test(t *testing.T) { sess.opts.Resolver = fakeResolver{} type fields struct { - Name string - FS map[string]*fstest.MapFile - Base string - Chunk string - BaseRef string - Registry Registry + Name string + FS map[string]*fstest.MapFile + Base string + Chunk string + BaseRef string + Compression Compression + Registry Registry } type args struct { ctx context.Context @@ -72,9 +73,10 @@ func TestProjectChunk_test(t *testing.T) { { name: "passes with no tests", fields: fields{ - Name: "no test chunk", - Base: "chunks", - Chunk: "notest", + Name: "no test chunk", + Base: "chunks", + Chunk: "notest", + Compression: Gzip, FS: map[string]*fstest.MapFile{ "chunks/notest/Dockerfile": { Data: []byte("FROM alpine"), @@ -91,9 +93,10 @@ func TestProjectChunk_test(t *testing.T) { { name: "fails when no base reference set", fields: fields{ - Name: "no base ref chunk", - Base: "chunks", - Chunk: "nobaseref", + Name: "no base ref chunk", + Base: "chunks", + Chunk: "nobaseref", + Compression: Gzip, FS: map[string]*fstest.MapFile{ "chunks/nobaseref/Dockerfile": { Data: []byte("FROM alpine"), @@ -118,9 +121,10 @@ func TestProjectChunk_test(t *testing.T) { { name: "does not build if tests have passed", fields: fields{ - Name: "a chunk", - Base: "chunks", - Chunk: "foobar", + Name: "a chunk", + Base: "chunks", + Chunk: "foobar", + Compression: Gzip, FS: map[string]*fstest.MapFile{ "chunks/foobar/Dockerfile": { Data: []byte("FROM alpine"), @@ -148,6 +152,26 @@ func TestProjectChunk_test(t *testing.T) { wantOk: true, wantErr: false, }, + { + name: "passes with no tests with zstd", + fields: fields{ + Name: "no test chunk", + Base: "chunks", + Chunk: "notest", + Compression: Zstd, + FS: map[string]*fstest.MapFile{ + "chunks/notest/Dockerfile": { + Data: []byte("FROM alpine"), + }, + }, + }, + args: args{ + ctx: ctx, + sess: sess, + }, + wantOk: true, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -175,7 +199,7 @@ func TestProjectChunk_test(t *testing.T) { if tt.fields.Registry != nil { sess.opts.Registry = tt.fields.Registry } - gotOk, _, err := chks[0].test(tt.args.ctx, tt.args.sess) + gotOk, _, err := chks[0].test(tt.args.ctx, tt.args.sess, tt.fields.Compression) if (err != nil) != tt.wantErr { t.Errorf("TestProjectChunk_test() error = %v, wantErr %v", err, tt.wantErr) return @@ -286,7 +310,7 @@ func TestProjectChunk_test_integration(t *testing.T) { } } - err = prj.Build(context.Background(), session) + err = prj.Build(context.Background(), session, Gzip) if err != nil { t.Errorf("TestProjectChunk_test_integration.test() unexpected Build error = %v", err) return @@ -318,7 +342,7 @@ func TestProjectChunk_test_integration(t *testing.T) { } // Re-running build should reuse existing images & tags - err = prj.Build(context.Background(), session) + err = prj.Build(context.Background(), session, Gzip) if err != nil { t.Errorf("TestProjectChunk_test_integration() unexpected rebuild 1 error = %v", err) return @@ -355,13 +379,13 @@ func TestProjectChunk_test_integration(t *testing.T) { // Individually check each chunk to ensure it doesn't rebuild for _, chk := range prj.Chunks { - ok, didRun, err := chk.test(ctx, session) + ok, didRun, err := chk.test(ctx, session, Gzip) if err != nil || !ok || didRun { t.Errorf("TestProjectChunk_test_integration() test() error:%v testing chunk: %s with results: %v:%v", err, chk.Name, ok, didRun) return } - _, didBuild, err := chk.build(ctx, session) + _, didBuild, err := chk.build(ctx, session, Gzip) if err != nil || didBuild { t.Errorf("TestProjectChunk_test_integration() build() error:%v building chunk: %s didBuild:%v", err, chk.Name, didBuild) return @@ -389,7 +413,7 @@ func TestProjectChunk_test_integration(t *testing.T) { } // Re-running build should create new test tags - err = prj.Build(context.Background(), session) + err = prj.Build(context.Background(), session, Gzip) if err != nil { t.Errorf("TestProjectChunk_test_integration() unexpected rebuild 2 error = %v", err) return diff --git a/pkg/dazzle/combiner.go b/pkg/dazzle/combiner.go index dfa7184..079b7ee 100644 --- a/pkg/dazzle/combiner.go +++ b/pkg/dazzle/combiner.go @@ -62,7 +62,7 @@ func asTempBuild(o *combinerOpts) error { // Combine combines a set of previously built chunks into a single image while maintaining // the layer identity. -func (p *Project) Combine(ctx context.Context, chunks []string, dest reference.Named, sess *BuildSession, opts ...CombinerOpt) (err error) { +func (p *Project) Combine(ctx context.Context, chunks []string, dest reference.Named, sess *BuildSession, compression Compression, opts ...CombinerOpt) (err error) { var options combinerOpts for _, o := range opts { err = o(&options) @@ -78,7 +78,7 @@ func (p *Project) Combine(ctx context.Context, chunks []string, dest reference.N if err != nil { return err } - err = p.Combine(ctx, chunks, tmpdest, sess, append(opts, asTempBuild)...) + err = p.Combine(ctx, chunks, tmpdest, sess, compression, append(opts, asTempBuild)...) if err != nil { return err } diff --git a/pkg/dazzle/compression.go b/pkg/dazzle/compression.go new file mode 100644 index 0000000..300e3f7 --- /dev/null +++ b/pkg/dazzle/compression.go @@ -0,0 +1,50 @@ +// Copyright © 2022 Gitpod + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package dazzle + +import ociv1 "github.com/opencontainers/image-spec/specs-go/v1" + +type Compression int + +const ( + Gzip Compression = iota + Zstd +) + +func (compression *Compression) Extension() string { + switch *compression { + case Gzip: + return ociv1.MediaTypeImageLayerGzip + case Zstd: + return ociv1.MediaTypeImageLayerZstd + } + return ociv1.MediaTypeImageLayerGzip +} + +func (compression *Compression) String() string { + switch *compression { + case Gzip: + return "gzip" + case Zstd: + return "zstd" + } + return "gzip" +} diff --git a/pkg/dazzle/project_test.go b/pkg/dazzle/project_test.go index ccb1dff..ee91a65 100644 --- a/pkg/dazzle/project_test.go +++ b/pkg/dazzle/project_test.go @@ -1,4 +1,4 @@ -// Copyright © 2020 Gitpod +// Copyright © 2022 Gitpod // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal