Skip to content

Commit

Permalink
Support for zstd compression
Browse files Browse the repository at this point in the history
  • Loading branch information
utam0k committed Oct 14, 2022
1 parent d889756 commit 838cfd0
Show file tree
Hide file tree
Showing 8 changed files with 148 additions and 41 deletions.
18 changes: 17 additions & 1 deletion cmd/core/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ package core

import (
"context"
"fmt"

"github.com/moby/buildkit/client"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -60,7 +61,21 @@ var buildCmd = &cobra.Command{
return err
}

err = prj.Build(context.Background(), session)
var compression dazzle.Compression
c, err := cmd.Flags().GetString("compression")
if err != nil {
return err
}
switch c {
case "gzip":
compression = dazzle.Gzip
case "zstd":
compression = dazzle.Zstd
default:
return fmt.Errorf("Unknow a compression type: %s", c)
}

err = prj.Build(context.Background(), session, compression)
if err != nil {
return err
}
Expand All @@ -77,4 +92,5 @@ func init() {
buildCmd.Flags().Bool("no-cache", false, "disables the buildkit build cache")
buildCmd.Flags().Bool("plain-output", false, "produce plain output")
buildCmd.Flags().Bool("chunked-without-hash", false, "disable hash qualification for chunked image")
buildCmd.Flags().String("compression", "gzip", "compression type for layers")
}
17 changes: 16 additions & 1 deletion cmd/core/combine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,20 @@ var combineCmd = &cobra.Command{
bldref = targetref.String()
}

var compression dazzle.Compression
c, err := cmd.Flags().GetString("compression")
if err != nil {
return err
}
switch c {
case "gzip":
compression = dazzle.Gzip
case "zstd":
compression = dazzle.Zstd
default:
return fmt.Errorf("Unknow a compression type: %s", c)
}

cl, err := client.New(context.Background(), rootCfg.BuildkitAddr, client.WithFailFast())
if err != nil {
return err
Expand Down Expand Up @@ -112,7 +126,7 @@ var combineCmd = &cobra.Command{
}

log.WithField("combination", cmb.Name).WithField("chunks", cmb.Chunks).WithField("ref", destref.String()).Warn("producing chunk combination")
err = prj.Combine(context.Background(), cmb.Chunks, destref, sess, opts...)
err = prj.Combine(context.Background(), cmb.Chunks, destref, sess, compression, opts...)
if err != nil {
return err
}
Expand All @@ -130,4 +144,5 @@ func init() {
combineCmd.Flags().String("combination", "", "build a specific combination")
combineCmd.Flags().Bool("all", false, "build all combinations")
combineCmd.Flags().String("build-ref", "", "use a different build-ref than the target-ref")
combineCmd.Flags().String("compression", "gzip", "compression type for layers")
}
4 changes: 2 additions & 2 deletions example.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@

# Build and run the example
gp await-port 5000
go run main.go build --context example localhost:5000/dazzle
go run main.go combine --context example localhost:5000/dazzle --all
go run main.go build --context example localhost:5000/dazzle --compression gzip
go run main.go combine --context example localhost:5000/dazzle --all --compression gzip
28 changes: 15 additions & 13 deletions pkg/dazzle/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ func WithChunkedWithoutHash(enable bool) BuildOpt {
}

// Build builds all images in a project
func (p *Project) Build(ctx context.Context, session *BuildSession) error {
func (p *Project) Build(ctx context.Context, session *BuildSession, compression Compression) error {
ctx = clog.WithLogger(ctx, log.NewEntry(log.New()))

// Relying on the buildkit cache alone does not result in fixed content hashes.
Expand All @@ -141,7 +141,7 @@ func (p *Project) Build(ctx context.Context, session *BuildSession) error {
}

log.WithField("ref", baseref.String()).Warn("building base image")
absbaseref, err := p.Base.buildAsBase(ctx, baseref, session)
absbaseref, err := p.Base.buildAsBase(ctx, baseref, session, compression)
if err != nil {
return fmt.Errorf("cannot build base image: %w", err)
}
Expand Down Expand Up @@ -169,12 +169,12 @@ func (p *Project) Build(ctx context.Context, session *BuildSession) error {
session.baseBuildFinished(absbaseref, basemf, basecfg)

for _, chk := range p.Chunks {
_, _, err := chk.test(ctx, session)
_, _, err := chk.test(ctx, session, compression)
if err != nil {
return fmt.Errorf("cannot test chunk %s: %w", chk.Name, err)
}

_, _, err = chk.build(ctx, session)
_, _, err = chk.build(ctx, session, compression)
if err != nil {
return fmt.Errorf("cannot build chunk %s: %w", chk.Name, err)
}
Expand Down Expand Up @@ -281,7 +281,7 @@ func (s *BuildSession) baseBuildFinished(ref reference.Digested, mf *ociv1.Manif
s.baseCfg = cfg
}

func removeBaseLayer(ctx context.Context, opts removeBaseLayerOpts) (chkmf *ociv1.Manifest, didbuild bool, err error) {
func removeBaseLayer(ctx context.Context, opts removeBaseLayerOpts, compression Compression) (chkmf *ociv1.Manifest, didbuild bool, err error) {
_, chkmf, chkcfg, err := getImageMetadata(ctx, opts.chunkref, opts.registry)
if err != nil {
return
Expand Down Expand Up @@ -334,7 +334,7 @@ func removeBaseLayer(ctx context.Context, opts removeBaseLayerOpts) (chkmf *ociv
}
chkmf.Layers = chkmf.Layers[len(opts.basemf.Layers):]
for i := range chkmf.Layers {
chkmf.Layers[i].MediaType = ociv1.MediaTypeImageLayerGzip
chkmf.Layers[i].MediaType = compression.Extension()
}
if chkmf.Annotations == nil {
chkmf.Annotations = make(map[string]string)
Expand Down Expand Up @@ -465,7 +465,7 @@ func (p *Project) BaseRef(build reference.Named) (reference.NamedTagged, error)
return reference.WithTag(build, fmt.Sprintf("base--%s", hash))
}

func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, sess *BuildSession) (absref reference.Digested, err error) {
func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, sess *BuildSession, compression Compression) (absref reference.Digested, err error) {
_, desc, err := sess.opts.Resolver.Resolve(ctx, dest.String())
if err == nil {
// if err == nil the image exists already
Expand Down Expand Up @@ -504,6 +504,7 @@ func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, se
"name": dest.String(),
"push": "true",
"oci-mediatypes": "true",
"compression": compression.String(),
},
},
},
Expand Down Expand Up @@ -552,7 +553,7 @@ func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, se
return resref, nil
}

func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession) (ok bool, didRun bool, err error) {
func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession, compression Compression) (ok bool, didRun bool, err error) {
if sess == nil {
return false, false, errors.New("cannot test without a session")
}
Expand All @@ -574,7 +575,7 @@ func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession) (ok bool, d
}

// build temp image for testing
testRef, _, err := p.buildImage(ctx, ImageTypeTest, sess)
testRef, _, err := p.buildImage(ctx, ImageTypeTest, sess, compression)
if err != nil {
return false, false, err
}
Expand All @@ -600,9 +601,9 @@ func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession) (ok bool, d
return true, true, nil
}

func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession) (chkRef reference.NamedTagged, didBuild bool, err error) {
func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession, compression Compression) (chkRef reference.NamedTagged, didBuild bool, err error) {
// build actual full image
fullRef, didBuild, err := p.buildImage(ctx, ImageTypeFull, sess)
fullRef, didBuild, err := p.buildImage(ctx, ImageTypeFull, sess, compression)
if err != nil {
return
}
Expand All @@ -618,7 +619,7 @@ func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession) (chkRef re
}
log.WithField("chunk", p.Name).WithField("ref", chkRef).Warn("building chunked image")
opts := removeBaseLayerOpts{sess.opts.Resolver, sess.opts.Registry, sess.baseRef, sess.baseMF, sess.baseCfg, fullRef, chkRef}
mf, didBuild, err := removeBaseLayer(ctx, opts)
mf, didBuild, err := removeBaseLayer(ctx, opts, compression)
if err != nil {
return
}
Expand All @@ -628,7 +629,7 @@ func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession) (chkRef re
return
}

func (p *ProjectChunk) buildImage(ctx context.Context, tpe ChunkImageType, sess *BuildSession) (tgt reference.Named, didBuild bool, err error) {
func (p *ProjectChunk) buildImage(ctx context.Context, tpe ChunkImageType, sess *BuildSession, compression Compression) (tgt reference.Named, didBuild bool, err error) {
tgt, err = p.ImageName(tpe, sess)
if err != nil {
return
Expand Down Expand Up @@ -690,6 +691,7 @@ func (p *ProjectChunk) buildImage(ctx context.Context, tpe ChunkImageType, sess
"name": tgt.String(),
"push": "true",
"oci-mediatypes": "true",
"compression": compression.String(),
},
},
},
Expand Down
66 changes: 45 additions & 21 deletions pkg/dazzle/build_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,13 @@ func TestProjectChunk_test(t *testing.T) {
sess.opts.Resolver = fakeResolver{}

type fields struct {
Name string
FS map[string]*fstest.MapFile
Base string
Chunk string
BaseRef string
Registry Registry
Name string
FS map[string]*fstest.MapFile
Base string
Chunk string
BaseRef string
Compression Compression
Registry Registry
}
type args struct {
ctx context.Context
Expand All @@ -72,9 +73,10 @@ func TestProjectChunk_test(t *testing.T) {
{
name: "passes with no tests",
fields: fields{
Name: "no test chunk",
Base: "chunks",
Chunk: "notest",
Name: "no test chunk",
Base: "chunks",
Chunk: "notest",
Compression: Gzip,
FS: map[string]*fstest.MapFile{
"chunks/notest/Dockerfile": {
Data: []byte("FROM alpine"),
Expand All @@ -91,9 +93,10 @@ func TestProjectChunk_test(t *testing.T) {
{
name: "fails when no base reference set",
fields: fields{
Name: "no base ref chunk",
Base: "chunks",
Chunk: "nobaseref",
Name: "no base ref chunk",
Base: "chunks",
Chunk: "nobaseref",
Compression: Gzip,
FS: map[string]*fstest.MapFile{
"chunks/nobaseref/Dockerfile": {
Data: []byte("FROM alpine"),
Expand All @@ -118,9 +121,10 @@ func TestProjectChunk_test(t *testing.T) {
{
name: "does not build if tests have passed",
fields: fields{
Name: "a chunk",
Base: "chunks",
Chunk: "foobar",
Name: "a chunk",
Base: "chunks",
Chunk: "foobar",
Compression: Gzip,
FS: map[string]*fstest.MapFile{
"chunks/foobar/Dockerfile": {
Data: []byte("FROM alpine"),
Expand Down Expand Up @@ -148,6 +152,26 @@ func TestProjectChunk_test(t *testing.T) {
wantOk: true,
wantErr: false,
},
{
name: "passes with no tests with zstd",
fields: fields{
Name: "no test chunk",
Base: "chunks",
Chunk: "notest",
Compression: Zstd,
FS: map[string]*fstest.MapFile{
"chunks/notest/Dockerfile": {
Data: []byte("FROM alpine"),
},
},
},
args: args{
ctx: ctx,
sess: sess,
},
wantOk: true,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down Expand Up @@ -175,7 +199,7 @@ func TestProjectChunk_test(t *testing.T) {
if tt.fields.Registry != nil {
sess.opts.Registry = tt.fields.Registry
}
gotOk, _, err := chks[0].test(tt.args.ctx, tt.args.sess)
gotOk, _, err := chks[0].test(tt.args.ctx, tt.args.sess, tt.fields.Compression)
if (err != nil) != tt.wantErr {
t.Errorf("TestProjectChunk_test() error = %v, wantErr %v", err, tt.wantErr)
return
Expand Down Expand Up @@ -286,7 +310,7 @@ func TestProjectChunk_test_integration(t *testing.T) {
}
}

err = prj.Build(context.Background(), session)
err = prj.Build(context.Background(), session, Gzip)
if err != nil {
t.Errorf("TestProjectChunk_test_integration.test() unexpected Build error = %v", err)
return
Expand Down Expand Up @@ -318,7 +342,7 @@ func TestProjectChunk_test_integration(t *testing.T) {
}

// Re-running build should reuse existing images & tags
err = prj.Build(context.Background(), session)
err = prj.Build(context.Background(), session, Gzip)
if err != nil {
t.Errorf("TestProjectChunk_test_integration() unexpected rebuild 1 error = %v", err)
return
Expand Down Expand Up @@ -355,13 +379,13 @@ func TestProjectChunk_test_integration(t *testing.T) {

// Individually check each chunk to ensure it doesn't rebuild
for _, chk := range prj.Chunks {
ok, didRun, err := chk.test(ctx, session)
ok, didRun, err := chk.test(ctx, session, Gzip)
if err != nil || !ok || didRun {
t.Errorf("TestProjectChunk_test_integration() test() error:%v testing chunk: %s with results: %v:%v", err, chk.Name, ok, didRun)
return
}

_, didBuild, err := chk.build(ctx, session)
_, didBuild, err := chk.build(ctx, session, Gzip)
if err != nil || didBuild {
t.Errorf("TestProjectChunk_test_integration() build() error:%v building chunk: %s didBuild:%v", err, chk.Name, didBuild)
return
Expand Down Expand Up @@ -389,7 +413,7 @@ func TestProjectChunk_test_integration(t *testing.T) {
}

// Re-running build should create new test tags
err = prj.Build(context.Background(), session)
err = prj.Build(context.Background(), session, Gzip)
if err != nil {
t.Errorf("TestProjectChunk_test_integration() unexpected rebuild 2 error = %v", err)
return
Expand Down
4 changes: 2 additions & 2 deletions pkg/dazzle/combiner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func asTempBuild(o *combinerOpts) error {

// Combine combines a set of previously built chunks into a single image while maintaining
// the layer identity.
func (p *Project) Combine(ctx context.Context, chunks []string, dest reference.Named, sess *BuildSession, opts ...CombinerOpt) (err error) {
func (p *Project) Combine(ctx context.Context, chunks []string, dest reference.Named, sess *BuildSession, compression Compression, opts ...CombinerOpt) (err error) {
var options combinerOpts
for _, o := range opts {
err = o(&options)
Expand All @@ -78,7 +78,7 @@ func (p *Project) Combine(ctx context.Context, chunks []string, dest reference.N
if err != nil {
return err
}
err = p.Combine(ctx, chunks, tmpdest, sess, append(opts, asTempBuild)...)
err = p.Combine(ctx, chunks, tmpdest, sess, compression, append(opts, asTempBuild)...)
if err != nil {
return err
}
Expand Down
Loading

0 comments on commit 838cfd0

Please sign in to comment.