Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support for zstd compression layer #58

Closed
wants to merge 1 commit into from
Closed

Support for zstd compression layer #58

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion cmd/core/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ package core

import (
"context"
"fmt"

"github.com/moby/buildkit/client"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -60,7 +61,21 @@ var buildCmd = &cobra.Command{
return err
}

err = prj.Build(context.Background(), session)
var compression dazzle.Compression
c, err := cmd.Flags().GetString("compression")
if err != nil {
return err
}
switch c {
case "gzip":
compression = dazzle.Gzip
case "zstd":
compression = dazzle.Zstd
default:
return fmt.Errorf("unknow a compression type: %s", c)
}

err = prj.Build(context.Background(), session, compression)
if err != nil {
return err
}
Expand All @@ -77,4 +92,5 @@ func init() {
buildCmd.Flags().Bool("no-cache", false, "disables the buildkit build cache")
buildCmd.Flags().Bool("plain-output", false, "produce plain output")
buildCmd.Flags().Bool("chunked-without-hash", false, "disable hash qualification for chunked image")
buildCmd.Flags().String("compression", "gzip", "compression type for layers")
}
17 changes: 16 additions & 1 deletion cmd/core/combine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,20 @@ var combineCmd = &cobra.Command{
bldref = targetref.String()
}

var compression dazzle.Compression
c, err := cmd.Flags().GetString("compression")
if err != nil {
return err
}
switch c {
case "gzip":
compression = dazzle.Gzip
case "zstd":
compression = dazzle.Zstd
default:
return fmt.Errorf("unknow a compression type: %s", c)
}

cl, err := client.New(context.Background(), rootCfg.BuildkitAddr, client.WithFailFast())
if err != nil {
return err
Expand Down Expand Up @@ -112,7 +126,7 @@ var combineCmd = &cobra.Command{
}

log.WithField("combination", cmb.Name).WithField("chunks", cmb.Chunks).WithField("ref", destref.String()).Warn("producing chunk combination")
err = prj.Combine(context.Background(), cmb.Chunks, destref, sess, opts...)
err = prj.Combine(context.Background(), cmb.Chunks, destref, sess, compression, opts...)
if err != nil {
return err
}
Expand All @@ -130,4 +144,5 @@ func init() {
combineCmd.Flags().String("combination", "", "build a specific combination")
combineCmd.Flags().Bool("all", false, "build all combinations")
combineCmd.Flags().String("build-ref", "", "use a different build-ref than the target-ref")
combineCmd.Flags().String("compression", "gzip", "compression type for layers")
}
4 changes: 2 additions & 2 deletions example.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@

# Build and run the example
gp await-port 5000
go run main.go build --context example localhost:5000/dazzle
go run main.go combine --context example localhost:5000/dazzle --all
go run main.go build --context example localhost:5000/dazzle --compression gzip
go run main.go combine --context example localhost:5000/dazzle --all --compression gzip
28 changes: 15 additions & 13 deletions pkg/dazzle/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ func WithChunkedWithoutHash(enable bool) BuildOpt {
}

// Build builds all images in a project
func (p *Project) Build(ctx context.Context, session *BuildSession) error {
func (p *Project) Build(ctx context.Context, session *BuildSession, compression Compression) error {
ctx = clog.WithLogger(ctx, log.NewEntry(log.New()))

// Relying on the buildkit cache alone does not result in fixed content hashes.
Expand All @@ -141,7 +141,7 @@ func (p *Project) Build(ctx context.Context, session *BuildSession) error {
}

log.WithField("ref", baseref.String()).Warn("building base image")
absbaseref, err := p.Base.buildAsBase(ctx, baseref, session)
absbaseref, err := p.Base.buildAsBase(ctx, baseref, session, compression)
if err != nil {
return fmt.Errorf("cannot build base image: %w", err)
}
Expand Down Expand Up @@ -169,12 +169,12 @@ func (p *Project) Build(ctx context.Context, session *BuildSession) error {
session.baseBuildFinished(absbaseref, basemf, basecfg)

for _, chk := range p.Chunks {
_, _, err := chk.test(ctx, session)
_, _, err := chk.test(ctx, session, compression)
if err != nil {
return fmt.Errorf("cannot test chunk %s: %w", chk.Name, err)
}

_, _, err = chk.build(ctx, session)
_, _, err = chk.build(ctx, session, compression)
if err != nil {
return fmt.Errorf("cannot build chunk %s: %w", chk.Name, err)
}
Expand Down Expand Up @@ -281,7 +281,7 @@ func (s *BuildSession) baseBuildFinished(ref reference.Digested, mf *ociv1.Manif
s.baseCfg = cfg
}

func removeBaseLayer(ctx context.Context, opts removeBaseLayerOpts) (chkmf *ociv1.Manifest, didbuild bool, err error) {
func removeBaseLayer(ctx context.Context, opts removeBaseLayerOpts, compression Compression) (chkmf *ociv1.Manifest, didbuild bool, err error) {
_, chkmf, chkcfg, err := getImageMetadata(ctx, opts.chunkref, opts.registry)
if err != nil {
return
Expand Down Expand Up @@ -334,7 +334,7 @@ func removeBaseLayer(ctx context.Context, opts removeBaseLayerOpts) (chkmf *ociv
}
chkmf.Layers = chkmf.Layers[len(opts.basemf.Layers):]
for i := range chkmf.Layers {
chkmf.Layers[i].MediaType = ociv1.MediaTypeImageLayerGzip
chkmf.Layers[i].MediaType = compression.Extension()
}
if chkmf.Annotations == nil {
chkmf.Annotations = make(map[string]string)
Expand Down Expand Up @@ -465,7 +465,7 @@ func (p *Project) BaseRef(build reference.Named) (reference.NamedTagged, error)
return reference.WithTag(build, fmt.Sprintf("base--%s", hash))
}

func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, sess *BuildSession) (absref reference.Digested, err error) {
func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, sess *BuildSession, compression Compression) (absref reference.Digested, err error) {
_, desc, err := sess.opts.Resolver.Resolve(ctx, dest.String())
if err == nil {
// if err == nil the image exists already
Expand Down Expand Up @@ -504,6 +504,7 @@ func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, se
"name": dest.String(),
"push": "true",
"oci-mediatypes": "true",
"compression": compression.String(),
},
},
},
Expand Down Expand Up @@ -552,7 +553,7 @@ func (p *ProjectChunk) buildAsBase(ctx context.Context, dest reference.Named, se
return resref, nil
}

func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession) (ok bool, didRun bool, err error) {
func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession, compression Compression) (ok bool, didRun bool, err error) {
if sess == nil {
return false, false, errors.New("cannot test without a session")
}
Expand All @@ -574,7 +575,7 @@ func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession) (ok bool, d
}

// build temp image for testing
testRef, _, err := p.buildImage(ctx, ImageTypeTest, sess)
testRef, _, err := p.buildImage(ctx, ImageTypeTest, sess, compression)
if err != nil {
return false, false, err
}
Expand All @@ -600,9 +601,9 @@ func (p *ProjectChunk) test(ctx context.Context, sess *BuildSession) (ok bool, d
return true, true, nil
}

func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession) (chkRef reference.NamedTagged, didBuild bool, err error) {
func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession, compression Compression) (chkRef reference.NamedTagged, didBuild bool, err error) {
// build actual full image
fullRef, didBuild, err := p.buildImage(ctx, ImageTypeFull, sess)
fullRef, didBuild, err := p.buildImage(ctx, ImageTypeFull, sess, compression)
if err != nil {
return
}
Expand All @@ -618,7 +619,7 @@ func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession) (chkRef re
}
log.WithField("chunk", p.Name).WithField("ref", chkRef).Warn("building chunked image")
opts := removeBaseLayerOpts{sess.opts.Resolver, sess.opts.Registry, sess.baseRef, sess.baseMF, sess.baseCfg, fullRef, chkRef}
mf, didBuild, err := removeBaseLayer(ctx, opts)
mf, didBuild, err := removeBaseLayer(ctx, opts, compression)
if err != nil {
return
}
Expand All @@ -628,7 +629,7 @@ func (p *ProjectChunk) build(ctx context.Context, sess *BuildSession) (chkRef re
return
}

func (p *ProjectChunk) buildImage(ctx context.Context, tpe ChunkImageType, sess *BuildSession) (tgt reference.Named, didBuild bool, err error) {
func (p *ProjectChunk) buildImage(ctx context.Context, tpe ChunkImageType, sess *BuildSession, compression Compression) (tgt reference.Named, didBuild bool, err error) {
tgt, err = p.ImageName(tpe, sess)
if err != nil {
return
Expand Down Expand Up @@ -690,6 +691,7 @@ func (p *ProjectChunk) buildImage(ctx context.Context, tpe ChunkImageType, sess
"name": tgt.String(),
"push": "true",
"oci-mediatypes": "true",
"compression": compression.String(),
},
},
},
Expand Down
66 changes: 45 additions & 21 deletions pkg/dazzle/build_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,13 @@ func TestProjectChunk_test(t *testing.T) {
sess.opts.Resolver = fakeResolver{}

type fields struct {
Name string
FS map[string]*fstest.MapFile
Base string
Chunk string
BaseRef string
Registry Registry
Name string
FS map[string]*fstest.MapFile
Base string
Chunk string
BaseRef string
Compression Compression
Registry Registry
}
type args struct {
ctx context.Context
Expand All @@ -72,9 +73,10 @@ func TestProjectChunk_test(t *testing.T) {
{
name: "passes with no tests",
fields: fields{
Name: "no test chunk",
Base: "chunks",
Chunk: "notest",
Name: "no test chunk",
Base: "chunks",
Chunk: "notest",
Compression: Gzip,
FS: map[string]*fstest.MapFile{
"chunks/notest/Dockerfile": {
Data: []byte("FROM alpine"),
Expand All @@ -91,9 +93,10 @@ func TestProjectChunk_test(t *testing.T) {
{
name: "fails when no base reference set",
fields: fields{
Name: "no base ref chunk",
Base: "chunks",
Chunk: "nobaseref",
Name: "no base ref chunk",
Base: "chunks",
Chunk: "nobaseref",
Compression: Gzip,
FS: map[string]*fstest.MapFile{
"chunks/nobaseref/Dockerfile": {
Data: []byte("FROM alpine"),
Expand All @@ -118,9 +121,10 @@ func TestProjectChunk_test(t *testing.T) {
{
name: "does not build if tests have passed",
fields: fields{
Name: "a chunk",
Base: "chunks",
Chunk: "foobar",
Name: "a chunk",
Base: "chunks",
Chunk: "foobar",
Compression: Gzip,
FS: map[string]*fstest.MapFile{
"chunks/foobar/Dockerfile": {
Data: []byte("FROM alpine"),
Expand Down Expand Up @@ -148,6 +152,26 @@ func TestProjectChunk_test(t *testing.T) {
wantOk: true,
wantErr: false,
},
{
name: "passes with no tests with zstd",
fields: fields{
Name: "no test chunk",
Base: "chunks",
Chunk: "notest",
Compression: Zstd,
FS: map[string]*fstest.MapFile{
"chunks/notest/Dockerfile": {
Data: []byte("FROM alpine"),
},
},
},
args: args{
ctx: ctx,
sess: sess,
},
wantOk: true,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down Expand Up @@ -175,7 +199,7 @@ func TestProjectChunk_test(t *testing.T) {
if tt.fields.Registry != nil {
sess.opts.Registry = tt.fields.Registry
}
gotOk, _, err := chks[0].test(tt.args.ctx, tt.args.sess)
gotOk, _, err := chks[0].test(tt.args.ctx, tt.args.sess, tt.fields.Compression)
if (err != nil) != tt.wantErr {
t.Errorf("TestProjectChunk_test() error = %v, wantErr %v", err, tt.wantErr)
return
Expand Down Expand Up @@ -286,7 +310,7 @@ func TestProjectChunk_test_integration(t *testing.T) {
}
}

err = prj.Build(context.Background(), session)
err = prj.Build(context.Background(), session, Gzip)
if err != nil {
t.Errorf("TestProjectChunk_test_integration.test() unexpected Build error = %v", err)
return
Expand Down Expand Up @@ -318,7 +342,7 @@ func TestProjectChunk_test_integration(t *testing.T) {
}

// Re-running build should reuse existing images & tags
err = prj.Build(context.Background(), session)
err = prj.Build(context.Background(), session, Gzip)
if err != nil {
t.Errorf("TestProjectChunk_test_integration() unexpected rebuild 1 error = %v", err)
return
Expand Down Expand Up @@ -355,13 +379,13 @@ func TestProjectChunk_test_integration(t *testing.T) {

// Individually check each chunk to ensure it doesn't rebuild
for _, chk := range prj.Chunks {
ok, didRun, err := chk.test(ctx, session)
ok, didRun, err := chk.test(ctx, session, Gzip)
if err != nil || !ok || didRun {
t.Errorf("TestProjectChunk_test_integration() test() error:%v testing chunk: %s with results: %v:%v", err, chk.Name, ok, didRun)
return
}

_, didBuild, err := chk.build(ctx, session)
_, didBuild, err := chk.build(ctx, session, Gzip)
if err != nil || didBuild {
t.Errorf("TestProjectChunk_test_integration() build() error:%v building chunk: %s didBuild:%v", err, chk.Name, didBuild)
return
Expand Down Expand Up @@ -389,7 +413,7 @@ func TestProjectChunk_test_integration(t *testing.T) {
}

// Re-running build should create new test tags
err = prj.Build(context.Background(), session)
err = prj.Build(context.Background(), session, Gzip)
if err != nil {
t.Errorf("TestProjectChunk_test_integration() unexpected rebuild 2 error = %v", err)
return
Expand Down
4 changes: 2 additions & 2 deletions pkg/dazzle/combiner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func asTempBuild(o *combinerOpts) error {

// Combine combines a set of previously built chunks into a single image while maintaining
// the layer identity.
func (p *Project) Combine(ctx context.Context, chunks []string, dest reference.Named, sess *BuildSession, opts ...CombinerOpt) (err error) {
func (p *Project) Combine(ctx context.Context, chunks []string, dest reference.Named, sess *BuildSession, compression Compression, opts ...CombinerOpt) (err error) {
var options combinerOpts
for _, o := range opts {
err = o(&options)
Expand All @@ -78,7 +78,7 @@ func (p *Project) Combine(ctx context.Context, chunks []string, dest reference.N
if err != nil {
return err
}
err = p.Combine(ctx, chunks, tmpdest, sess, append(opts, asTempBuild)...)
err = p.Combine(ctx, chunks, tmpdest, sess, compression, append(opts, asTempBuild)...)
if err != nil {
return err
}
Expand Down
Loading