Merge pull request #467 from dmcgowan/export-oci-media-types

Support creating and pushing OCI types

cache/remotecache/export.go

@@ -9,14 +9,14 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
-	"github.com/docker/distribution/manifest"
 	v1 "github.com/moby/buildkit/cache/remotecache/v1"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/progress"
 	"github.com/moby/buildkit/util/push"
 	digest "github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
@@ -46,7 +46,9 @@ func (ce *CacheExporter) Finalize(ctx context.Context, cc *v1.CacheChains, targe
 
 	// own type because oci type can't be pushed and docker type doesn't have annotations
 	type manifestList struct {
-		manifest.Versioned
+		specs.Versioned
+
+		MediaType string `json:"mediaType,omitempty"`
 
 		// Manifests references platform specific manifests.
 		Manifests []ocispec.Descriptor `json:"manifests"`

client/client_test.go

@@ -22,7 +22,6 @@ import (
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/containerd/continuity/fs/fstest"
-	"github.com/docker/distribution/manifest/schema2"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/util/testutil/httpserver"
@@ -612,11 +611,15 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
 	dt, err = content.ReadBlob(ctx, img.ContentStore(), img.Target())
 	require.NoError(t, err)
 
-	var mfst schema2.Manifest
+	var mfst = struct {
+		MediaType string `json:"mediaType,omitempty"`
+		ocispec.Manifest
+	}{}
+
 	err = json.Unmarshal(dt, &mfst)
 	require.NoError(t, err)
 
-	require.Equal(t, schema2.MediaTypeManifest, mfst.MediaType)
+	require.Equal(t, images.MediaTypeDockerSchema2Manifest, mfst.MediaType)
 	require.Equal(t, 2, len(mfst.Layers))
 
 	dt, err = content.ReadBlob(ctx, img.ContentStore(), ocispec.Descriptor{Digest: mfst.Layers[0].Digest})

exporter/containerimage/export.go

@@ -20,6 +20,7 @@ const (
 	keyPush             = "push"
 	keyInsecure         = "registry.insecure"
 	exporterImageConfig = "containerimage.config"
+	ociTypes            = "oci-mediatypes"
 )
 
 type Opt struct {
@@ -69,6 +70,16 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
 			i.insecure = b
 		case exporterImageConfig:
 			i.config = []byte(v)
+		case ociTypes:
+			if v == "" {
+				i.ociTypes = true
+				continue
+			}
+			b, err := strconv.ParseBool(v)
+			if err != nil {
+				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
+			}
+			i.ociTypes = b
 		default:
 			logrus.Warnf("image exporter: unknown option %s", k)
 		}
@@ -81,6 +92,7 @@ type imageExporterInstance struct {
 	targetName string
 	push       bool
 	insecure   bool
+	ociTypes   bool
 	config     []byte
 }
 
@@ -92,7 +104,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
 	if config, ok := opt[exporterImageConfig]; ok {
 		e.config = config
 	}
-	desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config)
+	desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config, e.ociTypes)
 	if err != nil {
 		return nil, err
 	}

exporter/containerimage/writer.go

@@ -10,14 +10,14 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/diff"
-	"github.com/docker/distribution"
-	"github.com/docker/distribution/manifest/schema2"
+	"github.com/containerd/containerd/images"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/cache/blobs"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/util/progress"
 	"github.com/moby/buildkit/util/system"
 	digest "github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -41,7 +41,7 @@ type ImageWriter struct {
 	opt WriterOpt
 }
 
-func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, config []byte) (*ocispec.Descriptor, error) {
+func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, config []byte, oci bool) (*ocispec.Descriptor, error) {
 	layersDone := oneOffProgress(ctx, "exporting layers")
 	diffPairs, err := blobs.GetDiffPairs(ctx, ic.opt.ContentStore, ic.opt.Snapshotter, ic.opt.Differ, ref, true)
 	if err != nil {
@@ -68,17 +68,39 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
 		return nil, err
 	}
 
-	configDigest := digest.FromBytes(config)
-
-	mfst := schema2.Manifest{
-		Config: distribution.Descriptor{
-			Digest:    configDigest,
-			Size:      int64(len(config)),
-			MediaType: schema2.MediaTypeImageConfig,
+	var (
+		configDigest = digest.FromBytes(config)
+		manifestType = ocispec.MediaTypeImageManifest
+		configType   = ocispec.MediaTypeImageConfig
+		layerType    = ocispec.MediaTypeImageLayerGzip
+	)
+
+	// Use docker media types for older Docker versions and registries
+	if !oci {
+		manifestType = images.MediaTypeDockerSchema2Manifest
+		configType = images.MediaTypeDockerSchema2Config
+		layerType = images.MediaTypeDockerSchema2LayerGzip
+	}
+
+	mfst := struct {
+		// MediaType is reserved in the OCI spec but
+		// excluded from go types.
+		MediaType string `json:"mediaType,omitempty"`
+
+		ocispec.Manifest
+	}{
+		MediaType: manifestType,
+		Manifest: ocispec.Manifest{
+			Versioned: specs.Versioned{
+				SchemaVersion: 2,
+			},
+			Config: ocispec.Descriptor{
+				Digest:    configDigest,
+				Size:      int64(len(config)),
+				MediaType: configType,
+			},
 		},
 	}
-	mfst.SchemaVersion = 2
-	mfst.MediaType = schema2.MediaTypeManifest
 
 	labels := map[string]string{
 		"containerd.io/gc.ref.content.0": configDigest.String(),
@@ -89,15 +111,15 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not find blob %s from contentstore", dp.Blobsum)
 		}
-		mfst.Layers = append(mfst.Layers, distribution.Descriptor{
+		mfst.Layers = append(mfst.Layers, ocispec.Descriptor{
 			Digest:    dp.Blobsum,
 			Size:      info.Size,
-			MediaType: schema2.MediaTypeLayer,
+			MediaType: layerType,
 		})
 		labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = dp.Blobsum.String()
 	}
 
-	mfstJSON, err := json.Marshal(mfst)
+	mfstJSON, err := json.MarshalIndent(mfst, "", "   ")
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to marshal manifest")
 	}
@@ -115,8 +137,9 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
 	mfstDone(nil)
 
 	configDesc := ocispec.Descriptor{
-		Digest: configDigest,
-		Size:   int64(len(config)),
+		Digest:    configDigest,
+		Size:      int64(len(config)),
+		MediaType: configType,
 	}
 	configDone := oneOffProgress(ctx, "exporting config "+configDigest.String())
 
@@ -133,7 +156,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
 	return &ocispec.Descriptor{
 		Digest:    mfstDigest,
 		Size:      int64(len(mfstJSON)),
-		MediaType: ocispec.MediaTypeImageManifest,
+		MediaType: manifestType,
 	}, nil
 }
 

exporter/oci/export.go

@@ -2,6 +2,7 @@ package oci
 
 import (
 	"context"
+	"strconv"
 	"time"
 
 	"github.com/containerd/containerd/images"
@@ -26,6 +27,7 @@ const (
 	keyImageName        = "name"
 	VariantOCI          = "oci"
 	VariantDocker       = "docker"
+	ociTypes            = "oci-mediatypes"
 )
 
 type Opt struct {
@@ -57,6 +59,7 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
 		return nil, err
 	}
 
+	var ot *bool
 	i := &imageExporterInstance{imageExporter: e, caller: caller}
 	for k, v := range opt {
 		switch k {
@@ -68,18 +71,35 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
 				return nil, errors.Wrapf(err, "failed to parse %s", v)
 			}
 			i.name = reference.TagNameOnly(parsed).String()
+		case ociTypes:
+			ot = new(bool)
+			if v == "" {
+				*ot = true
+				continue
+			}
+			b, err := strconv.ParseBool(v)
+			if err != nil {
+				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
+			}
+			*ot = b
 		default:
 			logrus.Warnf("oci exporter: unknown option %s", k)
 		}
 	}
+	if ot == nil {
+		i.ociTypes = e.opt.Variant == VariantOCI
+	} else {
+		i.ociTypes = *ot
+	}
 	return i, nil
 }
 
 type imageExporterInstance struct {
 	*imageExporter
-	config []byte
-	caller session.Caller
-	name   string
+	config   []byte
+	caller   session.Caller
+	name     string
+	ociTypes bool
 }
 
 func (e *imageExporterInstance) Name() string {
@@ -90,7 +110,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
 	if config, ok := opt[exporterImageConfig]; ok {
 		e.config = config
 	}
-	desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config)
+	desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config, e.ociTypes)
 	if err != nil {
 		return nil, err
 	}

util/imageutil/config.go

@@ -141,13 +141,17 @@ func DetectManifestMediaType(ra content.ReaderAt) (string, error) {
 	}
 
 	var mfst struct {
-		Config json.RawMessage `json:"config"`
+		MediaType string          `json:"mediaType"`
+		Config    json.RawMessage `json:"config"`
 	}
 
 	if err := json.Unmarshal(p, &mfst); err != nil {
 		return "", err
 	}
 
+	if mfst.MediaType != "" {
+		return mfst.MediaType, nil
+	}
 	if mfst.Config != nil {
 		return images.MediaTypeDockerSchema2Manifest, nil
 	}