Support creating and pushing OCI types

Adds image and oci exporter option "oci-mediatypes"
Ensures that the images created in the content store
have the correct type which matches the manifest.
Sets the correct media type on the descriptor in push from
reading the type specified in the manifest.
Removes use of distribution manifest packages.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

cache/remotecache/export.go

@@ -9,14 +9,14 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
-	"github.com/docker/distribution/manifest"
 	v1 "github.com/moby/buildkit/cache/remotecache/v1"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/progress"
 	"github.com/moby/buildkit/util/push"
 	digest "github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
@@ -46,7 +46,9 @@ func (ce *CacheExporter) Finalize(ctx context.Context, cc *v1.CacheChains, targe
 
 	// own type because oci type can't be pushed and docker type doesn't have annotations
 	type manifestList struct {
-		manifest.Versioned
+		specs.Versioned
+
+		MediaType string `json:"mediaType,omitempty"`
 
 		// Manifests references platform specific manifests.
 		Manifests []ocispec.Descriptor `json:"manifests"`

client/client_test.go

@@ -22,7 +22,6 @@ import (
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/containerd/continuity/fs/fstest"
-	"github.com/docker/distribution/manifest/schema2"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/util/testutil/httpserver"
@@ -612,11 +611,15 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
 	dt, err = content.ReadBlob(ctx, img.ContentStore(), img.Target())
 	require.NoError(t, err)
 
-	var mfst schema2.Manifest
+	var mfst = struct {
+		MediaType string `json:"mediaType,omitempty"`
+		ocispec.Manifest
+	}{}
+
 	err = json.Unmarshal(dt, &mfst)
 	require.NoError(t, err)
 
-	require.Equal(t, schema2.MediaTypeManifest, mfst.MediaType)
+	require.Equal(t, images.MediaTypeDockerSchema2Manifest, mfst.MediaType)
 	require.Equal(t, 2, len(mfst.Layers))
 
 	dt, err = content.ReadBlob(ctx, img.ContentStore(), ocispec.Descriptor{Digest: mfst.Layers[0].Digest})

exporter/containerimage/export.go

@@ -20,6 +20,7 @@ const (
 	keyPush             = "push"
 	keyInsecure         = "registry.insecure"
 	exporterImageConfig = "containerimage.config"
+	ociTypes            = "oci-mediatypes"
 )
 
 type Opt struct {
@@ -69,6 +70,16 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
 			i.insecure = b
 		case exporterImageConfig:
 			i.config = []byte(v)
+		case ociTypes:
+			if v == "" {
+				i.ociTypes = true
+				continue
+			}
+			b, err := strconv.ParseBool(v)
+			if err != nil {
+				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
+			}
+			i.ociTypes = b
 		default:
 			logrus.Warnf("image exporter: unknown option %s", k)
 		}
@@ -81,6 +92,7 @@ type imageExporterInstance struct {
 	targetName string
 	push       bool
 	insecure   bool
+	ociTypes   bool
 	config     []byte
 }
 
@@ -92,7 +104,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
 	if config, ok := opt[exporterImageConfig]; ok {
 		e.config = config
 	}
-	desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config)
+	desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config, e.ociTypes)
 	if err != nil {
 		return nil, err
 	}

exporter/containerimage/writer.go

@@ -10,14 +10,14 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/diff"
-	"github.com/docker/distribution"
-	"github.com/docker/distribution/manifest/schema2"
+	"github.com/containerd/containerd/images"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/cache/blobs"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/util/progress"
 	"github.com/moby/buildkit/util/system"
 	digest "github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -41,7 +41,7 @@ type ImageWriter struct {
 	opt WriterOpt
 }
 
-func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, config []byte) (*ocispec.Descriptor, error) {
+func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, config []byte, oci bool) (*ocispec.Descriptor, error) {
 	layersDone := oneOffProgress(ctx, "exporting layers")
 	diffPairs, err := blobs.GetDiffPairs(ctx, ic.opt.ContentStore, ic.opt.Snapshotter, ic.opt.Differ, ref, true)
 	if err != nil {
@@ -68,17 +68,39 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
 		return nil, err
 	}
 
-	configDigest := digest.FromBytes(config)
-
-	mfst := schema2.Manifest{
-		Config: distribution.Descriptor{
-			Digest:    configDigest,
-			Size:      int64(len(config)),
-			MediaType: schema2.MediaTypeImageConfig,
+	var (
+		configDigest = digest.FromBytes(config)
+		manifestType = ocispec.MediaTypeImageManifest
+		configType   = ocispec.MediaTypeImageConfig
+		layerType    = ocispec.MediaTypeImageLayerGzip
+	)
+
+	// Use docker media types for older Docker versions and registries
+	if !oci {
+		manifestType = images.MediaTypeDockerSchema2Manifest
+		configType = images.MediaTypeDockerSchema2Config
+		layerType = images.MediaTypeDockerSchema2LayerGzip
+	}
+
+	mfst := struct {
+		// MediaType is reserved in the OCI spec but
+		// excluded from go types.
+		MediaType string `json:"mediaType,omitempty"`
+
+		ocispec.Manifest
+	}{
+		MediaType: manifestType,
+		Manifest: ocispec.Manifest{
+			Versioned: specs.Versioned{
+				SchemaVersion: 2,
+			},
+			Config: ocispec.Descriptor{
+				Digest:    configDigest,
+				Size:      int64(len(config)),
+				MediaType: configType,
+			},
 		},
 	}
-	mfst.SchemaVersion = 2
-	mfst.MediaType = schema2.MediaTypeManifest
 
 	labels := map[string]string{
 		"containerd.io/gc.ref.content.0": configDigest.String(),
@@ -89,15 +111,15 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not find blob %s from contentstore", dp.Blobsum)
 		}
-		mfst.Layers = append(mfst.Layers, distribution.Descriptor{
+		mfst.Layers = append(mfst.Layers, ocispec.Descriptor{
 			Digest:    dp.Blobsum,
 			Size:      info.Size,
-			MediaType: schema2.MediaTypeLayer,
+			MediaType: layerType,
 		})
 		labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = dp.Blobsum.String()
 	}
 
-	mfstJSON, err := json.Marshal(mfst)
+	mfstJSON, err := json.MarshalIndent(mfst, "", "   ")
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to marshal manifest")
 	}
@@ -115,8 +137,9 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
 	mfstDone(nil)
 
 	configDesc := ocispec.Descriptor{
-		Digest: configDigest,
-		Size:   int64(len(config)),
+		Digest:    configDigest,
+		Size:      int64(len(config)),
+		MediaType: configType,
 	}
 	configDone := oneOffProgress(ctx, "exporting config "+configDigest.String())
 
@@ -133,7 +156,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
 	return &ocispec.Descriptor{
 		Digest:    mfstDigest,
 		Size:      int64(len(mfstJSON)),
-		MediaType: ocispec.MediaTypeImageManifest,
+		MediaType: manifestType,
 	}, nil
 }
 

exporter/oci/export.go

@@ -2,6 +2,7 @@ package oci
 
 import (
 	"context"
+	"strconv"
 	"time"
 
 	"github.com/containerd/containerd/images"
@@ -26,6 +27,7 @@ const (
 	keyImageName        = "name"
 	VariantOCI          = "oci"
 	VariantDocker       = "docker"
+	ociTypes            = "oci-mediatypes"
 )
 
 type Opt struct {
@@ -57,6 +59,7 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
 		return nil, err
 	}
 
+	var ot *bool
 	i := &imageExporterInstance{imageExporter: e, caller: caller}
 	for k, v := range opt {
 		switch k {
@@ -68,18 +71,35 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp
 				return nil, errors.Wrapf(err, "failed to parse %s", v)
 			}
 			i.name = reference.TagNameOnly(parsed).String()
+		case ociTypes:
+			ot = new(bool)
+			if v == "" {
+				*ot = true
+				continue
+			}
+			b, err := strconv.ParseBool(v)
+			if err != nil {
+				return nil, errors.Wrapf(err, "non-bool value specified for %s", k)
+			}
+			*ot = b
 		default:
 			logrus.Warnf("oci exporter: unknown option %s", k)
 		}
 	}
+	if ot == nil {
+		i.ociTypes = e.opt.Variant == VariantOCI
+	} else {
+		i.ociTypes = *ot
+	}
 	return i, nil
 }
 
 type imageExporterInstance struct {
 	*imageExporter
-	config []byte
-	caller session.Caller
-	name   string
+	config   []byte
+	caller   session.Caller
+	name     string
+	ociTypes bool
 }
 
 func (e *imageExporterInstance) Name() string {
@@ -90,7 +110,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, ref cache.ImmutableR
 	if config, ok := opt[exporterImageConfig]; ok {
 		e.config = config
 	}
-	desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config)
+	desc, err := e.opt.ImageWriter.Commit(ctx, ref, e.config, e.ociTypes)
 	if err != nil {
 		return nil, err
 	}

util/imageutil/config.go

@@ -139,13 +139,17 @@ func DetectManifestMediaType(ra content.ReaderAt) (string, error) {
 	}
 
 	var mfst struct {
-		Config json.RawMessage `json:"config"`
+		MediaType string          `json:"mediaType"`
+		Config    json.RawMessage `json:"config"`
 	}
 
 	if err := json.Unmarshal(p, &mfst); err != nil {
 		return "", err
 	}
 
+	if mfst.MediaType != "" {
+		return mfst.MediaType, nil
+	}
 	if mfst.Config != nil {
 		return images.MediaTypeDockerSchema2Manifest, nil
 	}