[Feat.] Converting ociv1 layer blob to turboOCI blob locally.

With proxyDiffer, containerd can decompress tgz layer and convert
it to turboOCI blob without unpack tarball

It is very effective in reducing disk I/O pressure because of
replacing unpack multiple files with generating ext4 fsmeta.

Signed-off-by: Yifan Yuan <tuji.yyf@alibaba-inc.com>

cmd/convertor/builder/builder_utils.go

@@ -40,7 +40,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
-	"github.com/containerd/accelerated-container-image/pkg/snapshot"
+	t "github.com/containerd/accelerated-container-image/pkg/types"
 )
 
 func fetch(ctx context.Context, fetcher remotes.Fetcher, desc specs.Descriptor, target any) error {
@@ -144,7 +144,7 @@ func downloadLayer(ctx context.Context, fetcher remotes.Fetcher, targetFile stri
 }
 
 // TODO maybe refactor this
-func writeConfig(dir string, configJSON *snapshot.OverlayBDBSConfig) error {
+func writeConfig(dir string, configJSON *t.OverlayBDBSConfig) error {
 	data, err := json.Marshal(configJSON)
 	if err != nil {
 		return err

cmd/convertor/builder/builder_utils_test.go

@@ -28,7 +28,7 @@ import (
 	"testing"
 
 	testingresources "github.com/containerd/accelerated-container-image/cmd/convertor/testingresources"
-	"github.com/containerd/accelerated-container-image/pkg/snapshot"
+	sn "github.com/containerd/accelerated-container-image/pkg/types"
 	"github.com/containerd/containerd/images"
 	_ "github.com/containerd/containerd/pkg/testutil" // Handle custom root flag
 	"github.com/containerd/containerd/remotes"
@@ -428,17 +428,17 @@ func Test_downloadLayer(t *testing.T) {
 func Test_writeConfig(t *testing.T) {
 	ctx := context.Background()
 	testingresources.RunTestWithTempDir(t, ctx, "writeConfigMinimal", func(t *testing.T, ctx context.Context, workdir string) {
-		configSample := snapshot.OverlayBDBSConfig{
+		configSample := sn.OverlayBDBSConfig{
 			ResultFile: "",
-			Lowers: []snapshot.OverlayBDBSConfigLower{
+			Lowers: []sn.OverlayBDBSConfigLower{
 				{
 					File: overlaybdBaseLayer,
 				},
 				{
 					File: path.Join(workdir, commitFile),
 				},
 			},
-			Upper: snapshot.OverlayBDBSConfigUpper{
+			Upper: sn.OverlayBDBSConfigUpper{
 				Data:  path.Join(workdir, "writable_data"),
 				Index: path.Join(workdir, "writable_index"),
 			},
@@ -455,7 +455,7 @@ func Test_writeConfig(t *testing.T) {
 		}
 		defer file.Close()
 
-		configRes := snapshot.OverlayBDBSConfig{}
+		configRes := sn.OverlayBDBSConfig{}
 
 		err = json.NewDecoder(file).Decode(&configRes)
 		if err != nil {

cmd/convertor/builder/overlaybd_builder.go

@@ -26,7 +26,7 @@ import (
 	"path"
 
 	"github.com/containerd/accelerated-container-image/pkg/label"
-	"github.com/containerd/accelerated-container-image/pkg/snapshot"
+	sn "github.com/containerd/accelerated-container-image/pkg/types"
 	"github.com/containerd/accelerated-container-image/pkg/utils"
 	"github.com/containerd/accelerated-container-image/pkg/version"
 	"github.com/containerd/containerd/errdefs"
@@ -50,17 +50,17 @@ type overlaybdConvertResult struct {
 
 type overlaybdBuilderEngine struct {
 	*builderEngineBase
-	overlaybdConfig *snapshot.OverlayBDBSConfig
+	overlaybdConfig *sn.OverlayBDBSConfig
 	overlaybdLayers []overlaybdConvertResult
 }
 
 func NewOverlayBDBuilderEngine(base *builderEngineBase) builderEngine {
-	config := &snapshot.OverlayBDBSConfig{
-		Lowers:     []snapshot.OverlayBDBSConfigLower{},
+	config := &sn.OverlayBDBSConfig{
+		Lowers:     []sn.OverlayBDBSConfigLower{},
 		ResultFile: "",
 	}
 	if !base.mkfs {
-		config.Lowers = append(config.Lowers, snapshot.OverlayBDBSConfigLower{
+		config.Lowers = append(config.Lowers, sn.OverlayBDBSConfigLower{
 			File: overlaybdBaseLayer,
 		})
 		logrus.Infof("using default baselayer")
@@ -111,7 +111,7 @@ func (e *overlaybdBuilderEngine) BuildLayer(ctx context.Context, idx int) error
 		if err := e.create(ctx, layerDir, mkfs, vsizeGB); err != nil {
 			return err
 		}
-		e.overlaybdConfig.Upper = snapshot.OverlayBDBSConfigUpper{
+		e.overlaybdConfig.Upper = sn.OverlayBDBSConfigUpper{
 			Data:  path.Join(layerDir, "writable_data"),
 			Index: path.Join(layerDir, "writable_index"),
 		}
@@ -130,7 +130,7 @@ func (e *overlaybdBuilderEngine) BuildLayer(ctx context.Context, idx int) error
 			os.Remove(path.Join(layerDir, "writable_index"))
 		}
 	}
-	e.overlaybdConfig.Lowers = append(e.overlaybdConfig.Lowers, snapshot.OverlayBDBSConfigLower{
+	e.overlaybdConfig.Lowers = append(e.overlaybdConfig.Lowers, sn.OverlayBDBSConfigLower{
 		File: path.Join(layerDir, commitFile),
 	})
 	return nil

cmd/convertor/builder/turboOCI_builder.go

@@ -23,14 +23,15 @@ import (
 	"path"
 
 	"github.com/containerd/accelerated-container-image/pkg/label"
-	"github.com/containerd/accelerated-container-image/pkg/snapshot"
+	sn "github.com/containerd/accelerated-container-image/pkg/types"
 	"github.com/containerd/accelerated-container-image/pkg/utils"
 	"github.com/containerd/accelerated-container-image/pkg/version"
 	"github.com/containerd/containerd/archive/compression"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
+
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -51,18 +52,18 @@ const (
 
 type turboOCIBuilderEngine struct {
 	*builderEngineBase
-	overlaybdConfig *snapshot.OverlayBDBSConfig
+	overlaybdConfig *sn.OverlayBDBSConfig
 	tociLayers      []specs.Descriptor
 	isGzip          []bool
 }
 
 func NewTurboOCIBuilderEngine(base *builderEngineBase) builderEngine {
-	config := &snapshot.OverlayBDBSConfig{
-		Lowers:     []snapshot.OverlayBDBSConfigLower{},
+	config := &sn.OverlayBDBSConfig{
+		Lowers:     []sn.OverlayBDBSConfigLower{},
 		ResultFile: "",
 	}
 	if !base.mkfs {
-		config.Lowers = append(config.Lowers, snapshot.OverlayBDBSConfigLower{
+		config.Lowers = append(config.Lowers, sn.OverlayBDBSConfigLower{
 			File: overlaybdBaseLayer,
 		})
 		logrus.Infof("using default baselayer")
@@ -91,7 +92,7 @@ func (e *turboOCIBuilderEngine) BuildLayer(ctx context.Context, idx int) error {
 	if err := e.create(ctx, layerDir, e.mkfs && (idx == 0)); err != nil {
 		return err
 	}
-	e.overlaybdConfig.Upper = snapshot.OverlayBDBSConfigUpper{
+	e.overlaybdConfig.Upper = sn.OverlayBDBSConfigUpper{
 		Data:   path.Join(layerDir, "writable_data"),
 		Index:  path.Join(layerDir, "writable_index"),
 		Target: path.Join(layerDir, "layer.tar"),
@@ -120,7 +121,7 @@ func (e *turboOCIBuilderEngine) BuildLayer(ctx context.Context, idx int) error {
 	if err := buildArchiveFromFiles(ctx, path.Join(layerDir, tociLayerTar), compression.Gzip, files...); err != nil {
 		return errors.Wrapf(err, "failed to create turboOCIv1 archive for layer %d", idx)
 	}
-	e.overlaybdConfig.Lowers = append(e.overlaybdConfig.Lowers, snapshot.OverlayBDBSConfigLower{
+	e.overlaybdConfig.Lowers = append(e.overlaybdConfig.Lowers, sn.OverlayBDBSConfigLower{
 		TargetFile:   path.Join(layerDir, "layer.tar"),
 		TargetDigest: string(e.manifest.Layers[idx].Digest), // TargetDigest should be set to work with gzip cache
 		File:         path.Join(layerDir, fsMetaFile),

pkg/label/label.go

@@ -92,6 +92,9 @@ const (
 
 	// OverlayBDVersion is the version number of overlaybd blob
 	OverlayBDVersion = "containerd.io/snapshot/overlaybd/version"
+
+	// LayerToTurboOCI is used to convert local layer to turboOCI with tar index
+	LayerToTurboOCI = "containerd.io/snapshot/overlaybd/convert2turbo-oci"
 )
 
 // used in filterAnnotationsForSave (https://github.com/moby/buildkit/blob/v0.11/cache/refs.go#L882)

pkg/snapshot/overlay.go

@@ -59,12 +59,19 @@ const (
 	// storageTypeRemoteBlock means that there is no unpacked layer data.
 	// But there are labels to mark data that will be pulling on demand.
 	storageTypeRemoteBlock
+
+	// storageTypeLocalLayer means that the unpacked layer data is in
+	// a tar file, which needs to generate overlaybd-turboOCI meta before
+	// create an overlaybd device
+	storageTypeLocalLayer
 )
 
 const (
 	RoDir = "overlayfs" // overlayfs as rootfs. upper + lower (overlaybd)
 	RwDir = "dir"       // mount overlaybd as rootfs
 	RwDev = "dev"       // use overlaybd directly
+
+	LayerBlob = "layer" // decompressed tgz layer (maybe compressed by ZFile)
 )
 
 type Registry struct {
@@ -574,8 +581,20 @@ func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind,
 	var m []mount.Mount
 	switch stype {
 	case storageTypeNormal:
-		m = o.normalOverlayMount(s)
-		log.G(ctx).Debugf("return mount point(R/W mode: %s): %v", writeType, m)
+		if _, ok := info.Labels[label.LayerToTurboOCI]; ok {
+			m = []mount.Mount{
+				{
+					Source: o.upperPath(s.ID),
+					Type:   "bind",
+					Options: []string{
+						"rw",
+						"rbind",
+					},
+				},
+			}
+		} else {
+			m = o.normalOverlayMount(s)
+		}
 	case storageTypeLocalBlock, storageTypeRemoteBlock:
 		m, err = o.basedOnBlockDeviceMount(ctx, s, writeType)
 		if err != nil {
@@ -584,6 +603,7 @@ func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind,
 	default:
 		panic("unreachable")
 	}
+	log.G(ctx).Debugf("return mount point(R/W mode: %s): %v", writeType, m)
 	return m, nil
 
 }
@@ -595,8 +615,15 @@ func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s
 	defer func() {
 		if retErr != nil {
 			metrics.GRPCErrCount.WithLabelValues("Prepare").Inc()
+		} else {
+			log.G(ctx).WithFields(logrus.Fields{
+				"d":      time.Since(start),
+				"key":    key,
+				"parent": parent,
+			}).Infof("Prepare")
 		}
 		metrics.GRPCLatency.WithLabelValues("Prepare").Observe(time.Since(start).Seconds())
+
 	}()
 	return o.createMountPoint(ctx, snapshots.KindActive, key, parent, opts...)
 }
@@ -626,6 +653,11 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) (_ []mount.Mount,
 	defer func() {
 		if retErr != nil {
 			metrics.GRPCErrCount.WithLabelValues("Mounts").Inc()
+		} else {
+			log.G(ctx).WithFields(logrus.Fields{
+				"d":   time.Since(start),
+				"key": key,
+			}).Infof("Mounts")
 		}
 		metrics.GRPCLatency.WithLabelValues("Mounts").Observe(time.Since(start).Seconds())
 	}()
@@ -691,6 +723,12 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap
 	defer func() {
 		if retErr != nil {
 			metrics.GRPCErrCount.WithLabelValues("Commit").Inc()
+		} else {
+			log.G(ctx).WithFields(logrus.Fields{
+				"d":    time.Since(start),
+				"name": name,
+				"key":  key,
+			}).Infof("Commit")
 		}
 		metrics.GRPCLatency.WithLabelValues("Commit").Observe(time.Since(start).Seconds())
 	}()
@@ -746,6 +784,15 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap
 	}
 
 	log.G(ctx).Debugf("Commit info (id: %s, info: %v, stype: %d)", id, info.Labels, stype)
+	// Firstly, try to convert an OCIv1 tarball to a turboOCI layer.
+	// then change stype to 'storageTypeLocalBlock' which can make it attach a overlaybd block
+	if stype == storageTypeLocalLayer {
+		log.G(ctx).Infof("convet a local blob to turboOCI layer (sn: %s)", id)
+		if err := o.constructOverlayBDSpec(ctx, name, false); err != nil {
+			return errors.Wrapf(err, "failed to construct overlaybd config")
+		}
+		stype = storageTypeLocalBlock
+	}
 
 	if stype == storageTypeLocalBlock {
 		if err := o.constructOverlayBDSpec(ctx, name, false); err != nil {
@@ -1158,6 +1205,13 @@ func (o *snapshotter) identifySnapshotStorageType(ctx context.Context, id string
 	if err == nil {
 		return st, nil
 	}
+
+	// check layer.tar if it should be converted to turboOCI
+	filePath = o.overlaybdOCILayerPath(id)
+	if _, err := os.Stat(filePath); err == nil {
+		log.G(ctx).Infof("uncompressed layer found in sn: %s", id)
+		return storageTypeLocalLayer, nil
+	}
 	if os.IsNotExist(err) {
 		// check config.v1.json
 		log.G(ctx).Debugf("failed to identify by writable_data(sn: %s), try to identify by config.v1.json", id)
@@ -1168,6 +1222,7 @@ func (o *snapshotter) identifySnapshotStorageType(ctx context.Context, id string
 		}
 		return storageTypeNormal, nil
 	}
+
 	log.G(ctx).Debugf("storageType(sn: %s): %d", id, st)
 
 	return st, err
@@ -1185,10 +1240,18 @@ func (o *snapshotter) workPath(id string) string {
 	return filepath.Join(o.root, "snapshots", id, "work")
 }
 
+func (o *snapshotter) convertTempdir(id string) string {
+	return filepath.Join(o.root, "snapshots", id, "temp")
+}
+
 func (o *snapshotter) blockPath(id string) string {
 	return filepath.Join(o.root, "snapshots", id, "block")
 }
 
+func (o *snapshotter) turboOCIFsMeta(id string) string {
+	return filepath.Join(o.root, "snapshots", id, "fs", "ext4.fs.meta")
+}
+
 func (o *snapshotter) magicFilePath(id string) string {
 	return filepath.Join(o.root, "snapshots", id, "fs", "overlaybd.commit")
 }
@@ -1217,12 +1280,16 @@ func (o *snapshotter) overlaybdWritableDataPath(id string) string {
 	return filepath.Join(o.root, "snapshots", id, "block", "writable_data")
 }
 
-func (o *snapshotter) overlaybdBackstoreMarkFile(id string) string {
-	return filepath.Join(o.root, "snapshots", id, "block", "backstore_mark")
+func (o *snapshotter) overlaybdOCILayerPath(id string) string {
+	return filepath.Join(o.root, "snapshots", id, "layer.tar")
 }
 
-func (o *snapshotter) turboOCIFsMeta(id string) string {
-	return filepath.Join(o.root, "snapshots", id, "fs", "ext4.fs.meta")
+func (o *snapshotter) overlaybdOCILayerMeta(id string) string {
+	return filepath.Join(o.root, "snapshots", id, "layer.metadata")
+}
+
+func (o *snapshotter) overlaybdBackstoreMarkFile(id string) string {
+	return filepath.Join(o.root, "snapshots", id, "block", "backstore_mark")
 }
 
 func (o *snapshotter) turboOCIGzipIdx(id string) string {