From a975fe954b3a6abbd57d0154147f77bffea62e66 Mon Sep 17 00:00:00 2001 From: Yifan Yuan Date: Thu, 1 Feb 2024 18:31:38 +0800 Subject: [PATCH] [Feat.] Converting ociv1 layer blob to turboOCI blob locally. With proxyDiffer, containerd can decompress tgz layer and convert it to turboOCI blob without unpack tarball It is very effective in reducing disk I/O pressure because of replacing unpack multiple files with generating ext4 fsmeta. Signed-off-by: Yifan Yuan --- cmd/convertor/builder/builder_utils.go | 4 +- cmd/convertor/builder/builder_utils_test.go | 10 +- cmd/convertor/builder/overlaybd_builder.go | 14 +-- cmd/convertor/builder/turboOCI_builder.go | 15 +-- pkg/label/label.go | 3 + pkg/snapshot/overlay.go | 75 +++++++++++- pkg/snapshot/storage.go | 108 ++++++++++------- pkg/types/types.go | 58 ++++++++++ pkg/utils/cmd.go | 122 +++++++++++++++++++- 9 files changed, 338 insertions(+), 71 deletions(-) create mode 100644 pkg/types/types.go diff --git a/cmd/convertor/builder/builder_utils.go b/cmd/convertor/builder/builder_utils.go index 3010d9b8..39e8c7ae 100644 --- a/cmd/convertor/builder/builder_utils.go +++ b/cmd/convertor/builder/builder_utils.go @@ -40,7 +40,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/containerd/accelerated-container-image/pkg/snapshot" + t "github.com/containerd/accelerated-container-image/pkg/types" ) func fetch(ctx context.Context, fetcher remotes.Fetcher, desc specs.Descriptor, target any) error { @@ -144,7 +144,7 @@ func downloadLayer(ctx context.Context, fetcher remotes.Fetcher, targetFile stri } // TODO maybe refactor this -func writeConfig(dir string, configJSON *snapshot.OverlayBDBSConfig) error { +func writeConfig(dir string, configJSON *t.OverlayBDBSConfig) error { data, err := json.Marshal(configJSON) if err != nil { return err diff --git a/cmd/convertor/builder/builder_utils_test.go b/cmd/convertor/builder/builder_utils_test.go index bf8ee9ac..6b38de3e 100644 --- a/cmd/convertor/builder/builder_utils_test.go +++ b/cmd/convertor/builder/builder_utils_test.go @@ -28,7 +28,7 @@ import ( "testing" testingresources "github.com/containerd/accelerated-container-image/cmd/convertor/testingresources" - "github.com/containerd/accelerated-container-image/pkg/snapshot" + sn "github.com/containerd/accelerated-container-image/pkg/types" "github.com/containerd/containerd/images" _ "github.com/containerd/containerd/pkg/testutil" // Handle custom root flag "github.com/containerd/containerd/remotes" @@ -428,9 +428,9 @@ func Test_downloadLayer(t *testing.T) { func Test_writeConfig(t *testing.T) { ctx := context.Background() testingresources.RunTestWithTempDir(t, ctx, "writeConfigMinimal", func(t *testing.T, ctx context.Context, workdir string) { - configSample := snapshot.OverlayBDBSConfig{ + configSample := sn.OverlayBDBSConfig{ ResultFile: "", - Lowers: []snapshot.OverlayBDBSConfigLower{ + Lowers: []sn.OverlayBDBSConfigLower{ { File: overlaybdBaseLayer, }, @@ -438,7 +438,7 @@ func Test_writeConfig(t *testing.T) { File: path.Join(workdir, commitFile), }, }, - Upper: snapshot.OverlayBDBSConfigUpper{ + Upper: sn.OverlayBDBSConfigUpper{ Data: path.Join(workdir, "writable_data"), Index: path.Join(workdir, "writable_index"), }, @@ -455,7 +455,7 @@ func Test_writeConfig(t *testing.T) { } defer file.Close() - configRes := snapshot.OverlayBDBSConfig{} + configRes := sn.OverlayBDBSConfig{} err = json.NewDecoder(file).Decode(&configRes) if err != nil { diff --git a/cmd/convertor/builder/overlaybd_builder.go b/cmd/convertor/builder/overlaybd_builder.go index 28aa35e5..315312d8 100644 --- a/cmd/convertor/builder/overlaybd_builder.go +++ b/cmd/convertor/builder/overlaybd_builder.go @@ -26,7 +26,7 @@ import ( "path" "github.com/containerd/accelerated-container-image/pkg/label" - "github.com/containerd/accelerated-container-image/pkg/snapshot" + sn "github.com/containerd/accelerated-container-image/pkg/types" "github.com/containerd/accelerated-container-image/pkg/utils" "github.com/containerd/accelerated-container-image/pkg/version" "github.com/containerd/containerd/errdefs" @@ -50,17 +50,17 @@ type overlaybdConvertResult struct { type overlaybdBuilderEngine struct { *builderEngineBase - overlaybdConfig *snapshot.OverlayBDBSConfig + overlaybdConfig *sn.OverlayBDBSConfig overlaybdLayers []overlaybdConvertResult } func NewOverlayBDBuilderEngine(base *builderEngineBase) builderEngine { - config := &snapshot.OverlayBDBSConfig{ - Lowers: []snapshot.OverlayBDBSConfigLower{}, + config := &sn.OverlayBDBSConfig{ + Lowers: []sn.OverlayBDBSConfigLower{}, ResultFile: "", } if !base.mkfs { - config.Lowers = append(config.Lowers, snapshot.OverlayBDBSConfigLower{ + config.Lowers = append(config.Lowers, sn.OverlayBDBSConfigLower{ File: overlaybdBaseLayer, }) logrus.Infof("using default baselayer") @@ -111,7 +111,7 @@ func (e *overlaybdBuilderEngine) BuildLayer(ctx context.Context, idx int) error if err := e.create(ctx, layerDir, mkfs, vsizeGB); err != nil { return err } - e.overlaybdConfig.Upper = snapshot.OverlayBDBSConfigUpper{ + e.overlaybdConfig.Upper = sn.OverlayBDBSConfigUpper{ Data: path.Join(layerDir, "writable_data"), Index: path.Join(layerDir, "writable_index"), } @@ -130,7 +130,7 @@ func (e *overlaybdBuilderEngine) BuildLayer(ctx context.Context, idx int) error os.Remove(path.Join(layerDir, "writable_index")) } } - e.overlaybdConfig.Lowers = append(e.overlaybdConfig.Lowers, snapshot.OverlayBDBSConfigLower{ + e.overlaybdConfig.Lowers = append(e.overlaybdConfig.Lowers, sn.OverlayBDBSConfigLower{ File: path.Join(layerDir, commitFile), }) return nil diff --git a/cmd/convertor/builder/turboOCI_builder.go b/cmd/convertor/builder/turboOCI_builder.go index 984b442d..4054434b 100644 --- a/cmd/convertor/builder/turboOCI_builder.go +++ b/cmd/convertor/builder/turboOCI_builder.go @@ -23,7 +23,7 @@ import ( "path" "github.com/containerd/accelerated-container-image/pkg/label" - "github.com/containerd/accelerated-container-image/pkg/snapshot" + sn "github.com/containerd/accelerated-container-image/pkg/types" "github.com/containerd/accelerated-container-image/pkg/utils" "github.com/containerd/accelerated-container-image/pkg/version" "github.com/containerd/containerd/archive/compression" @@ -31,6 +31,7 @@ import ( "github.com/containerd/containerd/images" "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -51,18 +52,18 @@ const ( type turboOCIBuilderEngine struct { *builderEngineBase - overlaybdConfig *snapshot.OverlayBDBSConfig + overlaybdConfig *sn.OverlayBDBSConfig tociLayers []specs.Descriptor isGzip []bool } func NewTurboOCIBuilderEngine(base *builderEngineBase) builderEngine { - config := &snapshot.OverlayBDBSConfig{ - Lowers: []snapshot.OverlayBDBSConfigLower{}, + config := &sn.OverlayBDBSConfig{ + Lowers: []sn.OverlayBDBSConfigLower{}, ResultFile: "", } if !base.mkfs { - config.Lowers = append(config.Lowers, snapshot.OverlayBDBSConfigLower{ + config.Lowers = append(config.Lowers, sn.OverlayBDBSConfigLower{ File: overlaybdBaseLayer, }) logrus.Infof("using default baselayer") @@ -91,7 +92,7 @@ func (e *turboOCIBuilderEngine) BuildLayer(ctx context.Context, idx int) error { if err := e.create(ctx, layerDir, e.mkfs && (idx == 0)); err != nil { return err } - e.overlaybdConfig.Upper = snapshot.OverlayBDBSConfigUpper{ + e.overlaybdConfig.Upper = sn.OverlayBDBSConfigUpper{ Data: path.Join(layerDir, "writable_data"), Index: path.Join(layerDir, "writable_index"), Target: path.Join(layerDir, "layer.tar"), @@ -120,7 +121,7 @@ func (e *turboOCIBuilderEngine) BuildLayer(ctx context.Context, idx int) error { if err := buildArchiveFromFiles(ctx, path.Join(layerDir, tociLayerTar), compression.Gzip, files...); err != nil { return errors.Wrapf(err, "failed to create turboOCIv1 archive for layer %d", idx) } - e.overlaybdConfig.Lowers = append(e.overlaybdConfig.Lowers, snapshot.OverlayBDBSConfigLower{ + e.overlaybdConfig.Lowers = append(e.overlaybdConfig.Lowers, sn.OverlayBDBSConfigLower{ TargetFile: path.Join(layerDir, "layer.tar"), TargetDigest: string(e.manifest.Layers[idx].Digest), // TargetDigest should be set to work with gzip cache File: path.Join(layerDir, fsMetaFile), diff --git a/pkg/label/label.go b/pkg/label/label.go index dda6abb6..ec4133f0 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -92,6 +92,9 @@ const ( // OverlayBDVersion is the version number of overlaybd blob OverlayBDVersion = "containerd.io/snapshot/overlaybd/version" + + // LayerToTurboOCI is used to convert local layer to turboOCI with tar index + LayerToTurboOCI = "containerd.io/snapshot/overlaybd/convert2turbo-oci" ) // used in filterAnnotationsForSave (https://github.com/moby/buildkit/blob/v0.11/cache/refs.go#L882) diff --git a/pkg/snapshot/overlay.go b/pkg/snapshot/overlay.go index 0ebd068a..bab80bee 100644 --- a/pkg/snapshot/overlay.go +++ b/pkg/snapshot/overlay.go @@ -59,12 +59,19 @@ const ( // storageTypeRemoteBlock means that there is no unpacked layer data. // But there are labels to mark data that will be pulling on demand. storageTypeRemoteBlock + + // storageTypeLocalLayer means that the unpacked layer data is in + // a tar file, which needs to generate overlaybd-turboOCI meta before + // create an overlaybd device + storageTypeLocalLayer ) const ( RoDir = "overlayfs" // overlayfs as rootfs. upper + lower (overlaybd) RwDir = "dir" // mount overlaybd as rootfs RwDev = "dev" // use overlaybd directly + + LayerBlob = "layer" // decompressed tgz layer (maybe compressed by ZFile) ) type Registry struct { @@ -574,8 +581,20 @@ func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind, var m []mount.Mount switch stype { case storageTypeNormal: - m = o.normalOverlayMount(s) - log.G(ctx).Debugf("return mount point(R/W mode: %s): %v", writeType, m) + if _, ok := info.Labels[label.LayerToTurboOCI]; ok { + m = []mount.Mount{ + { + Source: o.upperPath(s.ID), + Type: "bind", + Options: []string{ + "rw", + "rbind", + }, + }, + } + } else { + m = o.normalOverlayMount(s) + } case storageTypeLocalBlock, storageTypeRemoteBlock: m, err = o.basedOnBlockDeviceMount(ctx, s, writeType) if err != nil { @@ -584,6 +603,7 @@ func (o *snapshotter) createMountPoint(ctx context.Context, kind snapshots.Kind, default: panic("unreachable") } + log.G(ctx).Debugf("return mount point(R/W mode: %s): %v", writeType, m) return m, nil } @@ -595,8 +615,15 @@ func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s defer func() { if retErr != nil { metrics.GRPCErrCount.WithLabelValues("Prepare").Inc() + } else { + log.G(ctx).WithFields(logrus.Fields{ + "d": time.Since(start), + "key": key, + "parent": parent, + }).Infof("Prepare") } metrics.GRPCLatency.WithLabelValues("Prepare").Observe(time.Since(start).Seconds()) + }() return o.createMountPoint(ctx, snapshots.KindActive, key, parent, opts...) } @@ -626,6 +653,11 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) (_ []mount.Mount, defer func() { if retErr != nil { metrics.GRPCErrCount.WithLabelValues("Mounts").Inc() + } else { + log.G(ctx).WithFields(logrus.Fields{ + "d": time.Since(start), + "key": key, + }).Infof("Mounts") } metrics.GRPCLatency.WithLabelValues("Mounts").Observe(time.Since(start).Seconds()) }() @@ -691,6 +723,12 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap defer func() { if retErr != nil { metrics.GRPCErrCount.WithLabelValues("Commit").Inc() + } else { + log.G(ctx).WithFields(logrus.Fields{ + "d": time.Since(start), + "name": name, + "key": key, + }).Infof("Commit") } metrics.GRPCLatency.WithLabelValues("Commit").Observe(time.Since(start).Seconds()) }() @@ -746,6 +784,15 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap } log.G(ctx).Debugf("Commit info (id: %s, info: %v, stype: %d)", id, info.Labels, stype) + // Firstly, try to convert an OCIv1 tarball to a turboOCI layer. + // then change stype to 'storageTypeLocalBlock' which can make it attach a overlaybd block + if stype == storageTypeLocalLayer { + log.G(ctx).Infof("convet a local blob to turboOCI layer (sn: %s)", id) + if err := o.constructOverlayBDSpec(ctx, name, false); err != nil { + return errors.Wrapf(err, "failed to construct overlaybd config") + } + stype = storageTypeLocalBlock + } if stype == storageTypeLocalBlock { if err := o.constructOverlayBDSpec(ctx, name, false); err != nil { @@ -1158,6 +1205,13 @@ func (o *snapshotter) identifySnapshotStorageType(ctx context.Context, id string if err == nil { return st, nil } + + // check layer if it should be converted to turboOCI + filePath = o.overlaybdOCILayerPath(id) + if _, err := os.Stat(filePath); err == nil { + log.G(ctx).Infof("uncompressed layer found in sn: %s", id) + return storageTypeLocalLayer, nil + } if os.IsNotExist(err) { // check config.v1.json log.G(ctx).Debugf("failed to identify by writable_data(sn: %s), try to identify by config.v1.json", id) @@ -1168,6 +1222,7 @@ func (o *snapshotter) identifySnapshotStorageType(ctx context.Context, id string } return storageTypeNormal, nil } + log.G(ctx).Debugf("storageType(sn: %s): %d", id, st) return st, err @@ -1185,6 +1240,10 @@ func (o *snapshotter) workPath(id string) string { return filepath.Join(o.root, "snapshots", id, "work") } +func (o *snapshotter) convertTempdir(id string) string { + return filepath.Join(o.root, "snapshots", id, "temp") +} + func (o *snapshotter) blockPath(id string) string { return filepath.Join(o.root, "snapshots", id, "block") } @@ -1217,12 +1276,16 @@ func (o *snapshotter) overlaybdWritableDataPath(id string) string { return filepath.Join(o.root, "snapshots", id, "block", "writable_data") } -func (o *snapshotter) overlaybdBackstoreMarkFile(id string) string { - return filepath.Join(o.root, "snapshots", id, "block", "backstore_mark") +func (o *snapshotter) overlaybdOCILayerPath(id string) string { + return filepath.Join(o.root, "snapshots", id, "layer.tar") +} + +func (o *snapshotter) overlaybdOCILayerMeta(id string) string { + return filepath.Join(o.root, "snapshots", id, "layer.metadata") } -func (o *snapshotter) turboOCIFsMeta(id string) string { - return filepath.Join(o.root, "snapshots", id, "fs", "ext4.fs.meta") +func (o *snapshotter) overlaybdBackstoreMarkFile(id string) string { + return filepath.Join(o.root, "snapshots", id, "block", "backstore_mark") } func (o *snapshotter) turboOCIGzipIdx(id string) string { diff --git a/pkg/snapshot/storage.go b/pkg/snapshot/storage.go index d900092b..c3fd874e 100644 --- a/pkg/snapshot/storage.go +++ b/pkg/snapshot/storage.go @@ -31,6 +31,8 @@ import ( "strings" "time" + sn "github.com/containerd/accelerated-container-image/pkg/types" + "github.com/containerd/accelerated-container-image/pkg/label" "github.com/containerd/accelerated-container-image/pkg/utils" "github.com/containerd/containerd/images" @@ -65,34 +67,6 @@ const ( obdMaxDataAreaMB = 4 ) -// OverlayBDBSConfig is the config of overlaybd target. -type OverlayBDBSConfig struct { - RepoBlobURL string `json:"repoBlobUrl"` - Lowers []OverlayBDBSConfigLower `json:"lowers"` - Upper OverlayBDBSConfigUpper `json:"upper"` - ResultFile string `json:"resultFile"` - AccelerationLayer bool `json:"accelerationLayer,omitempty"` - RecordTracePath string `json:"recordTracePath,omitempty"` -} - -// OverlayBDBSConfigLower -type OverlayBDBSConfigLower struct { - GzipIndex string `json:"gzipIndex,omitempty"` - File string `json:"file,omitempty"` - Digest string `json:"digest,omitempty"` - TargetFile string `json:"targetFile,omitempty"` - TargetDigest string `json:"targetDigest,omitempty"` - Size int64 `json:"size,omitempty"` - Dir string `json:"dir,omitempty"` -} - -type OverlayBDBSConfigUpper struct { - Index string `json:"index,omitempty"` - Data string `json:"data,omitempty"` - Target string `json:"target,omitempty"` - GzipIndex string `json:"gzipIndex,omitempty"` -} - func (o *snapshotter) checkOverlaybdInUse(ctx context.Context, dir string) (bool, error) { f, err := os.Open("/proc/self/mountinfo") if err != nil { @@ -473,8 +447,8 @@ func (o *snapshotter) constructOverlayBDSpec(ctx context.Context, key string, wr return errors.Wrapf(err, "failed to identify storage of snapshot %s", key) } - configJSON := OverlayBDBSConfig{ - Lowers: []OverlayBDBSConfigLower{}, + configJSON := sn.OverlayBDBSConfig{ + Lowers: []sn.OverlayBDBSConfigLower{}, ResultFile: o.overlaybdInitDebuglogPath(id), } @@ -521,9 +495,9 @@ func (o *snapshotter) constructOverlayBDSpec(ctx context.Context, key string, wr configJSON.RepoBlobURL = blobPrefixURL if isTurboOCI, dataDgst, compType := o.checkTurboOCI(info.Labels); isTurboOCI { - lower := OverlayBDBSConfigLower{ + lower := sn.OverlayBDBSConfigLower{ Dir: o.upperPath(id), - File: o.turboOCIFsMeta(id), + File: o.magicFilePath(id), TargetDigest: dataDgst, } if isGzipLayerType(compType) { @@ -531,7 +505,7 @@ func (o *snapshotter) constructOverlayBDSpec(ctx context.Context, key string, wr } configJSON.Lowers = append(configJSON.Lowers, lower) } else { - configJSON.Lowers = append(configJSON.Lowers, OverlayBDBSConfigLower{ + configJSON.Lowers = append(configJSON.Lowers, sn.OverlayBDBSConfigLower{ Digest: blobDigest, Size: int64(blobSize), Dir: o.upperPath(id), @@ -542,11 +516,63 @@ func (o *snapshotter) constructOverlayBDSpec(ctx context.Context, key string, wr if writable { return errors.Errorf("local block device is readonly, not support writable") } + ociBlob := o.overlaybdOCILayerPath(id) + if _, err := os.Stat(ociBlob); err == nil { + log.G(ctx).Debugf("OCI layer blob found, construct overlaybd config with turboOCI (sn: %s)", id) + configJSON.Lowers = append(configJSON.Lowers, sn.OverlayBDBSConfigLower{ + TargetFile: o.overlaybdOCILayerPath(id), + File: o.magicFilePath(id), + }) + } else { + configJSON.Lowers = append(configJSON.Lowers, sn.OverlayBDBSConfigLower{ + Dir: o.upperPath(id), + // automatically find overlaybd.commit + }) + } + case storageTypeLocalLayer: + // 1. generate tar meta for oci layer blob + // 2. convert local layer.tarmeta to overlaybd + // 3. create layer's config + log.G(ctx).Infof("generate metadata of layer blob (sn: %s)", id) + if err := utils.GenerateTarMeta(ctx, o.overlaybdOCILayerPath(id), o.overlaybdOCILayerMeta(id)); err != nil { + log.G(ctx).Errorf("generate tar metadata failed. (sn: %s)", id) + return err + } + opt := &sn.ConvertOption{ + TarMetaPath: o.overlaybdOCILayerMeta(id), + Workdir: o.convertTempdir(id), + Ext4FSMetaPath: o.magicFilePath(id), + Config: configJSON, + } + log.G(ctx).Infof("convert layer to turboOCI (sn: %s)", id) - configJSON.Lowers = append(configJSON.Lowers, OverlayBDBSConfigLower{ - Dir: o.upperPath(id), + if err := utils.ConvertLayer(ctx, opt); err != nil { + log.G(ctx).Error(err.Error()) + os.Rename(o.snPath(id), o.snPath(id)+".err") + // os.RemoveAll(opt.Workdir) + // os.Remove(opt.Ext4FSMetaPath) + return err + } + // lb, _ := os.Open(o.overlaybdOCILayerPath(id)) + // hasher := sha256.New() + + // // 读取文件并更新哈希 + // if _, err := io.Copy(hasher, lb); err != nil { + // log.G(ctx).Errorf("Failed to copy file content to hasher: %s", err) + // return err + // } + // lb.Close() + + // // 计算最终哈希值 + // sum := fmt.Sprintf("sha256:%x.blob", hasher.Sum(nil)) + // targetFn := path.Join(o.snPath(id), sum) + // log.G(ctx).Debugf("rename layer.tar to %s", sum) + // os.Rename(o.overlaybdOCILayerPath(id), targetFn) + configJSON.Lowers = append(configJSON.Lowers, sn.OverlayBDBSConfigLower{ + TargetFile: o.overlaybdOCILayerPath(id), + File: opt.Ext4FSMetaPath, }) - + log.G(ctx).Debugf("generate config.json for %s:\n %+v", id, configJSON) default: if !writable { return errors.Errorf("unexpect storage %v of snapshot %v during construct overlaybd spec(writable=%v, parent=%s)", stype, key, writable, info.Parent) @@ -567,7 +593,7 @@ func (o *snapshotter) constructOverlayBDSpec(ctx context.Context, key string, wr return err } - configJSON.Upper = OverlayBDBSConfigUpper{ + configJSON.Upper = sn.OverlayBDBSConfigUpper{ Index: o.overlaybdWritableIndexPath(id), Data: o.overlaybdWritableDataPath(id), } @@ -582,7 +608,7 @@ func (o *snapshotter) constructSpecForAccelLayer(id, parentID string) error { if err != nil { return err } - accelLayerLower := OverlayBDBSConfigLower{Dir: o.upperPath(id)} + accelLayerLower := sn.OverlayBDBSConfigLower{Dir: o.upperPath(id)} config.Lowers = append(config.Lowers, accelLayerLower) return o.atomicWriteOverlaybdTargetConfig(id, config) } @@ -606,14 +632,14 @@ func (o *snapshotter) updateSpec(snID string, isAccelLayer bool, recordTracePath } // loadBackingStoreConfig loads overlaybd target config. -func (o *snapshotter) loadBackingStoreConfig(snID string) (*OverlayBDBSConfig, error) { +func (o *snapshotter) loadBackingStoreConfig(snID string) (*sn.OverlayBDBSConfig, error) { confPath := o.overlaybdConfPath(snID) data, err := os.ReadFile(confPath) if err != nil { return nil, errors.Wrapf(err, "failed to read config(path=%s) of snapshot %s", confPath, snID) } - var configJSON OverlayBDBSConfig + var configJSON sn.OverlayBDBSConfig if err := json.Unmarshal(data, &configJSON); err != nil { return nil, errors.Wrapf(err, "failed to unmarshal data(%s)", string(data)) } @@ -645,7 +671,7 @@ func (o *snapshotter) constructImageBlobURL(ref string) (string, error) { } // atomicWriteOverlaybdTargetConfig -func (o *snapshotter) atomicWriteOverlaybdTargetConfig(snID string, configJSON *OverlayBDBSConfig) error { +func (o *snapshotter) atomicWriteOverlaybdTargetConfig(snID string, configJSON *sn.OverlayBDBSConfig) error { data, err := json.Marshal(configJSON) if err != nil { return errors.Wrapf(err, "failed to marshal %+v configJSON into JSON", configJSON) diff --git a/pkg/types/types.go b/pkg/types/types.go new file mode 100644 index 00000000..feefd54b --- /dev/null +++ b/pkg/types/types.go @@ -0,0 +1,58 @@ +/* + Copyright The Accelerated Container Image Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package types + +// OverlayBDBSConfig is the config of overlaybd target. +type OverlayBDBSConfig struct { + RepoBlobURL string `json:"repoBlobUrl"` + Lowers []OverlayBDBSConfigLower `json:"lowers"` + Upper OverlayBDBSConfigUpper `json:"upper"` + ResultFile string `json:"resultFile"` + AccelerationLayer bool `json:"accelerationLayer,omitempty"` + RecordTracePath string `json:"recordTracePath,omitempty"` +} + +// OverlayBDBSConfigLower +type OverlayBDBSConfigLower struct { + GzipIndex string `json:"gzipIndex,omitempty"` + File string `json:"file,omitempty"` + Digest string `json:"digest,omitempty"` + TargetFile string `json:"targetFile,omitempty"` + TargetDigest string `json:"targetDigest,omitempty"` + Size int64 `json:"size,omitempty"` + Dir string `json:"dir,omitempty"` +} + +type OverlayBDBSConfigUpper struct { + Index string `json:"index,omitempty"` + Data string `json:"data,omitempty"` + Target string `json:"target,omitempty"` + GzipIndex string `json:"gzipIndex,omitempty"` +} + +type ConvertOption struct { + // src options + // (TODO) LayerPath string // path of layer.tgz or layer.tar + TarMetaPath string // path of layer.tar.meta + + Config OverlayBDBSConfig + Workdir string + + // output options + Ext4FSMetaPath string + // (TODO) GzipIndexPath string +} diff --git a/pkg/utils/cmd.go b/pkg/utils/cmd.go index a955ac10..efd2fbc6 100644 --- a/pkg/utils/cmd.go +++ b/pkg/utils/cmd.go @@ -18,19 +18,24 @@ package utils import ( "context" + "encoding/json" + "fmt" "os" "os/exec" "path" + "path/filepath" "strings" + sn "github.com/containerd/accelerated-container-image/pkg/types" "github.com/containerd/containerd/log" "github.com/pkg/errors" ) const ( - obdBinCreate = "/opt/overlaybd/bin/overlaybd-create" - obdBinCommit = "/opt/overlaybd/bin/overlaybd-commit" - obdBinApply = "/opt/overlaybd/bin/overlaybd-apply" + obdBinCreate = "/opt/overlaybd/bin/overlaybd-create" + obdBinCommit = "/opt/overlaybd/bin/overlaybd-commit" + obdBinApply = "/opt/overlaybd/bin/overlaybd-apply" + obdBinTurboOCIApply = "/opt/overlaybd/bin/turboOCI-apply" dataFile = "writable_data" idxFile = "writable_index" @@ -39,6 +44,31 @@ const ( commitFile = "overlaybd.commit" ) +var defaultServiceTemplate = ` +{ + "registryFsVersion": "v2", + "logPath": "", + "logLevel": 1, + "cacheConfig": { + "cacheType": "file", + "cacheDir": "%s", + "cacheSizeGB": 4 + }, + "gzipCacheConfig": { + "enable": false + }, + "credentialConfig": { + "mode": "file", + "path": "" + }, + "ioEngine": 0, + "download": { + "enable": false + }, + "enableAudit": false +} +` + func Create(ctx context.Context, dir string, opts ...string) error { dataPath := path.Join(dir, dataFile) indexPath := path.Join(dir, idxFile) @@ -123,3 +153,89 @@ func ApplyTurboOCI(ctx context.Context, dir, gzipMetaFile string, opts ...string } return nil } + +func GenerateTarMeta(ctx context.Context, srcTarFile string, dstTarMeta string) error { + + if _, err := os.Stat(srcTarFile); os.IsNotExist(err) { + return nil + } else if err != nil { + return fmt.Errorf("error stating tar file: %w", err) + } + log.G(ctx).Infof("generate layer meta for %s", srcTarFile) + if err := exec.Command(obdBinTurboOCIApply, srcTarFile, dstTarMeta, "--export").Run(); err != nil { + return fmt.Errorf("failed to convert tar file to overlaybd device: %w", err) + } + return nil +} + +// ConvertLayer produce a turbooci layer, target is path of ext4.fs.meta +func ConvertLayer(ctx context.Context, opt *sn.ConvertOption) error { + if opt.Workdir == "" { + opt.Workdir = "tmp_conv" + } + + if err := os.MkdirAll(opt.Workdir, 0755); err != nil { + return fmt.Errorf("failed to create work dir: %w", err) + } + + pathWritableData := filepath.Join(opt.Workdir, "writable_data") + pathWritableIndex := filepath.Join(opt.Workdir, "writable_index") + pathFakeTarget := filepath.Join(opt.Workdir, "fake_target") + pathService := filepath.Join(opt.Workdir, "service.json") + pathConfig := filepath.Join(opt.Workdir, "config.v1.json") + + // overlaybd-create + args := []string{pathWritableData, pathWritableIndex, "256", "-s", "--turboOCI"} + if len(opt.Config.Lowers) == 0 { + args = append(args, "--mkfs") + } + if out, err := exec.CommandContext(ctx, obdBinCreate, args...).CombinedOutput(); err != nil { + return fmt.Errorf("failed to overlaybd-create: %w, output: %s", err, out) + } + file, err := os.Create(pathFakeTarget) + if err != nil { + return fmt.Errorf("failed to create fake target: %w", err) + } + file.Close() + opt.Config.Upper = sn.OverlayBDBSConfigUpper{ + Data: pathWritableData, + Index: pathWritableIndex, + Target: pathFakeTarget, + } + + // turboOCI-apply + if err := os.WriteFile(pathService, []byte(fmt.Sprintf(defaultServiceTemplate, + filepath.Join(opt.Workdir, "cache"))), 0644, + ); err != nil { + return fmt.Errorf("failed to write service.json: %w", err) + } + configBytes, err := json.Marshal(opt.Config) + if err != nil { + return fmt.Errorf("failed to marshal overlaybd config: %w", err) + } + if err := os.WriteFile(pathConfig, configBytes, 0644); err != nil { + return fmt.Errorf("failed to write overlaybd config: %w", err) + } + args = []string{ + opt.TarMetaPath, pathConfig, + "--import", + "--service_config_path", pathService, + } + log.G(ctx).Debugf("%s %s", obdBinTurboOCIApply, strings.Join(args, " ")) + if out, err := exec.CommandContext(ctx, obdBinTurboOCIApply, + args..., + ).CombinedOutput(); err != nil { + return fmt.Errorf("failed to turboOCI-apply: %w, output: %s", err, out) + } + + // overlaybd-commit + if out, err := exec.CommandContext(ctx, obdBinCommit, + pathWritableData, + pathWritableIndex, + opt.Ext4FSMetaPath, + "-z", "--turboOCI", + ).CombinedOutput(); err != nil { + return fmt.Errorf("failed to overlaybd-commit: %w, output: %s", err, out) + } + return nil +}