Merge pull request #2 from ktock/estargz

Support estargz conversion

README.md

@@ -42,8 +42,16 @@ $ printf '<robot-name>:<robot-secret>' | base64
 
 1. Copy template config file
 
+nydus:
+
+``` shell
+$ cp misc/config/config.yaml.nydus.tmpl misc/config/config.yaml
+```
+
+estargz:
+
 ``` shell
-$ cp misc/config/config.yaml.tmpl misc/config/config.yaml
+$ cp misc/config/config.yaml.estargz.tmpl misc/config/config.yaml
 ```
 
 2. Edit config file

go.mod

@@ -4,13 +4,15 @@ go 1.16
 
 require (
 	github.com/containerd/containerd v1.5.7
+	github.com/containerd/stargz-snapshotter v0.9.0
+	github.com/containerd/stargz-snapshotter/estargz v0.9.0
 	github.com/docker/cli v20.10.9+incompatible
 	github.com/goharbor/harbor/src v0.0.0-20211021012518-bc6a7f65a6fa
 	github.com/labstack/echo-contrib v0.11.0
 	github.com/labstack/echo/v4 v4.6.1
 	github.com/labstack/gommon v0.3.0
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/opencontainers/image-spec v1.0.1
+	github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
 	github.com/sirupsen/logrus v1.8.1

misc/config/config.yaml.estargz.tmpl

@@ -0,0 +1,42 @@
+# Configuration file of Harbor Acceleration Service
+
+# http related config
+server:
+  name: API
+  # listened host for http
+  host: 0.0.0.0
+  # port for http
+  port: 2077
+
+metric:
+  # export metrics on `/metrics` endpoint
+  enabled: true
+
+provider:
+  source:
+    # hostname of harbor service
+    hub.harbor.com:
+      # base64 encoded `<robot-name>:<robot-secret>` for robot
+      # account created in harbor
+      auth: YTpiCg==
+      # use http registry communication
+      insecure: true
+      webhook:
+        # webhook request auth header configured in harbor
+        auth_header: header
+  containerd:
+    # ensure containerd service listening on this address
+    address: /run/containerd/containerd.sock
+    snapshotter: overlayfs
+
+converter:
+  # number of worker for executing conversion task
+  worker: 5
+  driver:
+    # accelerator driver type: `estargz`
+    type: estargz
+    config:
+      docker2oci: true
+  rules:
+    # add suffix to tag of source image reference as target image reference
+    - tag_suffix: -esgz

misc/config/config.yaml.tmpl → misc/config/config.yaml.nydus.tmpl

@@ -34,7 +34,6 @@ converter:
   worker: 5
   driver:
     # accelerator driver type: `nydus`
-    # estargz driver implementation is in progress
     type: nydus
     config:
       work_dir: /tmp

pkg/content/content.go

@@ -23,6 +23,7 @@ import (
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/snapshots"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -39,6 +40,8 @@ var logger = logrus.WithField("module", "content")
 // store for image conversion.
 type Provider interface {
 	// Pull pulls source image from remote registry by specified reference.
+	// This pulls all platforms of the image but Image() returns containerd.Image for
+	// the default platoform.
 	Pull(ctx context.Context, ref string) error
 	// Push pushes target image to remote registry by specified reference,
 	// the desc parameter represents the manifest of targe image.
@@ -92,8 +95,8 @@ func (pvd *LocalProvider) Pull(ctx context.Context, ref string) error {
 	}
 
 	opts := []containerd.RemoteOpt{
-		// TODO: support multi-platform source images.
 		// TODO: sets max concurrent downloaded layer limit by containerd.WithMaxConcurrentDownloads.
+		containerd.WithPlatformMatcher(platforms.All),
 		containerd.WithImageHandler(images.HandlerFunc(
 			func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
 				if images.IsLayerType(desc.MediaType) {
@@ -102,18 +105,18 @@ func (pvd *LocalProvider) Pull(ctx context.Context, ref string) error {
 				return nil, nil
 			},
 		)),
-		containerd.WithPullUnpack,
 		containerd.WithResolver(resolver),
 	}
 
 	// Pull the source image from remote registry.
-	image, err := pvd.client.Pull(ctx, ref, opts...)
+	image, err := pvd.client.Fetch(ctx, ref, opts...)
 	if err != nil {
 		return errors.Wrap(err, "pull source image")
 	}
-	pvd.image = image
+	pvd.image = containerd.NewImageWithPlatform(pvd.client, image, platforms.DefaultStrict())
 
-	return nil
+	// Unpack the image (default platform only)
+	return pvd.image.Unpack(ctx, "")
 }
 
 func (pvd *LocalProvider) Write(ctx context.Context, desc ocispec.Descriptor, reader io.Reader, labels map[string]string) error {

pkg/driver/driver.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/goharbor/acceleration-service/pkg/config"
 	"github.com/goharbor/acceleration-service/pkg/content"
+	"github.com/goharbor/acceleration-service/pkg/driver/estargz"
 	"github.com/goharbor/acceleration-service/pkg/driver/nydus"
 )
 
@@ -40,6 +41,8 @@ func NewLocalDriver(cfg *config.DriverConfig) (Driver, error) {
 	switch cfg.Type {
 	case "nydus":
 		return nydus.New(cfg.Config)
+	case "estargz":
+		return estargz.New(cfg.Config)
 	default:
 		return nil, fmt.Errorf("unsupported driver %s", cfg.Type)
 	}

pkg/driver/estargz/estargz.go

@@ -0,0 +1,57 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package estargz
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/containerd/containerd/images/converter"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/stargz-snapshotter/estargz"
+	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	"github.com/goharbor/acceleration-service/pkg/content"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+type Driver struct {
+	cfg map[string]string
+}
+
+func New(cfg map[string]string) (*Driver, error) {
+	return &Driver{cfg}, nil
+}
+
+func (d *Driver) Convert(ctx context.Context, p content.Provider) (*ocispec.Descriptor, error) {
+	opts, docker2oci, err := getESGZConvertOpts(d.cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "parse estargz conversion options")
+	}
+	platformMC := platforms.All // TODO: enable to configure the target platforms
+	return converter.DefaultIndexConvertFunc(estargzconvert.LayerConvertFunc(opts...), docker2oci, platformMC)(
+		ctx, p.ContentStore(), p.Image().Target())
+}
+
+func getESGZConvertOpts(cfg map[string]string) (opts []estargz.Option, docker2oci bool, err error) {
+	if s, ok := cfg["docker2oci"]; ok {
+		b, err := strconv.ParseBool(s)
+		if err != nil {
+			return nil, false, err
+		}
+		docker2oci = b
+	}
+	return
+}

test/integration_test.go

@@ -36,7 +36,7 @@ func (suite *IntegrationTestSuite) cleanUp() {
 func (suite *IntegrationTestSuite) SetupSuite() {
 	suite.cleanUp()
 
-	cfg, err := config.Parse("../misc/config/config.yaml.tmpl")
+	cfg, err := config.Parse("../misc/config/config.yaml.nydus.tmpl")
 	suite.Require().NoError(err)
 	suite.startDaemon(cfg)