Skip to content

Commit

Permalink
nydusify: introduce copy subcommand
Browse files Browse the repository at this point in the history 
`nydusify copy` copies an image from source registry to target
registry, it also supports to specify a backend storage.

Signed-off-by: Yan Song <imeoer@linux.alibaba.com>
  • Loading branch information
@imeoer
imeoer committed Jul 20, 2023
1 parent be52ebd commit e9cd067
Show file tree
Hide file tree
Showing 9 changed files with 505 additions and 10 deletions.
149 changes: 139 additions & 10 deletions contrib/nydusify/cmd/nydusify.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker/rule"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/copier"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/packer"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/provider"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
Expand Down Expand Up @@ -67,27 +68,27 @@ func parseBackendConfig(backendConfigJSON, backendConfigFile string) (string, er
return backendConfigJSON, nil
}

func getBackendConfig(c *cli.Context, required bool) (string, string, error) {
backendType := c.String("backend-type")
func getBackendConfig(c *cli.Context, suffix string, required bool) (string, string, error) {
backendType := c.String(suffix + "backend-type")
if backendType == "" {
if required {
return "", "", errors.Errorf("backend type is empty, please specify option '--backend-type'")
return "", "", errors.Errorf("backend type is empty, please specify option '--%sbackend-type'", suffix)
}
return "", "", nil
}

possibleBackendTypes := []string{"oss", "s3"}
if !isPossibleValue(possibleBackendTypes, backendType) {
return "", "", fmt.Errorf("--backend-type should be one of %v", possibleBackendTypes)
return "", "", fmt.Errorf("--%sbackend-type should be one of %v", suffix, possibleBackendTypes)
}

backendConfig, err := parseBackendConfig(
c.String("backend-config"), c.String("backend-config-file"),
c.String(suffix+"backend-config"), c.String(suffix+"backend-config-file"),
)
if err != nil {
return "", "", err
} else if (backendType == "oss" || backendType == "s3") && strings.TrimSpace(backendConfig) == "" {
return "", "", errors.Errorf("backend configuration is empty, please specify option '--backend-config'")
return "", "", errors.Errorf("backend configuration is empty, please specify option '--%sbackend-config'", suffix)
}

return backendType, backendConfig, nil
Expand Down Expand Up @@ -427,7 +428,7 @@ func main() {
return err
}

backendType, backendConfig, err := getBackendConfig(c, false)
backendType, backendConfig, err := getBackendConfig(c, "", false)
if err != nil {
return err
}
Expand Down Expand Up @@ -602,7 +603,7 @@ func main() {
Action: func(c *cli.Context) error {
setupLogLevel(c)

backendType, backendConfig, err := getBackendConfig(c, false)
backendType, backendConfig, err := getBackendConfig(c, "", false)
if err != nil {
return err
}
Expand Down Expand Up @@ -699,7 +700,7 @@ func main() {
Action: func(c *cli.Context) error {
setupLogLevel(c)

backendType, backendConfig, err := getBackendConfig(c, false)
backendType, backendConfig, err := getBackendConfig(c, "", false)
if err != nil {
return err
} else if backendConfig == "" {
Expand Down Expand Up @@ -875,7 +876,7 @@ func main() {

// if backend-push is specified, we should make sure backend-config-file exists
if c.Bool("backend-push") || c.Bool("compact") {
_backendType, _backendConfig, err := getBackendConfig(c, true)
_backendType, _backendConfig, err := getBackendConfig(c, "", true)
if err != nil {
return err
}
Expand Down Expand Up @@ -915,6 +916,134 @@ func main() {
return nil
},
},
{
Name: "copy",
Usage: "Copy an image from source to target",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "source",
Required: true,
Usage: "Source image reference",
EnvVars: []string{"SOURCE"},
},
&cli.StringFlag{
Name: "target",
Required: false,
Usage: "Target image reference",
EnvVars: []string{"TARGET"},
},
&cli.BoolFlag{
Name: "source-insecure",
Required: false,
Usage: "Skip verifying server certs for HTTPS source registry",
EnvVars: []string{"SOURCE_INSECURE"},
},
&cli.BoolFlag{
Name: "target-insecure",
Required: false,
Usage: "Skip verifying server certs for HTTPS target registry",
EnvVars: []string{"TARGET_INSECURE"},
},

&cli.StringFlag{
Name: "source-backend-type",
Value: "",
Usage: "Type of storage backend, possible values: 'oss', 's3'",
EnvVars: []string{"BACKEND_TYPE"},
},
&cli.StringFlag{
Name: "source-backend-config",
Value: "",
Usage: "Json configuration string for storage backend",
EnvVars: []string{"BACKEND_CONFIG"},
},
&cli.PathFlag{
Name: "source-backend-config-file",
Value: "",
TakesFile: true,
Usage: "Json configuration file for storage backend",
EnvVars: []string{"BACKEND_CONFIG_FILE"},
},

&cli.StringFlag{
Name: "target-backend-type",
Value: "",
Usage: "Type of storage backend, possible values: 'oss', 's3'",
EnvVars: []string{"BACKEND_TYPE"},
},
&cli.StringFlag{
Name: "target-backend-config",
Value: "",
Usage: "Json configuration string for storage backend",
EnvVars: []string{"BACKEND_CONFIG"},
},
&cli.PathFlag{
Name: "target-backend-config-file",
Value: "",
TakesFile: true,
Usage: "Json configuration file for storage backend",
EnvVars: []string{"BACKEND_CONFIG_FILE"},
},

&cli.BoolFlag{
Name: "all-platforms",
Value: false,
Usage: "Convert images for all platforms, conflicts with --platform",
},
&cli.StringFlag{
Name: "platform",
Value: "linux/" + runtime.GOARCH,
Usage: "Convert images for specific platforms, for example: 'linux/amd64,linux/arm64'",
},

&cli.StringFlag{
Name: "work-dir",
Value: "./tmp",
Usage: "Working directory for image conversion",
EnvVars: []string{"WORK_DIR"},
},
&cli.StringFlag{
Name: "nydus-image",
Value: "nydus-image",
Usage: "Path to the nydus-image binary, default to search in PATH",
EnvVars: []string{"NYDUS_IMAGE"},
},
},
Action: func(c *cli.Context) error {
setupLogLevel(c)

sourceBackendType, sourceBackendConfig, err := getBackendConfig(c, "source-", false)
if err != nil {
return err
}

targetBackendType, targetBackendConfig, err := getBackendConfig(c, "target-", false)
if err != nil {
return err
}

opt := copier.Opt{
WorkDir: c.String("work-dir"),
NydusImagePath: c.String("nydus-image"),

Source: c.String("source"),
Target: c.String("target"),
SourceInsecure: c.Bool("source-insecure"),
TargetInsecure: c.Bool("target-insecure"),

SourceBackendType: sourceBackendType,
SourceBackendConfig: sourceBackendConfig,

TargetBackendType: targetBackendType,
TargetBackendConfig: targetBackendConfig,

AllPlatforms: c.Bool("all-platforms"),
Platforms: c.String("platform"),
}

return copier.Copy(context.Background(), opt)
},
},
}

if !utils.IsSupportedArch(runtime.GOARCH) {
Expand Down
3 changes: 3 additions & 0 deletions contrib/nydusify/pkg/backend/backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package backend
import (
"context"
"fmt"
"io"

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
Expand All @@ -25,6 +26,8 @@ type Backend interface {
Finalize(cancel bool) error
Check(blobID string) (bool, error)
Type() Type
Reader(blobID string) (io.ReadCloser, error)
Size(blobID string) (int64, error)
}

// TODO: Directly forward blob data to storage backend
Expand Down
20 changes: 20 additions & 0 deletions contrib/nydusify/pkg/backend/oss.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,26 @@ func (b *OSSBackend) Type() Type {
return OssBackend
}

func (b *OSSBackend) Reader(blobID string) (io.ReadCloser, error) {
blobID = b.objectPrefix + blobID
rc, err := b.bucket.GetObject(blobID)
return rc, err
}

func (b *OSSBackend) Size(blobID string) (int64, error) {
blobID = b.objectPrefix + blobID
headers, err := b.bucket.GetObjectMeta(blobID)
if err != nil {
return 0, errors.Wrap(err, "get object size")
}
sizeStr := headers.Get("Content-Length")
size, err := strconv.ParseInt(sizeStr, 10, 0)
if err != nil {
return 0, errors.Wrap(err, "parse content-length header")
}
return size, nil
}

func (b *OSSBackend) remoteID(blobID string) string {
return fmt.Sprintf("oss://%s/%s%s", b.bucket.BucketName, b.objectPrefix, blobID)
}
9 changes: 9 additions & 0 deletions contrib/nydusify/pkg/backend/registry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package backend

import (
"context"
"io"
"os"

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
Expand Down Expand Up @@ -46,6 +47,14 @@ func (r *Registry) Type() Type {
return RegistryBackend
}

func (r *Registry) Reader(blobID string) (io.ReadCloser, error) {
panic("not implemented")
}

func (r *Registry) Size(blobID string) (int64, error) {
panic("not implemented")
}

func newRegistryBackend(rawConfig []byte, remote *remote.Remote) (Backend, error) {
return &Registry{remote: remote}, nil
}
9 changes: 9 additions & 0 deletions contrib/nydusify/pkg/backend/s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
Expand Down Expand Up @@ -140,6 +141,14 @@ func (b *S3Backend) Type() Type {
return S3backend
}

func (b *S3Backend) Reader(blobID string) (io.ReadCloser, error) {
panic("not implemented")
}

func (b *S3Backend) Size(blobID string) (int64, error) {
panic("not implemented")
}

func (b *S3Backend) existObject(ctx context.Context, objectKey string) (bool, error) {
_, err := b.client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &b.bucketName,
Expand Down
4 changes: 4 additions & 0 deletions contrib/nydusify/pkg/converter/provider/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,3 +100,7 @@ func (pvd *Provider) Image(ctx context.Context, ref string) (*ocispec.Descriptor
func (pvd *Provider) ContentStore() content.Store {
return pvd.store
}

func (pvd *Provider) SetContentStore(store content.Store) {
pvd.store = store
}
Loading

0 comments on commit e9cd067

Please sign in to comment.