From 9f958465e71540e8450fc3fd04d71356a96a097f Mon Sep 17 00:00:00 2001 From: lihuahua123 <771725652@qq.com> Date: Tue, 13 Jun 2023 00:56:45 +0800 Subject: [PATCH] Nydusify: fix some bug about the subcommand mount of nydusify - The `nydusify mount` subcomend don't require `--backend-type` and `--backend-config` options when the backend is registry. - The methord to resolve it is we can get the `--backend-type` and `--backend-config` options from the docker configuration. - Also, we have refractor the code of checker module in order to reuse the code Signed-off-by: lihuahua123 <771725652@qq.com> --- contrib/nydusify/cmd/nydusify.go | 27 +++++++++-- .../nydusify/pkg/checker/rule/filesystem.go | 41 ++++++++++------ contrib/nydusify/pkg/viewer/viewer.go | 48 +++++++++++++------ 3 files changed, 84 insertions(+), 32 deletions(-) diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index dd0b1859151..58643eaef24 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -9,6 +9,7 @@ package main import ( "context" + "encoding/json" "fmt" "io" "os" @@ -16,11 +17,13 @@ import ( "strings" "github.com/containerd/containerd/reference/docker" + "github.com/docker/distribution/reference" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker" + "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker/rule" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/packer" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/provider" @@ -635,7 +638,7 @@ func main() { &cli.StringFlag{ Name: "backend-type", Value: "", - Required: true, + Required: false, Usage: "Type of storage backend, possible values: 'oss', 's3'", EnvVars: []string{"BACKEND_TYPE"}, }, @@ -685,8 +688,26 @@ func main() { if err != nil { return err } else if backendConfig == "" { - // TODO get auth from docker configuration file - return errors.Errorf("backend configuration is empty, please specify option '--backend-config'") + + backendType = "registry" + parsed, err := reference.ParseNormalizedNamed(c.String("target")) + if err != nil { + return err + } + + backendConfigStruct, err := rule.NewRegistryBackendConfig(parsed) + if err != nil { + return errors.Wrap(err, "parse registry backend configuration") + } + + backendConfigStruct.SkipVerify = c.Bool("target-insecure") + + bytes, err := json.Marshal(backendConfigStruct) + if err != nil { + return errors.Wrap(err, "marshal registry backend configuration") + } + backendConfig = string(bytes) + } _, arch, err := provider.ExtractOsArch(c.String("platform")) diff --git a/contrib/nydusify/pkg/checker/rule/filesystem.go b/contrib/nydusify/pkg/checker/rule/filesystem.go index 690f3f77b39..c40c8e2d76b 100644 --- a/contrib/nydusify/pkg/checker/rule/filesystem.go +++ b/contrib/nydusify/pkg/checker/rule/filesystem.go @@ -221,6 +221,28 @@ func (rule *FilesystemRule) mountSourceImage() (*tool.Image, error) { return image, nil } +func NewRegistryBackendConfig(parsed reference.Named) (RegistryBackendConfig, error) { + + backendConfig := RegistryBackendConfig{ + Scheme: "https", + Host: reference.Domain(parsed), + Repo: reference.Path(parsed), + } + + config := dockerconfig.LoadDefaultConfigFile(os.Stderr) + authConfig, err := config.GetAuthConfig(backendConfig.Host) + if err != nil { + return backendConfig, errors.Wrap(err, "get docker registry auth config") + } + var auth string + if authConfig.Username != "" && authConfig.Password != "" { + auth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", authConfig.Username, authConfig.Password))) + } + backendConfig.Auth = auth + + return backendConfig, nil +} + func (rule *FilesystemRule) mountNydusImage() (*tool.Nydusd, error) { logrus.Infof("Mounting Nydus image to %s", rule.NydusdConfig.MountPath) @@ -237,32 +259,23 @@ func (rule *FilesystemRule) mountNydusImage() (*tool.Nydusd, error) { return nil, err } - host := reference.Domain(parsed) - repo := reference.Path(parsed) if rule.NydusdConfig.BackendType == "" { rule.NydusdConfig.BackendType = "registry" if rule.NydusdConfig.BackendConfig == "" { - config := dockerconfig.LoadDefaultConfigFile(os.Stderr) - authConfig, err := config.GetAuthConfig(host) + backendConfig, err := NewRegistryBackendConfig(parsed) if err != nil { - return nil, errors.Wrap(err, "get docker registry auth config") + return nil, errors.Wrap(err, "failed to parse backend configuration") } - var auth string - if authConfig.Username != "" && authConfig.Password != "" { - auth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", authConfig.Username, authConfig.Password))) - } - skipVerify := false if rule.TargetInsecure { - skipVerify = true + backendConfig.SkipVerify = true } - scheme := "https" + if rule.PlainHTTP { - scheme = "http" + backendConfig.Scheme = "http" } - backendConfig := RegistryBackendConfig{scheme, host, repo, auth, skipVerify} bytes, err := json.Marshal(backendConfig) if err != nil { return nil, errors.Wrap(err, "parse registry backend config") diff --git a/contrib/nydusify/pkg/viewer/viewer.go b/contrib/nydusify/pkg/viewer/viewer.go index f36d3ec548e..fcd9a82a0ec 100644 --- a/contrib/nydusify/pkg/viewer/viewer.go +++ b/contrib/nydusify/pkg/viewer/viewer.go @@ -61,23 +61,17 @@ func New(opt Opt) (*FsViewer, error) { } mode := "cached" - digestValidate := true - if opt.FsVersion == "6" { - mode = "direct" - digestValidate = false - } nydusdConfig := tool.NydusdConfig{ - NydusdPath: opt.NydusdPath, - BackendType: opt.BackendType, - BackendConfig: opt.BackendConfig, - BootstrapPath: filepath.Join(opt.WorkDir, "nydus_bootstrap"), - ConfigPath: filepath.Join(opt.WorkDir, "fs/nydusd_config.json"), - BlobCacheDir: filepath.Join(opt.WorkDir, "fs/nydus_blobs"), - MountPath: opt.MountPath, - APISockPath: filepath.Join(opt.WorkDir, "fs/nydus_api.sock"), - Mode: mode, - DigestValidate: digestValidate, + NydusdPath: opt.NydusdPath, + BackendType: opt.BackendType, + BackendConfig: opt.BackendConfig, + BootstrapPath: filepath.Join(opt.WorkDir, "nydus_bootstrap"), + ConfigPath: filepath.Join(opt.WorkDir, "fs/nydusd_config.json"), + BlobCacheDir: filepath.Join(opt.WorkDir, "fs/nydus_blobs"), + MountPath: opt.MountPath, + APISockPath: filepath.Join(opt.WorkDir, "fs/nydus_api.sock"), + Mode: mode, } fsViewer := &FsViewer{ @@ -157,6 +151,18 @@ func (fsViewer *FsViewer) MountImage() error { // It includes two steps, pull the boostrap of the image, and mount the // image under specified path. func (fsViewer *FsViewer) View(ctx context.Context) error { + if err := fsViewer.view(ctx); err != nil { + if utils.RetryWithHTTP(err) { + fsViewer.Parser.Remote.MaybeWithHTTP(err) + return fsViewer.view(ctx) + } + return err + + } + return nil +} + +func (fsViewer *FsViewer) view(ctx context.Context) error { // Pull bootstrap targetParsed, err := fsViewer.Parser.Parse(ctx) if err != nil { @@ -167,6 +173,18 @@ func (fsViewer *FsViewer) View(ctx context.Context) error { return errors.Wrap(err, "failed to pull Nydus image bootstrap") } + // Adjust nydused parameters(DigestValidate) according to rafs format + nydusManifest := parser.FindNydusBootstrapDesc(&targetParsed.NydusImage.Manifest) + if nydusManifest != nil { + v := utils.GetNydusFsVersionOrDefault(nydusManifest.Annotations, utils.V5) + if v == utils.V5 { + // Digest validate is not currently supported for v6, + // but v5 supports it. In order to make the check more sufficient, + // this validate needs to be turned on for v5. + fsViewer.NydusdConfig.DigestValidate = true + } + } + err = fsViewer.MountImage() if err != nil { return err