Nydusify: fix some bug about the subcommand mount of nydusify

contrib/nydusify/cmd/nydusify.go

@@ -9,18 +9,21 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
 	"runtime"
 	"strings"
 
 	"github.com/containerd/containerd/reference/docker"
+	"github.com/docker/distribution/reference"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker"
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker/rule"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/packer"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/provider"
@@ -635,7 +638,7 @@ func main() {
 				&cli.StringFlag{
 					Name:     "backend-type",
 					Value:    "",
-					Required: true,
+					Required: false,
 					Usage:    "Type of storage backend, possible values: 'oss', 's3'",
 					EnvVars:  []string{"BACKEND_TYPE"},
 				},
@@ -685,8 +688,26 @@ func main() {
 				if err != nil {
 					return err
 				} else if backendConfig == "" {
-					// TODO get auth from docker configuration file
-					return errors.Errorf("backend configuration is empty, please specify option '--backend-config'")
+
+					backendType = "registry"
+					parsed, err := reference.ParseNormalizedNamed(c.String("target"))
+					if err != nil {
+						return err
+					}
+
+					backendConfigStruct, err := rule.NewRegistryBackendConfig(parsed)
+					if err != nil {
+						return errors.Wrap(err, "parse registry backend configuration")
+					}
+
+					backendConfigStruct.SkipVerify = c.Bool("target-insecure")
+
+					bytes, err := json.Marshal(backendConfigStruct)
+					if err != nil {
+						return errors.Wrap(err, "marshal registry backend configuration")
+					}
+					backendConfig = string(bytes)
+
 				}
 
 				_, arch, err := provider.ExtractOsArch(c.String("platform"))

contrib/nydusify/pkg/checker/rule/filesystem.go

@@ -221,6 +221,28 @@ func (rule *FilesystemRule) mountSourceImage() (*tool.Image, error) {
 	return image, nil
 }
 
+func NewRegistryBackendConfig(parsed reference.Named) (RegistryBackendConfig, error) {
+
+	backendConfig := RegistryBackendConfig{
+		Scheme: "https",
+		Host:   reference.Domain(parsed),
+		Repo:   reference.Path(parsed),
+	}
+
+	config := dockerconfig.LoadDefaultConfigFile(os.Stderr)
+	authConfig, err := config.GetAuthConfig(backendConfig.Host)
+	if err != nil {
+		return backendConfig, errors.Wrap(err, "get docker registry auth config")
+	}
+	var auth string
+	if authConfig.Username != "" && authConfig.Password != "" {
+		auth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", authConfig.Username, authConfig.Password)))
+	}
+	backendConfig.Auth = auth
+
+	return backendConfig, nil
+}
+
 func (rule *FilesystemRule) mountNydusImage() (*tool.Nydusd, error) {
 	logrus.Infof("Mounting Nydus image to %s", rule.NydusdConfig.MountPath)
 
@@ -237,32 +259,23 @@ func (rule *FilesystemRule) mountNydusImage() (*tool.Nydusd, error) {
 		return nil, err
 	}
 
-	host := reference.Domain(parsed)
-	repo := reference.Path(parsed)
 	if rule.NydusdConfig.BackendType == "" {
 		rule.NydusdConfig.BackendType = "registry"
 
 		if rule.NydusdConfig.BackendConfig == "" {
-			config := dockerconfig.LoadDefaultConfigFile(os.Stderr)
-			authConfig, err := config.GetAuthConfig(host)
+			backendConfig, err := NewRegistryBackendConfig(parsed)
 			if err != nil {
-				return nil, errors.Wrap(err, "get docker registry auth config")
+				return nil, errors.Wrap(err, "failed to parse backend configuration")
 			}
 
-			var auth string
-			if authConfig.Username != "" && authConfig.Password != "" {
-				auth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", authConfig.Username, authConfig.Password)))
-			}
-			skipVerify := false
 			if rule.TargetInsecure {
-				skipVerify = true
+				backendConfig.SkipVerify = true
 			}
-			scheme := "https"
+
 			if rule.PlainHTTP {
-				scheme = "http"
+				backendConfig.Scheme = "http"
 			}
 
-			backendConfig := RegistryBackendConfig{scheme, host, repo, auth, skipVerify}
 			bytes, err := json.Marshal(backendConfig)
 			if err != nil {
 				return nil, errors.Wrap(err, "parse registry backend config")

contrib/nydusify/pkg/viewer/viewer.go

@@ -61,23 +61,17 @@ func New(opt Opt) (*FsViewer, error) {
 	}
 
 	mode := "cached"
-	digestValidate := true
-	if opt.FsVersion == "6" {
-		mode = "direct"
-		digestValidate = false
-	}
 
 	nydusdConfig := tool.NydusdConfig{
-		NydusdPath:     opt.NydusdPath,
-		BackendType:    opt.BackendType,
-		BackendConfig:  opt.BackendConfig,
-		BootstrapPath:  filepath.Join(opt.WorkDir, "nydus_bootstrap"),
-		ConfigPath:     filepath.Join(opt.WorkDir, "fs/nydusd_config.json"),
-		BlobCacheDir:   filepath.Join(opt.WorkDir, "fs/nydus_blobs"),
-		MountPath:      opt.MountPath,
-		APISockPath:    filepath.Join(opt.WorkDir, "fs/nydus_api.sock"),
-		Mode:           mode,
-		DigestValidate: digestValidate,
+		NydusdPath:    opt.NydusdPath,
+		BackendType:   opt.BackendType,
+		BackendConfig: opt.BackendConfig,
+		BootstrapPath: filepath.Join(opt.WorkDir, "nydus_bootstrap"),
+		ConfigPath:    filepath.Join(opt.WorkDir, "fs/nydusd_config.json"),
+		BlobCacheDir:  filepath.Join(opt.WorkDir, "fs/nydus_blobs"),
+		MountPath:     opt.MountPath,
+		APISockPath:   filepath.Join(opt.WorkDir, "fs/nydus_api.sock"),
+		Mode:          mode,
 	}
 
 	fsViewer := &FsViewer{
@@ -157,6 +151,18 @@ func (fsViewer *FsViewer) MountImage() error {
 // It includes two steps, pull the boostrap of the image, and mount the
 // image under specified path.
 func (fsViewer *FsViewer) View(ctx context.Context) error {
+	if err := fsViewer.view(ctx); err != nil {
+		if utils.RetryWithHTTP(err) {
+			fsViewer.Parser.Remote.MaybeWithHTTP(err)
+			return fsViewer.view(ctx)
+		}
+		return err
+
+	}
+	return nil
+}
+
+func (fsViewer *FsViewer) view(ctx context.Context) error {
 	// Pull bootstrap
 	targetParsed, err := fsViewer.Parser.Parse(ctx)
 	if err != nil {
@@ -167,6 +173,18 @@ func (fsViewer *FsViewer) View(ctx context.Context) error {
 		return errors.Wrap(err, "failed to pull Nydus image bootstrap")
 	}
 
+	// Adjust nydused parameters(DigestValidate) according to rafs format
+	nydusManifest := parser.FindNydusBootstrapDesc(&targetParsed.NydusImage.Manifest)
+	if nydusManifest != nil {
+		v := utils.GetNydusFsVersionOrDefault(nydusManifest.Annotations, utils.V5)
+		if v == utils.V5 {
+			// Digest validate is not currently supported for v6,
+			// but v5 supports it. In order to make the check more sufficient,
+			// this validate needs to be turned on for v5.
+			fsViewer.NydusdConfig.DigestValidate = true
+		}
+	}
+
 	err = fsViewer.MountImage()
 	if err != nil {
 		return err