converter: detect features by checking help message, close #482

1. detect features by checking `nydus-image create -h`.
2. Add feature detection for `--batch-size`.

Signed-off-by: Wenhao Ren <wenhaoren@mail.dlut.edu.cn>

pkg/converter/convert_unix.go

@@ -321,7 +321,17 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 	}
 
 	builderPath := getBuilder(opt.BuilderPath)
-	opt.features = tool.DetectFeatures(builderPath, []tool.Feature{tool.FeatureTar2Rafs})
+
+	requiredFeatures := tool.NewFeatures(tool.FeatureTar2Rafs)
+	if opt.BatchSize != "" && opt.BatchSize != "0" {
+		requiredFeatures.Add(tool.FeatureBatchSize)
+	}
+
+	detectedFeatures, err := tool.DetectFeatures(builderPath, requiredFeatures, tool.GetHelp)
+	if err != nil {
+		return nil, err
+	}
+	opt.features = detectedFeatures
 
 	if opt.OCIRef {
 		if opt.FsVersion == "6" {
@@ -330,6 +340,10 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser,
 		return nil, fmt.Errorf("oci ref can only be supported by fs version 6")
 	}
 
+	if opt.features.Contains(tool.FeatureBatchSize) && opt.FsVersion != "6" {
+		return nil, fmt.Errorf("'--batch-size' can only be supported by fs version 6")
+	}
+
 	if opt.features.Contains(tool.FeatureTar2Rafs) {
 		return packFromTar(ctx, dest, opt)
 	}

pkg/converter/tool/builder.go

@@ -133,7 +133,7 @@ func buildPackArgs(option PackOption) []string {
 	if option.ChunkSize != "" {
 		args = append(args, "--chunk-size", option.ChunkSize)
 	}
-	if option.BatchSize != "" {
+	if option.Features.Contains(FeatureBatchSize) {
 		args = append(args, "--batch-size", option.BatchSize)
 	}
 	args = append(args, option.SourcePath)

pkg/converter/tool/feature.go

@@ -8,106 +8,141 @@ package tool
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/exec"
 	"regexp"
+	"strings"
 	"sync"
 
 	"github.com/sirupsen/logrus"
-	"golang.org/x/mod/semver"
 )
 
-const envNydusDisableTar2Rafs = "NYDUS_DISABLE_TAR2RAFS"
+type Feature string
+type Features map[Feature]struct{}
 
-var currentVersion string
-var currentVersionDetectOnce sync.Once
-var disableTar2Rafs = os.Getenv(envNydusDisableTar2Rafs) != ""
+const envNydusDisableTar2Rafs string = "NYDUS_DISABLE_TAR2RAFS"
 
 const (
 	// The option `--type tar-rafs` enables converting OCI tar blob
 	// stream into nydus blob directly, the tar2rafs eliminates the
 	// need to decompress it to a local directory first, thus greatly
 	// accelerating the pack process.
 	FeatureTar2Rafs Feature = "--type tar-rafs"
+	// The option `--batch-size` enables merging multiple small chunks
+	// into a big batch chunk, which can reduce the the size of the image
+	// and accelerate the runtime file loading.
+	FeatureBatchSize Feature = "--batch-size"
 )
 
-var featureMap = map[Feature]string{
-	FeatureTar2Rafs: "v2.2",
+// Customized regular expressions for detecting features.
+// Just for some features that cannot be detected by string matching.
+var featureMap = map[Feature]*regexp.Regexp{
+	FeatureTar2Rafs: regexp.MustCompile(`--type.+\r?\n(.+deprecated.+\r?\n)?.+possible values.+tar-rafs`),
 }
 
-type Feature string
-type Features []Feature
+var requiredFeatures Features
+var detectedFeatures Features
+var detectFeaturesOnce sync.Once
+var disableTar2Rafs = os.Getenv(envNydusDisableTar2Rafs) != ""
 
-func (features *Features) Contains(feature Feature) bool {
-	for _, feat := range *features {
-		if feat == feature {
-			return true
-		}
-	}
-	return false
+func NewFeatures(items ...Feature) Features {
+	features := Features{}
+	features.Add(items...)
+	return features
 }
 
-func (features *Features) Remove(feature Feature) {
-	found := -1
-	for idx, feat := range *features {
-		if feat == feature {
-			found = idx
-			break
-		}
-	}
-	if found != -1 {
-		*features = append((*features)[:found], (*features)[found+1:]...)
+func (features *Features) Add(items ...Feature) {
+	for _, item := range items {
+		(*features)[item] = struct{}{}
 	}
 }
 
-func detectVersion(msg []byte) string {
-	re := regexp.MustCompile(`Version:\s*v*(\d+.\d+.\d+)`)
-	matches := re.FindSubmatch(msg)
-	if len(matches) > 1 {
-		return string(matches[1])
+func (features *Features) Remove(items ...Feature) {
+	for _, item := range items {
+		delete(*features, item)
 	}
-	return ""
 }
 
-// DetectFeatures returns supported feature list from required feature list.
-func DetectFeatures(builder string, required Features) Features {
-	currentVersionDetectOnce.Do(func() {
-		if required.Contains(FeatureTar2Rafs) && disableTar2Rafs {
-			logrus.Warnf("the feature '%s' is disabled by env '%s'", FeatureTar2Rafs, envNydusDisableTar2Rafs)
-		}
+func (features *Features) Contains(feature Feature) bool {
+	_, ok := (*features)[feature]
+	return ok
+}
+
+func (features *Features) Equals(other Features) bool {
+	if len(*features) != len(other) {
+		return false
+	}
 
-		cmd := exec.CommandContext(context.Background(), builder, "--version")
-		output, err := cmd.Output()
-		if err != nil {
-			return
+	for f := range *features {
+		if !other.Contains(f) {
+			return false
 		}
+	}
 
-		currentVersion = detectVersion(output)
-	})
+	return true
+}
+
+// GetHelp returns the help message of `nydus-image create`.
+func GetHelp(builder string) []byte {
+	cmd := exec.CommandContext(context.Background(), builder, "create", "-h")
+	output, err := cmd.Output()
+	if err != nil {
+		return nil
+	}
+
+	return output
+}
+
+// detectFeature returns true if the feature is detected in the help message.
+func detectFeature(msg []byte, feature Feature) bool {
+	if feature == "" {
+		return false
+	}
 
-	if currentVersion == "" {
-		return Features{}
+	if strings.Contains(string(msg), string(feature)) {
+		return true
 	}
 
-	detectedFeatures := Features{}
-	for _, feature := range required {
-		requiredVersion := featureMap[feature]
-		if requiredVersion == "" {
-			detectedFeatures = append(detectedFeatures, feature)
-			continue
+	if re, ok := featureMap[feature]; ok {
+		if re.MatchString(string(msg)) {
+			return true
 		}
+	}
+
+	return false
+}
 
-		// The feature is supported by current version
-		supported := semver.Compare(requiredVersion, "v"+currentVersion) <= 0
-		if supported {
-			// It is an experimental feature, so we still provide an env
-			// variable to allow users to disable it.
-			if feature == FeatureTar2Rafs && disableTar2Rafs {
-				continue
+// DetectFeatures returns supported feature list from required feature list.
+// The supported feature list is detected from the help message of `nydus-image create`.
+func DetectFeatures(builder string, required Features, getHelp func(string) []byte) (Features, error) {
+	detectFeaturesOnce.Do(func() {
+		requiredFeatures = required
+		detectedFeatures = Features{}
+
+		helpMsg := getHelp(builder)
+
+		for feature := range required {
+			// The feature is supported by current version of nydus-image.
+			supported := detectFeature(helpMsg, feature)
+			if supported {
+				// It is an experimental feature, so we still provide an env
+				// variable to allow users to disable it.
+				if feature == FeatureTar2Rafs && disableTar2Rafs {
+					logrus.Warnf("the feature '%s' is disabled by env '%s'", FeatureTar2Rafs, envNydusDisableTar2Rafs)
+					continue
+				}
+				detectedFeatures.Add(feature)
+			} else {
+				logrus.Warnf("the feature '%s' is ignored, it requires higher version of nydus-image", feature)
 			}
-			detectedFeatures = append(detectedFeatures, feature)
 		}
+	})
+
+	// Return Error if required features changed in different calls.
+	if !requiredFeatures.Equals(required) {
+		return nil, fmt.Errorf("features changed: %v -> %v", requiredFeatures, required)
 	}
 
-	return detectedFeatures
+	return detectedFeatures, nil
 }