From 7ae7f5d67b7d3bcf81e7f2cbcf40f1f566ebd41b Mon Sep 17 00:00:00 2001 From: Wenhao Ren Date: Tue, 6 Jun 2023 18:18:37 +0800 Subject: [PATCH] converter: detect features by checking help message, close #482 1. detect features by checking `nydus-image create -h`. 2. Add feature detection for `--batch-size`. Signed-off-by: Wenhao Ren --- pkg/converter/convert_unix.go | 16 +- pkg/converter/tool/builder.go | 2 +- pkg/converter/tool/feature.go | 157 +++--- pkg/converter/tool/feature_test.go | 740 +++++++++++++++++++++++++++-- 4 files changed, 824 insertions(+), 91 deletions(-) diff --git a/pkg/converter/convert_unix.go b/pkg/converter/convert_unix.go index 649e1e2f3f..e0bbd05561 100644 --- a/pkg/converter/convert_unix.go +++ b/pkg/converter/convert_unix.go @@ -321,7 +321,17 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser, } builderPath := getBuilder(opt.BuilderPath) - opt.features = tool.DetectFeatures(builderPath, []tool.Feature{tool.FeatureTar2Rafs}) + + requiredFeatures := tool.NewFeatures(tool.FeatureTar2Rafs) + if opt.BatchSize != "" && opt.BatchSize != "0" { + requiredFeatures.Add(tool.FeatureBatchSize) + } + + detectedFeatures, err := tool.DetectFeatures(builderPath, requiredFeatures, tool.GetHelp) + if err != nil { + return nil, err + } + opt.features = detectedFeatures if opt.OCIRef { if opt.FsVersion == "6" { @@ -330,6 +340,10 @@ func Pack(ctx context.Context, dest io.Writer, opt PackOption) (io.WriteCloser, return nil, fmt.Errorf("oci ref can only be supported by fs version 6") } + if opt.features.Contains(tool.FeatureBatchSize) && opt.FsVersion != "6" { + return nil, fmt.Errorf("'--batch-size' can only be supported by fs version 6") + } + if opt.features.Contains(tool.FeatureTar2Rafs) { return packFromTar(ctx, dest, opt) } diff --git a/pkg/converter/tool/builder.go b/pkg/converter/tool/builder.go index 5c8284e95a..ce808951a7 100644 --- a/pkg/converter/tool/builder.go +++ b/pkg/converter/tool/builder.go @@ -133,7 +133,7 @@ func buildPackArgs(option PackOption) []string { if option.ChunkSize != "" { args = append(args, "--chunk-size", option.ChunkSize) } - if option.BatchSize != "" { + if option.Features.Contains(FeatureBatchSize) { args = append(args, "--batch-size", option.BatchSize) } args = append(args, option.SourcePath) diff --git a/pkg/converter/tool/feature.go b/pkg/converter/tool/feature.go index f20c11b73c..cfa439a8f8 100644 --- a/pkg/converter/tool/feature.go +++ b/pkg/converter/tool/feature.go @@ -8,20 +8,20 @@ package tool import ( "context" + "fmt" "os" "os/exec" "regexp" + "strings" "sync" "github.com/sirupsen/logrus" - "golang.org/x/mod/semver" ) -const envNydusDisableTar2Rafs = "NYDUS_DISABLE_TAR2RAFS" +type Feature string +type Features map[Feature]struct{} -var currentVersion string -var currentVersionDetectOnce sync.Once -var disableTar2Rafs = os.Getenv(envNydusDisableTar2Rafs) != "" +const envNydusDisableTar2Rafs string = "NYDUS_DISABLE_TAR2RAFS" const ( // The option `--type tar-rafs` enables converting OCI tar blob @@ -29,85 +29,120 @@ const ( // need to decompress it to a local directory first, thus greatly // accelerating the pack process. FeatureTar2Rafs Feature = "--type tar-rafs" + // The option `--batch-size` enables merging multiple small chunks + // into a big batch chunk, which can reduce the the size of the image + // and accelerate the runtime file loading. + FeatureBatchSize Feature = "--batch-size" ) -var featureMap = map[Feature]string{ - FeatureTar2Rafs: "v2.2", +// Customized regular expressions for detecting features. +// Just for some features that cannot be detected by string matching. +var featureMap = map[Feature]*regexp.Regexp{ + FeatureTar2Rafs: regexp.MustCompile(`--type.+\r?\n(.+deprecated.+\r?\n)?.+possible values.+tar-rafs`), } -type Feature string -type Features []Feature +var requiredFeatures Features +var detectedFeatures Features +var detectFeaturesOnce sync.Once +var disableTar2Rafs = os.Getenv(envNydusDisableTar2Rafs) != "" -func (features *Features) Contains(feature Feature) bool { - for _, feat := range *features { - if feat == feature { - return true - } - } - return false +func NewFeatures(items ...Feature) Features { + features := Features{} + features.Add(items...) + return features } -func (features *Features) Remove(feature Feature) { - found := -1 - for idx, feat := range *features { - if feat == feature { - found = idx - break - } - } - if found != -1 { - *features = append((*features)[:found], (*features)[found+1:]...) +func (features *Features) Add(items ...Feature) { + for _, item := range items { + (*features)[item] = struct{}{} } } -func detectVersion(msg []byte) string { - re := regexp.MustCompile(`Version:\s*v*(\d+.\d+.\d+)`) - matches := re.FindSubmatch(msg) - if len(matches) > 1 { - return string(matches[1]) +func (features *Features) Remove(items ...Feature) { + for _, item := range items { + delete(*features, item) } - return "" } -// DetectFeatures returns supported feature list from required feature list. -func DetectFeatures(builder string, required Features) Features { - currentVersionDetectOnce.Do(func() { - if required.Contains(FeatureTar2Rafs) && disableTar2Rafs { - logrus.Warnf("the feature '%s' is disabled by env '%s'", FeatureTar2Rafs, envNydusDisableTar2Rafs) - } +func (features *Features) Contains(feature Feature) bool { + _, ok := (*features)[feature] + return ok +} + +func (features *Features) Equals(other Features) bool { + if len(*features) != len(other) { + return false + } - cmd := exec.CommandContext(context.Background(), builder, "--version") - output, err := cmd.Output() - if err != nil { - return + for f := range *features { + if !other.Contains(f) { + return false } + } - currentVersion = detectVersion(output) - }) + return true +} + +// GetHelp returns the help message of `nydus-image create`. +func GetHelp(builder string) []byte { + cmd := exec.CommandContext(context.Background(), builder, "create", "-h") + output, err := cmd.Output() + if err != nil { + return nil + } + + return output +} + +// detectFeature returns true if the feature is detected in the help message. +func detectFeature(msg []byte, feature Feature) bool { + if feature == "" { + return false + } - if currentVersion == "" { - return Features{} + if strings.Contains(string(msg), string(feature)) { + return true } - detectedFeatures := Features{} - for _, feature := range required { - requiredVersion := featureMap[feature] - if requiredVersion == "" { - detectedFeatures = append(detectedFeatures, feature) - continue + if re, ok := featureMap[feature]; ok { + if re.MatchString(string(msg)) { + return true } + } + + return false +} - // The feature is supported by current version - supported := semver.Compare(requiredVersion, "v"+currentVersion) <= 0 - if supported { - // It is an experimental feature, so we still provide an env - // variable to allow users to disable it. - if feature == FeatureTar2Rafs && disableTar2Rafs { - continue +// DetectFeatures returns supported feature list from required feature list. +// The supported feature list is detected from the help message of `nydus-image create`. +func DetectFeatures(builder string, required Features, getHelp func(string) []byte) (Features, error) { + detectFeaturesOnce.Do(func() { + requiredFeatures = required + detectedFeatures = Features{} + + helpMsg := getHelp(builder) + + for feature := range required { + // The feature is supported by current version of nydus-image. + supported := detectFeature(helpMsg, feature) + if supported { + // It is an experimental feature, so we still provide an env + // variable to allow users to disable it. + if feature == FeatureTar2Rafs && disableTar2Rafs { + logrus.Warnf("the feature '%s' is disabled by env '%s'", FeatureTar2Rafs, envNydusDisableTar2Rafs) + continue + } + detectedFeatures.Add(feature) + } else { + logrus.Warnf("the feature '%s' is ignored, it requires higher version of nydus-image", feature) } - detectedFeatures = append(detectedFeatures, feature) } + }) + + // Return Error if required features changed in different calls. + if !requiredFeatures.Equals(required) { + return nil, fmt.Errorf("features changed: %v -> %v", requiredFeatures, required) } - return detectedFeatures + return detectedFeatures, nil } diff --git a/pkg/converter/tool/feature_test.go b/pkg/converter/tool/feature_test.go index 82e099639b..aa49e537a8 100644 --- a/pkg/converter/tool/feature_test.go +++ b/pkg/converter/tool/feature_test.go @@ -7,41 +7,725 @@ package tool import ( + "sync" "testing" "github.com/stretchr/testify/require" ) func TestFeature(t *testing.T) { - features := Features{FeatureTar2Rafs} - require.True(t, features.Contains(FeatureTar2Rafs)) + testsAdd := []struct { + name string + features Features + items []Feature + expect Features + }{ + { + name: "should successfully add items", + features: Features{FeatureBatchSize: {}}, + items: []Feature{FeatureTar2Rafs}, + expect: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}}, + }, + { + name: "should add nothing if duplicated", + features: Features{FeatureBatchSize: {}}, + items: []Feature{FeatureBatchSize}, + expect: Features{FeatureBatchSize: {}}, + }, + { + name: "add should accept nil", + features: Features{FeatureBatchSize: {}}, + items: nil, + expect: Features{FeatureBatchSize: {}}, + }, + } + for _, tt := range testsAdd { + t.Run(tt.name, func(t *testing.T) { + tt.features.Add(tt.items...) + require.Equal(t, tt.expect, tt.features) + }) + } - features.Remove(FeatureTar2Rafs) - require.False(t, features.Contains(FeatureTar2Rafs)) + testsNew := []struct { + name string + items []Feature + expect Features + }{ + { + name: "should successfully new Features", + items: []Feature{FeatureTar2Rafs, FeatureBatchSize}, + expect: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}}, + }, + { + name: "should duplicate same items", + items: []Feature{FeatureBatchSize, FeatureBatchSize}, + expect: Features{FeatureBatchSize: {}}, + }, + { + name: "New should accept nil", + items: nil, + expect: Features{}, + }, + } + for _, tt := range testsNew { + t.Run(tt.name, func(t *testing.T) { + features := NewFeatures(tt.items...) + require.Equal(t, tt.expect, features) + }) + } + + testsRemove := []struct { + name string + features Features + items []Feature + expect Features + }{ + { + name: "should successfully remove items", + features: Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}}, + items: []Feature{FeatureTar2Rafs}, + expect: Features{FeatureBatchSize: {}}, + }, + { + name: "should remove item iff exists", + features: Features{FeatureBatchSize: {}}, + items: []Feature{FeatureBatchSize, FeatureTar2Rafs}, + expect: Features{}, + }, + { + name: "Remove should accept nil", + features: Features{FeatureBatchSize: {}}, + items: nil, + expect: Features{FeatureBatchSize: {}}, + }, + } + for _, tt := range testsRemove { + t.Run(tt.name, func(t *testing.T) { + tt.features.Remove(tt.items...) + require.Equal(t, tt.expect, tt.features) + }) + } + + testsContains := []struct { + name string + features Features + item Feature + expect bool + }{ + { + name: "should return contains", + features: Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}}, + item: FeatureTar2Rafs, + expect: true, + }, + { + name: "should return not contains", + features: Features{FeatureBatchSize: {}}, + item: FeatureTar2Rafs, + expect: false, + }, + { + name: "Contains should accept empty string", + features: Features{FeatureBatchSize: {}}, + item: "", + expect: false, + }, + } + for _, tt := range testsContains { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expect, tt.features.Contains(tt.item)) + }) + } + + testsEquals := []struct { + name string + features Features + other Features + expect bool + }{ + { + name: "should successfully check equality", + features: Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}}, + other: Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}}, + expect: true, + }, + { + name: "should successfully check inequality with different length", + features: Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}}, + other: Features{FeatureBatchSize: {}}, + expect: false, + }, + { + name: "should successfully check inequality with different items", + features: Features{FeatureTar2Rafs: {}}, + other: Features{FeatureBatchSize: {}}, + expect: false, + }, + { + name: "should ignore order", + features: Features{FeatureBatchSize: {}, FeatureTar2Rafs: {}}, + other: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}}, + expect: true, + }, + { + name: "Equals should accept nil", + features: Features{FeatureBatchSize: {}}, + other: nil, + expect: false, + }, + } + for _, tt := range testsEquals { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expect, tt.features.Equals(tt.other)) + }) + } +} + +func TestDetectFeature(t *testing.T) { + tests := []struct { + name string + feature Feature + helpMsg []byte + expect bool + }{ + { + name: "'--type tar-rafs' is supported in v2.2.0-239-gf5c08fcf", + feature: FeatureTar2Rafs, + expect: true, + helpMsg: []byte(` + Create RAFS filesystems from directories, tar files or OCI images + + Usage: nydus-image create [OPTIONS] + + Arguments: + source from which to build the RAFS filesystem + + Options: + -L, --log-file + Log file path + -t, --type + Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index] + -B, --bootstrap + File path to save the generated RAFS metadata blob + -l, --log-level + Log level: [default: info] [possible values: trace, debug, info, warn, error] + -D, --blob-dir + Directory path to save generated RAFS metadata and data blobs + -b, --blob + File path to save the generated RAFS data blob + --blob-inline-meta + Inline RAFS metadata and blob metadata into the data blob + --blob-id + OSS object id for the generated RAFS data blob + --blob-data-size + Set data blob size for 'estargztoc-ref' conversion + --chunk-size + Set the size of data chunks, must be power of two and between 0x1000-0x1000000: + --batch-size + Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0] + --compressor + Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd] + --digester + Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256] + -C, --config + Configuration file for storage backend, cache and RAFS FUSE filesystem. + -v, --fs-version + Set RAFS format version number: [default: 6] [possible values: 5, 6] + --features + Enable/disable features [possible values: blob-toc] + --chunk-dict + File path of chunk dictionary for data deduplication + --parent-bootstrap + File path of the parent/referenced RAFS metadata blob (optional) + --aligned-chunk + Align uncompressed data chunks to 4K, only for RAFS V5 + --repeatable + Generate reproducible RAFS metadata + --whiteout-spec + Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none] + --prefetch-policy + Set data prefetch policy [default: none] [possible values: fs, blob, none] + -J, --output-json + File path to save operation result in JSON format + -h, --help + Print help information + `), + }, + { + name: "'--batch-size' is supported in v2.2.0-239-gf5c08fcf", + feature: FeatureBatchSize, + expect: true, + helpMsg: []byte(` + Create RAFS filesystems from directories, tar files or OCI images + + Usage: nydus-image create [OPTIONS] + + Arguments: + source from which to build the RAFS filesystem + + Options: + -L, --log-file + Log file path + -t, --type + Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index] + -B, --bootstrap + File path to save the generated RAFS metadata blob + -l, --log-level + Log level: [default: info] [possible values: trace, debug, info, warn, error] + -D, --blob-dir + Directory path to save generated RAFS metadata and data blobs + -b, --blob + File path to save the generated RAFS data blob + --blob-inline-meta + Inline RAFS metadata and blob metadata into the data blob + --blob-id + OSS object id for the generated RAFS data blob + --blob-data-size + Set data blob size for 'estargztoc-ref' conversion + --chunk-size + Set the size of data chunks, must be power of two and between 0x1000-0x1000000: + --batch-size + Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0] + --compressor + Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd] + --digester + Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256] + -C, --config + Configuration file for storage backend, cache and RAFS FUSE filesystem. + -v, --fs-version + Set RAFS format version number: [default: 6] [possible values: 5, 6] + --features + Enable/disable features [possible values: blob-toc] + --chunk-dict + File path of chunk dictionary for data deduplication + --parent-bootstrap + File path of the parent/referenced RAFS metadata blob (optional) + --aligned-chunk + Align uncompressed data chunks to 4K, only for RAFS V5 + --repeatable + Generate reproducible RAFS metadata + --whiteout-spec + Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none] + --prefetch-policy + Set data prefetch policy [default: none] [possible values: fs, blob, none] + -J, --output-json + File path to save operation result in JSON format + -h, --help + Print help information + `), + }, + { + name: "'--batch-size' is not supported in v2.2.0-163-g180f6d2c", + feature: FeatureBatchSize, + expect: false, + helpMsg: []byte(` + Create RAFS filesystems from directories, tar files or OCI images + + Usage: nydus-image create [OPTIONS] + + Arguments: + source from which to build the RAFS filesystem + + Options: + -L, --log-file + Log file path + -t, --type + Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index] + -B, --bootstrap + File path to save the generated RAFS metadata blob + -l, --log-level + Log level: [default: info] [possible values: trace, debug, info, warn, error] + -D, --blob-dir + Directory path to save generated RAFS metadata and data blobs + -b, --blob + File path to save the generated RAFS data blob + --blob-inline-meta + Inline RAFS metadata and blob metadata into the data blob + --blob-id + OSS object id for the generated RAFS data blob + --blob-data-size + Set data blob size for 'estargztoc-ref' conversion + --chunk-size + Set the size of data chunks, must be power of two and between 0x1000-0x1000000: + --compressor + Algorithm to compress data chunks: [default: zstd] [possible values: none, lz4_block, zstd] + --digester + Algorithm to digest data chunks: [default: blake3] [possible values: blake3, sha256] + -C, --config + Configuration file for storage backend, cache and RAFS FUSE filesystem. + -v, --fs-version + Set RAFS format version number: [default: 6] [possible values: 5, 6] + --features + Enable/disable features [possible values: blob-toc] + --chunk-dict + File path of chunk dictionary for data deduplication + --parent-bootstrap + File path of the parent/referenced RAFS metadata blob (optional) + --aligned-chunk + Align uncompressed data chunks to 4K, only for RAFS V5 + --repeatable + Generate reproducible RAFS metadata + --whiteout-spec + Set the type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none] + --prefetch-policy + Set data prefetch policy [default: none] [possible values: fs, blob, none] + -J, --output-json + File path to save operation result in JSON format + -h, --help + Print help information + `), + }, + + { + name: "'--type tar-rafs' is not supported in v2.1.4", + feature: FeatureTar2Rafs, + expect: false, + helpMsg: []byte(` + nydus-image-create + Creates a nydus image from source + + USAGE: + nydus-image create [FLAGS] [OPTIONS] ... --blob --bootstrap --fs-version --whiteout-spec + + FLAGS: + -A, --aligned-chunk Align data chunks to 4K + --disable-check disable validation of metadata after building + -h, --help Prints help information + --inline-bootstrap append bootstrap data to blob + -R, --repeatable generate reproducible nydus image + -V, --version Prints version information + + OPTIONS: + --backend-config + [deprecated!] Blob storage backend config - JSON string, only support localfs for compatibility + + --backend-type + [deprecated!] Blob storage backend type, only support localfs for compatibility. Try use --blob instead. + [possible values: localfs] + -b, --blob path to store nydus image's data blob + -D, --blob-dir directory to store nydus image's metadata and data blob + --blob-id blob id (as object id in backend/oss) + --blob-meta path to store nydus blob metadata + --blob-offset + add an offset for compressed blob (is only used to put the blob in the tarball) [default: 0] + + -B, --bootstrap path to store the nydus image's metadata blob + -M, --chunk-dict Specify a chunk dictionary for chunk deduplication + -S, --chunk-size + size of nydus image data chunk, must be power of two and between 0x1000-0x100000: [default: 0x100000] + + -c, --compressor + algorithm to compress image data blob: [default: lz4_block] [possible values: none, lz4_block, gzip, zstd] + + -d, --digester + algorithm to digest inodes and data chunks: [default: blake3] [possible values: blake3, sha256] + + -v, --fs-version + version number of nydus image format: [default: 5] [possible values: 5, 6] + + -o, --log-file Specify log file name + -l, --log-level + Specify log level: [default: info] [possible values: trace, debug, info, warn, error] + + -J, --output-json JSON file output path for result + -p, --parent-bootstrap path to parent/referenced image's metadata blob (optional) + -P, --prefetch-policy + blob data prefetch policy [default: none] [possible values: fs, blob, none] + + -t, --source-type + type of the source: [default: directory] [possible values: directory, stargz_index] + + -W, --whiteout-spec + type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none] + + + ARGS: + ... source path to build the nydus image from + `), + }, + { + name: "'--batch-size' is not supported in v1.1.2", + feature: FeatureBatchSize, + expect: false, + helpMsg: []byte(` + nydus-image-create + Create a nydus format accelerated container image + + USAGE: + nydus-image create [FLAGS] [OPTIONS] --blob --bootstrap --whiteout-spec + + FLAGS: + --aligned-chunk Whether to align chunks into blobcache + --disable-check Disable to validate bootstrap file after building + -h, --help Prints help information + --repeatable Produce environment independent image + -V, --version Prints version information + + OPTIONS: + --backend-config + [deprecated!] Blob storage backend config - JSON string, only support localfs for compatibility + + --backend-type + [deprecated!] Blob storage backend type, only support localfs for compatibility. Try use --blob instead. + [possible values: localfs] + --blob A path to blob file which stores nydus image data portion + --blob-dir + A directory where blob files are saved named as their sha256 digest. It's very useful when multiple layers + are built at the same time. + --blob-id blob id (as object id in backend/oss) + --bootstrap A path to bootstrap file which stores nydus image metadata portion + --chunk-dict + specify a chunk dictionary file in bootstrap/db format for chunk deduplication. + + --compressor + how blob will be compressed: none, lz4_block (default) [default: lz4_block] + + --digester + how inode and blob chunk will be digested: blake3 (default), sha256 [default: blake3] + + --log-level + Specify log level: trace, debug, info, warn, error [default: info] [possible values: trace, debug, info, + warn, error] + --output-json JSON output path for build result + --parent-bootstrap bootstrap file path of parent (optional) + --prefetch-policy + Prefetch policy: fs(issued from Fs layer), blob(issued from backend/blob layer), none(no readahead is + needed) [default: none] + --source-type + source type [default: directory] [possible values: directory, stargz_index] + + --whiteout-spec + decide which whiteout spec to follow: "oci" or "overlayfs" [default: oci] [possible values: oci, overlayfs] + + + ARGS: + source path + `), + }, + { + name: "'--type tar-rafs' is not supported in v0.1.0", + feature: FeatureTar2Rafs, + expect: false, + helpMsg: []byte(` + nydus-image-create + dump image bootstrap and upload blob to storage backend + + USAGE: + nydus-image create [FLAGS] [OPTIONS] --bootstrap --whiteout-spec + + FLAGS: + --aligned-chunk Whether to align chunks into blobcache + --disable-check Disable to validate bootstrap file after building + -h, --help Prints help information + --repeatable Produce environment independent image + -V, --version Prints version information + + OPTIONS: + --backend-config blob storage backend config (JSON string) + --backend-config-file blob storage backend config (JSON file) + --backend-type blob storage backend type (enable blob upload if specified) + --blob blob file path + --blob-id blob id (as object id in backend) + --bootstrap bootstrap file path (required) + --compressor + how blob will be compressed: none, lz4_block (default) [default: lz4_block] + + --digester + how inode and blob chunk will be digested: blake3 (default), sha256 [default: blake3] + + --log-level + Specify log level: trace, debug, info, warn, error [default: info] [possible values: trace, debug, info, + warn, error] + --output-json JSON output path for build result + --parent-bootstrap bootstrap file path of parent (optional) + --prefetch-policy + Prefetch policy: fs(issued from Fs layer), blob(issued from backend/blob layer), none(no readahead is + needed) [default: none] + --source-type + source type [default: directory] [possible values: directory, stargz_index] + + --whiteout-spec + decide which whiteout spec to follow: "oci" or "overlayfs" [default: oci] [possible values: oci, overlayfs] + + + ARGS: + source path + `), + }, + { + name: "'--type tar-rafs' should not be detected in separate args", + feature: FeatureTar2Rafs, + expect: false, + helpMsg: []byte(` + Options: + -t, --type + Conversion type: [default: dir-rafs] [possible values: directory] + -l, --log-level + Log level: [default: info] [possible values: tar-rafs] + `), + }, + { + name: "'--type tar-rafs' should be detected if deprecated", + feature: FeatureTar2Rafs, + expect: true, + helpMsg: []byte(` + OPTIONS: + --type + [deprecated!] Conversion type. + [possible values: tar-rafs] + `), + }, + { + name: "detectFeature should support empty input", + feature: "", + expect: false, + helpMsg: []byte(` + OPTIONS: + --type + [deprecated!] Conversion type. + [possible values: tar-rafs] + `), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expect, detectFeature(tt.helpMsg, tt.feature)) + }) + } } -func TestVersion(t *testing.T) { - require.Equal(t, "0.1.0", detectVersion([]byte(` - Version: 0.1.0 - Git Commit: 57a5ae40e91f82eb9d1e9934dee98358bcf822eb - Build Time: Fri, 19 Mar 2021 10:45:00 +0000 - Profile: release - Rustc: rustc 1.49.0 (e1884a8e3 2020-12-29) - `))) - - require.Equal(t, "2.1.3", detectVersion([]byte(` - Version: v2.1.3-rc1 - Git Commit: 24c3bb9ab213ab94dfbf9ba4106042b34034a390 - Build Time: 2023-01-19T02:26:07.782135583Z - Profile: release - Rustc: rustc 1.61.0 (fe5b13d68 2022-05-18) - `))) - - require.Equal(t, "", detectVersion([]byte(` - Version: unknown - Git Commit: 96efc2cf7e75174b49942fd41b84d672f921f9b4 - Build Time: 2023-02-16T13:20:59.102548977Z - Profile: release - Rustc: rustc 1.66.1 (90743e729 2023-01-10) - `))) +func TestDetectFeatures(t *testing.T) { + testsCompare := []struct { + name string + resetGlobal bool + disableTar2Rafs bool + helpText []byte + required Features + detected Features + expectErr bool + }{ + { + name: "should satisfy required features in v2.2.0-239-gf5c08fcf", + resetGlobal: true, + disableTar2Rafs: false, + helpText: []byte(` + Options: + -t, --type + Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index] + --batch-size + Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0] + `), + required: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}}, + detected: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}}, + expectErr: false, + }, + { + name: "should not support '--batch-size' and '--type tar-rafs' in v2.1.4", + resetGlobal: true, + disableTar2Rafs: true, + helpText: []byte(` + nydus-image-create + Creates a nydus image from source + + USAGE: + nydus-image create [FLAGS] [OPTIONS] ... --blob --bootstrap --fs-version --whiteout-spec + + FLAGS: + -A, --aligned-chunk Align data chunks to 4K + --disable-check disable validation of metadata after building + -h, --help Prints help information + --inline-bootstrap append bootstrap data to blob + -R, --repeatable generate reproducible nydus image + -V, --version Prints version information + + OPTIONS: + --backend-config + [deprecated!] Blob storage backend config - JSON string, only support localfs for compatibility + + --backend-type + [deprecated!] Blob storage backend type, only support localfs for compatibility. Try use --blob instead. + [possible values: localfs] + -b, --blob path to store nydus image's data blob + -D, --blob-dir directory to store nydus image's metadata and data blob + --blob-id blob id (as object id in backend/oss) + --blob-meta path to store nydus blob metadata + --blob-offset + add an offset for compressed blob (is only used to put the blob in the tarball) [default: 0] + + -B, --bootstrap path to store the nydus image's metadata blob + -M, --chunk-dict Specify a chunk dictionary for chunk deduplication + -S, --chunk-size + size of nydus image data chunk, must be power of two and between 0x1000-0x100000: [default: 0x100000] + + -c, --compressor + algorithm to compress image data blob: [default: lz4_block] [possible values: none, lz4_block, gzip, zstd] + + -d, --digester + algorithm to digest inodes and data chunks: [default: blake3] [possible values: blake3, sha256] + + -v, --fs-version + version number of nydus image format: [default: 5] [possible values: 5, 6] + + -o, --log-file Specify log file name + -l, --log-level + Specify log level: [default: info] [possible values: trace, debug, info, warn, error] + + -J, --output-json JSON file output path for result + -p, --parent-bootstrap path to parent/referenced image's metadata blob (optional) + -P, --prefetch-policy + blob data prefetch policy [default: none] [possible values: fs, blob, none] + + -t, --source-type + type of the source: [default: directory] [possible values: directory, stargz_index] + + -W, --whiteout-spec + type of whiteout specification: [default: oci] [possible values: oci, overlayfs, none] + + + ARGS: + ... source path to build the nydus image from + `), + required: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}}, + detected: Features{}, + expectErr: false, + }, + { + name: "should ignore '--type tar-rafs' if disabled", + resetGlobal: true, + disableTar2Rafs: true, + helpText: []byte(` + Options: + -t, --type + Conversion type: [default: dir-rafs] [possible values: directory, dir-rafs, estargz-rafs, estargz-ref, estargztoc-ref, tar-rafs, tar-tarfs, targz-rafs, targz-ref, stargz_index] + --batch-size + Set the batch size to merge small chunks, must be power of two, between 0x1000-0x1000000 or be zero: [default: 0] + `), + required: Features{FeatureTar2Rafs: {}, FeatureBatchSize: {}}, + detected: Features{FeatureBatchSize: {}}, + expectErr: false, + }, + { + name: "should return error if required features changed in different calls", + resetGlobal: false, + disableTar2Rafs: false, + helpText: nil, + required: Features{}, + detected: nil, + expectErr: true, + }, + } + for _, tt := range testsCompare { + t.Run(tt.name, func(t *testing.T) { + if tt.resetGlobal { + // Reset global variables. + requiredFeatures = Features{} + detectedFeatures = Features{} + detectFeaturesOnce = sync.Once{} + disableTar2Rafs = tt.disableTar2Rafs + } + detected, err := DetectFeatures("", tt.required, func(_ string) []byte { return tt.helpText }) + require.Equal(t, tt.expectErr, err != nil) + require.Equal(t, tt.detected, detected) + }) + } }