From 4407d10109b2e1f8b012e8ce5ea6d2024ed3ecbc Mon Sep 17 00:00:00 2001 From: huangzy Date: Tue, 21 Feb 2023 10:42:28 +0800 Subject: [PATCH] add http zstd reader for cdi Signed-off-by: huangzy --- pkg/image/filefmt.go | 6 ++++++ pkg/image/validate.go | 2 ++ pkg/importer/BUILD.bazel | 1 + pkg/importer/format-readers.go | 17 +++++++++++++++ tests/utils/fileConversion.go | 25 ++++++++++++++++++++++ tools/cdi-func-test-file-host-init/main.go | 2 ++ 6 files changed, 53 insertions(+) diff --git a/pkg/image/filefmt.go b/pkg/image/filefmt.go index 4cf8b38fe3..87b078ba1d 100644 --- a/pkg/image/filefmt.go +++ b/pkg/image/filefmt.go @@ -25,6 +25,12 @@ var knownHeaders = Headers{ SizeOff: 0, SizeLen: 0, }, + "zst": Header{ + Format: "zst", + magicNumber: []byte{0x28, 0xb5, 0x2f, 0xfd}, + SizeOff: 0, + SizeLen: 0, + }, "qcow2": Header{ Format: "qcow2", magicNumber: []byte{'Q', 'F', 'I', 0xfb}, diff --git a/pkg/image/validate.go b/pkg/image/validate.go index b44fe941d6..845c4834d2 100644 --- a/pkg/image/validate.go +++ b/pkg/image/validate.go @@ -21,6 +21,8 @@ const ( ExtTar = ".tar" // ExtXz is a constant for the .xz extenstion ExtXz = ".xz" + // ExtZst is a constant for the .zst extenstion + ExtZst = ".zst" // ExtTarXz is a constant for the .tar.xz extenstion ExtTarXz = ExtTar + ExtXz // ExtTarGz is a constant for the .tar.gz extenstion diff --git a/pkg/importer/BUILD.bazel b/pkg/importer/BUILD.bazel index 4683ef1f21..b80975c06a 100644 --- a/pkg/importer/BUILD.bazel +++ b/pkg/importer/BUILD.bazel @@ -34,6 +34,7 @@ go_library( "//vendor/github.com/containers/image/v5/oci/archive:go_default_library", "//vendor/github.com/containers/image/v5/pkg/blobinfocache:go_default_library", "//vendor/github.com/containers/image/v5/types:go_default_library", + "//vendor/github.com/klauspost/compress/zstd:go_default_library", "//vendor/github.com/ovirt/go-ovirt:go_default_library", "//vendor/github.com/ovirt/go-ovirt-client:go_default_library", "//vendor/github.com/ovirt/go-ovirt-client-log-klog:go_default_library", diff --git a/pkg/importer/format-readers.go b/pkg/importer/format-readers.go index 3c87c26092..d6d8162e4a 100644 --- a/pkg/importer/format-readers.go +++ b/pkg/importer/format-readers.go @@ -23,6 +23,7 @@ import ( "io" "strconv" + "github.com/klauspost/compress/zstd" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/ulikunitz/xz" @@ -72,6 +73,7 @@ type FormatReaders struct { Archived bool ArchiveXz bool ArchiveGz bool + ArchiveZstd bool progressReader *prometheusutil.ProgressReader } @@ -167,6 +169,12 @@ func (fr *FormatReaders) fileFormatSelector(hdr *image.Header) { fr.Archived = true fr.ArchiveGz = true } + case "zst": + r, err = fr.zstReader() + if err == nil { + fr.Archived = true + fr.ArchiveZstd = true + } case "qcow2": r, err = fr.qcow2NopReader(hdr) fr.Convert = true @@ -208,6 +216,15 @@ func (fr *FormatReaders) gzReader() (io.ReadCloser, error) { return gz, nil } +// Return the zst reader. +func (fr *FormatReaders) zstReader() (io.ReadCloser, error) { + zst, err := zstd.NewReader(fr.TopReader()) + if err != nil { + return nil, errors.Wrap(err, "could not create zst reader") + } + return zst.IOReadCloser(), nil +} + // Return the size of the endpoint "through the eye" of the previous reader. Note: there is no // qcow2 reader so nil is returned so that nothing is appended to the reader stack. // Note: size is stored at offset 24 in the qcow2 header. diff --git a/tests/utils/fileConversion.go b/tests/utils/fileConversion.go index 495954d947..d955f8bb96 100644 --- a/tests/utils/fileConversion.go +++ b/tests/utils/fileConversion.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" + "github.com/klauspost/compress/zstd" "github.com/pkg/errors" "github.com/ulikunitz/xz" @@ -18,6 +19,7 @@ import ( var formatTable = map[string]func(string, string, string) (string, error){ image.ExtGz: toGz, image.ExtXz: toXz, + image.ExtZst: toZst, image.ExtTar: toTar, image.ExtQcow2: convertUsingQemuImg, image.ExtVmdk: convertUsingQemuImg, @@ -111,6 +113,29 @@ func toGz(src, tgtDir, ext string) (string, error) { return tgtPath, nil } +func toZst(src, tgtDir, ext string) (string, error) { + tgtFile, tgtPath, _ := createTargetFile(src, tgtDir, image.ExtGz) + defer tgtFile.Close() + + w, err := zstd.NewWriter(tgtFile) + if err != nil { + return "", errors.Wrapf(err, "Error getting zst writer for file %s", tgtPath) + } + defer w.Close() + + srcFile, err := os.Open(src) + if err != nil { + return "", errors.Wrapf(err, "Error opening file %s", src) + } + defer srcFile.Close() + + _, err = io.Copy(w, srcFile) + if err != nil { + return "", errors.Wrapf(err, "Error writing to file %s", tgtPath) + } + return tgtPath, nil +} + func toXz(src, tgtDir, ext string) (string, error) { tgtFile, tgtPath, _ := createTargetFile(src, tgtDir, image.ExtXz) defer tgtFile.Close() diff --git a/tools/cdi-func-test-file-host-init/main.go b/tools/cdi-func-test-file-host-init/main.go index df26f5c79d..e9123336a5 100644 --- a/tools/cdi-func-test-file-host-init/main.go +++ b/tools/cdi-func-test-file-host-init/main.go @@ -44,12 +44,14 @@ func main() { []string{""}, []string{".gz"}, []string{".xz"}, + []string{".zst"}, []string{".qcow2"}, []string{".vmdk"}, []string{".vhd"}, []string{".vhdx"}, []string{".qcow2", ".gz"}, []string{".qcow2", ".xz"}, + []string{".qcow2", ".zst"}, } if err := utils.CreateCertForTestService(util.GetNamespace(), serviceName, configMapName, *certDir, certFile, keyFile); err != nil {