From b4a9ea2c6d826b260a775ba0890d6225521c6c84 Mon Sep 17 00:00:00 2001 From: Forest Eckhardt Date: Wed, 9 Jun 2021 17:47:05 +0000 Subject: [PATCH] Adds support for bzip2 file format - Adds new Bzip2Archive type - Adds support for bzip2 in generic Archive type --- go.mod | 1 + go.sum | 6 ++ vacation/example_test.go | 75 ++++++++++++++++++ vacation/init_test.go | 1 + vacation/vacation.go | 17 ++++ vacation/vacation_archive_test.go | 54 +++++++++++++ vacation/vacation_bzip2_test.go | 126 ++++++++++++++++++++++++++++++ 7 files changed, 280 insertions(+) create mode 100644 vacation/vacation_bzip2_test.go diff --git a/go.mod b/go.mod index 2ed8ce22..8e55cc66 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/cheggaaa/pb/v3 v3.0.8 github.com/docker/distribution v2.7.1+incompatible + github.com/dsnet/compress v0.0.1 github.com/gabriel-vasile/mimetype v1.3.0 github.com/google/go-containerregistry v0.5.1 github.com/onsi/gomega v1.13.0 diff --git a/go.sum b/go.sum index efaffa9e..9cb23470 100644 --- a/go.sum +++ b/go.sum @@ -83,6 +83,9 @@ github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avu github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -211,6 +214,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -339,6 +344,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= diff --git a/vacation/example_test.go b/vacation/example_test.go index a233f8be..be6da4d4 100644 --- a/vacation/example_test.go +++ b/vacation/example_test.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" + dsnetBzip2 "github.com/dsnet/compress/bzip2" "github.com/paketo-buildpacks/packit/vacation" "github.com/ulikunitz/xz" ) @@ -629,3 +630,77 @@ func ExampleZipArchive() { // some-dir/some-other-dir/some-file // third } + +func ExampleBzip2Archive() { + buffer := bytes.NewBuffer(nil) + + // Using the dsnet library because the Go compression library does not + // have a writer. There is recent discussion on this issue + // https://github.com/golang/go/issues/4828 to add an encoder. The + // library should be removed once there is a native encoder + bz, err := dsnetBzip2.NewWriter(buffer, nil) + if err != nil { + log.Fatal(err) + } + + zw := zip.NewWriter(bz) + + files := []ArchiveFile{ + {Name: "some-dir/"}, + {Name: "some-dir/some-other-dir/"}, + {Name: "some-dir/some-other-dir/some-file", Content: []byte("some-dir/some-other-dir/some-file")}, + {Name: "first", Content: []byte("first")}, + {Name: "second", Content: []byte("second")}, + {Name: "third", Content: []byte("third")}, + } + + for _, file := range files { + header := &zip.FileHeader{Name: file.Name} + header.SetMode(0755) + + f, err := zw.CreateHeader(header) + if err != nil { + log.Fatal(err) + } + + if _, err := f.Write(file.Content); err != nil { + log.Fatal(err) + } + } + + zw.Close() + bz.Close() + + destination, err := os.MkdirTemp("", "destination") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(destination) + + archive := vacation.NewBzip2Archive(bytes.NewReader(buffer.Bytes())) + if err := archive.Decompress(destination); err != nil { + log.Fatal(err) + } + + err = filepath.Walk(destination, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + rel, err := filepath.Rel(destination, path) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", rel) + return nil + } + return nil + }) + if err != nil { + log.Fatal(err) + } + + // Output: + // first + // second + // some-dir/some-other-dir/some-file + // third +} diff --git a/vacation/init_test.go b/vacation/init_test.go index c0e33e4c..7f679b9b 100644 --- a/vacation/init_test.go +++ b/vacation/init_test.go @@ -10,6 +10,7 @@ import ( func TestVacation(t *testing.T) { suite := spec.New("vacation", spec.Report(report.Terminal{})) suite("VacationArchive", testVacationArchive) + suite("VacationBzip2", testVacationBzip2) suite("VacationSymlinkSorting", testVacationSymlinkSorting) suite("VacationTar", testVacationTar) suite("VacationTarGzip", testVacationTarGzip) diff --git a/vacation/vacation.go b/vacation/vacation.go index b09c1600..44726d09 100644 --- a/vacation/vacation.go +++ b/vacation/vacation.go @@ -8,6 +8,7 @@ import ( "archive/tar" "archive/zip" "bufio" + "compress/bzip2" "compress/gzip" "fmt" "io" @@ -247,6 +248,8 @@ func (a Archive) Decompress(destination string) error { return NewTarXZArchive(bufferedReader).StripComponents(a.components).Decompress(destination) case "application/zip": return NewZipArchive(bufferedReader).Decompress(destination) + case "application/x-bzip2": + return NewBzip2Archive(bufferedReader).Decompress(destination) case "text/plain; charset=utf-8": // This function will write the contents of the reader to file called // "artifact" in the destination directory @@ -327,11 +330,21 @@ type ZipArchive struct { reader io.Reader } +// A Bzip2Archive decompresses bzip2 files from an input stream. +type Bzip2Archive struct { + reader io.Reader +} + // NewZipArchive returns a new ZipArchive that reads from inputReader. func NewZipArchive(inputReader io.Reader) ZipArchive { return ZipArchive{reader: inputReader} } +// NewBzip2Archive returns a new Bzip2Archive that reads from inputReader. +func NewBzip2Archive(inputReader io.Reader) Bzip2Archive { + return Bzip2Archive{reader: inputReader} +} + // Decompress reads from ZipArchive and writes files into the destination // specified. func (z ZipArchive) Decompress(destination string) error { @@ -475,6 +488,10 @@ func (z ZipArchive) Decompress(destination string) error { return nil } +func (bz Bzip2Archive) Decompress(destination string) error { + return NewZipArchive(bzip2.NewReader(bz.reader)).Decompress(destination) +} + // This function checks to see that the given path is within the destination // directory func checkExtractPath(tarFilePath string, destination string) error { diff --git a/vacation/vacation_archive_test.go b/vacation/vacation_archive_test.go index a52e773b..b5946299 100644 --- a/vacation/vacation_archive_test.go +++ b/vacation/vacation_archive_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "testing" + dsnetBzip2 "github.com/dsnet/compress/bzip2" "github.com/paketo-buildpacks/packit/vacation" "github.com/sclevine/spec" "github.com/ulikunitz/xz" @@ -252,6 +253,59 @@ func testVacationArchive(t *testing.T, context spec.G, it spec.S) { }) }) + context("when passed the reader of a bzip2 file", func() { + var ( + archive vacation.Archive + tempDir string + ) + + it.Before(func() { + var err error + tempDir, err = os.MkdirTemp("", "vacation") + Expect(err).NotTo(HaveOccurred()) + + buffer := bytes.NewBuffer(nil) + + // Using the dsnet library because the Go compression library does not + // have a writer. There is recent discussion on this issue + // https://github.com/golang/go/issues/4828 to add an encoder. The + // library should be removed once there is a native encoder + bz, err := dsnetBzip2.NewWriter(buffer, nil) + Expect(err).NotTo(HaveOccurred()) + + zw := zip.NewWriter(bz) + + header := &zip.FileHeader{Name: "some-file"} + header.SetMode(0755) + + f, err := zw.CreateHeader(header) + Expect(err).NotTo(HaveOccurred()) + + _, err = f.Write([]byte("some-file")) + Expect(err).NotTo(HaveOccurred()) + + Expect(zw.Close()).To(Succeed()) + Expect(bz.Close()).To(Succeed()) + + archive = vacation.NewArchive(buffer) + }) + + it.After(func() { + Expect(os.RemoveAll(tempDir)).To(Succeed()) + }) + + it("unpackages the archive into the path", func() { + err := archive.Decompress(tempDir) + Expect(err).NotTo(HaveOccurred()) + + files, err := filepath.Glob(filepath.Join(tempDir, "*")) + Expect(err).NotTo(HaveOccurred()) + Expect(files).To(ConsistOf([]string{ + filepath.Join(tempDir, "some-file"), + })) + }) + }) + context("failure cases", func() { context("the buffer passed is of are unknown type", func() { var ( diff --git a/vacation/vacation_bzip2_test.go b/vacation/vacation_bzip2_test.go new file mode 100644 index 00000000..ee1e9515 --- /dev/null +++ b/vacation/vacation_bzip2_test.go @@ -0,0 +1,126 @@ +package vacation_test + +import ( + "archive/zip" + "bytes" + "fmt" + "os" + "path/filepath" + "testing" + + dsnetBzip2 "github.com/dsnet/compress/bzip2" + "github.com/paketo-buildpacks/packit/vacation" + "github.com/sclevine/spec" + + . "github.com/onsi/gomega" +) + +func testVacationBzip2(t *testing.T, context spec.G, it spec.S) { + var ( + Expect = NewWithT(t).Expect + ) + + context("Bzip2Archive.Decompress", func() { + var ( + tempDir string + bzip2Archive vacation.Bzip2Archive + ) + + it.Before(func() { + var err error + tempDir, err = os.MkdirTemp("", "vacation") + Expect(err).NotTo(HaveOccurred()) + + buffer := bytes.NewBuffer(nil) + + // Using the dsnet library because the Go compression library does not + // have a writer. There is recent discussion on this issue + // https://github.com/golang/go/issues/4828 to add an encoder. The + // library should be removed once there is a native encoder + bz, err := dsnetBzip2.NewWriter(buffer, nil) + Expect(err).NotTo(HaveOccurred()) + + zw := zip.NewWriter(bz) + + fileHeader := &zip.FileHeader{Name: "symlink"} + fileHeader.SetMode(0755 | os.ModeSymlink) + + symlink, err := zw.CreateHeader(fileHeader) + Expect(err).NotTo(HaveOccurred()) + + _, err = symlink.Write([]byte(filepath.Join("some-dir", "some-other-dir", "some-file"))) + Expect(err).NotTo(HaveOccurred()) + + // Some archive files will make a relative top level path directory these + // should still successfully decompress. + _, err = zw.Create("./") + Expect(err).NotTo(HaveOccurred()) + + _, err = zw.Create("some-dir/") + Expect(err).NotTo(HaveOccurred()) + + _, err = zw.Create(fmt.Sprintf("%s/", filepath.Join("some-dir", "some-other-dir"))) + Expect(err).NotTo(HaveOccurred()) + + fileHeader = &zip.FileHeader{Name: filepath.Join("some-dir", "some-other-dir", "some-file")} + fileHeader.SetMode(0644) + + nestedFile, err := zw.CreateHeader(fileHeader) + Expect(err).NotTo(HaveOccurred()) + + _, err = nestedFile.Write([]byte("nested file")) + Expect(err).NotTo(HaveOccurred()) + + for _, name := range []string{"first", "second", "third"} { + fileHeader := &zip.FileHeader{Name: name} + fileHeader.SetMode(0755) + + f, err := zw.CreateHeader(fileHeader) + Expect(err).NotTo(HaveOccurred()) + + _, err = f.Write([]byte(name)) + Expect(err).NotTo(HaveOccurred()) + } + + Expect(zw.Close()).To(Succeed()) + Expect(bz.Close()).To(Succeed()) + + bzip2Archive = vacation.NewBzip2Archive(bytes.NewReader(buffer.Bytes())) + }) + + it.After(func() { + Expect(os.RemoveAll(tempDir)).To(Succeed()) + }) + + it("unpackages the archive into the path", func() { + var err error + err = bzip2Archive.Decompress(tempDir) + Expect(err).ToNot(HaveOccurred()) + + files, err := filepath.Glob(fmt.Sprintf("%s/*", tempDir)) + Expect(err).NotTo(HaveOccurred()) + Expect(files).To(ConsistOf([]string{ + filepath.Join(tempDir, "first"), + filepath.Join(tempDir, "second"), + filepath.Join(tempDir, "third"), + filepath.Join(tempDir, "some-dir"), + filepath.Join(tempDir, "symlink"), + })) + + info, err := os.Stat(filepath.Join(tempDir, "first")) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode()).To(Equal(os.FileMode(0755))) + + Expect(filepath.Join(tempDir, "some-dir", "some-other-dir")).To(BeADirectory()) + Expect(filepath.Join(tempDir, "some-dir", "some-other-dir", "some-file")).To(BeARegularFile()) + + link, err := os.Readlink(filepath.Join(tempDir, "symlink")) + Expect(err).NotTo(HaveOccurred()) + Expect(link).To(Equal("some-dir/some-other-dir/some-file")) + + data, err := os.ReadFile(filepath.Join(tempDir, "symlink")) + Expect(err).NotTo(HaveOccurred()) + Expect(data).To(Equal([]byte("nested file"))) + }) + }) +}