diff --git a/go.mod b/go.mod index 2ed8ce22..8e55cc66 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/cheggaaa/pb/v3 v3.0.8 github.com/docker/distribution v2.7.1+incompatible + github.com/dsnet/compress v0.0.1 github.com/gabriel-vasile/mimetype v1.3.0 github.com/google/go-containerregistry v0.5.1 github.com/onsi/gomega v1.13.0 diff --git a/go.sum b/go.sum index efaffa9e..9cb23470 100644 --- a/go.sum +++ b/go.sum @@ -83,6 +83,9 @@ github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avu github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -211,6 +214,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -339,6 +344,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= diff --git a/vacation/example_test.go b/vacation/example_test.go index a233f8be..c75c7132 100644 --- a/vacation/example_test.go +++ b/vacation/example_test.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" + dsnetBzip2 "github.com/dsnet/compress/bzip2" "github.com/paketo-buildpacks/packit/vacation" "github.com/ulikunitz/xz" ) @@ -567,6 +568,147 @@ func ExampleTarXZArchive_StripComponents() { // some-other-dir/some-file } +func ExampleTarBzip2Archive() { + buffer := bytes.NewBuffer(nil) + + // Using the dsnet library because the Go compression library does not + // have a writer. There is recent discussion on this issue + // https://github.com/golang/go/issues/4828 to add an encoder. The + // library should be removed once there is a native encoder + bz, err := dsnetBzip2.NewWriter(buffer, nil) + if err != nil { + log.Fatal(err) + } + + tw := tar.NewWriter(bz) + + files := []ArchiveFile{ + {Name: "some-dir/"}, + {Name: "some-dir/some-other-dir/"}, + {Name: "some-dir/some-other-dir/some-file", Content: []byte("some-dir/some-other-dir/some-file")}, + {Name: "first", Content: []byte("first")}, + {Name: "second", Content: []byte("second")}, + {Name: "third", Content: []byte("third")}, + } + + for _, file := range files { + err := tw.WriteHeader(&tar.Header{Name: file.Name, Mode: 0755, Size: int64(len(file.Content))}) + if err != nil { + log.Fatal(err) + } + + _, err = tw.Write(file.Content) + if err != nil { + log.Fatal(err) + } + } + + tw.Close() + bz.Close() + + destination, err := os.MkdirTemp("", "destination") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(destination) + + archive := vacation.NewTarBzip2Archive(bytes.NewReader(buffer.Bytes())) + if err := archive.Decompress(destination); err != nil { + log.Fatal(err) + } + + err = filepath.Walk(destination, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + rel, err := filepath.Rel(destination, path) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", rel) + return nil + } + return nil + }) + if err != nil { + log.Fatal(err) + } + + // Output: + // first + // second + // some-dir/some-other-dir/some-file + // third +} + +func ExampleTarBzip2Archive_StripComponents() { + buffer := bytes.NewBuffer(nil) + + // Using the dsnet library because the Go compression library does not + // have a writer. There is recent discussion on this issue + // https://github.com/golang/go/issues/4828 to add an encoder. The + // library should be removed once there is a native encoder + bz, err := dsnetBzip2.NewWriter(buffer, nil) + if err != nil { + log.Fatal(err) + } + + tw := tar.NewWriter(bz) + + files := []ArchiveFile{ + {Name: "some-dir/"}, + {Name: "some-dir/some-other-dir/"}, + {Name: "some-dir/some-other-dir/some-file", Content: []byte("some-dir/some-other-dir/some-file")}, + {Name: "first", Content: []byte("first")}, + {Name: "second", Content: []byte("second")}, + {Name: "third", Content: []byte("third")}, + } + + for _, file := range files { + err := tw.WriteHeader(&tar.Header{Name: file.Name, Mode: 0755, Size: int64(len(file.Content))}) + if err != nil { + log.Fatal(err) + } + + _, err = tw.Write(file.Content) + if err != nil { + log.Fatal(err) + } + } + + tw.Close() + bz.Close() + + destination, err := os.MkdirTemp("", "destination") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(destination) + + archive := vacation.NewTarBzip2Archive(bytes.NewReader(buffer.Bytes())).StripComponents(1) + if err := archive.Decompress(destination); err != nil { + log.Fatal(err) + } + + err = filepath.Walk(destination, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + rel, err := filepath.Rel(destination, path) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", rel) + return nil + } + return nil + }) + if err != nil { + log.Fatal(err) + } + + // Output: + // some-other-dir/some-file +} + func ExampleZipArchive() { buffer := bytes.NewBuffer(nil) zw := zip.NewWriter(buffer) diff --git a/vacation/init_test.go b/vacation/init_test.go index c0e33e4c..80c3fe81 100644 --- a/vacation/init_test.go +++ b/vacation/init_test.go @@ -10,6 +10,7 @@ import ( func TestVacation(t *testing.T) { suite := spec.New("vacation", spec.Report(report.Terminal{})) suite("VacationArchive", testVacationArchive) + suite("VacationTarBzip2", testVacationTarBzip2) suite("VacationSymlinkSorting", testVacationSymlinkSorting) suite("VacationTar", testVacationTar) suite("VacationTarGzip", testVacationTarGzip) diff --git a/vacation/vacation.go b/vacation/vacation.go index b09c1600..b0d4077d 100644 --- a/vacation/vacation.go +++ b/vacation/vacation.go @@ -8,6 +8,7 @@ import ( "archive/tar" "archive/zip" "bufio" + "compress/bzip2" "compress/gzip" "fmt" "io" @@ -45,6 +46,12 @@ type TarXZArchive struct { components int } +// A TarBzip2Archive decompresses bzip2 files from an input stream. +type TarBzip2Archive struct { + reader io.Reader + components int +} + // NewArchive returns a new Archive that reads from inputReader. func NewArchive(inputReader io.Reader) Archive { return Archive{reader: inputReader} @@ -65,6 +72,11 @@ func NewTarXZArchive(inputReader io.Reader) TarXZArchive { return TarXZArchive{reader: inputReader} } +// NewTarBzip2Archive returns a new Bzip2Archive that reads from inputReader. +func NewTarBzip2Archive(inputReader io.Reader) TarBzip2Archive { + return TarBzip2Archive{reader: inputReader} +} + // Decompress reads from TarArchive and writes files into the // destination specified. func (ta TarArchive) Decompress(destination string) error { @@ -245,6 +257,8 @@ func (a Archive) Decompress(destination string) error { return NewTarGzipArchive(bufferedReader).StripComponents(a.components).Decompress(destination) case "application/x-xz": return NewTarXZArchive(bufferedReader).StripComponents(a.components).Decompress(destination) + case "application/x-bzip2": + return NewTarBzip2Archive(bufferedReader).StripComponents(a.components).Decompress(destination) case "application/zip": return NewZipArchive(bufferedReader).Decompress(destination) case "text/plain; charset=utf-8": @@ -278,6 +292,12 @@ func (txz TarXZArchive) Decompress(destination string) error { return NewTarArchive(xzr).StripComponents(txz.components).Decompress(destination) } +// Decompress reads from TarBzip2Archive and writes files into the destination +// specified. +func (tbz TarBzip2Archive) Decompress(destination string) error { + return NewTarArchive(bzip2.NewReader(tbz.reader)).StripComponents(tbz.components).Decompress(destination) +} + func writeTextFile(reader io.Reader, destination string) error { file, err := os.Create(filepath.Join(destination, "artifact")) if err != nil { @@ -322,6 +342,13 @@ func (txz TarXZArchive) StripComponents(components int) TarXZArchive { return txz } +// StripComponents behaves like the --strip-components flag on tar command +// removing the first n levels from the final decompression destination. +func (tbz TarBzip2Archive) StripComponents(components int) TarBzip2Archive { + tbz.components = components + return tbz +} + // A ZipArchive decompresses zip files from an input stream. type ZipArchive struct { reader io.Reader diff --git a/vacation/vacation_archive_test.go b/vacation/vacation_archive_test.go index a52e773b..c075e467 100644 --- a/vacation/vacation_archive_test.go +++ b/vacation/vacation_archive_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "testing" + dsnetBzip2 "github.com/dsnet/compress/bzip2" "github.com/paketo-buildpacks/packit/vacation" "github.com/sclevine/spec" "github.com/ulikunitz/xz" @@ -208,6 +209,75 @@ func testVacationArchive(t *testing.T, context spec.G, it spec.S) { }) }) + context("when passed the reader of a bzip2 file", func() { + var ( + archive vacation.Archive + tempDir string + ) + + it.Before(func() { + var err error + tempDir, err = os.MkdirTemp("", "vacation") + Expect(err).NotTo(HaveOccurred()) + + buffer := bytes.NewBuffer(nil) + + // Using the dsnet library because the Go compression library does not + // have a writer. There is recent discussion on this issue + // https://github.com/golang/go/issues/4828 to add an encoder. The + // library should be removed once there is a native encoder + bz, err := dsnetBzip2.NewWriter(buffer, nil) + Expect(err).NotTo(HaveOccurred()) + + tw := tar.NewWriter(bz) + + Expect(tw.WriteHeader(&tar.Header{Name: "some-dir", Mode: 0755, Typeflag: tar.TypeDir})).To(Succeed()) + _, err = tw.Write(nil) + Expect(err).NotTo(HaveOccurred()) + + nestedFile := filepath.Join("some-dir", "some-nested-file") + Expect(tw.WriteHeader(&tar.Header{Name: nestedFile, Mode: 0755, Size: int64(len(nestedFile))})).To(Succeed()) + _, err = tw.Write([]byte(nestedFile)) + Expect(err).NotTo(HaveOccurred()) + + Expect(tw.WriteHeader(&tar.Header{Name: "some-file", Mode: 0755, Size: int64(len("some-file"))})).To(Succeed()) + _, err = tw.Write([]byte("some-file")) + Expect(err).NotTo(HaveOccurred()) + + Expect(tw.Close()).To(Succeed()) + Expect(bz.Close()).To(Succeed()) + + archive = vacation.NewArchive(buffer) + }) + + it.After(func() { + Expect(os.RemoveAll(tempDir)).To(Succeed()) + }) + + it("unpackages the archive into the path", func() { + err := archive.Decompress(tempDir) + Expect(err).NotTo(HaveOccurred()) + + files, err := filepath.Glob(filepath.Join(tempDir, "*")) + Expect(err).NotTo(HaveOccurred()) + Expect(files).To(ConsistOf([]string{ + filepath.Join(tempDir, "some-dir"), + filepath.Join(tempDir, "some-file"), + })) + }) + + it("unpackages the archive into the path but also strips the first component", func() { + err := archive.StripComponents(1).Decompress(tempDir) + Expect(err).NotTo(HaveOccurred()) + + files, err := filepath.Glob(filepath.Join(tempDir, "*")) + Expect(err).NotTo(HaveOccurred()) + Expect(files).To(ConsistOf([]string{ + filepath.Join(tempDir, "some-nested-file"), + })) + }) + }) + context("when passed the reader of a zip file", func() { var ( archive vacation.Archive diff --git a/vacation/vacation_tar_bzip2_test.go b/vacation/vacation_tar_bzip2_test.go new file mode 100644 index 00000000..1d21f2d3 --- /dev/null +++ b/vacation/vacation_tar_bzip2_test.go @@ -0,0 +1,120 @@ +package vacation_test + +import ( + "archive/tar" + "bytes" + "fmt" + "os" + "path/filepath" + "testing" + + dsnetBzip2 "github.com/dsnet/compress/bzip2" + "github.com/paketo-buildpacks/packit/vacation" + "github.com/sclevine/spec" + + . "github.com/onsi/gomega" +) + +func testVacationTarBzip2(t *testing.T, context spec.G, it spec.S) { + var ( + Expect = NewWithT(t).Expect + ) + + context("TarBzip2Archive.Decompress", func() { + var ( + tempDir string + tarBzip2Archive vacation.TarBzip2Archive + ) + + it.Before(func() { + var err error + tempDir, err = os.MkdirTemp("", "vacation") + Expect(err).NotTo(HaveOccurred()) + + buffer := bytes.NewBuffer(nil) + + // Using the dsnet library because the Go compression library does not + // have a writer. There is recent discussion on this issue + // https://github.com/golang/go/issues/4828 to add an encoder. The + // library should be removed once there is a native encoder + bz, err := dsnetBzip2.NewWriter(buffer, nil) + Expect(err).NotTo(HaveOccurred()) + + tw := tar.NewWriter(bz) + + Expect(tw.WriteHeader(&tar.Header{Name: "some-dir", Mode: 0755, Typeflag: tar.TypeDir})).To(Succeed()) + _, err = tw.Write(nil) + Expect(err).NotTo(HaveOccurred()) + + Expect(tw.WriteHeader(&tar.Header{Name: filepath.Join("some-dir", "some-other-dir"), Mode: 0755, Typeflag: tar.TypeDir})).To(Succeed()) + _, err = tw.Write(nil) + Expect(err).NotTo(HaveOccurred()) + + nestedFile := filepath.Join("some-dir", "some-other-dir", "some-file") + Expect(tw.WriteHeader(&tar.Header{Name: nestedFile, Mode: 0755, Size: int64(len(nestedFile))})).To(Succeed()) + _, err = tw.Write([]byte(nestedFile)) + Expect(err).NotTo(HaveOccurred()) + + for _, file := range []string{"first", "second", "third"} { + Expect(tw.WriteHeader(&tar.Header{Name: file, Mode: 0755, Size: int64(len(file))})).To(Succeed()) + _, err = tw.Write([]byte(file)) + Expect(err).NotTo(HaveOccurred()) + } + + Expect(tw.WriteHeader(&tar.Header{Name: "symlink", Mode: 0777, Size: int64(0), Typeflag: tar.TypeSymlink, Linkname: "first"})).To(Succeed()) + _, err = tw.Write([]byte{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(tw.Close()).To(Succeed()) + Expect(bz.Close()).To(Succeed()) + + tarBzip2Archive = vacation.NewTarBzip2Archive(bytes.NewReader(buffer.Bytes())) + }) + + it.After(func() { + Expect(os.RemoveAll(tempDir)).To(Succeed()) + }) + + it("unpackages the archive into the path", func() { + var err error + err = tarBzip2Archive.Decompress(tempDir) + Expect(err).ToNot(HaveOccurred()) + + files, err := filepath.Glob(fmt.Sprintf("%s/*", tempDir)) + Expect(err).NotTo(HaveOccurred()) + Expect(files).To(ConsistOf([]string{ + filepath.Join(tempDir, "first"), + filepath.Join(tempDir, "second"), + filepath.Join(tempDir, "third"), + filepath.Join(tempDir, "some-dir"), + filepath.Join(tempDir, "symlink"), + })) + + info, err := os.Stat(filepath.Join(tempDir, "first")) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode()).To(Equal(os.FileMode(0755))) + + Expect(filepath.Join(tempDir, "some-dir", "some-other-dir")).To(BeADirectory()) + Expect(filepath.Join(tempDir, "some-dir", "some-other-dir", "some-file")).To(BeARegularFile()) + + data, err := os.ReadFile(filepath.Join(tempDir, "symlink")) + Expect(err).NotTo(HaveOccurred()) + Expect(data).To(Equal([]byte(`first`))) + }) + + it("unpackages the archive into the path but also strips the first component", func() { + var err error + err = tarBzip2Archive.StripComponents(1).Decompress(tempDir) + Expect(err).ToNot(HaveOccurred()) + + files, err := filepath.Glob(fmt.Sprintf("%s/*", tempDir)) + Expect(err).NotTo(HaveOccurred()) + Expect(files).To(ConsistOf([]string{ + filepath.Join(tempDir, "some-other-dir"), + })) + + Expect(filepath.Join(tempDir, "some-other-dir")).To(BeADirectory()) + Expect(filepath.Join(tempDir, "some-other-dir", "some-file")).To(BeARegularFile()) + }) + }) +}