Skip to content

Commit

Permalink
Adds support for bzip2 file format
Browse files Browse the repository at this point in the history
- Adds new Bzip2Archive type
- Adds support for bzip2 in generic Archive type
  • Loading branch information
ForestEckhardt committed Jun 9, 2021
1 parent eea8806 commit b4a9ea2
Show file tree
Hide file tree
Showing 7 changed files with 280 additions and 0 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/Masterminds/semver/v3 v3.1.1
github.com/cheggaaa/pb/v3 v3.0.8
github.com/docker/distribution v2.7.1+incompatible
github.com/dsnet/compress v0.0.1
github.com/gabriel-vasile/mimetype v1.3.0
github.com/google/go-containerregistry v0.5.1
github.com/onsi/gomega v1.13.0
Expand Down
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avu
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
Expand Down Expand Up @@ -211,6 +214,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
Expand Down Expand Up @@ -339,6 +344,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
Expand Down
75 changes: 75 additions & 0 deletions vacation/example_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"os"
"path/filepath"

dsnetBzip2 "github.com/dsnet/compress/bzip2"
"github.com/paketo-buildpacks/packit/vacation"
"github.com/ulikunitz/xz"
)
Expand Down Expand Up @@ -629,3 +630,77 @@ func ExampleZipArchive() {
// some-dir/some-other-dir/some-file
// third
}

func ExampleBzip2Archive() {
buffer := bytes.NewBuffer(nil)

// Using the dsnet library because the Go compression library does not
// have a writer. There is recent discussion on this issue
// https://github.com/golang/go/issues/4828 to add an encoder. The
// library should be removed once there is a native encoder
bz, err := dsnetBzip2.NewWriter(buffer, nil)
if err != nil {
log.Fatal(err)
}

zw := zip.NewWriter(bz)

files := []ArchiveFile{
{Name: "some-dir/"},
{Name: "some-dir/some-other-dir/"},
{Name: "some-dir/some-other-dir/some-file", Content: []byte("some-dir/some-other-dir/some-file")},
{Name: "first", Content: []byte("first")},
{Name: "second", Content: []byte("second")},
{Name: "third", Content: []byte("third")},
}

for _, file := range files {
header := &zip.FileHeader{Name: file.Name}
header.SetMode(0755)

f, err := zw.CreateHeader(header)
if err != nil {
log.Fatal(err)
}

if _, err := f.Write(file.Content); err != nil {
log.Fatal(err)
}
}

zw.Close()
bz.Close()

destination, err := os.MkdirTemp("", "destination")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(destination)

archive := vacation.NewBzip2Archive(bytes.NewReader(buffer.Bytes()))
if err := archive.Decompress(destination); err != nil {
log.Fatal(err)
}

err = filepath.Walk(destination, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
rel, err := filepath.Rel(destination, path)
if err != nil {
log.Fatal(err)
}

fmt.Printf("%s\n", rel)
return nil
}
return nil
})
if err != nil {
log.Fatal(err)
}

// Output:
// first
// second
// some-dir/some-other-dir/some-file
// third
}
1 change: 1 addition & 0 deletions vacation/init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
func TestVacation(t *testing.T) {
suite := spec.New("vacation", spec.Report(report.Terminal{}))
suite("VacationArchive", testVacationArchive)
suite("VacationBzip2", testVacationBzip2)
suite("VacationSymlinkSorting", testVacationSymlinkSorting)
suite("VacationTar", testVacationTar)
suite("VacationTarGzip", testVacationTarGzip)
Expand Down
17 changes: 17 additions & 0 deletions vacation/vacation.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"archive/tar"
"archive/zip"
"bufio"
"compress/bzip2"
"compress/gzip"
"fmt"
"io"
Expand Down Expand Up @@ -247,6 +248,8 @@ func (a Archive) Decompress(destination string) error {
return NewTarXZArchive(bufferedReader).StripComponents(a.components).Decompress(destination)
case "application/zip":
return NewZipArchive(bufferedReader).Decompress(destination)
case "application/x-bzip2":
return NewBzip2Archive(bufferedReader).Decompress(destination)
case "text/plain; charset=utf-8":
// This function will write the contents of the reader to file called
// "artifact" in the destination directory
Expand Down Expand Up @@ -327,11 +330,21 @@ type ZipArchive struct {
reader io.Reader
}

// A Bzip2Archive decompresses bzip2 files from an input stream.
type Bzip2Archive struct {
reader io.Reader
}

// NewZipArchive returns a new ZipArchive that reads from inputReader.
func NewZipArchive(inputReader io.Reader) ZipArchive {
return ZipArchive{reader: inputReader}
}

// NewBzip2Archive returns a new Bzip2Archive that reads from inputReader.
func NewBzip2Archive(inputReader io.Reader) Bzip2Archive {
return Bzip2Archive{reader: inputReader}
}

// Decompress reads from ZipArchive and writes files into the destination
// specified.
func (z ZipArchive) Decompress(destination string) error {
Expand Down Expand Up @@ -475,6 +488,10 @@ func (z ZipArchive) Decompress(destination string) error {
return nil
}

func (bz Bzip2Archive) Decompress(destination string) error {
return NewZipArchive(bzip2.NewReader(bz.reader)).Decompress(destination)
}

// This function checks to see that the given path is within the destination
// directory
func checkExtractPath(tarFilePath string, destination string) error {
Expand Down
54 changes: 54 additions & 0 deletions vacation/vacation_archive_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"path/filepath"
"testing"

dsnetBzip2 "github.com/dsnet/compress/bzip2"
"github.com/paketo-buildpacks/packit/vacation"
"github.com/sclevine/spec"
"github.com/ulikunitz/xz"
Expand Down Expand Up @@ -252,6 +253,59 @@ func testVacationArchive(t *testing.T, context spec.G, it spec.S) {
})
})

context("when passed the reader of a bzip2 file", func() {
var (
archive vacation.Archive
tempDir string
)

it.Before(func() {
var err error
tempDir, err = os.MkdirTemp("", "vacation")
Expect(err).NotTo(HaveOccurred())

buffer := bytes.NewBuffer(nil)

// Using the dsnet library because the Go compression library does not
// have a writer. There is recent discussion on this issue
// https://github.com/golang/go/issues/4828 to add an encoder. The
// library should be removed once there is a native encoder
bz, err := dsnetBzip2.NewWriter(buffer, nil)
Expect(err).NotTo(HaveOccurred())

zw := zip.NewWriter(bz)

header := &zip.FileHeader{Name: "some-file"}
header.SetMode(0755)

f, err := zw.CreateHeader(header)
Expect(err).NotTo(HaveOccurred())

_, err = f.Write([]byte("some-file"))
Expect(err).NotTo(HaveOccurred())

Expect(zw.Close()).To(Succeed())
Expect(bz.Close()).To(Succeed())

archive = vacation.NewArchive(buffer)
})

it.After(func() {
Expect(os.RemoveAll(tempDir)).To(Succeed())
})

it("unpackages the archive into the path", func() {
err := archive.Decompress(tempDir)
Expect(err).NotTo(HaveOccurred())

files, err := filepath.Glob(filepath.Join(tempDir, "*"))
Expect(err).NotTo(HaveOccurred())
Expect(files).To(ConsistOf([]string{
filepath.Join(tempDir, "some-file"),
}))
})
})

context("failure cases", func() {
context("the buffer passed is of are unknown type", func() {
var (
Expand Down
126 changes: 126 additions & 0 deletions vacation/vacation_bzip2_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
package vacation_test

import (
"archive/zip"
"bytes"
"fmt"
"os"
"path/filepath"
"testing"

dsnetBzip2 "github.com/dsnet/compress/bzip2"
"github.com/paketo-buildpacks/packit/vacation"
"github.com/sclevine/spec"

. "github.com/onsi/gomega"
)

func testVacationBzip2(t *testing.T, context spec.G, it spec.S) {
var (
Expect = NewWithT(t).Expect
)

context("Bzip2Archive.Decompress", func() {
var (
tempDir string
bzip2Archive vacation.Bzip2Archive
)

it.Before(func() {
var err error
tempDir, err = os.MkdirTemp("", "vacation")
Expect(err).NotTo(HaveOccurred())

buffer := bytes.NewBuffer(nil)

// Using the dsnet library because the Go compression library does not
// have a writer. There is recent discussion on this issue
// https://github.com/golang/go/issues/4828 to add an encoder. The
// library should be removed once there is a native encoder
bz, err := dsnetBzip2.NewWriter(buffer, nil)
Expect(err).NotTo(HaveOccurred())

zw := zip.NewWriter(bz)

fileHeader := &zip.FileHeader{Name: "symlink"}
fileHeader.SetMode(0755 | os.ModeSymlink)

symlink, err := zw.CreateHeader(fileHeader)
Expect(err).NotTo(HaveOccurred())

_, err = symlink.Write([]byte(filepath.Join("some-dir", "some-other-dir", "some-file")))
Expect(err).NotTo(HaveOccurred())

// Some archive files will make a relative top level path directory these
// should still successfully decompress.
_, err = zw.Create("./")
Expect(err).NotTo(HaveOccurred())

_, err = zw.Create("some-dir/")
Expect(err).NotTo(HaveOccurred())

_, err = zw.Create(fmt.Sprintf("%s/", filepath.Join("some-dir", "some-other-dir")))
Expect(err).NotTo(HaveOccurred())

fileHeader = &zip.FileHeader{Name: filepath.Join("some-dir", "some-other-dir", "some-file")}
fileHeader.SetMode(0644)

nestedFile, err := zw.CreateHeader(fileHeader)
Expect(err).NotTo(HaveOccurred())

_, err = nestedFile.Write([]byte("nested file"))
Expect(err).NotTo(HaveOccurred())

for _, name := range []string{"first", "second", "third"} {
fileHeader := &zip.FileHeader{Name: name}
fileHeader.SetMode(0755)

f, err := zw.CreateHeader(fileHeader)
Expect(err).NotTo(HaveOccurred())

_, err = f.Write([]byte(name))
Expect(err).NotTo(HaveOccurred())
}

Expect(zw.Close()).To(Succeed())
Expect(bz.Close()).To(Succeed())

bzip2Archive = vacation.NewBzip2Archive(bytes.NewReader(buffer.Bytes()))
})

it.After(func() {
Expect(os.RemoveAll(tempDir)).To(Succeed())
})

it("unpackages the archive into the path", func() {
var err error
err = bzip2Archive.Decompress(tempDir)
Expect(err).ToNot(HaveOccurred())

files, err := filepath.Glob(fmt.Sprintf("%s/*", tempDir))
Expect(err).NotTo(HaveOccurred())
Expect(files).To(ConsistOf([]string{
filepath.Join(tempDir, "first"),
filepath.Join(tempDir, "second"),
filepath.Join(tempDir, "third"),
filepath.Join(tempDir, "some-dir"),
filepath.Join(tempDir, "symlink"),
}))

info, err := os.Stat(filepath.Join(tempDir, "first"))
Expect(err).NotTo(HaveOccurred())
Expect(info.Mode()).To(Equal(os.FileMode(0755)))

Expect(filepath.Join(tempDir, "some-dir", "some-other-dir")).To(BeADirectory())
Expect(filepath.Join(tempDir, "some-dir", "some-other-dir", "some-file")).To(BeARegularFile())

link, err := os.Readlink(filepath.Join(tempDir, "symlink"))
Expect(err).NotTo(HaveOccurred())
Expect(link).To(Equal("some-dir/some-other-dir/some-file"))

data, err := os.ReadFile(filepath.Join(tempDir, "symlink"))
Expect(err).NotTo(HaveOccurred())
Expect(data).To(Equal([]byte("nested file")))
})
})
}

0 comments on commit b4a9ea2

Please sign in to comment.