From 49ef04d60cce97064084cca9ed710d32ec46385e Mon Sep 17 00:00:00 2001 From: Daniel Mikusa Date: Wed, 26 Jan 2022 09:07:54 -0500 Subject: [PATCH] Jjbustamante feature/support multiple artifacts of different types (#100) Add support to handle multiple artifacts during a build - Adds single new ArtifactResolver method, ResolveMany & tests - Uses ResolveMany in Application to check for artifacts - If a single artifact is returned, check if it's a file then proceed using the existing procedure. If it's a directory, then copy directory contents to the layer. - If multiple artifacts are returned, check and persist all files and all directory contents - A directory is persisted to the layer. The behavior is such that files copied out of directory A are stored under the layer in a directory named A and files out of a directory B are stored under the layer in a directory B. When files are then restored back out of the layer they will be in sub-directories as well. - Adds test cases around file matching scenarios - Adds test cases around file copies Co-authored-by: Juan Bustamante --- application.go | 123 +++++++++++++++++++++++++++++------ application_test.go | 154 ++++++++++++++++++++++++++++++++++++++++++++ resolvers.go | 33 +++++++++- resolvers_test.go | 69 +++++++++++++++++++- 4 files changed, 359 insertions(+), 20 deletions(-) diff --git a/application.go b/application.go index d869c4a..23f678b 100644 --- a/application.go +++ b/application.go @@ -51,6 +51,7 @@ func (a Application) Contribute(layer libcnb.Layer) (libcnb.Layer, error) { a.LayerContributor.Logger = a.Logger layer, err := a.LayerContributor.Contribute(layer, func() (libcnb.Layer, error) { + // Build a.Logger.Bodyf("Executing %s %s", filepath.Base(a.Command), strings.Join(a.Arguments, " ")) if err := a.Executor.Execute(effect.Execution{ Command: a.Command, @@ -62,27 +63,58 @@ func (a Application) Contribute(layer libcnb.Layer) (libcnb.Layer, error) { return libcnb.Layer{}, fmt.Errorf("error running build\n%w", err) } - artifact, err := a.ArtifactResolver.Resolve(a.ApplicationPath) + // Persist Artifacts + artifacts, err := a.ArtifactResolver.ResolveMany(a.ApplicationPath) if err != nil { - return libcnb.Layer{}, fmt.Errorf("unable to resolve artifact\n%w", err) + return libcnb.Layer{}, fmt.Errorf("unable to resolve artifacts\n%w", err) } - - in, err := os.Open(artifact) - if err != nil { - return libcnb.Layer{}, fmt.Errorf("unable to open %s\n%w", artifact, err) + a.Logger.Debugf("Found artifacts: %s", artifacts) + + if len(artifacts) == 1 { + artifact := artifacts[0] + + fileInfo, err := os.Stat(artifact) + if err != nil { + return libcnb.Layer{}, fmt.Errorf("unable to resolve artifact %s\n%w", artifact, err) + } + + if fileInfo.IsDir() { + if err := copyDirectory(artifact, filepath.Join(layer.Path, filepath.Base(artifact))); err != nil { + return libcnb.Layer{}, fmt.Errorf("unable to copy the directory\n%w", err) + } + } else { + file := filepath.Join(layer.Path, "application.zip") + if err := copyFile(artifact, file); err != nil { + return libcnb.Layer{}, fmt.Errorf("unable to copy the file %s to %s\n%w", artifact, file, err) + } + } + } else { + for _, artifact := range artifacts { + fileInfo, err := os.Stat(artifact) + if err != nil { + return libcnb.Layer{}, fmt.Errorf("unable to resolve artifact %s\n%w", artifact, err) + } + + if fileInfo.IsDir() { + if err := copyDirectory(artifact, filepath.Join(layer.Path, filepath.Base(artifact))); err != nil { + return libcnb.Layer{}, fmt.Errorf("unable to copy a directory\n%w", err) + } + } else { + dest := filepath.Join(layer.Path, fileInfo.Name()) + if err := copyFile(artifact, dest); err != nil { + return libcnb.Layer{}, fmt.Errorf("unable to copy a file %s to %s\n%w", artifact, dest, err) + } + } + } } - defer in.Close() - file := filepath.Join(layer.Path, "application.zip") - if err := sherpa.CopyFile(in, file); err != nil { - return libcnb.Layer{}, fmt.Errorf("unable to copy %s to %s\n%w", artifact, file, err) - } return layer, nil }) if err != nil { return libcnb.Layer{}, fmt.Errorf("unable to contribute application layer\n%w", err) } + // Create SBOM if err := a.SBOMScanner.ScanBuild(a.ApplicationPath, libcnb.CycloneDXJSON, libcnb.SyftJSON); err != nil { return libcnb.Layer{}, fmt.Errorf("unable to create Build SBoM \n%w", err) } @@ -96,6 +128,7 @@ func (a Application) Contribute(layer libcnb.Layer) (libcnb.Layer, error) { a.BOM.Entries = append(a.BOM.Entries, entry) } + // Purge Workspace a.Logger.Header("Removing source code") cs, err := ioutil.ReadDir(a.ApplicationPath) if err != nil { @@ -108,15 +141,26 @@ func (a Application) Contribute(layer libcnb.Layer) (libcnb.Layer, error) { } } + // Restore compiled artifacts file := filepath.Join(layer.Path, "application.zip") - in, err := os.Open(file) - if err != nil { - return libcnb.Layer{}, fmt.Errorf("unable to open %s\n%w", file, err) - } - defer in.Close() + if _, err := os.Stat(file); err == nil { + in, err := os.Open(file) + if err != nil { + return libcnb.Layer{}, fmt.Errorf("unable to open %s\n%w", file, err) + } + defer in.Close() - if err := crush.ExtractZip(in, a.ApplicationPath, 0); err != nil { - return libcnb.Layer{}, fmt.Errorf("unable to extract %s\n%w", file, err) + if err := crush.ExtractZip(in, a.ApplicationPath, 0); err != nil { + return libcnb.Layer{}, fmt.Errorf("unable to extract %s\n%w", file, err) + } + } else if err != nil && os.IsNotExist(err) { + a.Logger.Infof("Restoring multiple artifacts") + err := copyDirectory(layer.Path, a.ApplicationPath) + if err != nil { + return libcnb.Layer{}, fmt.Errorf("unable to restore multiple artifacts\n%w", err) + } + } else { + return libcnb.Layer{}, fmt.Errorf("unable to restore artifacts\n%w", err) } return layer, nil @@ -125,3 +169,46 @@ func (a Application) Contribute(layer libcnb.Layer) (libcnb.Layer, error) { func (Application) Name() string { return "application" } + +func copyDirectory(from, to string) error { + files, err := ioutil.ReadDir(from) + if err != nil { + return err + } + + for _, file := range files { + sourcePath := filepath.Join(from, file.Name()) + destPath := filepath.Join(to, file.Name()) + + fileInfo, err := os.Stat(sourcePath) + if err != nil { + return err + } + + if fileInfo.IsDir() { + if err := copyDirectory(sourcePath, destPath); err != nil { + return err + } + } else { + if err := copyFile(sourcePath, destPath); err != nil { + return err + } + } + } + + return nil +} + +func copyFile(from string, to string) error { + in, err := os.Open(from) + if err != nil { + return fmt.Errorf("unable to open file%s\n%w", from, err) + } + defer in.Close() + + if err := sherpa.CopyFile(in, to); err != nil { + return fmt.Errorf("unable to copy %s to %s\n%w", from, to, err) + } + + return nil +} diff --git a/application_test.go b/application_test.go index 2e3822e..2604fd7 100644 --- a/application_test.go +++ b/application_test.go @@ -17,6 +17,7 @@ package libbs_test import ( + "fmt" "io" "io/ioutil" "os" @@ -135,4 +136,157 @@ func testApplication(t *testing.T, context spec.G, it spec.S) { Expect(bom.Entries).To(HaveLen(0)) }) + context("contributes layer with ", func() { + context("folder with multiple files", func() { + it.Before(func() { + folder := filepath.Join(ctx.Application.Path, "target", "native-sources") + os.MkdirAll(folder, os.ModePerm) + + files := []string{"stub-application.jar", "stub-executable.jar"} + for _, file := range files { + in, err := os.Open(filepath.Join("testdata", file)) + Expect(err).NotTo(HaveOccurred()) + + out, err := os.OpenFile(filepath.Join(folder, file), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + Expect(err).NotTo(HaveOccurred()) + + _, err = io.Copy(out, in) + Expect(err).NotTo(HaveOccurred()) + Expect(in.Close()).To(Succeed()) + Expect(out.Close()).To(Succeed()) + } + }) + + it("matches multiple files", func() { + artifactResolver := libbs.ArtifactResolver{ + ConfigurationResolver: libpak.ConfigurationResolver{ + Configurations: []libpak.BuildpackConfiguration{{Default: "target/native-sources/*.jar"}}, + }, + } + application.ArtifactResolver = artifactResolver + + application.Logger = bard.NewLogger(ioutil.Discard) + executor.On("Execute", mock.Anything).Return(nil) + + layer, err := ctx.Layers.Layer("test-layer") + Expect(err).NotTo(HaveOccurred()) + + layer, err = application.Contribute(layer) + + Expect(err).NotTo(HaveOccurred()) + + e := executor.Calls[0].Arguments[0].(effect.Execution) + Expect(e.Command).To(Equal("test-command")) + Expect(e.Args).To(Equal([]string{"test-argument"})) + Expect(e.Dir).To(Equal(ctx.Application.Path)) + Expect(e.Stdout).NotTo(BeNil()) + Expect(e.Stderr).NotTo(BeNil()) + + Expect(filepath.Join(layer.Path, "application.zip")).NotTo(BeAnExistingFile()) + Expect(filepath.Join(ctx.Application.Path, "stub-application.jar")).To(BeAnExistingFile()) + Expect(filepath.Join(ctx.Application.Path, "stub-executable.jar")).To(BeAnExistingFile()) + }) + + it("matches a folder", func() { + artifactResolver := libbs.ArtifactResolver{ + ConfigurationResolver: libpak.ConfigurationResolver{ + Configurations: []libpak.BuildpackConfiguration{{Default: "target/native-sources"}}, + }, + } + application.ArtifactResolver = artifactResolver + + application.Logger = bard.NewLogger(ioutil.Discard) + executor.On("Execute", mock.Anything).Return(nil) + + layer, err := ctx.Layers.Layer("test-layer") + Expect(err).NotTo(HaveOccurred()) + + layer, err = application.Contribute(layer) + + Expect(err).NotTo(HaveOccurred()) + + e := executor.Calls[0].Arguments[0].(effect.Execution) + Expect(e.Command).To(Equal("test-command")) + Expect(e.Args).To(Equal([]string{"test-argument"})) + Expect(e.Dir).To(Equal(ctx.Application.Path)) + Expect(e.Stdout).NotTo(BeNil()) + Expect(e.Stderr).NotTo(BeNil()) + + Expect(filepath.Join(layer.Path, "application.zip")).NotTo(BeAnExistingFile()) + Expect(filepath.Join(ctx.Application.Path, "native-sources", "stub-application.jar")).To(BeAnExistingFile()) + Expect(filepath.Join(ctx.Application.Path, "native-sources", "stub-executable.jar")).To(BeAnExistingFile()) + }) + }) + + context("multiple folders", func() { + it.Before(func() { + folder := filepath.Join(ctx.Application.Path, "target", "native-sources") + os.MkdirAll(folder, os.ModePerm) + + files := []string{"stub-application.jar", "stub-executable.jar"} + for _, file := range files { + in, err := os.Open(filepath.Join("testdata", file)) + Expect(err).NotTo(HaveOccurred()) + + out, err := os.OpenFile(filepath.Join(folder, file), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + Expect(err).NotTo(HaveOccurred()) + + _, err = io.Copy(out, in) + Expect(err).NotTo(HaveOccurred()) + Expect(in.Close()).To(Succeed()) + Expect(out.Close()).To(Succeed()) + } + + folder = filepath.Join(ctx.Application.Path, "target", "code-sources") + os.MkdirAll(folder, os.ModePerm) + + files = []string{"stub-application.jar", "stub-executable.jar"} + for _, file := range files { + in, err := os.Open(filepath.Join("testdata", file)) + Expect(err).NotTo(HaveOccurred()) + + out, err := os.OpenFile(filepath.Join(folder, fmt.Sprintf("source-%s", file)), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + Expect(err).NotTo(HaveOccurred()) + + _, err = io.Copy(out, in) + Expect(err).NotTo(HaveOccurred()) + Expect(in.Close()).To(Succeed()) + Expect(out.Close()).To(Succeed()) + } + }) + + it("matches multiple folders", func() { + artifactResolver := libbs.ArtifactResolver{ + ConfigurationResolver: libpak.ConfigurationResolver{ + Configurations: []libpak.BuildpackConfiguration{{Default: "target/*"}}, + }, + } + application.ArtifactResolver = artifactResolver + + application.Logger = bard.NewLogger(ioutil.Discard) + executor.On("Execute", mock.Anything).Return(nil) + + layer, err := ctx.Layers.Layer("test-layer") + Expect(err).NotTo(HaveOccurred()) + + layer, err = application.Contribute(layer) + + Expect(err).NotTo(HaveOccurred()) + + e := executor.Calls[0].Arguments[0].(effect.Execution) + Expect(e.Command).To(Equal("test-command")) + Expect(e.Args).To(Equal([]string{"test-argument"})) + Expect(e.Dir).To(Equal(ctx.Application.Path)) + Expect(e.Stdout).NotTo(BeNil()) + Expect(e.Stderr).NotTo(BeNil()) + + Expect(filepath.Join(layer.Path, "application.zip")).NotTo(BeAnExistingFile()) + Expect(filepath.Join(ctx.Application.Path, "native-sources", "stub-application.jar")).To(BeAnExistingFile()) + Expect(filepath.Join(ctx.Application.Path, "native-sources", "stub-executable.jar")).To(BeAnExistingFile()) + Expect(filepath.Join(ctx.Application.Path, "code-sources", "source-stub-application.jar")).To(BeAnExistingFile()) + Expect(filepath.Join(ctx.Application.Path, "code-sources", "source-stub-executable.jar")).To(BeAnExistingFile()) + + }) + }) + }) } diff --git a/resolvers.go b/resolvers.go index 39bb60b..2cd6ec6 100644 --- a/resolvers.go +++ b/resolvers.go @@ -20,6 +20,7 @@ import ( "archive/zip" "fmt" "io/ioutil" + "os" "path/filepath" "sort" @@ -131,7 +132,6 @@ func (a *ArtifactResolver) Pattern() string { // Resolve resolves the artifact that was created by the build system. func (a *ArtifactResolver) Resolve(applicationPath string) (string, error) { pattern := a.Pattern() - file := filepath.Join(applicationPath, pattern) candidates, err := filepath.Glob(file) if err != nil { @@ -163,6 +163,37 @@ func (a *ArtifactResolver) Resolve(applicationPath string) (string, error) { return "", fmt.Errorf(helpMsg) } +func (a *ArtifactResolver) ResolveMany(applicationPath string) ([]string, error) { + pattern := a.Pattern() + file := filepath.Join(applicationPath, pattern) + candidates, err := filepath.Glob(file) + if err != nil { + return []string{}, fmt.Errorf("unable to find files with %s\n%w", pattern, err) + } + + if len(candidates) > 0 { + return candidates, nil + } + + entries, err := os.ReadDir(filepath.Dir(pattern)) + if err != nil && os.IsNotExist(err) { + return []string{}, fmt.Errorf("unable to find directory referened by pattern: %s", pattern) + } else if err != nil { + return []string{}, fmt.Errorf("unable to read directory\n%w", err) + } + + contents := []string{} + for _, entry := range entries { + contents = append(contents, entry.Name()) + } + + helpMsg := fmt.Sprintf("unable to find any built artifacts in %s, directory contains: %s", pattern, contents) + if len(a.AdditionalHelpMessage) > 0 { + helpMsg = fmt.Sprintf("%s. %s", helpMsg, a.AdditionalHelpMessage) + } + return []string{}, fmt.Errorf(helpMsg) +} + // ResolveArguments resolves the arguments that should be passed to a build system. func ResolveArguments(configurationKey string, configurationResolver libpak.ConfigurationResolver) ([]string, error) { s, _ := configurationResolver.Resolve(configurationKey) diff --git a/resolvers_test.go b/resolvers_test.go index 98aa5b2..1d9a29e 100644 --- a/resolvers_test.go +++ b/resolvers_test.go @@ -60,7 +60,7 @@ func testResolvers(t *testing.T, context spec.G, it spec.S) { }) }) - context("ArtifactResolver", func() { + context("Resolve", func() { var ( detector *mocks.InterestingFileDetector path string @@ -204,4 +204,71 @@ func testResolvers(t *testing.T, context spec.G, it spec.S) { }) }) }) + + context("ResolveMany", func() { + var ( + detector *mocks.InterestingFileDetector + path string + resolver libbs.ArtifactResolver + ) + + it.Before(func() { + var err error + + detector = &mocks.InterestingFileDetector{} + + path, err = ioutil.TempDir("", "multiple-artifact-resolver") + Expect(err).NotTo(HaveOccurred()) + + resolver = libbs.ArtifactResolver{ + ArtifactConfigurationKey: "TEST_ARTIFACT_CONFIGURATION_KEY", + ConfigurationResolver: libpak.ConfigurationResolver{ + Configurations: []libpak.BuildpackConfiguration{ + {Name: "TEST_ARTIFACT_CONFIGURATION_KEY", Default: "test-*"}, + }, + }, + ModuleConfigurationKey: "TEST_MODULE_CONFIGURATION_KEY", + InterestingFileDetector: detector, + } + }) + + it.After(func() { + Expect(os.RemoveAll(path)).To(Succeed()) + }) + + it("passes with a single candidate", func() { + Expect(ioutil.WriteFile(filepath.Join(path, "test-file"), []byte{}, 0644)).To(Succeed()) + Expect(resolver.ResolveMany(path)).To(Equal([]string{filepath.Join(path, "test-file")})) + }) + + it("passes with multiple candidates", func() { + Expect(ioutil.WriteFile(filepath.Join(path, "test-file"), []byte{}, 0644)).To(Succeed()) + Expect(ioutil.WriteFile(filepath.Join(path, "test-file-1"), []byte{}, 0644)).To(Succeed()) + + Expect(resolver.ResolveMany(path)).To(ContainElements(filepath.Join(path, "test-file"), filepath.Join(path, "test-file-1"))) + }) + + it("passes with a single folder candidate", func() { + Expect(os.Mkdir(filepath.Join(path, "test-folder"), os.ModePerm)).To(Succeed()) + Expect(resolver.ResolveMany(path)).To(ContainElement(filepath.Join(path, "test-folder"))) + }) + + it("passes with multiple folders", func() { + Expect(os.Mkdir(filepath.Join(path, "test-folder-1"), os.ModePerm)).To(Succeed()) + Expect(os.Mkdir(filepath.Join(path, "test-folder-2"), os.ModePerm)).To(Succeed()) + Expect(resolver.ResolveMany(path)).To(ContainElements(filepath.Join(path, "test-folder-1"), filepath.Join(path, "test-folder-2"))) + }) + + it("passes with a file and a folder", func() { + Expect(ioutil.WriteFile(filepath.Join(path, "test-file"), []byte{}, 0644)).To(Succeed()) + Expect(os.Mkdir(filepath.Join(path, "test-folder-1"), os.ModePerm)).To(Succeed()) + Expect(resolver.ResolveMany(path)).To(ContainElements(filepath.Join(path, "test-file"), filepath.Join(path, "test-folder-1"))) + }) + + it("fails with zero candidates", func() { + _, err := resolver.ResolveMany(path) + + Expect(err).To(MatchError(HavePrefix("unable to find any built artifacts in test-*, directory contains:"))) + }) + }) }