Jjbustamante feature/support multiple artifacts of different types (#100

)

Add support to handle multiple artifacts during a build

- Adds single new ArtifactResolver method, ResolveMany & tests
- Uses ResolveMany in Application to check for artifacts
- If a single artifact is returned, check if it's a file then proceed using the existing procedure. If it's a directory, then copy directory contents to the layer.
- If multiple artifacts are returned, check and persist all files and all directory contents
- A directory is persisted to the layer. The behavior is such that files copied out of directory A are stored under the layer in a directory named A and files out of a directory B are stored under the layer in a directory B. When files are then restored back out of the layer they will be in sub-directories as well.
- Adds test cases around file matching scenarios 
- Adds test cases around file copies 

Co-authored-by: Juan Bustamante <jbustamante@vmware.com>

application.go

@@ -51,6 +51,7 @@ func (a Application) Contribute(layer libcnb.Layer) (libcnb.Layer, error) {
 	a.LayerContributor.Logger = a.Logger
 
 	layer, err := a.LayerContributor.Contribute(layer, func() (libcnb.Layer, error) {
+		// Build
 		a.Logger.Bodyf("Executing %s %s", filepath.Base(a.Command), strings.Join(a.Arguments, " "))
 		if err := a.Executor.Execute(effect.Execution{
 			Command: a.Command,
@@ -62,27 +63,58 @@ func (a Application) Contribute(layer libcnb.Layer) (libcnb.Layer, error) {
 			return libcnb.Layer{}, fmt.Errorf("error running build\n%w", err)
 		}
 
-		artifact, err := a.ArtifactResolver.Resolve(a.ApplicationPath)
+		// Persist Artifacts
+		artifacts, err := a.ArtifactResolver.ResolveMany(a.ApplicationPath)
 		if err != nil {
-			return libcnb.Layer{}, fmt.Errorf("unable to resolve artifact\n%w", err)
+			return libcnb.Layer{}, fmt.Errorf("unable to resolve artifacts\n%w", err)
 		}
-
-		in, err := os.Open(artifact)
-		if err != nil {
-			return libcnb.Layer{}, fmt.Errorf("unable to open %s\n%w", artifact, err)
+		a.Logger.Debugf("Found artifacts: %s", artifacts)
+
+		if len(artifacts) == 1 {
+			artifact := artifacts[0]
+
+			fileInfo, err := os.Stat(artifact)
+			if err != nil {
+				return libcnb.Layer{}, fmt.Errorf("unable to resolve artifact %s\n%w", artifact, err)
+			}
+
+			if fileInfo.IsDir() {
+				if err := copyDirectory(artifact, filepath.Join(layer.Path, filepath.Base(artifact))); err != nil {
+					return libcnb.Layer{}, fmt.Errorf("unable to copy the directory\n%w", err)
+				}
+			} else {
+				file := filepath.Join(layer.Path, "application.zip")
+				if err := copyFile(artifact, file); err != nil {
+					return libcnb.Layer{}, fmt.Errorf("unable to copy the file %s to %s\n%w", artifact, file, err)
+				}
+			}
+		} else {
+			for _, artifact := range artifacts {
+				fileInfo, err := os.Stat(artifact)
+				if err != nil {
+					return libcnb.Layer{}, fmt.Errorf("unable to resolve artifact %s\n%w", artifact, err)
+				}
+
+				if fileInfo.IsDir() {
+					if err := copyDirectory(artifact, filepath.Join(layer.Path, filepath.Base(artifact))); err != nil {
+						return libcnb.Layer{}, fmt.Errorf("unable to copy a directory\n%w", err)
+					}
+				} else {
+					dest := filepath.Join(layer.Path, fileInfo.Name())
+					if err := copyFile(artifact, dest); err != nil {
+						return libcnb.Layer{}, fmt.Errorf("unable to copy a file %s to %s\n%w", artifact, dest, err)
+					}
+				}
+			}
 		}
-		defer in.Close()
 
-		file := filepath.Join(layer.Path, "application.zip")
-		if err := sherpa.CopyFile(in, file); err != nil {
-			return libcnb.Layer{}, fmt.Errorf("unable to copy %s to %s\n%w", artifact, file, err)
-		}
 		return layer, nil
 	})
 	if err != nil {
 		return libcnb.Layer{}, fmt.Errorf("unable to contribute application layer\n%w", err)
 	}
 
+	// Create SBOM
 	if err := a.SBOMScanner.ScanBuild(a.ApplicationPath, libcnb.CycloneDXJSON, libcnb.SyftJSON); err != nil {
 		return libcnb.Layer{}, fmt.Errorf("unable to create Build SBoM \n%w", err)
 	}
@@ -96,6 +128,7 @@ func (a Application) Contribute(layer libcnb.Layer) (libcnb.Layer, error) {
 		a.BOM.Entries = append(a.BOM.Entries, entry)
 	}
 
+	// Purge Workspace
 	a.Logger.Header("Removing source code")
 	cs, err := ioutil.ReadDir(a.ApplicationPath)
 	if err != nil {
@@ -108,15 +141,26 @@ func (a Application) Contribute(layer libcnb.Layer) (libcnb.Layer, error) {
 		}
 	}
 
+	// Restore compiled artifacts
 	file := filepath.Join(layer.Path, "application.zip")
-	in, err := os.Open(file)
-	if err != nil {
-		return libcnb.Layer{}, fmt.Errorf("unable to open %s\n%w", file, err)
-	}
-	defer in.Close()
+	if _, err := os.Stat(file); err == nil {
+		in, err := os.Open(file)
+		if err != nil {
+			return libcnb.Layer{}, fmt.Errorf("unable to open %s\n%w", file, err)
+		}
+		defer in.Close()
 
-	if err := crush.ExtractZip(in, a.ApplicationPath, 0); err != nil {
-		return libcnb.Layer{}, fmt.Errorf("unable to extract %s\n%w", file, err)
+		if err := crush.ExtractZip(in, a.ApplicationPath, 0); err != nil {
+			return libcnb.Layer{}, fmt.Errorf("unable to extract %s\n%w", file, err)
+		}
+	} else if err != nil && os.IsNotExist(err) {
+		a.Logger.Infof("Restoring multiple artifacts")
+		err := copyDirectory(layer.Path, a.ApplicationPath)
+		if err != nil {
+			return libcnb.Layer{}, fmt.Errorf("unable to restore multiple artifacts\n%w", err)
+		}
+	} else {
+		return libcnb.Layer{}, fmt.Errorf("unable to restore artifacts\n%w", err)
 	}
 
 	return layer, nil
@@ -125,3 +169,46 @@ func (a Application) Contribute(layer libcnb.Layer) (libcnb.Layer, error) {
 func (Application) Name() string {
 	return "application"
 }
+
+func copyDirectory(from, to string) error {
+	files, err := ioutil.ReadDir(from)
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		sourcePath := filepath.Join(from, file.Name())
+		destPath := filepath.Join(to, file.Name())
+
+		fileInfo, err := os.Stat(sourcePath)
+		if err != nil {
+			return err
+		}
+
+		if fileInfo.IsDir() {
+			if err := copyDirectory(sourcePath, destPath); err != nil {
+				return err
+			}
+		} else {
+			if err := copyFile(sourcePath, destPath); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func copyFile(from string, to string) error {
+	in, err := os.Open(from)
+	if err != nil {
+		return fmt.Errorf("unable to open file%s\n%w", from, err)
+	}
+	defer in.Close()
+
+	if err := sherpa.CopyFile(in, to); err != nil {
+		return fmt.Errorf("unable to copy %s to %s\n%w", from, to, err)
+	}
+
+	return nil
+}

application_test.go

@@ -17,6 +17,7 @@
 package libbs_test
 
 import (
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -135,4 +136,157 @@ func testApplication(t *testing.T, context spec.G, it spec.S) {
 		Expect(bom.Entries).To(HaveLen(0))
 	})
 
+	context("contributes layer with ", func() {
+		context("folder with multiple files", func() {
+			it.Before(func() {
+				folder := filepath.Join(ctx.Application.Path, "target", "native-sources")
+				os.MkdirAll(folder, os.ModePerm)
+
+				files := []string{"stub-application.jar", "stub-executable.jar"}
+				for _, file := range files {
+					in, err := os.Open(filepath.Join("testdata", file))
+					Expect(err).NotTo(HaveOccurred())
+
+					out, err := os.OpenFile(filepath.Join(folder, file), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
+					Expect(err).NotTo(HaveOccurred())
+
+					_, err = io.Copy(out, in)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(in.Close()).To(Succeed())
+					Expect(out.Close()).To(Succeed())
+				}
+			})
+
+			it("matches multiple files", func() {
+				artifactResolver := libbs.ArtifactResolver{
+					ConfigurationResolver: libpak.ConfigurationResolver{
+						Configurations: []libpak.BuildpackConfiguration{{Default: "target/native-sources/*.jar"}},
+					},
+				}
+				application.ArtifactResolver = artifactResolver
+
+				application.Logger = bard.NewLogger(ioutil.Discard)
+				executor.On("Execute", mock.Anything).Return(nil)
+
+				layer, err := ctx.Layers.Layer("test-layer")
+				Expect(err).NotTo(HaveOccurred())
+
+				layer, err = application.Contribute(layer)
+
+				Expect(err).NotTo(HaveOccurred())
+
+				e := executor.Calls[0].Arguments[0].(effect.Execution)
+				Expect(e.Command).To(Equal("test-command"))
+				Expect(e.Args).To(Equal([]string{"test-argument"}))
+				Expect(e.Dir).To(Equal(ctx.Application.Path))
+				Expect(e.Stdout).NotTo(BeNil())
+				Expect(e.Stderr).NotTo(BeNil())
+
+				Expect(filepath.Join(layer.Path, "application.zip")).NotTo(BeAnExistingFile())
+				Expect(filepath.Join(ctx.Application.Path, "stub-application.jar")).To(BeAnExistingFile())
+				Expect(filepath.Join(ctx.Application.Path, "stub-executable.jar")).To(BeAnExistingFile())
+			})
+
+			it("matches a folder", func() {
+				artifactResolver := libbs.ArtifactResolver{
+					ConfigurationResolver: libpak.ConfigurationResolver{
+						Configurations: []libpak.BuildpackConfiguration{{Default: "target/native-sources"}},
+					},
+				}
+				application.ArtifactResolver = artifactResolver
+
+				application.Logger = bard.NewLogger(ioutil.Discard)
+				executor.On("Execute", mock.Anything).Return(nil)
+
+				layer, err := ctx.Layers.Layer("test-layer")
+				Expect(err).NotTo(HaveOccurred())
+
+				layer, err = application.Contribute(layer)
+
+				Expect(err).NotTo(HaveOccurred())
+
+				e := executor.Calls[0].Arguments[0].(effect.Execution)
+				Expect(e.Command).To(Equal("test-command"))
+				Expect(e.Args).To(Equal([]string{"test-argument"}))
+				Expect(e.Dir).To(Equal(ctx.Application.Path))
+				Expect(e.Stdout).NotTo(BeNil())
+				Expect(e.Stderr).NotTo(BeNil())
+
+				Expect(filepath.Join(layer.Path, "application.zip")).NotTo(BeAnExistingFile())
+				Expect(filepath.Join(ctx.Application.Path, "native-sources", "stub-application.jar")).To(BeAnExistingFile())
+				Expect(filepath.Join(ctx.Application.Path, "native-sources", "stub-executable.jar")).To(BeAnExistingFile())
+			})
+		})
+
+		context("multiple folders", func() {
+			it.Before(func() {
+				folder := filepath.Join(ctx.Application.Path, "target", "native-sources")
+				os.MkdirAll(folder, os.ModePerm)
+
+				files := []string{"stub-application.jar", "stub-executable.jar"}
+				for _, file := range files {
+					in, err := os.Open(filepath.Join("testdata", file))
+					Expect(err).NotTo(HaveOccurred())
+
+					out, err := os.OpenFile(filepath.Join(folder, file), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
+					Expect(err).NotTo(HaveOccurred())
+
+					_, err = io.Copy(out, in)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(in.Close()).To(Succeed())
+					Expect(out.Close()).To(Succeed())
+				}
+
+				folder = filepath.Join(ctx.Application.Path, "target", "code-sources")
+				os.MkdirAll(folder, os.ModePerm)
+
+				files = []string{"stub-application.jar", "stub-executable.jar"}
+				for _, file := range files {
+					in, err := os.Open(filepath.Join("testdata", file))
+					Expect(err).NotTo(HaveOccurred())
+
+					out, err := os.OpenFile(filepath.Join(folder, fmt.Sprintf("source-%s", file)), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
+					Expect(err).NotTo(HaveOccurred())
+
+					_, err = io.Copy(out, in)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(in.Close()).To(Succeed())
+					Expect(out.Close()).To(Succeed())
+				}
+			})
+
+			it("matches multiple folders", func() {
+				artifactResolver := libbs.ArtifactResolver{
+					ConfigurationResolver: libpak.ConfigurationResolver{
+						Configurations: []libpak.BuildpackConfiguration{{Default: "target/*"}},
+					},
+				}
+				application.ArtifactResolver = artifactResolver
+
+				application.Logger = bard.NewLogger(ioutil.Discard)
+				executor.On("Execute", mock.Anything).Return(nil)
+
+				layer, err := ctx.Layers.Layer("test-layer")
+				Expect(err).NotTo(HaveOccurred())
+
+				layer, err = application.Contribute(layer)
+
+				Expect(err).NotTo(HaveOccurred())
+
+				e := executor.Calls[0].Arguments[0].(effect.Execution)
+				Expect(e.Command).To(Equal("test-command"))
+				Expect(e.Args).To(Equal([]string{"test-argument"}))
+				Expect(e.Dir).To(Equal(ctx.Application.Path))
+				Expect(e.Stdout).NotTo(BeNil())
+				Expect(e.Stderr).NotTo(BeNil())
+
+				Expect(filepath.Join(layer.Path, "application.zip")).NotTo(BeAnExistingFile())
+				Expect(filepath.Join(ctx.Application.Path, "native-sources", "stub-application.jar")).To(BeAnExistingFile())
+				Expect(filepath.Join(ctx.Application.Path, "native-sources", "stub-executable.jar")).To(BeAnExistingFile())
+				Expect(filepath.Join(ctx.Application.Path, "code-sources", "source-stub-application.jar")).To(BeAnExistingFile())
+				Expect(filepath.Join(ctx.Application.Path, "code-sources", "source-stub-executable.jar")).To(BeAnExistingFile())
+
+			})
+		})
+	})
 }

resolvers.go

@@ -20,6 +20,7 @@ import (
 	"archive/zip"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"sort"
 
@@ -131,7 +132,6 @@ func (a *ArtifactResolver) Pattern() string {
 // Resolve resolves the artifact that was created by the build system.
 func (a *ArtifactResolver) Resolve(applicationPath string) (string, error) {
 	pattern := a.Pattern()
-
 	file := filepath.Join(applicationPath, pattern)
 	candidates, err := filepath.Glob(file)
 	if err != nil {
@@ -163,6 +163,37 @@ func (a *ArtifactResolver) Resolve(applicationPath string) (string, error) {
 	return "", fmt.Errorf(helpMsg)
 }
 
+func (a *ArtifactResolver) ResolveMany(applicationPath string) ([]string, error) {
+	pattern := a.Pattern()
+	file := filepath.Join(applicationPath, pattern)
+	candidates, err := filepath.Glob(file)
+	if err != nil {
+		return []string{}, fmt.Errorf("unable to find files with %s\n%w", pattern, err)
+	}
+
+	if len(candidates) > 0 {
+		return candidates, nil
+	}
+
+	entries, err := os.ReadDir(filepath.Dir(pattern))
+	if err != nil && os.IsNotExist(err) {
+		return []string{}, fmt.Errorf("unable to find directory referened by pattern: %s", pattern)
+	} else if err != nil {
+		return []string{}, fmt.Errorf("unable to read directory\n%w", err)
+	}
+
+	contents := []string{}
+	for _, entry := range entries {
+		contents = append(contents, entry.Name())
+	}
+
+	helpMsg := fmt.Sprintf("unable to find any built artifacts in %s, directory contains: %s", pattern, contents)
+	if len(a.AdditionalHelpMessage) > 0 {
+		helpMsg = fmt.Sprintf("%s. %s", helpMsg, a.AdditionalHelpMessage)
+	}
+	return []string{}, fmt.Errorf(helpMsg)
+}
+
 // ResolveArguments resolves the arguments that should be passed to a build system.
 func ResolveArguments(configurationKey string, configurationResolver libpak.ConfigurationResolver) ([]string, error) {
 	s, _ := configurationResolver.Resolve(configurationKey)