Switch to using RFC Utilities RFC #5

Utilities RFC 0005 defines how source file deletion should be performed, what key environment variables should be considered, provides the core logic and a buildpack which implements the RFC.

This PR utilizes the logic contributed by RFC 0005 as a Go package and the environment variables defined in it to specify included/excluded files.

It also deprecates using the 'BP_CARGO_EXCLUDE_FOLDERS' environment variable. If this environment variable is used, the values in it are converted to the format expected by the RFC 0005 implementation, so it should not be a breaking change. A warning message will also be emitted. 'BP_CARGO_EXCLUDE_FOLDERS' will be removed before the 1.0 GA release.

Signed-off-by: Daniel Mikusa <dmikusa@vmware.com>

README.md

@@ -33,7 +33,8 @@ The buildpack will do the following:
 | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | `$BP_CARGO_INSTALL_ARGS`       | Additional arguments for `cargo install`. By default, `--locked`. The buildpack will also add `--color=never`, `--root=<destination layer>`, and `--path=<path-to-member>` for each workspace member. You cannot override those values. See more details below.                                                                                                                                        |
 | `$BP_CARGO_WORKSPACE_MEMBERS`  | A comma delimited list of the workspace package names (this is the package name in the member's `Cargo.toml`, not what is in the workspace's `Cargo.toml`'s member list) to install. If the project is not using workspaces, this is not used. By default, for projects with a workspace, the buildpack will build all members in a workspace. See more details below.                                 |
-| `$BP_CARGO_EXCLUDE_FOLDERS`    | A comma delimited list of the top-level folders that should not be deleted, which means they will persist through to the generated image. This *only* applies to top level folders. You only need the folder name, not a full path.                                                                                                                                                                    |
+| `$BP_INCLUDE_FILES`            | Colon separated list of glob patterns to match source files. Any matched file will be retained in the final image. Defaults to `static/*:templates/*:public/*:html/*`.                                                                                                                                                                                                                                 |
+| `$BP_EXCLUDE_FILES`            | Colon separated list of glob patterns to match source files. Any matched file will be specifically removed from the final image. If include patterns are also specified, then they are applied first and exclude patterns can be used to further reduce the fileset.                                                                                                                                   |
 | `$BP_CARGO_TINI_DISABLED`      | Disable using `tini` to launch binary targets. Defaults to `false`, so `tini` is installed and used by default. Set to `true` and `tini` will not be installed or used.                                                                                                                                                                                                                                |
 | `$BP_DISABLE_SBOM`             | Disable running the SBOM scanner. Defaults to `false`, so the scan runs. With larger projects this can take time and disabling the scan will speed up builds. You may want to disable this scane when building locally for a bit of a faster build, but you should not disable this in CI/CD pipelines or when you generate your production images.                                                    |
 | `$BP_CARGO_INSTALL_TOOLS`      | Additional tools that should be installed by running `cargo install`. This should be a space separated list, and each item should contain the name of the tool to install like `cargo-bloat` or `diesel_cli`. Tools installed will be installed prior to compiling application source code and will be available on `$PATH` during build execution (but are not installed into the runtime container). |

buildpack.toml

@@ -57,9 +57,15 @@ api = "0.7"
 
   [[metadata.configurations]]
     build = true
-    default = "static, templates, public, html"
-    description = "folders that should not be deleted and should persist to the generated image"
-    name = "BP_CARGO_EXCLUDE_FOLDERS"
+    default = "static/*:templates/*:public/*:html/*"
+    description = "colon separated list of glob patterns, matched source files are included"
+    name = "BP_INCLUDE_FILES"
+
+  [[metadata.configurations]]
+    build = true
+    default = ""
+    description = "colon separated list of glob patterns, matched source files are removed"
+    name = "BP_EXCLUDE_FILES"
 
   [[metadata.configurations]]
     build = true

cargo/build.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"github.com/buildpacks/libcnb"
+	"github.com/heroku/color"
 	"github.com/mattn/go-shellwords"
 	"github.com/paketo-buildpacks/libpak"
 	"github.com/paketo-buildpacks/libpak/bard"
@@ -77,12 +78,17 @@ func (b Build) Build(context libcnb.BuildContext) (libcnb.BuildResult, error) {
 			return libcnb.BuildResult{}, fmt.Errorf("unable to locate cargo home")
 		}
 
-		excludeFoldersRaw, _ := cr.Resolve("BP_CARGO_EXCLUDE_FOLDERS")
-		var excludeFolders []string
-		for _, excludeFolder := range strings.Split(excludeFoldersRaw, ",") {
-			excludeFolders = append(excludeFolders, strings.TrimSpace(excludeFolder))
+		includeFolders, _ := cr.Resolve("BP_INCLUDE_FILES")
+
+		// Deprecated: to be removed before the cargo 1.0.0 release
+		deprecatedExcludeFolders, usedDeprecatedExclude := cr.Resolve("BP_CARGO_EXCLUDE_FOLDERS")
+		if usedDeprecatedExclude {
+			b.Logger.Infof("%s: `BP_CARGO_EXCLUDE_FOLDERS` has been deprecated and will be removed before the paketo-community/cargo 1.0 GA release. Use `BP_INCLUDE_FILES` instead.", color.YellowString("Warning"))
+			includeFolders = fmt.Sprintf("%s:%s", includeFolders, strings.ReplaceAll(deprecatedExcludeFolders, ",", ":"))
 		}
 
+		excludeFolders, _ := cr.Resolve("BP_EXCLUDE_FILES")
+
 		cargoWorkspaceMembers, _ := cr.Resolve("BP_CARGO_WORKSPACE_MEMBERS")
 		cargoInstallArgs, _ := cr.Resolve("BP_CARGO_INSTALL_ARGS")
 		skipSBOMScan := cr.ResolveBool("BP_DISABLE_SBOM")
@@ -121,6 +127,7 @@ func (b Build) Build(context libcnb.BuildContext) (libcnb.BuildResult, error) {
 		cargoLayer, err := NewCargo(
 			WithApplicationPath(context.Application.Path),
 			WithCargoService(service),
+			WithIncludeFolders(includeFolders),
 			WithExcludeFolders(excludeFolders),
 			WithInstallArgs(cargoInstallArgs),
 			WithLogger(b.Logger),

cargo/cargo.go

@@ -18,7 +18,6 @@ package cargo
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -28,6 +27,7 @@ import (
 	"github.com/paketo-buildpacks/libpak/bard"
 	"github.com/paketo-buildpacks/libpak/sbom"
 	"github.com/paketo-buildpacks/libpak/sherpa"
+	"github.com/paketo-buildpacks/source-removal/logic"
 	"github.com/paketo-community/cargo/mtimes"
 	"github.com/paketo-community/cargo/runner"
 )
@@ -59,8 +59,16 @@ func WithCargoService(s runner.CargoService) Option {
 	}
 }
 
+// WithIncludeFolders sets logger
+func WithIncludeFolders(f string) Option {
+	return func(cargo Cargo) Cargo {
+		cargo.IncludeFolders = f
+		return cargo
+	}
+}
+
 // WithExcludeFolders sets logger
-func WithExcludeFolders(f []string) Option {
+func WithExcludeFolders(f string) Option {
 	return func(cargo Cargo) Cargo {
 		cargo.ExcludeFolders = f
 		return cargo
@@ -136,7 +144,8 @@ type Cargo struct {
 	ApplicationPath    string
 	Cache              Cache
 	CargoService       runner.CargoService
-	ExcludeFolders     []string
+	IncludeFolders     string
+	ExcludeFolders     string
 	InstallArgs        string
 	LayerContributor   libpak.LayerContributor
 	Logger             bard.Logger
@@ -272,23 +281,14 @@ func (c Cargo) Contribute(layer libcnb.Layer) (libcnb.Layer, error) {
 	}
 
 	c.Logger.Header("Removing source code")
-	fs, err := ioutil.ReadDir(c.ApplicationPath)
+	err = logic.Include(c.ApplicationPath, c.IncludeFolders)
 	if err != nil {
-		return libcnb.Layer{}, fmt.Errorf("unable to list children of %s\n%w", c.ApplicationPath, err)
+		return libcnb.Layer{}, err
 	}
 
-DELETE:
-	for _, f := range fs {
-		for _, excludeFolder := range c.ExcludeFolders {
-			if f.Name() == excludeFolder {
-				continue DELETE
-			}
-		}
-
-		file := filepath.Join(c.ApplicationPath, f.Name())
-		if err := os.RemoveAll(file); err != nil {
-			return libcnb.Layer{}, fmt.Errorf("unable to remove %s\n%w", file, err)
-		}
+	err = logic.Exclude(c.ApplicationPath, c.ExcludeFolders)
+	if err != nil {
+		return libcnb.Layer{}, err
 	}
 
 	if err := os.MkdirAll(filepath.Join(c.ApplicationPath, "bin"), 0755); err != nil {

cargo/cargo_test.go

@@ -625,7 +625,7 @@ func testCargo(t *testing.T, context spec.G, it spec.S) {
 				c, err = cargo.NewCargo(
 					cargo.WithApplicationPath(ctx.Application.Path),
 					cargo.WithCargoService(service),
-					cargo.WithExcludeFolders([]string{"static", "templates"}),
+					cargo.WithIncludeFolders("static/*:templates/*"),
 					cargo.WithSBOMScanner(sbomScanner))
 
 				Expect(err).ToNot(HaveOccurred())

go.mod

@@ -4,9 +4,11 @@ go 1.17
 
 require (
 	github.com/buildpacks/libcnb v1.26.0
+	github.com/heroku/color v0.0.6
 	github.com/mattn/go-shellwords v1.0.12
 	github.com/onsi/gomega v1.19.0
 	github.com/paketo-buildpacks/libpak v1.60.1
+	github.com/paketo-buildpacks/source-removal v0.2.0
 	github.com/sclevine/spec v1.4.0
 	github.com/stretchr/testify v1.8.0
 )
@@ -16,16 +18,15 @@ require (
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/creack/pty v1.1.18 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/heroku/color v0.0.6 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/stretchr/objx v0.4.0 // indirect
-	golang.org/x/net v0.0.0-20220524220425-1d687d428aca // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/net v0.0.0-20220706163947-c90051bbdb60 // indirect
+	golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e // indirect
 	golang.org/x/text v0.3.7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect