diff --git a/postal/fakes/mirror_resolver.go b/postal/fakes/mirror_resolver.go new file mode 100644 index 00000000..19c83bfd --- /dev/null +++ b/postal/fakes/mirror_resolver.go @@ -0,0 +1,31 @@ +package fakes + +import "sync" + +type MirrorResolver struct { + FindDependencyMirrorCall struct { + mutex sync.Mutex + CallCount int + Receives struct { + Uri string + PlatformDir string + } + Returns struct { + String string + Error error + } + Stub func(string, string) (string, error) + } +} + +func (f *MirrorResolver) FindDependencyMirror(param1 string, param2 string) (string, error) { + f.FindDependencyMirrorCall.mutex.Lock() + defer f.FindDependencyMirrorCall.mutex.Unlock() + f.FindDependencyMirrorCall.CallCount++ + f.FindDependencyMirrorCall.Receives.Uri = param1 + f.FindDependencyMirrorCall.Receives.PlatformDir = param2 + if f.FindDependencyMirrorCall.Stub != nil { + return f.FindDependencyMirrorCall.Stub(param1, param2) + } + return f.FindDependencyMirrorCall.Returns.String, f.FindDependencyMirrorCall.Returns.Error +} diff --git a/postal/internal/dependency_mirror.go b/postal/internal/dependency_mirror.go new file mode 100644 index 00000000..d021b106 --- /dev/null +++ b/postal/internal/dependency_mirror.go @@ -0,0 +1,138 @@ +package internal + +import ( + "fmt" + "net/url" + "os" + "strings" +) + +type DependencyMirrorResolver struct { + bindingResolver BindingResolver +} + +func NewDependencyMirrorResolver(bindingResolver BindingResolver) DependencyMirrorResolver { + return DependencyMirrorResolver{ + bindingResolver: bindingResolver, + } +} + +func formatAndVerifyMirror(mirror, uri string) (string, error) { + mirrorURL, err := url.Parse(mirror) + if err != nil { + return "", err + } + + uriURL, err := url.Parse(uri) + if err != nil { + return "", err + } + + if mirrorURL.Scheme != "https" && mirrorURL.Scheme != "file" { + return "", fmt.Errorf("invalid mirror scheme") + } + + mirrorURL.Path = strings.Replace(mirrorURL.Path, "{originalHost}", uriURL.Host+uriURL.Path, 1) + return mirrorURL.String(), nil +} + +func (d DependencyMirrorResolver) FindDependencyMirror(uri, platformDir string) (string, error) { + mirror, err := d.findMirrorFromEnv(uri) + if err != nil { + return "", err + } + + if mirror != "" { + return formatAndVerifyMirror(mirror, uri) + } + + mirror, err = d.findMirrorFromBinding(uri, platformDir) + if err != nil { + return "", err + } + + if mirror != "" { + return formatAndVerifyMirror(mirror, uri) + } + + return "", nil +} + +func (d DependencyMirrorResolver) findMirrorFromEnv(uri string) (string, error) { + const DefaultMirror = "BP_DEPENDENCY_MIRROR" + const NonDefaultMirrorPrefix = "BP_DEPENDENCY_MIRROR_" + mirrors := make(map[string]string) + environmentVariables := os.Environ() + for _, ev := range environmentVariables { + pair := strings.SplitN(ev, "=", 2) + key := pair[0] + value := pair[1] + + if !strings.Contains(key, DefaultMirror) { + continue + } + + if key == DefaultMirror { + mirrors["default"] = value + continue + } + + // convert key + hostname := strings.SplitN(key, NonDefaultMirrorPrefix, 2)[1] + hostname = strings.ReplaceAll(strings.ReplaceAll(hostname, "__", "-"), "_", ".") + hostname = strings.ToLower(hostname) + mirrors[hostname] = value + + if !strings.Contains(uri, hostname) { + continue + } + + return value, nil + } + + if mirrorUri, ok := mirrors["default"]; ok { + return mirrorUri, nil + } + + return "", nil +} + +func (d DependencyMirrorResolver) findMirrorFromBinding(uri, platformDir string) (string, error) { + bindings, err := d.bindingResolver.Resolve("dependency-mirror", "", platformDir) + if err != nil { + return "", fmt.Errorf("failed to resolve 'dependency-mirror' binding: %w", err) + } + + if len(bindings) > 1 { + return "", fmt.Errorf("cannot have multiple bindings of type 'dependency-mirror'") + } + + if len(bindings) == 0 { + return "", nil + } + + mirror := "" + entries := bindings[0].Entries + for hostname, entry := range entries { + if hostname == "default" { + mirror, err = entry.ReadString() + if err != nil { + return "", err + } + continue + } + + if !strings.Contains(uri, hostname) { + continue + } + + mirror, err = entry.ReadString() + if err != nil { + return "", err + } + + return mirror, nil + } + + return mirror, nil +} diff --git a/postal/internal/dependency_mirror_test.go b/postal/internal/dependency_mirror_test.go new file mode 100644 index 00000000..e871d8f5 --- /dev/null +++ b/postal/internal/dependency_mirror_test.go @@ -0,0 +1,285 @@ +package internal_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/paketo-buildpacks/packit/v2/postal/internal" + "github.com/paketo-buildpacks/packit/v2/postal/internal/fakes" + "github.com/paketo-buildpacks/packit/v2/servicebindings" + "github.com/sclevine/spec" + + . "github.com/onsi/gomega" +) + +func testDependencyMirror(t *testing.T, context spec.G, it spec.S) { + var ( + Expect = NewWithT(t).Expect + tmpDir string + resolver internal.DependencyMirrorResolver + bindingResolver *fakes.BindingResolver + err error + ) + + context("FindDependencyMirror", func() { + context("via binding", func() { + it.Before(func() { + tmpDir, err = os.MkdirTemp("", "dependency-mirror") + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(tmpDir, "default"), []byte("https://mirror.example.org/{originalHost}"), os.ModePerm)) + Expect(os.WriteFile(filepath.Join(tmpDir, "type"), []byte("dependency-mirror"), os.ModePerm)) + + bindingResolver = &fakes.BindingResolver{} + resolver = internal.NewDependencyMirrorResolver(bindingResolver) + + bindingResolver.ResolveCall.Returns.BindingSlice = []servicebindings.Binding{ + { + Name: "some-binding", + Path: "some-path", + Type: "dependency-mirror", + Entries: map[string]*servicebindings.Entry{ + "default": servicebindings.NewEntry(filepath.Join(tmpDir, "default")), + }, + }, + } + }) + + it.After(func() { + Expect(os.RemoveAll(tmpDir)).To(Succeed()) + }) + + context("given a default mirror binding", func() { + it("finds a matching dependency mirror in the platform bindings if there is one", func() { + boundDependency, err := resolver.FindDependencyMirror("some-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(bindingResolver.ResolveCall.Receives.Typ).To(Equal("dependency-mirror")) + Expect(bindingResolver.ResolveCall.Receives.Provider).To(BeEmpty()) + Expect(bindingResolver.ResolveCall.Receives.PlatformDir).To(Equal("some-platform-dir")) + Expect(boundDependency).To(Equal("https://mirror.example.org/some-uri")) + }) + }) + + context("given default mirror and specific hostname bindings", func() { + it.Before(func() { + Expect(os.WriteFile(filepath.Join(tmpDir, "github.com"), []byte("https://mirror.example.org/public-github"), os.ModePerm)) + Expect(os.WriteFile(filepath.Join(tmpDir, "nodejs.org"), []byte("https://mirror.example.org/node-dist"), os.ModePerm)) + + bindingResolver.ResolveCall.Returns.BindingSlice[0].Entries = map[string]*servicebindings.Entry{ + "default": servicebindings.NewEntry(filepath.Join(tmpDir, "default")), + "github.com": servicebindings.NewEntry(filepath.Join(tmpDir, "github.com")), + "nodejs.org": servicebindings.NewEntry(filepath.Join(tmpDir, "nodejs.org")), + } + }) + + it("finds the default mirror when given uri does not match a specific hostname", func() { + boundDependency, err := resolver.FindDependencyMirror("some-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(bindingResolver.ResolveCall.Receives.Typ).To(Equal("dependency-mirror")) + Expect(bindingResolver.ResolveCall.Receives.Provider).To(BeEmpty()) + Expect(bindingResolver.ResolveCall.Receives.PlatformDir).To(Equal("some-platform-dir")) + Expect(boundDependency).To(Equal("https://mirror.example.org/some-uri")) + }) + + it("finds the mirror matching the specific hostname in the given uri", func() { + boundDependency, err := resolver.FindDependencyMirror("some-github.com-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(bindingResolver.ResolveCall.Receives.Typ).To(Equal("dependency-mirror")) + Expect(bindingResolver.ResolveCall.Receives.Provider).To(BeEmpty()) + Expect(bindingResolver.ResolveCall.Receives.PlatformDir).To(Equal("some-platform-dir")) + Expect(boundDependency).To(Equal("https://mirror.example.org/public-github")) + }) + }) + + context("given a specific hostname binding and no default mirror binding", func() { + it.Before(func() { + Expect(os.Remove(filepath.Join(tmpDir, "default"))) + Expect(os.WriteFile(filepath.Join(tmpDir, "github.com"), []byte("https://mirror.example.org/public-github"), os.ModePerm)) + Expect(os.WriteFile(filepath.Join(tmpDir, "nodejs.org"), []byte("https://mirror.example.org/node-dist"), os.ModePerm)) + + bindingResolver.ResolveCall.Returns.BindingSlice[0].Entries = map[string]*servicebindings.Entry{ + "github.com": servicebindings.NewEntry(filepath.Join(tmpDir, "github.com")), + "nodejs.org": servicebindings.NewEntry(filepath.Join(tmpDir, "nodejs.org")), + } + }) + + it("return empty string for non specific hostnames", func() { + boundDependency, err := resolver.FindDependencyMirror("some-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(bindingResolver.ResolveCall.Receives.Typ).To(Equal("dependency-mirror")) + Expect(bindingResolver.ResolveCall.Receives.Provider).To(BeEmpty()) + Expect(bindingResolver.ResolveCall.Receives.PlatformDir).To(Equal("some-platform-dir")) + Expect(boundDependency).To(Equal("")) + }) + + it("finds the mirror matching the specific hostname in the given uri", func() { + boundDependency, err := resolver.FindDependencyMirror("some-nodejs.org-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(bindingResolver.ResolveCall.Receives.Typ).To(Equal("dependency-mirror")) + Expect(bindingResolver.ResolveCall.Receives.Provider).To(BeEmpty()) + Expect(bindingResolver.ResolveCall.Receives.PlatformDir).To(Equal("some-platform-dir")) + Expect(boundDependency).To(Equal("https://mirror.example.org/node-dist")) + }) + }) + }) + + context("via environment variables", func() { + it.Before(func() { + Expect(os.Setenv("BP_DEPENDENCY_MIRROR", "https://mirror.example.org/{originalHost}")) + + bindingResolver = &fakes.BindingResolver{} + resolver = internal.NewDependencyMirrorResolver(bindingResolver) + }) + + it.After(func() { + Expect(os.Unsetenv("BP_DEPENDENCY_MIRROR")) + }) + + context("given the default mirror environment variable is set", func() { + it("finds the matching dependency mirror", func() { + boundDependency, err := resolver.FindDependencyMirror("some-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(boundDependency).To(Equal("https://mirror.example.org/some-uri")) + }) + }) + + context("given environment variables for a default mirror and specific hostname mirrors", func() { + it.Before(func() { + Expect(os.Setenv("BP_DEPENDENCY_MIRROR_GITHUB_COM", "https://mirror.example.org/public-github")) + Expect(os.Setenv("BP_DEPENDENCY_MIRROR_TESTING_123__ABC", "https://mirror.example.org/testing")) + }) + + it.After(func() { + Expect(os.Unsetenv("BP_DEPENDENCY_MIRROR_GITHUB_COM")) + Expect(os.Unsetenv("BP_DEPENDENCY_MIRROR_TESTING_123__ABC")) + }) + + it("finds the default mirror when given uri does not match a specific hostname", func() { + boundDependency, err := resolver.FindDependencyMirror("some-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(boundDependency).To(Equal("https://mirror.example.org/some-uri")) + }) + + it("finds the mirror matching the specific hostname in the given uri", func() { + boundDependency, err := resolver.FindDependencyMirror("some-github.com-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(boundDependency).To(Equal("https://mirror.example.org/public-github")) + }) + + it("properly decodes the hostname", func() { + boundDependency, err := resolver.FindDependencyMirror("testing.123-abc-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(boundDependency).To(Equal("https://mirror.example.org/testing")) + }) + }) + + context("given environment variables for a specific hostname and none for a default mirror", func() { + it.Before(func() { + Expect(os.Unsetenv("BP_DEPENDENCY_MIRROR")) + Expect(os.Setenv("BP_DEPENDENCY_MIRROR_GITHUB_COM", "https://mirror.example.org/public-github")) + }) + + it.After(func() { + Expect(os.Unsetenv("BP_DEPENDENCY_MIRROR_GITHUB_COM")) + }) + + it("return empty string for non specific hostnames", func() { + boundDependency, err := resolver.FindDependencyMirror("some-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(boundDependency).To(Equal("")) + }) + + it("finds the mirror matching the specific hostname in the given uri", func() { + boundDependency, err := resolver.FindDependencyMirror("some-github.com-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(boundDependency).To(Equal("https://mirror.example.org/public-github")) + }) + }) + }) + + context("when mirror is provided by both bindings and environment variables", func() { + it.Before(func() { + tmpDir, err = os.MkdirTemp("", "dependency-mirror") + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(tmpDir, "default"), []byte("https://mirror.example.org/{originalHost}"), os.ModePerm)) + Expect(os.WriteFile(filepath.Join(tmpDir, "type"), []byte("dependency-mirror"), os.ModePerm)) + + Expect(os.Setenv("BP_DEPENDENCY_MIRROR", "https://mirror.other-example.org/{originalHost}")) + }) + + it.After(func() { + Expect(os.RemoveAll(tmpDir)).To(Succeed()) + Expect(os.Unsetenv("BP_DEPENDENCY_MIRROR")) + }) + + it("defaults to environment variable and ignores binding", func() { + boundDependency, err := resolver.FindDependencyMirror("some-uri", "some-platform-dir") + Expect(err).ToNot(HaveOccurred()) + Expect(boundDependency).NotTo(Equal("https://mirror.example.org/some-uri")) + Expect(boundDependency).To(Equal("https://mirror.other-example.org/some-uri")) + + }) + }) + + context("failure cases", func() { + context("when more than one dependency mirror binding exists", func() { + it.Before(func() { + tmpDir, err = os.MkdirTemp("", "dependency-mirror") + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(tmpDir, "default"), []byte("https://mirror.example.org/{originalHost}"), os.ModePerm)) + Expect(os.WriteFile(filepath.Join(tmpDir, "github.com"), []byte("https://mirror.example.org/public-github"), os.ModePerm)) + Expect(os.WriteFile(filepath.Join(tmpDir, "type"), []byte("dependency-mirror"), os.ModePerm)) + + bindingResolver = &fakes.BindingResolver{} + resolver = internal.NewDependencyMirrorResolver(bindingResolver) + }) + + it.After(func() { + Expect(os.RemoveAll(tmpDir)).To(Succeed()) + }) + + it("returns an error", func() { + bindingResolver.ResolveCall.Returns.BindingSlice = []servicebindings.Binding{ + { + Name: "some-binding", + Path: "some-path", + Type: "dependency-mirror", + Entries: map[string]*servicebindings.Entry{ + "default": servicebindings.NewEntry(filepath.Join(tmpDir, "default")), + }, + }, + { + Name: "some-other-binding", + Path: "some-other-path", + Type: "dependency-mirror", + Entries: map[string]*servicebindings.Entry{ + "github.com": servicebindings.NewEntry(filepath.Join(tmpDir, "github.com")), + }, + }, + } + + _, err = resolver.FindDependencyMirror("some-uri", "some-platform-dir") + Expect(err).To(MatchError(ContainSubstring("cannot have multiple bindings of type 'dependency-mirror'"))) + }) + }) + + context("when mirror contains invalid scheme", func() { + it.Before(func() { + Expect(os.Setenv("BP_DEPENDENCY_MIRROR", "http://mirror.example.org/{originalHost}")) + + bindingResolver = &fakes.BindingResolver{} + resolver = internal.NewDependencyMirrorResolver(bindingResolver) + }) + + it.After(func() { + Expect(os.Unsetenv("BP_DEPENDENCY_MIRROR")) + }) + + it("returns an error", func() { + _, err := resolver.FindDependencyMirror("some-uri", "some-platform-dir") + Expect(err).To(MatchError(ContainSubstring("invalid mirror scheme"))) + }) + }) + }) + }) +} diff --git a/postal/internal/init_test.go b/postal/internal/init_test.go index 08b32ef1..56969873 100644 --- a/postal/internal/init_test.go +++ b/postal/internal/init_test.go @@ -10,6 +10,7 @@ import ( func TestUnitPostalInternal(t *testing.T) { suite := spec.New("packit/postal/internal", spec.Report(report.Terminal{})) suite("DependencyMappings", testDependencyMappings) + suite("DependencyMirror", testDependencyMirror) suite.Run(t) } diff --git a/postal/service.go b/postal/service.go index 07fa8fe3..3147b059 100644 --- a/postal/service.go +++ b/postal/service.go @@ -37,6 +37,14 @@ type MappingResolver interface { FindDependencyMapping(checksum, platformDir string) (string, error) } +// MirrorResolver serves as the interface that looks for a dependency mirror via +// environment variable or binding +// +//go:generate faux --interface MirrorResolver --output fakes/mirror_resolver.go +type MirrorResolver interface { + FindDependencyMirror(uri, platformDir string) (string, error) +} + // ErrNoDeps is a typed error indicating that no dependencies were resolved during Service.Resolve() // // errors can be tested against this type with: errors.As() @@ -62,6 +70,7 @@ func (e *ErrNoDeps) Error() string { type Service struct { transport Transport mappingResolver MappingResolver + mirrorResolver MirrorResolver } // NewService creates an instance of a Service given a Transport. @@ -71,6 +80,9 @@ func NewService(transport Transport) Service { mappingResolver: internal.NewDependencyMappingResolver( servicebindings.NewResolver(), ), + mirrorResolver: internal.NewDependencyMirrorResolver( + servicebindings.NewResolver(), + ), } } @@ -79,6 +91,11 @@ func (s Service) WithDependencyMappingResolver(mappingResolver MappingResolver) return s } +func (s Service) WithDependencyMirrorResolver(mirrorResolver MirrorResolver) Service { + s.mirrorResolver = mirrorResolver + return s +} + // Resolve will pick the best matching dependency given a path to a // buildpack.toml file, and the id, version, and stack value of a dependency. // The version value is treated as a SemVer constraint and will pick the @@ -228,15 +245,23 @@ func stringSliceElementCount(slice []string, str string) int { // location of the CNBPath is given so that dependencies that may be included // in a buildpack when packaged for offline consumption can be retrieved. If // there is a dependency mapping for the specified dependency, Deliver will use -// the given dependency mapping URI to fetch the dependency. The dependency is -// validated against the checksum value provided on the Dependency and will -// error if there are inconsistencies in the fetched result. +// the given dependency mapping URI to fetch the dependency. If there is a +// dependency mirror for the specified dependency, Deliver will use the mirror +// URI to fetch the dependency. If both a dependency mapping and mirror are BOTH +// present, the mapping will take precedence over the mirror.The dependency is +// validated against the checksum value provided on the Dependency and will error +// if there are inconsistencies in the fetched result. func (s Service) Deliver(dependency Dependency, cnbPath, layerPath, platformPath string) error { dependencyChecksum := dependency.Checksum if dependency.SHA256 != "" { dependencyChecksum = fmt.Sprintf("sha256:%s", dependency.SHA256) } + dependencyMirrorURI, err := s.mirrorResolver.FindDependencyMirror(dependency.URI, platformPath) + if err != nil { + return fmt.Errorf("failure checking for dependency mirror: %s", err) + } + dependencyMappingURI, err := s.mappingResolver.FindDependencyMapping(dependencyChecksum, platformPath) if err != nil { return fmt.Errorf("failure checking for dependency mappings: %s", err) @@ -244,6 +269,8 @@ func (s Service) Deliver(dependency Dependency, cnbPath, layerPath, platformPath if dependencyMappingURI != "" { dependency.URI = dependencyMappingURI + } else if dependencyMirrorURI != "" { + dependency.URI = dependencyMirrorURI } bundle, err := s.transport.Drop(cnbPath, dependency.URI) diff --git a/postal/service_test.go b/postal/service_test.go index f46af900..25e54b2c 100644 --- a/postal/service_test.go +++ b/postal/service_test.go @@ -34,6 +34,7 @@ func testService(t *testing.T, context spec.G, it spec.S) { transport *fakes.Transport mappingResolver *fakes.MappingResolver + mirrorResolver *fakes.MirrorResolver service postal.Service ) @@ -109,7 +110,11 @@ strip-components = 1 mappingResolver = &fakes.MappingResolver{} - service = postal.NewService(transport).WithDependencyMappingResolver(mappingResolver) + mirrorResolver = &fakes.MirrorResolver{} + + service = postal.NewService(transport). + WithDependencyMappingResolver(mappingResolver). + WithDependencyMirrorResolver(mirrorResolver) }) context("Resolve", func() { @@ -795,6 +800,37 @@ version = "1.2.3" }) }) + context("when there is a dependency mirror", func() { + it.Before(func() { + mirrorResolver.FindDependencyMirrorCall.Returns.String = "dependency-mirror-url" + }) + + it("downloads dependency from mirror", func() { + err := deliver() + + Expect(err).NotTo(HaveOccurred()) + + Expect(mirrorResolver.FindDependencyMirrorCall.Receives.Uri).To(Equal("some-entry.tgz")) + Expect(mirrorResolver.FindDependencyMirrorCall.Receives.PlatformDir).To(Equal("some-platform-dir")) + Expect(transport.DropCall.Receives.Root).To(Equal("some-cnb-path")) + Expect(transport.DropCall.Receives.Uri).To(Equal("dependency-mirror-url")) + + files, err := filepath.Glob(fmt.Sprintf("%s/*", layerPath)) + Expect(err).NotTo(HaveOccurred()) + Expect(files).To(ConsistOf([]string{ + filepath.Join(layerPath, "first"), + filepath.Join(layerPath, "second"), + filepath.Join(layerPath, "third"), + filepath.Join(layerPath, "some-dir"), + filepath.Join(layerPath, "symlink"), + })) + + info, err := os.Stat(filepath.Join(layerPath, "first")) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode()).To(Equal(os.FileMode(0755))) + }) + }) + context("failure cases", func() { context("when dependency mapping resolver fails", func() { it.Before(func() { @@ -807,6 +843,17 @@ version = "1.2.3" }) }) + context("when dependency mirror resolver fails", func() { + it.Before(func() { + mirrorResolver.FindDependencyMirrorCall.Returns.Error = fmt.Errorf("some dependency mirror error") + }) + it("fails to find dependency mirror", func() { + err := deliver() + + Expect(err).To(MatchError(ContainSubstring("some dependency mirror error"))) + }) + }) + context("when the transport cannot fetch a dependency", func() { it.Before(func() { transport.DropCall.Returns.Error = errors.New("there was an error")