diff --git a/dependency_cache.go b/dependency_cache.go index 5dda859..b6c702e 100644 --- a/dependency_cache.go +++ b/dependency_cache.go @@ -69,28 +69,29 @@ type DependencyCache struct { // httpClientTimeouts contains the timeout values used by HTTP client HttpClientTimeouts HttpClientTimeouts - // Alternative source used for downloading dependencies. - DependencyMirror string + // Alternative sources used for downloading dependencies. + DependencyMirrors map[string]string } // NewDependencyCache creates a new instance setting the default cache path (/dependencies) and user // agent (/). // Mappings will be read from any libcnb.Binding in the context with type "dependency-mappings". // -// In some air-gapped environments, dependencies might not be download directly but need to be pulled from a local mirror registry. -// In such cases, an alternative URI can either be provided as environment variable "BP_DEPENDENCY_MIRROR", or by a binding of type "dependency-mirror" -// where a file named "uri" holds the desired location. -// The two schemes https:// and file:// are supported in mirror URIs where the expected format is (optional parts in "[]"): -// ://[:@][:][/] -// The optional path part of the provided URI is used as a prefix that might be necessary in some setups. -// This (prefix) path may also include a placeholder of "{originalHost}" at any level (in sub-paths or at top-level) and is replaced with the -// hostname of the original download URI at build time. A sample mirror URI might look like this: https://local-mirror.example.com/buildpacks-dependencies/{originalHost} +// In some environments, many dependencies might need to be downloaded from a (local) mirror registry or filesystem. +// Such alternative locations can be configured using bindings of type "dependency-mirror", avoiding too many "dependency-mapping" bindings. +// Environment variables named "BP_DEPENDENCY_MIRROR" (default) or "BP_DEPENDENCY_MIRROR_" (hostname-specific mirror) +// can also be used for the same purpose. func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) { cache := DependencyCache{ - CachePath: filepath.Join(context.Buildpack.Path, "dependencies"), - DownloadPath: os.TempDir(), - UserAgent: fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version), - Mappings: map[string]string{}, + CachePath: filepath.Join(context.Buildpack.Path, "dependencies"), + DownloadPath: os.TempDir(), + UserAgent: fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version), + Mappings: map[string]string{}, + DependencyMirrors: map[string]string{}, + // We create the logger here because the initialization process may log some warnings that should be visible to users. + // This goes against the usual pattern, which has the user supply the Logger after initialization. + // There's no choice though, if we want the warning messages to be visible to users. We should clean this up in v2. + Logger: bard.NewLogger(os.Stdout), } mappings, err := filterBindingsByType(context.Platform.Bindings, "dependency-mapping") if err != nil { @@ -104,11 +105,11 @@ func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) { } cache.HttpClientTimeouts = *clientTimeouts - dependencyMirror, err := getDependencyMirror(context.Platform.Bindings) + bindingMirrors, err := filterBindingsByType(context.Platform.Bindings, "dependency-mirror") if err != nil { - return DependencyCache{}, err + return DependencyCache{}, fmt.Errorf("unable to process dependency-mirror bindings\n%w", err) } - cache.DependencyMirror = dependencyMirror + cache.setDependencyMirrors(bindingMirrors) return cache, nil } @@ -153,22 +154,44 @@ func customizeHttpClientTimeouts() (*HttpClientTimeouts, error) { }, nil } -// Returns the URI of a dependency mirror (optional). -// Such mirror location can be defined in a binding of type 'dependency-mirror' with filename 'uri' -// or using the environment variable 'BP_DEPENDENCY_MIRROR'. The latter takes precedence in case both are found. -func getDependencyMirror(bindings libcnb.Bindings) (string, error) { - dependencyMirror := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR", "") - // If no mirror was found in environment variables, try to find one in bindings. - if dependencyMirror == "" { - dependencyMirrorBindings, err := filterBindingsByType(bindings, "dependency-mirror") - if err == nil { - // Use the content of the file named "uri" as the mirror's URI. - dependencyMirror = dependencyMirrorBindings["uri"] - } else { - return "", err +func (d *DependencyCache) setDependencyMirrors(bindingMirrors map[string]string) { + // Initialize with mirrors from bindings. + d.DependencyMirrors = bindingMirrors + // Add mirrors from env variables and override duplicate hostnames set in bindings. + envs := os.Environ() + for _, env := range envs { + envPair := strings.SplitN(env, "=", 2) + if len(envPair) != 2 { + continue + } + hostnameSuffix, isMirror := strings.CutPrefix(envPair[0], "BP_DEPENDENCY_MIRROR") + if isMirror { + hostnameEncoded, _ := strings.CutPrefix(hostnameSuffix, "_") + if strings.ToLower(hostnameEncoded) == "default" { + d.Logger.Bodyf("%s with illegal hostname 'default'. Please use BP_DEPENDENCY_MIRROR to set a default.", + color.YellowString("Ignored dependency mirror")) + continue + } + d.DependencyMirrors[decodeHostnameEnv(hostnameEncoded, d)] = envPair[1] } } - return dependencyMirror, nil +} + +// Takes an encoded hostname (from env key) and returns the decoded version in lower case. +// Replaces double underscores (__) with one dash (-) and single underscores (_) with one period (.). +func decodeHostnameEnv(encodedHostname string, d *DependencyCache) string { + if strings.ContainsAny(encodedHostname, "-.") || encodedHostname != strings.ToUpper(encodedHostname) { + d.Logger.Bodyf("%s These will be allowed but for best results across different shells, you should replace . characters with _ characters "+ + "and - characters with __, and use all upper case letters. The buildpack will convert these back before using the mirror.", + color.YellowString("You have invalid characters in your mirror host environment variable.")) + } + var decodedHostname string + if encodedHostname == "" { + decodedHostname = "default" + } else { + decodedHostname = strings.ReplaceAll(strings.ReplaceAll(encodedHostname, "__", "-"), "_", ".") + } + return strings.ToLower(decodedHostname) } // Returns a key/value map with all entries for a given binding type. @@ -178,10 +201,10 @@ func filterBindingsByType(bindings libcnb.Bindings, bindingType string) (map[str for _, binding := range bindings { if strings.ToLower(binding.Type) == bindingType { for key, value := range binding.Secret { - if _, ok := filteredBindings[key]; ok { + if _, ok := filteredBindings[strings.ToLower(key)]; ok { return nil, fmt.Errorf("multiple %s bindings found with duplicate keys %s", binding.Type, key) } - filteredBindings[key] = value + filteredBindings[strings.ToLower(key)] = value } } } @@ -225,11 +248,17 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque return nil, fmt.Errorf("unable to parse URI. see DEBUG log level") } - if isBinding && d.DependencyMirror != "" { + mirror := d.DependencyMirrors["default"] + mirrorHostSpecific := d.DependencyMirrors[urlP.Hostname()] + if mirrorHostSpecific != "" { + mirror = mirrorHostSpecific + } + + if isBinding && mirror != "" { d.Logger.Bodyf("Both dependency mirror and bindings are present. %s Please remove dependency map bindings if you wish to use the mirror.", color.YellowString("Mirror is being ignored.")) } else { - d.setDependencyMirror(urlP) + d.setDependencyMirror(urlP, mirror) } if dependency.SHA256 == "" { @@ -422,10 +451,10 @@ func (DependencyCache) verify(path string, expected string) error { return nil } -func (d DependencyCache) setDependencyMirror(urlD *url.URL) { - if d.DependencyMirror != "" { +func (d DependencyCache) setDependencyMirror(urlD *url.URL, mirror string) { + if mirror != "" { d.Logger.Bodyf("%s Download URIs will be overridden.", color.GreenString("Dependency mirror found.")) - urlOverride, err := url.ParseRequestURI(d.DependencyMirror) + urlOverride, err := url.ParseRequestURI(mirror) if strings.ToLower(urlOverride.Scheme) == "https" || strings.ToLower(urlOverride.Scheme) == "file" { urlD.Scheme = urlOverride.Scheme @@ -433,7 +462,7 @@ func (d DependencyCache) setDependencyMirror(urlD *url.URL) { urlD.Path = strings.Replace(urlOverride.Path, "{originalHost}", urlD.Hostname(), 1) + urlD.Path urlD.Host = urlOverride.Host } else { - d.Logger.Debugf("Dependency mirror URI is invalid: %s\n%w", d.DependencyMirror, err) + d.Logger.Debugf("Dependency mirror URI is invalid: %s\n%w", mirror, err) d.Logger.Bodyf("%s is ignored. Have you used one of the supported schemes https:// or file://?", color.YellowString("Invalid dependency mirror")) } } diff --git a/dependency_cache_test.go b/dependency_cache_test.go index fd9ce7a..37178ac 100644 --- a/dependency_cache_test.go +++ b/dependency_cache_test.go @@ -154,21 +154,25 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { context("dependency mirror from environment variable", func() { it.Before(func() { t.Setenv("BP_DEPENDENCY_MIRROR", "https://env-var-mirror.acme.com") + t.Setenv("BP_DEPENDENCY_MIRROR_EXAMP__LE_COM", "https://examp-le.com") }) it("uses BP_DEPENDENCY_MIRROR environment variable", func() { dependencyCache, err := libpak.NewDependencyCache(ctx) Expect(err).NotTo(HaveOccurred()) - Expect(dependencyCache.DependencyMirror).To(Equal("https://env-var-mirror.acme.com")) + Expect(dependencyCache.DependencyMirrors["default"]).To(Equal("https://env-var-mirror.acme.com")) + Expect(dependencyCache.DependencyMirrors["examp-le.com"]).To(Equal("https://examp-le.com")) }) }) - context("dependency mirror from binding", func() { + context("dependency mirror from binding and environment variable", func() { it.Before(func() { + t.Setenv("BP_DEPENDENCY_MIRROR_EXAMP__LE_COM", "https://examp-le.com") ctx.Platform.Bindings = append(ctx.Platform.Bindings, libcnb.Binding{ Type: "dependency-mirror", Secret: map[string]string{ - "uri": "https://bindings-mirror.acme.com", + "default": "https://bindings-mirror.acme.com", + "examp-le.com": "https://invalid.com", }, }) }) @@ -176,7 +180,13 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { it("uses dependency-mirror binding", func() { dependencyCache, err := libpak.NewDependencyCache(ctx) Expect(err).NotTo(HaveOccurred()) - Expect(dependencyCache.DependencyMirror).To(Equal("https://bindings-mirror.acme.com")) + Expect(dependencyCache.DependencyMirrors["default"]).To(Equal("https://bindings-mirror.acme.com")) + }) + + it("environment variable overrides binding", func() { + dependencyCache, err := libpak.NewDependencyCache(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(dependencyCache.DependencyMirrors["examp-le.com"]).To(Equal("https://examp-le.com")) }) }) }) @@ -332,6 +342,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { it.Before(func() { mirrorServer = ghttp.NewTLSServer() + dependencyCache.DependencyMirrors = map[string]string{} }) it.After(func() { @@ -347,7 +358,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { ghttp.RespondWith(http.StatusOK, "test-fixture"), )) - dependencyCache.DependencyMirror = url.Scheme + "://" + "username:password@" + url.Host + "/foo/bar" + dependencyCache.DependencyMirrors["default"] = url.Scheme + "://" + "username:password@" + url.Host + "/foo/bar" a, err := dependencyCache.Artifact(dependency) Expect(err).NotTo(HaveOccurred()) @@ -362,7 +373,22 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { ghttp.RespondWith(http.StatusOK, "test-fixture"), )) - dependencyCache.DependencyMirror = url.Scheme + "://" + url.Host + "/{originalHost}" + dependencyCache.DependencyMirrors["default"] = url.Scheme + "://" + url.Host + "/{originalHost}" + a, err := dependencyCache.Artifact(dependency) + Expect(err).NotTo(HaveOccurred()) + + Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture"))) + }) + + it("downloads from https mirror host specific", func() { + url, err := url.Parse(mirrorServer.URL()) + Expect(err).NotTo(HaveOccurred()) + mirrorServer.AppendHandlers(ghttp.CombineHandlers( + ghttp.VerifyRequest(http.MethodGet, "/host-specific/test-path", ""), + ghttp.RespondWith(http.StatusOK, "test-fixture"), + )) + + dependencyCache.DependencyMirrors["127.0.0.1"] = url.Scheme + "://" + url.Host + "/host-specific" a, err := dependencyCache.Artifact(dependency) Expect(err).NotTo(HaveOccurred()) @@ -384,6 +410,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { Expect(err).NotTo(HaveOccurred()) mirrorPathPreservedHost = filepath.Join(mirrorPath, originalUrl.Hostname(), "prefix") Expect(os.MkdirAll(mirrorPathPreservedHost, os.ModePerm)).NotTo(HaveOccurred()) + dependencyCache.DependencyMirrors = map[string]string{} }) it.After(func() { @@ -394,7 +421,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { mirrorFile := filepath.Join(mirrorPath, "test-path") Expect(os.WriteFile(mirrorFile, []byte("test-fixture"), 0644)).ToNot(HaveOccurred()) - dependencyCache.DependencyMirror = "file://" + mirrorPath + dependencyCache.DependencyMirrors["default"] = "file://" + mirrorPath a, err := dependencyCache.Artifact(dependency) Expect(err).NotTo(HaveOccurred()) @@ -405,7 +432,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { mirrorFilePreservedHost := filepath.Join(mirrorPathPreservedHost, "test-path") Expect(os.WriteFile(mirrorFilePreservedHost, []byte("test-fixture"), 0644)).ToNot(HaveOccurred()) - dependencyCache.DependencyMirror = "file://" + mirrorPath + "/{originalHost}" + "/prefix" + dependencyCache.DependencyMirrors["default"] = "file://" + mirrorPath + "/{originalHost}" + "/prefix" a, err := dependencyCache.Artifact(dependency) Expect(err).NotTo(HaveOccurred())