Skip to content

Commit

Permalink
added support for setting dependency mirror as binding and using plac…
Browse files Browse the repository at this point in the history 
…eholder {originalHost} in mirror path.
  • Loading branch information
@bitgully
bitgully committed Mar 3, 2024
1 parent a7ae498 commit f3079a3
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 35 deletions.
74 changes: 49 additions & 25 deletions dependency_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ type DependencyCache struct {

// httpClientTimeouts contains the timeout values used by HTTP client
HttpClientTimeouts HttpClientTimeouts

// Alternative source used for downloading dependencies.
DependencyMirror string
}

// NewDependencyCache creates a new instance setting the default cache path (<BUILDPACK_PATH>/dependencies) and user
Expand All @@ -80,7 +83,7 @@ func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) {
UserAgent: fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version),
Mappings: map[string]string{},
}
mappings, err := mappingsFromBindings(context.Platform.Bindings)
mappings, err := filterBindingsByType(context.Platform.Bindings, "dependency-mapping")
if err != nil {
return DependencyCache{}, fmt.Errorf("unable to process dependency-mapping bindings\n%w", err)
}
Expand All @@ -92,6 +95,12 @@ func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) {
}
cache.HttpClientTimeouts = *clientTimeouts

dependencyMirror, err := getDependencyMirror(context.Platform.Bindings)
if err != nil {
return DependencyCache{}, err
}
cache.DependencyMirror = dependencyMirror

return cache, nil
}

Expand Down Expand Up @@ -135,19 +144,39 @@ func customizeHttpClientTimeouts() (*HttpClientTimeouts, error) {
}, nil
}

func mappingsFromBindings(bindings libcnb.Bindings) (map[string]string, error) {
mappings := map[string]string{}
// Returns the URI of a dependency mirror (optional).
// Such mirror location can be defined in a binding of type 'dependency-mirror' with filename 'uri'
// or using the environment variable 'BP_DEPENDENCY_MIRROR'. The latter takes precedence in case both are found.
func getDependencyMirror(bindings libcnb.Bindings) (string, error) {
dependencyMirror := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR", "")
// If no mirror was found in environment variables, try to find one in bindings.
if dependencyMirror == "" {
dependencyMirrorBindings, err := filterBindingsByType(bindings, "dependency-mirror")
if err == nil {
// Use the content of the file named "uri" as the mirror's URI.
dependencyMirror = dependencyMirrorBindings["uri"]
} else {
return "", err
}
}
return dependencyMirror, nil
}

// Returns a key/value map with all entries for a given binding type.
// An error is returned if multiple entries are found using the same key (e.g. duplicate digests in dependency mappings).
func filterBindingsByType(bindings libcnb.Bindings, bindingType string) (map[string]string, error) {
filteredBindings := map[string]string{}
for _, binding := range bindings {
if strings.ToLower(binding.Type) == "dependency-mapping" {
for digest, uri := range binding.Secret {
if _, ok := mappings[digest]; ok {
return nil, fmt.Errorf("multiple mappings for digest %q", digest)
if strings.ToLower(binding.Type) == bindingType {
for key, value := range binding.Secret {
if _, ok := filteredBindings[key]; ok {
return nil, fmt.Errorf("multiple %s bindings found with duplicate keys %s", binding.Type, key)
}
mappings[digest] = uri
filteredBindings[key] = value
}
}
}
return mappings, nil
return filteredBindings, nil
}

// RequestModifierFunc is a callback that enables modification of a download request before it is sent. It is often
Expand Down Expand Up @@ -187,7 +216,10 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque
return nil, fmt.Errorf("unable to parse URI. see DEBUG log level")
}

if !isBinding {
if isBinding && d.DependencyMirror != "" {
d.Logger.Bodyf("Both dependency mirror and bindings are present. %s Please remove dependency map bindings if you wish to use the mirror.",
color.YellowString("Mirror is being ignored."))
} else {
d.setDependencyMirror(urlP)
}

Expand Down Expand Up @@ -382,26 +414,18 @@ func (DependencyCache) verify(path string, expected string) error {
}

func (d DependencyCache) setDependencyMirror(urlD *url.URL) {
dependencyMirror := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR", "")
dependencyMirrorPreserveHost := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR_PRESERVE_HOST", "false")
if dependencyMirror != "" {
var originalHost string

d.Logger.Infof("variable BP_DEPENDENCY_MIRROR found. overriding download uri.")
urlOverride, err := url.ParseRequestURI(dependencyMirror)

if strings.EqualFold(dependencyMirrorPreserveHost, "true") && urlD.Hostname() != "" {
originalHost = "/" + urlD.Hostname()
}
if d.DependencyMirror != "" {
d.Logger.Bodyf("%s Download URIs will be overridden.", color.GreenString("Dependency mirror found."))
urlOverride, err := url.ParseRequestURI(d.DependencyMirror)

if strings.EqualFold(urlOverride.Scheme, "https") || strings.EqualFold(urlOverride.Scheme, "file") {
if strings.ToLower(urlOverride.Scheme) == "https" || strings.ToLower(urlOverride.Scheme) == "file" {
urlD.Scheme = urlOverride.Scheme
urlD.User = urlOverride.User
urlD.Path = strings.Replace(urlOverride.Path, "{originalHost}", urlD.Hostname(), 1) + urlD.Path
urlD.Host = urlOverride.Host
urlD.Path = urlOverride.Path + originalHost + urlD.Path
} else {
d.Logger.Debugf("environment variable BP_DEPENDENCY_MIRROR has invalid value: %s\n%w", dependencyMirror, err)
d.Logger.Infof("ignoring invalid variable BP_DEPENDENCY_MIRROR. have you used one of the supported schemes 'https://' or 'file://'? continuing without override.")
d.Logger.Debugf("Dependency mirror URI is invalid: %s\n%w", d.DependencyMirror, err)
d.Logger.Bodyf("%s is ignored. Have you used one of the supported schemes https:// or file://?", color.YellowString("Invalid dependency mirror"))
}
}
}
47 changes: 37 additions & 10 deletions dependency_cache_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,35 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
})
})
})

context("dependency mirror from environment variable", func() {
it.Before(func() {
t.Setenv("BP_DEPENDENCY_MIRROR", "https://env-var-mirror.acme.com")
})

it("uses BP_DEPENDENCY_MIRROR environment variable", func() {
dependencyCache, err := libpak.NewDependencyCache(ctx)
Expect(err).NotTo(HaveOccurred())
Expect(dependencyCache.DependencyMirror).To(Equal("https://env-var-mirror.acme.com"))
})
})

context("dependency mirror from binding", func() {
it.Before(func() {
ctx.Platform.Bindings = append(ctx.Platform.Bindings, libcnb.Binding{
Type: "dependency-mirror",
Secret: map[string]string{
"uri": "https://bindings-mirror.acme.com",
},
})
})

it("uses dependency-mirror binding", func() {
dependencyCache, err := libpak.NewDependencyCache(ctx)
Expect(err).NotTo(HaveOccurred())
Expect(dependencyCache.DependencyMirror).To(Equal("https://bindings-mirror.acme.com"))
})
})
})

context("artifacts", func() {
Expand Down Expand Up @@ -312,13 +341,13 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
it("downloads from https mirror", func() {
url, err := url.Parse(mirrorServer.URL())
Expect(err).NotTo(HaveOccurred())
t.Setenv("BP_DEPENDENCY_MIRROR", url.Scheme+"://"+"username:password@"+url.Host+"/foo/bar")
mirrorServer.AppendHandlers(ghttp.CombineHandlers(
ghttp.VerifyBasicAuth("username", "password"),
ghttp.VerifyRequest(http.MethodGet, "/foo/bar/test-path", ""),
ghttp.RespondWith(http.StatusOK, "test-fixture"),
))

dependencyCache.DependencyMirror = url.Scheme + "://" + "username:password@" + url.Host + "/foo/bar"
a, err := dependencyCache.Artifact(dependency)
Expect(err).NotTo(HaveOccurred())

Expand All @@ -328,13 +357,12 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
it("downloads from https mirror preserving hostname", func() {
url, err := url.Parse(mirrorServer.URL())
Expect(err).NotTo(HaveOccurred())
t.Setenv("BP_DEPENDENCY_MIRROR", url.Scheme+"://"+url.Host)
t.Setenv("BP_DEPENDENCY_MIRROR_PRESERVE_HOST", "true")
mirrorServer.AppendHandlers(ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodGet, "/"+url.Hostname()+"/test-path", ""),
ghttp.RespondWith(http.StatusOK, "test-fixture"),
))

dependencyCache.DependencyMirror = url.Scheme + "://" + url.Host + "/{originalHost}"
a, err := dependencyCache.Artifact(dependency)
Expect(err).NotTo(HaveOccurred())

Expand All @@ -352,8 +380,10 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
var err error
mirrorPath, err = os.MkdirTemp("", "mirror-path")
Expect(err).NotTo(HaveOccurred())
mirrorPathPreservedHost = mirrorPath + "/127.0.0.1"
Expect(os.Mkdir(mirrorPathPreservedHost, os.ModePerm)).NotTo(HaveOccurred())
originalUrl, err := url.Parse(dependency.URI)
Expect(err).NotTo(HaveOccurred())
mirrorPathPreservedHost = filepath.Join(mirrorPath, originalUrl.Hostname(), "prefix")
Expect(os.MkdirAll(mirrorPathPreservedHost, os.ModePerm)).NotTo(HaveOccurred())
})

it.After(func() {
Expand All @@ -364,8 +394,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
mirrorFile := filepath.Join(mirrorPath, "test-path")
Expect(os.WriteFile(mirrorFile, []byte("test-fixture"), 0644)).ToNot(HaveOccurred())

t.Setenv("BP_DEPENDENCY_MIRROR", "file://"+mirrorPath)

dependencyCache.DependencyMirror = "file://" + mirrorPath
a, err := dependencyCache.Artifact(dependency)
Expect(err).NotTo(HaveOccurred())

Expand All @@ -376,9 +405,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
mirrorFilePreservedHost := filepath.Join(mirrorPathPreservedHost, "test-path")
Expect(os.WriteFile(mirrorFilePreservedHost, []byte("test-fixture"), 0644)).ToNot(HaveOccurred())

t.Setenv("BP_DEPENDENCY_MIRROR", "file://"+mirrorPath)
t.Setenv("BP_DEPENDENCY_MIRROR_PRESERVE_HOST", "true")

dependencyCache.DependencyMirror = "file://" + mirrorPath + "/{originalHost}" + "/prefix"
a, err := dependencyCache.Artifact(dependency)
Expect(err).NotTo(HaveOccurred())

Expand Down

0 comments on commit f3079a3

Please sign in to comment.