From 50ea11a32e99b99f50e8344f3131c78d9b19afe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 14 Sep 2020 17:27:28 +0200 Subject: [PATCH] Add vendoring to Google Cloud Functions again (#21070) ## What does this PR do? This PR adds the vendor folder back the Cloud Functions of Functionbeat, so they can be used on GCP. This PR basically reverts the removal of the vendor builder functionality. ## Why is it important? The functions are currently broken, as there is no go.mod file and vendor folder. However, we cannot use go modules, because the Golang environment on GCP runs go version 1.13. Thus, we have to fall back to vendoring. ## Checklist - [x] My code follows the style guidelines of this project - [x] I have commented my code, particularly in hard-to-understand areas ~~- [ ] I have made corresponding changes to the documentation~~ ~~- [ ] I have made corresponding change to the default configuration files~~ ~~- [ ] I have added tests that prove my fix is effective or that my feature works~~ - [x] I have added an entry in `CHANGELOG.next.asciidoc` or `CHANGELOG-developer.next.asciidoc`. ## Related issues Closes #20830 --- CHANGELOG.next.asciidoc | 1 + dev-tools/mage/gotool/go.go | 19 +++++++++ .../dev-tools/packaging/packages.yml | 6 +++ x-pack/functionbeat/magefile.go | 8 ++-- .../manager/gcp/template_builder.go | 3 +- x-pack/functionbeat/scripts/mage/update.go | 42 ++++++++++++++++++- 6 files changed, 74 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7c5a9ee1112..b62b9187856 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -357,6 +357,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix timeout option of GCP functions. {issue}16282[16282] {pull}16287[16287] - Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072] +- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070] ==== Added diff --git a/dev-tools/mage/gotool/go.go b/dev-tools/mage/gotool/go.go index 9022e0c0137..bb7066a4f3e 100644 --- a/dev-tools/mage/gotool/go.go +++ b/dev-tools/mage/gotool/go.go @@ -85,6 +85,25 @@ func ListDeps(pkg string) ([]string, error) { return getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg)) } +// ListDepsLocation calls `go list -dep` for every package spec given. +func ListDepsLocation(pkg string) (map[string]string, error) { + const tmpl = `{{if not .Standard}}{{.ImportPath}};{{.Dir}}{{end}}` + + lines, err := getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg)) + if err != nil { + return nil, err + } + deps := make(map[string]string, len(lines)) + for _, l := range lines { + parts := strings.Split(l, ";") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid number of parts") + } + deps[parts[0]] = parts[1] + } + return deps, nil +} + // ListTestFiles lists all go and cgo test files available in a package. func ListTestFiles(pkg string) ([]string, error) { const tmpl = `{{ range .TestGoFiles }}{{ printf "%s\n" . }}{{ end }}` + diff --git a/x-pack/functionbeat/dev-tools/packaging/packages.yml b/x-pack/functionbeat/dev-tools/packaging/packages.yml index 47379c8793e..e6c3346bf1a 100644 --- a/x-pack/functionbeat/dev-tools/packaging/packages.yml +++ b/x-pack/functionbeat/dev-tools/packaging/packages.yml @@ -65,6 +65,12 @@ shared: pkg/functionbeat-aws: source: 'provider/aws/build/golang-crossbuild/aws-linux-amd64' mode: 0755 + pkg/pubsub/vendor: + source: 'provider/gcp/build/pubsub/vendor' + mode: 0644 + pkg/storage/vendor: + source: 'provider/gcp/build/storage/vendor' + mode: 0644 pkg/pubsub/pubsub.go: source: 'provider/gcp/pubsub/pubsub.go' mode: 0655 diff --git a/x-pack/functionbeat/magefile.go b/x-pack/functionbeat/magefile.go index e1982b66400..a317bd0cb71 100644 --- a/x-pack/functionbeat/magefile.go +++ b/x-pack/functionbeat/magefile.go @@ -159,9 +159,11 @@ func BuildPkgForFunctions() error { err := os.RemoveAll("pkg") filesToCopy := map[string]string{ - filepath.Join("provider", "aws", "functionbeat-aws"): filepath.Join("pkg", "functionbeat-aws"), - filepath.Join("provider", "gcp", "pubsub", "pubsub.go"): filepath.Join("pkg", "pubsub", "pubsub.go"), - filepath.Join("provider", "gcp", "storage", "storage.go"): filepath.Join("pkg", "storage", "storage.go"), + filepath.Join("provider", "aws", "functionbeat-aws"): filepath.Join("pkg", "functionbeat-aws"), + filepath.Join("provider", "gcp", "pubsub", "pubsub.go"): filepath.Join("pkg", "pubsub", "pubsub.go"), + filepath.Join("provider", "gcp", "storage", "storage.go"): filepath.Join("pkg", "storage", "storage.go"), + filepath.Join("provider", "gcp", "build", "pubsub", "vendor"): filepath.Join("pkg", "pubsub", "vendor"), + filepath.Join("provider", "gcp", "build", "storage", "vendor"): filepath.Join("pkg", "storage", "vendor"), } for src, dest := range filesToCopy { c := &devtools.CopyTask{ diff --git a/x-pack/functionbeat/manager/gcp/template_builder.go b/x-pack/functionbeat/manager/gcp/template_builder.go index da468abee30..c6f1292ce15 100644 --- a/x-pack/functionbeat/manager/gcp/template_builder.go +++ b/x-pack/functionbeat/manager/gcp/template_builder.go @@ -190,5 +190,6 @@ func zipResources() map[string][]bundle.Resource { func zipResourcesOfFunc(typeName string) []bundle.Resource { root := filepath.Join("pkg", typeName) - return []bundle.Resource{&bundle.LocalFile{Path: filepath.Join(root, typeName+".go"), FileMode: 0755}} + vendor := bundle.Folder(filepath.Join("pkg", typeName, "vendor"), filepath.Join("pkg", typeName), 0644) + return append(vendor, &bundle.LocalFile{Path: filepath.Join(root, typeName+".go"), FileMode: 0755}) } diff --git a/x-pack/functionbeat/scripts/mage/update.go b/x-pack/functionbeat/scripts/mage/update.go index 468bdafbe0f..59b56cb6bed 100644 --- a/x-pack/functionbeat/scripts/mage/update.go +++ b/x-pack/functionbeat/scripts/mage/update.go @@ -5,9 +5,13 @@ package mage import ( + "os" + "path/filepath" + "github.com/magefile/mage/mg" devtools "github.com/elastic/beats/v7/dev-tools/mage" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) // Update target namespace. @@ -20,7 +24,7 @@ var Aliases = map[string]interface{}{ // All updates all generated content. func (Update) All() { - mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs) + mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs, Update.VendorBeats) } // Config generates both the short and reference configs. @@ -46,3 +50,39 @@ func (Update) IncludeFields() error { return devtools.GenerateAllInOneFieldsGo() } + +// VendorBeats collects the vendor folder required to deploy the function for GCP. +func (Update) VendorBeats() error { + for _, f := range []string{"pubsub", "storage"} { + gcpVendorPath := filepath.Join("provider", "gcp", "build", f, "vendor") + err := os.RemoveAll(gcpVendorPath) + if err != nil { + return err + } + + deps, err := gotool.ListDepsLocation("github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/" + f) + if err != nil { + return err + } + + for importPath, location := range deps { + cp := &devtools.CopyTask{ + Source: location, + Dest: filepath.Join(gcpVendorPath, importPath), + Mode: 0600, + DirMode: os.ModeDir | 0750, + Exclude: []string{ + ".*_test.go$", + ".*.yml", + }, + } + err = cp.Execute() + if err != nil { + return err + } + } + + } + + return nil +}