From 55c5e0899d66739e3224e359bbb8aa4d90f0a722 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 5 Feb 2019 08:14:08 -0500 Subject: [PATCH] Allow a keystore to be send with functionbeat files (#10263) (#10528) * Allow a keystore to be send with functionbeat files Add a new interface in the keystore package named Packager, this allow a keystore to return the raw bytes of the keys and values store in it. This is used with core.MakeZip() to create the package that will be send to the lambda function. Fixes: #9009 (cherry picked from commit db8a6472d1b28aaa9f905962ccd5bf72f8446dd0) --- CHANGELOG.next.asciidoc | 1 + libbeat/cmd/instance/beat.go | 13 +++++--- libbeat/keystore/file_keystore.go | 45 ++++++++++++++++++-------- libbeat/keystore/keystore.go | 5 +++ x-pack/functionbeat/core/makezip.go | 49 +++++++++++++++++++++++++++-- 5 files changed, 92 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 68f2114388b..7426920ba3f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -47,6 +47,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff] - The CLI will now log CloudFormation Stack events. {issue}8912[8912] - Correctly normalize Cloudformation resource name. {issue}10087[10087] - Functionbeat can now deploy a function for Kinesis. {10116}10116[10116] +- Allow functionbeat to use the keystore. {issue}9009[9009] ==== Bugfixes diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index d9af56010ed..9789d91c7e2 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -28,7 +28,6 @@ import ( "math/big" "math/rand" "os" - "path/filepath" "runtime" "strings" "time" @@ -565,10 +564,7 @@ func (b *Beat) configure(settings Settings) error { // We have to initialize the keystore before any unpack or merging the cloud // options. - keystoreCfg, _ := cfg.Child("keystore", -1) - defaultPathConfig, _ := cfg.String("path.config", -1) - defaultPathConfig = filepath.Join(defaultPathConfig, fmt.Sprintf("%s.keystore", b.Info.Beat)) - store, err := keystore.Factory(keystoreCfg, defaultPathConfig) + store, err := LoadKeystore(cfg, b.Info.Beat) if err != nil { return fmt.Errorf("could not initialize the keystore: %v", err) } @@ -1002,3 +998,10 @@ func obfuscateConfigOpts() []ucfg.Option { ucfg.ResolveNOOP, } } + +// LoadKeystore returns the appropriate keystore based on the configuration. +func LoadKeystore(cfg *common.Config, name string) (keystore.Keystore, error) { + keystoreCfg, _ := cfg.Child("keystore", -1) + defaultPathConfig := paths.Resolve(paths.Data, fmt.Sprintf("%s.keystore", name)) + return keystore.Factory(keystoreCfg, defaultPathConfig) +} diff --git a/libbeat/keystore/file_keystore.go b/libbeat/keystore/file_keystore.go index 0a7e03080a1..3f530a58120 100644 --- a/libbeat/keystore/file_keystore.go +++ b/libbeat/keystore/file_keystore.go @@ -234,41 +234,53 @@ func (k *FileKeystore) doSave(override bool) error { return nil } -func (k *FileKeystore) load() error { - k.Lock() - defer k.Unlock() - +func (k *FileKeystore) loadRaw() ([]byte, error) { f, err := os.OpenFile(k.Path, os.O_RDONLY, filePermission) if err != nil { if os.IsNotExist(err) { - return nil + return nil, nil } - return err + return nil, err } defer f.Close() if common.IsStrictPerms() { if err := k.checkPermissions(k.Path); err != nil { - return err + return nil, err } } raw, err := ioutil.ReadAll(f) if err != nil { - return err + return nil, err } v := raw[0:len(version)] if !bytes.Equal(v, version) { - return fmt.Errorf("keystore format doesn't match expected version: '%s' got '%s'", version, v) + return nil, fmt.Errorf("keystore format doesn't match expected version: '%s' got '%s'", version, v) } - base64Content := raw[len(version):] - if len(base64Content) == 0 { - return fmt.Errorf("corrupt or empty keystore") + if len(raw) <= len(version) { + return nil, fmt.Errorf("corrupt or empty keystore") } - base64Decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader(base64Content)) + return raw, nil +} + +func (k *FileKeystore) load() error { + k.Lock() + defer k.Unlock() + + raw, err := k.loadRaw() + if err != nil { + return err + } + + if len(raw) == 0 { + return nil + } + + base64Decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader(raw[len(version):])) plaintext, err := k.decrypt(base64Decoder) if err != nil { return fmt.Errorf("could not decrypt the keystore: %v", err) @@ -396,6 +408,13 @@ func (k *FileKeystore) checkPermissions(f string) error { return nil } +// Package returns the bytes of the encrypted keystore. +func (k *FileKeystore) Package() ([]byte, error) { + k.Lock() + defer k.Unlock() + return k.loadRaw() +} + func (k *FileKeystore) hashPassword(password, salt []byte) []byte { return pbkdf2.Key(password, salt, iterationsCount, keyLength, sha512.New) } diff --git a/libbeat/keystore/keystore.go b/libbeat/keystore/keystore.go index eb95c3e041d..f8d2da5c38c 100644 --- a/libbeat/keystore/keystore.go +++ b/libbeat/keystore/keystore.go @@ -64,6 +64,11 @@ type Keystore interface { Save() error } +// Packager defines a keystore that we can read the raw bytes and be packaged in an artifact. +type Packager interface { + Package() ([]byte, error) +} + // Factory Create the right keystore with the configured options. func Factory(cfg *common.Config, defaultPath string) (Keystore, error) { config := defaultConfig diff --git a/x-pack/functionbeat/core/makezip.go b/x-pack/functionbeat/core/makezip.go index 0d1a88d00d9..5ac5127c646 100644 --- a/x-pack/functionbeat/core/makezip.go +++ b/x-pack/functionbeat/core/makezip.go @@ -5,9 +5,16 @@ package core import ( + "fmt" + yaml "gopkg.in/yaml.v2" + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/cfgfile" + "github.com/elastic/beats/libbeat/cmd/instance" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/keystore" "github.com/elastic/beats/x-pack/functionbeat/config" "github.com/elastic/beats/x-pack/functionbeat/core/bundle" ) @@ -43,12 +50,29 @@ func MakeZip() ([]byte, error) { if err != nil { return nil, err } + + resources := []bundle.Resource{ + &bundle.MemoryFile{Path: "functionbeat.yml", Raw: rawConfig, FileMode: 0766}, + &bundle.LocalFile{Path: "pkg/functionbeat", FileMode: 0755}, + } + + rawKeystore, err := keystoreRaw() + if err != nil { + return nil, err + } + + if len(rawKeystore) > 0 { + resources = append(resources, &bundle.MemoryFile{ + Path: "data/functionbeat.keystore", + Raw: rawKeystore, + FileMode: 0600, + }) + } + bundle := bundle.NewZipWithLimits( packageUncompressedLimit, packageCompressedLimit, - &bundle.MemoryFile{Path: "functionbeat.yml", Raw: rawConfig, FileMode: 0766}, - &bundle.LocalFile{Path: "pkg/functionbeat", FileMode: 0755}, - ) + resources...) content, err := bundle.Bytes() if err != nil { @@ -56,3 +80,22 @@ func MakeZip() ([]byte, error) { } return content, nil } + +func keystoreRaw() ([]byte, error) { + cfg, err := cfgfile.Load("", common.NewConfig()) + if err != nil { + return nil, fmt.Errorf("error loading config file: %v", err) + } + + store, err := instance.LoadKeystore(cfg, "functionbeat") + if err != nil { + return nil, errors.Wrapf(err, "cannot load the keystore for packaging") + } + + packager, ok := store.(keystore.Packager) + if !ok { + return nil, fmt.Errorf("the configured keystore cannot be packaged") + } + + return packager.Package() +}