Allow a keystore to be send with functionbeat files (#10263) (#10528)

* Allow a keystore to be send with functionbeat files

Add a new interface in the keystore package named Packager, this allow a
keystore to return the raw bytes of the keys and values store in it.

This is used with core.MakeZip() to create the package that will be send
to the lambda function.

Fixes: #9009

(cherry picked from commit db8a647)

CHANGELOG.next.asciidoc

@@ -47,6 +47,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff]
 - The CLI will now log CloudFormation Stack events. {issue}8912[8912]
 - Correctly normalize Cloudformation resource name. {issue}10087[10087]
 - Functionbeat can now deploy a function for Kinesis. {10116}10116[10116]
+- Allow functionbeat to use the keystore. {issue}9009[9009]
 
 ==== Bugfixes
 

libbeat/cmd/instance/beat.go

@@ -28,7 +28,6 @@ import (
 	"math/big"
 	"math/rand"
 	"os"
-	"path/filepath"
 	"runtime"
 	"strings"
 	"time"
@@ -565,10 +564,7 @@ func (b *Beat) configure(settings Settings) error {
 
 	// We have to initialize the keystore before any unpack or merging the cloud
 	// options.
-	keystoreCfg, _ := cfg.Child("keystore", -1)
-	defaultPathConfig, _ := cfg.String("path.config", -1)
-	defaultPathConfig = filepath.Join(defaultPathConfig, fmt.Sprintf("%s.keystore", b.Info.Beat))
-	store, err := keystore.Factory(keystoreCfg, defaultPathConfig)
+	store, err := LoadKeystore(cfg, b.Info.Beat)
 	if err != nil {
 		return fmt.Errorf("could not initialize the keystore: %v", err)
 	}
@@ -1002,3 +998,10 @@ func obfuscateConfigOpts() []ucfg.Option {
 		ucfg.ResolveNOOP,
 	}
 }
+
+// LoadKeystore returns the appropriate keystore based on the configuration.
+func LoadKeystore(cfg *common.Config, name string) (keystore.Keystore, error) {
+	keystoreCfg, _ := cfg.Child("keystore", -1)
+	defaultPathConfig := paths.Resolve(paths.Data, fmt.Sprintf("%s.keystore", name))
+	return keystore.Factory(keystoreCfg, defaultPathConfig)
+}

libbeat/keystore/file_keystore.go

@@ -234,41 +234,53 @@ func (k *FileKeystore) doSave(override bool) error {
 	return nil
 }
 
-func (k *FileKeystore) load() error {
-	k.Lock()
-	defer k.Unlock()
-
+func (k *FileKeystore) loadRaw() ([]byte, error) {
 	f, err := os.OpenFile(k.Path, os.O_RDONLY, filePermission)
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil
+			return nil, nil
 		}
-		return err
+		return nil, err
 	}
 	defer f.Close()
 
 	if common.IsStrictPerms() {
 		if err := k.checkPermissions(k.Path); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
 	raw, err := ioutil.ReadAll(f)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	v := raw[0:len(version)]
 	if !bytes.Equal(v, version) {
-		return fmt.Errorf("keystore format doesn't match expected version: '%s' got '%s'", version, v)
+		return nil, fmt.Errorf("keystore format doesn't match expected version: '%s' got '%s'", version, v)
 	}
 
-	base64Content := raw[len(version):]
-	if len(base64Content) == 0 {
-		return fmt.Errorf("corrupt or empty keystore")
+	if len(raw) <= len(version) {
+		return nil, fmt.Errorf("corrupt or empty keystore")
 	}
 
-	base64Decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader(base64Content))
+	return raw, nil
+}
+
+func (k *FileKeystore) load() error {
+	k.Lock()
+	defer k.Unlock()
+
+	raw, err := k.loadRaw()
+	if err != nil {
+		return err
+	}
+
+	if len(raw) == 0 {
+		return nil
+	}
+
+	base64Decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader(raw[len(version):]))
 	plaintext, err := k.decrypt(base64Decoder)
 	if err != nil {
 		return fmt.Errorf("could not decrypt the keystore: %v", err)
@@ -396,6 +408,13 @@ func (k *FileKeystore) checkPermissions(f string) error {
 	return nil
 }
 
+// Package returns the bytes of the encrypted keystore.
+func (k *FileKeystore) Package() ([]byte, error) {
+	k.Lock()
+	defer k.Unlock()
+	return k.loadRaw()
+}
+
 func (k *FileKeystore) hashPassword(password, salt []byte) []byte {
 	return pbkdf2.Key(password, salt, iterationsCount, keyLength, sha512.New)
 }

libbeat/keystore/keystore.go

@@ -64,6 +64,11 @@ type Keystore interface {
 	Save() error
 }
 
+// Packager defines a keystore that we can read the raw bytes and be packaged in an artifact.
+type Packager interface {
+	Package() ([]byte, error)
+}
+
 // Factory Create the right keystore with the configured options.
 func Factory(cfg *common.Config, defaultPath string) (Keystore, error) {
 	config := defaultConfig

x-pack/functionbeat/core/makezip.go

@@ -5,9 +5,16 @@
 package core
 
 import (
+	"fmt"
+
 	yaml "gopkg.in/yaml.v2"
 
+	"github.com/pkg/errors"
+
 	"github.com/elastic/beats/libbeat/cfgfile"
+	"github.com/elastic/beats/libbeat/cmd/instance"
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/keystore"
 	"github.com/elastic/beats/x-pack/functionbeat/config"
 	"github.com/elastic/beats/x-pack/functionbeat/core/bundle"
 )
@@ -43,16 +50,52 @@ func MakeZip() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	resources := []bundle.Resource{
+		&bundle.MemoryFile{Path: "functionbeat.yml", Raw: rawConfig, FileMode: 0766},
+		&bundle.LocalFile{Path: "pkg/functionbeat", FileMode: 0755},
+	}
+
+	rawKeystore, err := keystoreRaw()
+	if err != nil {
+		return nil, err
+	}
+
+	if len(rawKeystore) > 0 {
+		resources = append(resources, &bundle.MemoryFile{
+			Path:     "data/functionbeat.keystore",
+			Raw:      rawKeystore,
+			FileMode: 0600,
+		})
+	}
+
 	bundle := bundle.NewZipWithLimits(
 		packageUncompressedLimit,
 		packageCompressedLimit,
-		&bundle.MemoryFile{Path: "functionbeat.yml", Raw: rawConfig, FileMode: 0766},
-		&bundle.LocalFile{Path: "pkg/functionbeat", FileMode: 0755},
-	)
+		resources...)
 
 	content, err := bundle.Bytes()
 	if err != nil {
 		return nil, err
 	}
 	return content, nil
 }
+
+func keystoreRaw() ([]byte, error) {
+	cfg, err := cfgfile.Load("", common.NewConfig())
+	if err != nil {
+		return nil, fmt.Errorf("error loading config file: %v", err)
+	}
+
+	store, err := instance.LoadKeystore(cfg, "functionbeat")
+	if err != nil {
+		return nil, errors.Wrapf(err, "cannot load the keystore for packaging")
+	}
+
+	packager, ok := store.(keystore.Packager)
+	if !ok {
+		return nil, fmt.Errorf("the configured keystore cannot be packaged")
+	}
+
+	return packager.Package()
+}