From e16e041fb7432a14bfaec3a7d2e6aba885ba3a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 24 Apr 2019 09:37:57 +0200 Subject: [PATCH 01/13] add new subcommand to export: function --- x-pack/functionbeat/cmd/provider_cmd.go | 36 ++- x-pack/functionbeat/cmd/root.go | 1 + x-pack/functionbeat/provider/aws/aws.go | 2 +- .../functionbeat/provider/aws/cli_manager.go | 257 ++------------- .../provider/aws/template_builder.go | 298 ++++++++++++++++++ .../functionbeat/provider/default_provider.go | 46 ++- x-pack/functionbeat/provider/local/local.go | 2 +- x-pack/functionbeat/provider/provider.go | 1 + x-pack/functionbeat/provider/template.go | 19 ++ 9 files changed, 411 insertions(+), 251 deletions(-) create mode 100644 x-pack/functionbeat/provider/aws/template_builder.go create mode 100644 x-pack/functionbeat/provider/template.go diff --git a/x-pack/functionbeat/cmd/provider_cmd.go b/x-pack/functionbeat/cmd/provider_cmd.go index 66864af45766..8f428b790484 100644 --- a/x-pack/functionbeat/cmd/provider_cmd.go +++ b/x-pack/functionbeat/cmd/provider_cmd.go @@ -5,6 +5,7 @@ package cmd import ( + "fmt" "os" "path/filepath" @@ -18,8 +19,7 @@ import ( var output string -// TODO: Add List() subcommand. -func handler() (*cliHandler, error) { +func initProvider() (provider.Provider, error) { b, err := instance.NewInitializedBeat(instance.Settings{Name: Name}) if err != nil { return nil, err @@ -35,7 +35,12 @@ func handler() (*cliHandler, error) { return nil, err } - provider, err := provider.NewProvider(cfg) + return provider.NewProvider(cfg) +} + +// TODO: Add List() subcommand. +func handler() (*cliHandler, error) { + provider, err := initProvider() if err != nil { return nil, err } @@ -99,3 +104,28 @@ func genPackageCmd() *cobra.Command { cmd.Flags().StringVarP(&output, "output", "o", "", "full path to the package") return cmd } + +func genExportTemplateCmd() *cobra.Command { + return &cobra.Command{ + Use: "function", + Short: "Export function template", + Run: cli.RunWith(func(_ *cobra.Command, args []string) error { + p, err := initProvider() + if err != nil { + return err + } + builder, err := p.TemplateBuilder() + if err != nil { + return err + } + for _, name := range args { + template, err := builder.RawTemplate(name) + if err != nil { + return fmt.Errorf("error generating raw template for %s: %+v", name, err) + } + fmt.Println(template) + } + return nil + }), + } +} diff --git a/x-pack/functionbeat/cmd/root.go b/x-pack/functionbeat/cmd/root.go index 043e303fb815..8e2e57b1a91b 100644 --- a/x-pack/functionbeat/cmd/root.go +++ b/x-pack/functionbeat/cmd/root.go @@ -27,4 +27,5 @@ func init() { RootCmd.AddCommand(genUpdateCmd()) RootCmd.AddCommand(genRemoveCmd()) RootCmd.AddCommand(genPackageCmd()) + RootCmd.ExportCmd.AddCommand(genExportTemplateCmd()) } diff --git a/x-pack/functionbeat/provider/aws/aws.go b/x-pack/functionbeat/provider/aws/aws.go index 1949eac8df01..32d412690ec2 100644 --- a/x-pack/functionbeat/provider/aws/aws.go +++ b/x-pack/functionbeat/provider/aws/aws.go @@ -12,7 +12,7 @@ import ( // Bundle exposes the trigger supported by the AWS provider. var Bundle = provider.MustCreate( "aws", - provider.NewDefaultProvider("aws", NewCLI), + provider.NewDefaultProvider("aws", NewCLI, NewTemplateBuilder), feature.NewDetails("AWS Lambda", "listen to events on AWS lambda", feature.Stable), ).MustAddFunction("cloudwatch_logs", NewCloudwatchLogs, diff --git a/x-pack/functionbeat/provider/aws/cli_manager.go b/x-pack/functionbeat/provider/aws/cli_manager.go index eed1be860fa5..45ab67c76fa9 100644 --- a/x-pack/functionbeat/provider/aws/cli_manager.go +++ b/x-pack/functionbeat/provider/aws/cli_manager.go @@ -5,9 +5,6 @@ package aws import ( - "crypto/sha256" - "encoding/base64" - "errors" "fmt" "regexp" @@ -19,7 +16,6 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/x-pack/functionbeat/core" "github.com/elastic/beats/x-pack/functionbeat/provider" ) @@ -49,154 +45,10 @@ type installer interface { // It will take care of creating the main lambda function and ask for each function type for the // operation that need to be executed to connect the lambda to the triggers. type CLIManager struct { - provider provider.Provider - awsCfg aws.Config - log *logp.Logger - config *Config -} - -func (c *CLIManager) findFunction(name string) (installer, error) { - fn, err := c.provider.FindFunctionByName(name) - if err != nil { - return nil, err - } - - function, ok := fn.(installer) - if !ok { - return nil, errors.New("incompatible type received, expecting: 'functionManager'") - } - - return function, nil -} - -func (c *CLIManager) template(function installer, name, codeLoc string) *cloudformation.Template { - lambdaConfig := function.LambdaConfig() - - prefix := func(s string) string { - return normalizeResourceName("fnb" + name + s) - } - - // AWS variables references:. - // AWS::Partition: aws, aws-cn, aws-gov. - // AWS::Region: us-east-1, us-east-2, ap-northeast-3, - // AWS::AccountId: account id for the current request. - // AWS::URLSuffix: amazonaws.com - // - // Documentation: https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/Welcome.html - // Intrinsic function reference: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/intrinsic-function-reference.html - - template := cloudformation.NewTemplate() - - role := lambdaConfig.Role - dependsOn := make([]string, 0) - if lambdaConfig.Role == "" { - c.log.Infof("No role is configured for function %s, creating a custom role.", name) - - roleRes := prefix("") + "IAMRoleLambdaExecution" - template.Resources[roleRes] = c.roleTemplate(function, name) - role = cloudformation.GetAtt(roleRes, "Arn") - dependsOn = []string{roleRes} - } - - // Configure the Dead letter, any failed events will be send to the configured amazon resource name. - var dlc *cloudformation.AWSLambdaFunction_DeadLetterConfig - if lambdaConfig.DeadLetterConfig != nil && len(lambdaConfig.DeadLetterConfig.TargetArn) != 0 { - dlc = &cloudformation.AWSLambdaFunction_DeadLetterConfig{ - TargetArn: lambdaConfig.DeadLetterConfig.TargetArn, - } - } - - // Configure VPC - var vcpConf *cloudformation.AWSLambdaFunction_VpcConfig - if lambdaConfig.VPCConfig != nil && len(lambdaConfig.VPCConfig.SecurityGroupIDs) != 0 && len(lambdaConfig.VPCConfig.SubnetIDs) != 0 { - vcpConf = &cloudformation.AWSLambdaFunction_VpcConfig{ - SecurityGroupIds: lambdaConfig.VPCConfig.SecurityGroupIDs, - SubnetIds: lambdaConfig.VPCConfig.SubnetIDs, - } - } - - // Create the lambda - // Doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-function.html - template.Resources[prefix("")] = &AWSLambdaFunction{ - AWSLambdaFunction: &cloudformation.AWSLambdaFunction{ - Code: &cloudformation.AWSLambdaFunction_Code{ - S3Bucket: c.bucket(), - S3Key: codeLoc, - }, - Description: lambdaConfig.Description, - Environment: &cloudformation.AWSLambdaFunction_Environment{ - // Configure which function need to be run by the lambda function. - Variables: map[string]string{ - "BEAT_STRICT_PERMS": "false", // Disable any check on disk, we are running with really differents permission on lambda. - "ENABLED_FUNCTIONS": name, - }, - }, - DeadLetterConfig: dlc, - VpcConfig: vcpConf, - FunctionName: name, - Role: role, - Runtime: runtime, - Handler: handlerName, - MemorySize: lambdaConfig.MemorySize.Megabytes(), - ReservedConcurrentExecutions: lambdaConfig.Concurrency, - Timeout: int(lambdaConfig.Timeout.Seconds()), - }, - DependsOn: dependsOn, - } - - // Create the log group for the specific function lambda. - template.Resources[prefix("LogGroup")] = &cloudformation.AWSLogsLogGroup{ - LogGroupName: "/aws/lambda/" + name, - } - - return template -} - -func (c *CLIManager) roleTemplate(function installer, name string) *cloudformation.AWSIAMRole { - // Default policies to writes logs from the Lambda. - policies := []cloudformation.AWSIAMRole_Policy{ - cloudformation.AWSIAMRole_Policy{ - PolicyName: cloudformation.Join("-", []string{"fnb", "lambda", name}), - PolicyDocument: map[string]interface{}{ - "Statement": []map[string]interface{}{ - map[string]interface{}{ - "Action": []string{"logs:CreateLogStream", "logs:PutLogEvents"}, - "Effect": "Allow", - "Resource": []string{ - cloudformation.Sub("arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/" + name + ":*"), - }, - }, - }, - }, - }, - } - - // Merge any specific policies from the service. - policies = append(policies, function.Policies()...) - - // Create the roles for the lambda. - // doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html - return &cloudformation.AWSIAMRole{ - AssumeRolePolicyDocument: map[string]interface{}{ - "Statement": []interface{}{ - map[string]interface{}{ - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": map[string]interface{}{ - "Service": cloudformation.Join("", []string{ - "lambda.", - cloudformation.Ref("AWS::URLSuffix"), - }), - }, - }, - }, - }, - Path: "/", - RoleName: "functionbeat-lambda-" + name, - // Allow the lambda to write log to cloudwatch logs. - // doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html - Policies: policies, - } + templateBuilder *defaultTemplateBuilder + awsCfg aws.Config + log *logp.Logger + config *Config } // stackName cloudformation stack are unique per function. @@ -205,38 +57,12 @@ func (c *CLIManager) stackName(name string) string { } func (c *CLIManager) deployTemplate(update bool, name string) error { - c.log.Debug("Compressing all assets into an artifact") - content, err := core.MakeZip() - if err != nil { - return err - } - c.log.Debugf("Compression is successful (zip size: %d bytes)", len(content)) - - function, err := c.findFunction(name) + templateData, err := c.templateBuilder.execute(name) if err != nil { return err } - fnTemplate := function.Template() - - zipChecksum := checksum(content) - codeKey := "functionbeat-deployment/" + name + "/" + zipChecksum + "/functionbeat.zip" - - to := c.template(function, name, codeKey) - if err := mergeTemplate(to, fnTemplate); err != nil { - return err - } - - json, err := to.JSON() - if err != nil { - return err - } - - templateChecksum := checksum(json) - templateKey := "functionbeat-deployment/" + name + "/" + templateChecksum + "/cloudformation-template-create.json" - templateURL := "https://s3.amazonaws.com/" + c.bucket() + "/" + templateKey - - c.log.Debugf("Using cloudformation template:\n%s", json) + c.log.Debugf("Using cloudformation template:\n%s", templateData.json) svcCF := cf.New(c.awsCfg) executer := newExecutor(c.log) @@ -245,34 +71,34 @@ func (c *CLIManager) deployTemplate(update bool, name string) error { c.log, c.awsCfg, c.bucket(), - codeKey, - content, + templateData.codeKey, + templateData.zip.content, )) executer.Add(newOpUploadToBucket( c.log, c.awsCfg, c.bucket(), - templateKey, - json, + templateData.key, + templateData.json, )) if update { executer.Add(newOpUpdateCloudFormation( c.log, svcCF, - templateURL, + templateData.url, c.stackName(name), )) } else { executer.Add(newOpCreateCloudFormation( c.log, svcCF, - templateURL, + templateData.url, c.stackName(name), )) } executer.Add(newOpWaitCloudFormation(c.log, cf.New(c.awsCfg))) - executer.Add(newOpDeleteFileBucket(c.log, c.awsCfg, c.bucket(), codeKey)) + executer.Add(newOpDeleteFileBucket(c.log, c.awsCfg, c.bucket(), templateData.codeKey)) ctx := newStackContext() if err := executer.Execute(ctx); err != nil { @@ -349,59 +175,24 @@ func NewCLI( return nil, err } - return &CLIManager{ - config: config, - provider: provider, - awsCfg: awsCfg, - log: logp.NewLogger("aws"), - }, nil -} - -// mergeTemplate takes two cloudformation and merge them, if a key already exist we return an error. -func mergeTemplate(to, from *cloudformation.Template) error { - merge := func(m1 map[string]interface{}, m2 map[string]interface{}) error { - for k, v := range m2 { - if _, ok := m1[k]; ok { - return fmt.Errorf("key %s already exist in the template map", k) - } - m1[k] = v - } - return nil - } - - err := merge(to.Parameters, from.Parameters) + builder, err := provider.TemplateBuilder() if err != nil { - return err - } - - err = merge(to.Mappings, from.Mappings) - if err != nil { - return err - } - - err = merge(to.Conditions, from.Conditions) - if err != nil { - return err - } - - err = merge(to.Resources, from.Resources) - if err != nil { - return err + return nil, err } - err = merge(to.Outputs, from.Outputs) - if err != nil { - return err + templateBuilder, ok := builder.(*defaultTemplateBuilder) + if !ok { + return nil, fmt.Errorf("not defaultTemplateBuilder") } - return nil + return &CLIManager{ + config: config, + awsCfg: awsCfg, + log: logp.NewLogger("aws"), + templateBuilder: templateBuilder, + }, nil } func normalizeResourceName(s string) string { return validChars.ReplaceAllString(s, "") } - -func checksum(data []byte) string { - sha := sha256.Sum256(data) - return base64.RawURLEncoding.EncodeToString(sha[:]) -} diff --git a/x-pack/functionbeat/provider/aws/template_builder.go b/x-pack/functionbeat/provider/aws/template_builder.go new file mode 100644 index 000000000000..7ac06fb01921 --- /dev/null +++ b/x-pack/functionbeat/provider/aws/template_builder.go @@ -0,0 +1,298 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package aws + +import ( + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + + "github.com/awslabs/goformation/cloudformation" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/x-pack/functionbeat/core" + "github.com/elastic/beats/x-pack/functionbeat/provider" +) + +// zipData stores the data on the zip to be deployed +type zipData struct { + content []byte + checksum string +} + +// templateData stores the template and its metadata required to deploy it +type templateData struct { + json []byte + checksum string + key string + url string + codeKey string + zip zipData +} + +type defaultTemplateBuilder struct { + provider provider.Provider + log *logp.Logger + bucket string +} + +const ( + keyPrefix = "functionbeat-deployment/" +) + +func NewTemplateBuilder(log *logp.Logger, cfg *common.Config, p provider.Provider) (provider.TemplateBuilder, error) { + config := &Config{} + if err := cfg.Unpack(config); err != nil { + return nil, err + } + + return &defaultTemplateBuilder{ + provider: p, + log: log, + bucket: string(config.DeployBucket), + }, nil +} + +func (d *defaultTemplateBuilder) findFunction(name string) (installer, error) { + fn, err := d.provider.FindFunctionByName(name) + if err != nil { + return nil, err + } + + function, ok := fn.(installer) + if !ok { + return nil, errors.New("incompatible type received, expecting: 'functionManager'") + } + + return function, nil +} + +// execute generates a template +func (d *defaultTemplateBuilder) execute(name string) (templateData, error) { + d.log.Debug("Compressing all assets into an artifact") + content, err := core.MakeZip() + if err != nil { + return templateData{}, err + } + d.log.Debugf("Compression is successful (zip size: %d bytes)", len(content)) + + function, err := d.findFunction(name) + if err != nil { + return templateData{}, err + } + + fnTemplate := function.Template() + + zipChecksum := checksum(content) + codeKey := keyPrefix + name + "/" + zipChecksum + "/functionbeat.zip" + to := d.template(function, name, codeKey) + if err := mergeTemplate(to, fnTemplate); err != nil { + return templateData{}, err + } + + templateJSON, err := to.JSON() + if err != nil { + return templateData{}, err + } + + templateChecksum := checksum(templateJSON) + templateKey := keyPrefix + name + "/" + templateChecksum + "/cloudformation-template-create.json" + templateURL := "https://s3.amazonaws.com/" + d.bucket + "/" + templateKey + + return templateData{ + json: templateJSON, + checksum: templateChecksum, + key: templateKey, + url: templateURL, + codeKey: codeKey, + zip: zipData{ + checksum: zipChecksum, + content: content, + }, + }, nil +} + +func (d *defaultTemplateBuilder) template(function installer, name, codeLoc string) *cloudformation.Template { + lambdaConfig := function.LambdaConfig() + + prefix := func(s string) string { + return normalizeResourceName("fnb" + name + s) + } + + // AWS variables references:. + // AWS::Partition: aws, aws-cn, aws-gov. + // AWS::Region: us-east-1, us-east-2, ap-northeast-3, + // AWS::AccountId: account id for the current request. + // AWS::URLSuffix: amazonaws.com + // + // Documentation: https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/Welcome.html + // Intrinsic function reference: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/intrinsic-function-reference.html + + template := cloudformation.NewTemplate() + + role := lambdaConfig.Role + dependsOn := make([]string, 0) + if lambdaConfig.Role == "" { + d.log.Infof("No role is configured for function %s, creating a custom role.", name) + + roleRes := prefix("") + "IAMRoleLambdaExecution" + template.Resources[roleRes] = d.roleTemplate(function, name) + role = cloudformation.GetAtt(roleRes, "Arn") + dependsOn = []string{roleRes} + } + + // Configure the Dead letter, any failed events will be send to the configured amazon resource name. + var dlc *cloudformation.AWSLambdaFunction_DeadLetterConfig + if lambdaConfig.DeadLetterConfig != nil && len(lambdaConfig.DeadLetterConfig.TargetArn) != 0 { + dlc = &cloudformation.AWSLambdaFunction_DeadLetterConfig{ + TargetArn: lambdaConfig.DeadLetterConfig.TargetArn, + } + } + + // Configure VPC + var vcpConf *cloudformation.AWSLambdaFunction_VpcConfig + if lambdaConfig.VPCConfig != nil && len(lambdaConfig.VPCConfig.SecurityGroupIDs) != 0 && len(lambdaConfig.VPCConfig.SubnetIDs) != 0 { + vcpConf = &cloudformation.AWSLambdaFunction_VpcConfig{ + SecurityGroupIds: lambdaConfig.VPCConfig.SecurityGroupIDs, + SubnetIds: lambdaConfig.VPCConfig.SubnetIDs, + } + } + + // Create the lambda + // Doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-function.html + template.Resources[prefix("")] = &AWSLambdaFunction{ + AWSLambdaFunction: &cloudformation.AWSLambdaFunction{ + Code: &cloudformation.AWSLambdaFunction_Code{ + S3Bucket: d.bucket, + S3Key: codeLoc, + }, + Description: lambdaConfig.Description, + Environment: &cloudformation.AWSLambdaFunction_Environment{ + // Configure which function need to be run by the lambda function. + Variables: map[string]string{ + "BEAT_STRICT_PERMS": "false", // Disable any check on disk, we are running with really differents permission on lambda. + "ENABLED_FUNCTIONS": name, + }, + }, + DeadLetterConfig: dlc, + VpcConfig: vcpConf, + FunctionName: name, + Role: role, + Runtime: runtime, + Handler: handlerName, + MemorySize: lambdaConfig.MemorySize.Megabytes(), + ReservedConcurrentExecutions: lambdaConfig.Concurrency, + Timeout: int(lambdaConfig.Timeout.Seconds()), + }, + DependsOn: dependsOn, + } + + // Create the log group for the specific function lambda. + template.Resources[prefix("LogGroup")] = &cloudformation.AWSLogsLogGroup{ + LogGroupName: "/aws/lambda/" + name, + } + + return template +} + +func (d *defaultTemplateBuilder) roleTemplate(function installer, name string) *cloudformation.AWSIAMRole { + // Default policies to writes logs from the Lambda. + policies := []cloudformation.AWSIAMRole_Policy{ + cloudformation.AWSIAMRole_Policy{ + PolicyName: cloudformation.Join("-", []string{"fnb", "lambda", name}), + PolicyDocument: map[string]interface{}{ + "Statement": []map[string]interface{}{ + map[string]interface{}{ + "Action": []string{"logs:CreateLogStream", "logs:PutLogEvents"}, + "Effect": "Allow", + "Resource": []string{ + cloudformation.Sub("arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/" + name + ":*"), + }, + }, + }, + }, + }, + } + + // Merge any specific policies from the service. + policies = append(policies, function.Policies()...) + + // Create the roles for the lambda. + // doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html + return &cloudformation.AWSIAMRole{ + AssumeRolePolicyDocument: map[string]interface{}{ + "Statement": []interface{}{ + map[string]interface{}{ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": map[string]interface{}{ + "Service": cloudformation.Join("", []string{ + "lambda.", + cloudformation.Ref("AWS::URLSuffix"), + }), + }, + }, + }, + }, + Path: "/", + RoleName: "functionbeat-lambda-" + name, + // Allow the lambda to write log to cloudwatch logs. + // doc: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-policy.html + Policies: policies, + } +} + +// RawTemplate generates a template and returns it in a string +func (d *defaultTemplateBuilder) RawTemplate(name string) (string, error) { + data, err := d.execute(name) + return string(data.json), err +} + +// mergeTemplate takes two cloudformation and merge them, if a key already exist we return an error. +func mergeTemplate(to, from *cloudformation.Template) error { + merge := func(m1 map[string]interface{}, m2 map[string]interface{}) error { + for k, v := range m2 { + if _, ok := m1[k]; ok { + return fmt.Errorf("key %s already exist in the template map", k) + } + m1[k] = v + } + return nil + } + + err := merge(to.Parameters, from.Parameters) + if err != nil { + return err + } + + err = merge(to.Mappings, from.Mappings) + if err != nil { + return err + } + + err = merge(to.Conditions, from.Conditions) + if err != nil { + return err + } + + err = merge(to.Resources, from.Resources) + if err != nil { + return err + } + + err = merge(to.Outputs, from.Outputs) + if err != nil { + return err + } + + return nil +} + +func checksum(data []byte) string { + sha := sha256.Sum256(data) + return base64.RawURLEncoding.EncodeToString(sha[:]) +} diff --git a/x-pack/functionbeat/provider/default_provider.go b/x-pack/functionbeat/provider/default_provider.go index 060273691417..688db666d899 100644 --- a/x-pack/functionbeat/provider/default_provider.go +++ b/x-pack/functionbeat/provider/default_provider.go @@ -15,16 +15,21 @@ import ( // DefaultProvider implements the minimal required to retrieve and start functions. type DefaultProvider struct { - rawConfig *common.Config - config *config.ProviderConfig - registry *Registry - name string - log *logp.Logger - managerFactory CLIManagerFactory + rawConfig *common.Config + config *config.ProviderConfig + registry *Registry + name string + log *logp.Logger + managerFactory CLIManagerFactory + templateFactory TemplateBuilderFactory } // NewDefaultProvider returns factory methods to handle generic provider. -func NewDefaultProvider(name string, manager CLIManagerFactory) func(*logp.Logger, *Registry, *common.Config) (Provider, error) { +func NewDefaultProvider( + name string, + manager CLIManagerFactory, + templater TemplateBuilderFactory, +) func(*logp.Logger, *Registry, *common.Config) (Provider, error) { return func(log *logp.Logger, registry *Registry, cfg *common.Config) (Provider, error) { c := &config.ProviderConfig{} err := cfg.Unpack(c) @@ -37,12 +42,13 @@ func NewDefaultProvider(name string, manager CLIManagerFactory) func(*logp.Logge } return &DefaultProvider{ - rawConfig: cfg, - config: c, - registry: registry, - name: name, - log: log, - managerFactory: manager, + rawConfig: cfg, + config: c, + registry: registry, + name: name, + log: log, + managerFactory: manager, + templateFactory: templater, }, nil } } @@ -68,6 +74,10 @@ func (d *DefaultProvider) CLIManager() (CLIManager, error) { return d.managerFactory(nil, d.rawConfig, d) } +func (d *DefaultProvider) TemplateBuilder() (TemplateBuilder, error) { + return d.templateFactory(d.log, d.rawConfig, d) +} + // nullCLI is used when a provider doesn't implement the CLI to manager functions on the service provider. type nullCLI struct{} @@ -79,3 +89,13 @@ func NewNullCli(_ *logp.Logger, _ *common.Config, _ Provider) (CLIManager, error func (*nullCLI) Deploy(_ string) error { return fmt.Errorf("deploy not implemented") } func (*nullCLI) Update(_ string) error { return fmt.Errorf("update not implemented") } func (*nullCLI) Remove(_ string) error { return fmt.Errorf("remove not implemented") } + +type nullTemplateBuilder struct{} + +func NewNullTemplateBuilder(_ *logp.Logger, _ *common.Config, _ Provider) (TemplateBuilder, error) { + return (*nullTemplateBuilder)(nil), nil +} + +func (*nullTemplateBuilder) RawTemplate(_ string) (string, error) { + return "", fmt.Errorf("raw temaplate not implemented") +} diff --git a/x-pack/functionbeat/provider/local/local.go b/x-pack/functionbeat/provider/local/local.go index 9954047583d5..bf4318c46e72 100644 --- a/x-pack/functionbeat/provider/local/local.go +++ b/x-pack/functionbeat/provider/local/local.go @@ -22,7 +22,7 @@ const stdinName = "stdin" // Bundle exposes the local provider and the STDIN function. var Bundle = provider.MustCreate( "local", - provider.NewDefaultProvider("local", provider.NewNullCli), + provider.NewDefaultProvider("local", provider.NewNullCli, provider.NewNullTemplateBuilder), feature.NewDetails("local events", "allows to trigger events locally.", feature.Experimental), ).MustAddFunction( stdinName, diff --git a/x-pack/functionbeat/provider/provider.go b/x-pack/functionbeat/provider/provider.go index 90669ae3fbe6..2a7c6d013aa6 100644 --- a/x-pack/functionbeat/provider/provider.go +++ b/x-pack/functionbeat/provider/provider.go @@ -32,6 +32,7 @@ type Provider interface { CreateFunctions(clientFactory, []string) ([]core.Runner, error) FindFunctionByName(string) (Function, error) CLIManager() (CLIManager, error) + TemplateBuilder() (TemplateBuilder, error) Name() string } diff --git a/x-pack/functionbeat/provider/template.go b/x-pack/functionbeat/provider/template.go new file mode 100644 index 000000000000..dfa0651ed3fa --- /dev/null +++ b/x-pack/functionbeat/provider/template.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package provider + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" +) + +// TemplateBuilderFactory factory method to call to create a new template builder. +type TemplateBuilderFactory func(*logp.Logger, *common.Config, Provider) (TemplateBuilder, error) + +// TemplateBuilder generates templates for a given provider. +type TemplateBuilder interface { + // RawTemplate returns a deployable template string. + RawTemplate(string) (string, error) +} From bd5a53e9c02a3fa0177024cc7ce59d8c0d79ee5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 25 Apr 2019 09:08:29 +0200 Subject: [PATCH 02/13] feed the hound --- x-pack/functionbeat/provider/default_provider.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/functionbeat/provider/default_provider.go b/x-pack/functionbeat/provider/default_provider.go index 688db666d899..413e44fe24b5 100644 --- a/x-pack/functionbeat/provider/default_provider.go +++ b/x-pack/functionbeat/provider/default_provider.go @@ -74,6 +74,7 @@ func (d *DefaultProvider) CLIManager() (CLIManager, error) { return d.managerFactory(nil, d.rawConfig, d) } +// TemplateBuilder returns a TemplateBuilder returns a the type responsible to generate templates. func (d *DefaultProvider) TemplateBuilder() (TemplateBuilder, error) { return d.templateFactory(d.log, d.rawConfig, d) } @@ -90,8 +91,10 @@ func (*nullCLI) Deploy(_ string) error { return fmt.Errorf("deploy not implement func (*nullCLI) Update(_ string) error { return fmt.Errorf("update not implemented") } func (*nullCLI) Remove(_ string) error { return fmt.Errorf("remove not implemented") } +// nullTemplateBuilder is used when a provider does not implement a template builder functionality. type nullTemplateBuilder struct{} +// NewNullTemplateBuilder returns a NOOP TemplateBuilder. func NewNullTemplateBuilder(_ *logp.Logger, _ *common.Config, _ Provider) (TemplateBuilder, error) { return (*nullTemplateBuilder)(nil), nil } From ad3a5c31ca01126edc54a55439528001b3888397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 6 May 2019 10:26:32 +0200 Subject: [PATCH 03/13] do not use path-style uri --- x-pack/functionbeat/provider/aws/template_builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/functionbeat/provider/aws/template_builder.go b/x-pack/functionbeat/provider/aws/template_builder.go index 7ac06fb01921..e3bf136b07eb 100644 --- a/x-pack/functionbeat/provider/aws/template_builder.go +++ b/x-pack/functionbeat/provider/aws/template_builder.go @@ -101,7 +101,7 @@ func (d *defaultTemplateBuilder) execute(name string) (templateData, error) { templateChecksum := checksum(templateJSON) templateKey := keyPrefix + name + "/" + templateChecksum + "/cloudformation-template-create.json" - templateURL := "https://s3.amazonaws.com/" + d.bucket + "/" + templateKey + templateURL := "https://" + d.bucket + "s3.amazonaws.com/" + templateKey return templateData{ json: templateJSON, From a30c13b8e355018c3c8a044fb10aa17b3ea34be3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 6 May 2019 10:26:44 +0200 Subject: [PATCH 04/13] add missing function to mock --- x-pack/functionbeat/provider/registry_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/functionbeat/provider/registry_test.go b/x-pack/functionbeat/provider/registry_test.go index 9340a5917e2b..847214c4a581 100644 --- a/x-pack/functionbeat/provider/registry_test.go +++ b/x-pack/functionbeat/provider/registry_test.go @@ -35,6 +35,8 @@ func (m *mockProvider) Name() string { return m.name } func (m *mockProvider) CLIManager() (CLIManager, error) { return nil, nil } +func (m *mockProvider) TemplateBuilder() (TemplateBuilder, error) { return nil, nil } + func TestRegistry(t *testing.T) { t.Run("provider", testProviderLookup) t.Run("functions", testFunctionLookup) From 9838b9426c35fb7358656d18e2d239b75c3dfe83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 6 May 2019 11:03:29 +0200 Subject: [PATCH 05/13] add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index bfe7f82f09e7..16afb648a0d4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -216,6 +216,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Functionbeat* - New options to configure roles and VPC. {pull}11779[11779] +- Export function templates. {pull}11923[11923] *Winlogbeat* From e88832afb1e154ebca629bcd01514f35c483ec5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Tue, 7 May 2019 12:03:29 +0200 Subject: [PATCH 06/13] add system test to check exported function --- libbeat/tests/system/beat/beat.py | 12 +++ .../tests/system/config/functionbeat.yml.j2 | 20 +++++ x-pack/functionbeat/tests/system/test_base.py | 81 ++++++++++++++++++- 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 587c2692caf2..98f1afe4e492 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -360,6 +360,18 @@ def get_log(self, logfile=None): return data + def get_log_lines(self, logfile=None): + """ + Returns the log lines as a list of strings + """ + if logfile is None: + logfile = self.beat_name + ".log" + + with open(os.path.join(self.working_dir, logfile), 'r') as f: + data = f.readlines() + + return data + def wait_log_contains(self, msg, logfile=None, max_timeout=10, poll_interval=0.1, name="log_contains", diff --git a/x-pack/functionbeat/tests/system/config/functionbeat.yml.j2 b/x-pack/functionbeat/tests/system/config/functionbeat.yml.j2 index 9ae2bd2d9c4b..178aab2b3a6b 100644 --- a/x-pack/functionbeat/tests/system/config/functionbeat.yml.j2 +++ b/x-pack/functionbeat/tests/system/config/functionbeat.yml.j2 @@ -1,8 +1,28 @@ ################### Beat Configuration ######################### +{% if local %} functionbeat.provider.local: functions: - type: stdin enabled: true +{% endif %} + +{% if cloudwatch %} +functionbeat.provider.aws.deploy_bucket: {{ cloudwatch.bucket | default("functionbeat-deploy") }} +functionbeat.provider.aws.functions: + - name: {{ cloudwatch.name }} + enabled: true + type: cloudwatch_logs + description: "lambda function for cloudwatch logs" + {% if cloudwatch.role %}role: {{ cloudwatch.role }}{% endif %} + {% if cloudwatch.virtual_private_cloud %} + virtual_private_cloud: + security_group_ids: {{ cloudwatch.virtual_private_cloud.security_group_ids }} + subnet_ids: {{ cloudwatch.virtual_private_cloud.subnet_ids }} + {% endif %} + + triggers: + - log_group_name: {{ cloudwatch.log_group | default("/aws/lambda/functionbeat-cloudwatch") }} +{% endif %} ############################# Output ########################################## diff --git a/x-pack/functionbeat/tests/system/test_base.py b/x-pack/functionbeat/tests/system/test_base.py index 30402e377f3b..cd86cac5c7d0 100644 --- a/x-pack/functionbeat/tests/system/test_base.py +++ b/x-pack/functionbeat/tests/system/test_base.py @@ -1,5 +1,6 @@ from functionbeat import BaseTest +import json import os import unittest @@ -12,10 +13,88 @@ def test_base(self): Basic test with exiting Functionbeat normally """ self.render_config_template( - path=os.path.abspath(self.working_dir) + "/log/*" + path=os.path.abspath(self.working_dir) + "/log/*", + local=True, ) functionbeat_proc = self.start_beat() self.wait_until(lambda: self.log_contains("functionbeat is running")) exit_code = functionbeat_proc.kill_and_wait() assert exit_code == 0 + + def test_export_function_template(self): + """ + Test if the template can be exported + """ + + function_name = "testcloudwatchlogs" + bucket_name = "my-bucket-name" + fnb_name = "fnb" + function_name + role = "arn:aws:iam::123456789012:role/MyFunction" + security_group_ids = ["sg-ABCDEFGHIJKL"] + subnet_ids = ["subnet-ABCDEFGHIJKL"] + log_group = "/aws/lambda/functionbeat-cloudwatch" + + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/log/*", + cloudwatch={ + "name": function_name, + "bucket": bucket_name, + "role": role, + "virtual_private_cloud": { + "security_group_ids": security_group_ids, + "subnet_ids": subnet_ids, + }, + "log_group": log_group, + }, + ) + functionbeat_proc = self.start_beat( + logging_args=["-d", "*"], + extra_args=["export", "function", function_name] + ) + + self.wait_until(lambda: self.log_contains("PASS")) + exit_code = functionbeat_proc.kill_and_wait() + assert exit_code == 0 + + function_template = self._get_generated_function_template() + function_properties = function_template["Resources"][fnb_name]["Properties"] + + assert function_properties["FunctionName"] == function_name + assert function_properties["Code"]["S3Bucket"] == bucket_name + assert function_properties["Role"] == role + assert function_properties["VpcConfig"]["SecurityGroupIds"] == security_group_ids + assert function_properties["VpcConfig"]["SubnetIds"] == subnet_ids + + def test_export_function_template_with_invalid_configuration(self): + """ + Test if invalid configuration is exportable + """ + function_name = "INVALID_$_FUNCTION_$_NAME" + bucket_name = "my-bucket-name" + + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/log/*", + cloudwatch={ + "name": function_name, + "bucket": bucket_name, + }, + ) + functionbeat_proc = self.start_beat( + logging_args=["-d", "*"], + extra_args=["export", "function", function_name] + ) + + self.wait_until( + lambda: self.log_contains("error generating raw template for {}: invalid name".format(function_name)) + ) + + exit_code = functionbeat_proc.kill_and_wait() + assert exit_code != 0 + + def _get_generated_function_template(self): + logs = self.get_log_lines() + function_template_lines = logs[:-2] + raw_function_temaplate = "".join(function_template_lines) + function_template = json.loads(raw_function_temaplate) + return function_template From 066b5d33ac0e7c75f326435641828b5ff0cdb1c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 8 May 2019 11:30:21 +0200 Subject: [PATCH 07/13] mention functions in the help of export command --- x-pack/functionbeat/cmd/root.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/functionbeat/cmd/root.go b/x-pack/functionbeat/cmd/root.go index 8e2e57b1a91b..9bdc022239ff 100644 --- a/x-pack/functionbeat/cmd/root.go +++ b/x-pack/functionbeat/cmd/root.go @@ -27,5 +27,11 @@ func init() { RootCmd.AddCommand(genUpdateCmd()) RootCmd.AddCommand(genRemoveCmd()) RootCmd.AddCommand(genPackageCmd()) + + addBeatSpecificSubcommands() +} + +func addBeatSpecificSubcommands() { + RootCmd.ExportCmd.Short = "Export current config, index template or function" RootCmd.ExportCmd.AddCommand(genExportTemplateCmd()) } From 7086fab078121fa0eb9ce2d7cc443e5e78953db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 8 May 2019 13:06:32 +0200 Subject: [PATCH 08/13] rename command generator function --- x-pack/functionbeat/cmd/provider_cmd.go | 2 +- x-pack/functionbeat/cmd/root.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/functionbeat/cmd/provider_cmd.go b/x-pack/functionbeat/cmd/provider_cmd.go index 8f428b790484..f740ca3c1466 100644 --- a/x-pack/functionbeat/cmd/provider_cmd.go +++ b/x-pack/functionbeat/cmd/provider_cmd.go @@ -105,7 +105,7 @@ func genPackageCmd() *cobra.Command { return cmd } -func genExportTemplateCmd() *cobra.Command { +func genExportFunctionCmd() *cobra.Command { return &cobra.Command{ Use: "function", Short: "Export function template", diff --git a/x-pack/functionbeat/cmd/root.go b/x-pack/functionbeat/cmd/root.go index 9bdc022239ff..c10b9315ffaf 100644 --- a/x-pack/functionbeat/cmd/root.go +++ b/x-pack/functionbeat/cmd/root.go @@ -33,5 +33,5 @@ func init() { func addBeatSpecificSubcommands() { RootCmd.ExportCmd.Short = "Export current config, index template or function" - RootCmd.ExportCmd.AddCommand(genExportTemplateCmd()) + RootCmd.ExportCmd.AddCommand(genExportFunctionCmd()) } From fcc00d22f185f27b0754a002e2e420db1f93c530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 23 May 2019 15:22:09 +0200 Subject: [PATCH 09/13] reword changelog --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 16afb648a0d4..e7739ee91808 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -216,7 +216,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Functionbeat* - New options to configure roles and VPC. {pull}11779[11779] -- Export function templates. {pull}11923[11923] +- Export automation templates used to create functions. {pull}11923[11923] *Winlogbeat* From eb1233b5457f987b84f90330b171a9d969bbc702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 27 May 2019 13:47:34 +0200 Subject: [PATCH 10/13] make tests pass with dummy binary --- x-pack/functionbeat/tests/system/test_base.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/functionbeat/tests/system/test_base.py b/x-pack/functionbeat/tests/system/test_base.py index cd86cac5c7d0..6c8895c25600 100644 --- a/x-pack/functionbeat/tests/system/test_base.py +++ b/x-pack/functionbeat/tests/system/test_base.py @@ -35,6 +35,8 @@ def test_export_function_template(self): subnet_ids = ["subnet-ABCDEFGHIJKL"] log_group = "/aws/lambda/functionbeat-cloudwatch" + self._generate_dummy_binary_for_template_checksum() + self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", cloudwatch={ @@ -73,6 +75,8 @@ def test_export_function_template_with_invalid_configuration(self): function_name = "INVALID_$_FUNCTION_$_NAME" bucket_name = "my-bucket-name" + self._generate_dummy_binary_for_template_checksum() + self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/*", cloudwatch={ @@ -92,6 +96,10 @@ def test_export_function_template_with_invalid_configuration(self): exit_code = functionbeat_proc.kill_and_wait() assert exit_code != 0 + def _generate_dummy_binary_for_template_checksum(self): + with open('pkg/functionbeat', 'wb') as f: + f.write("my dummy functionbeat binary") + def _get_generated_function_template(self): logs = self.get_log_lines() function_template_lines = logs[:-2] From 71e00a3e4c96841b28abf0ac0874c0df37635de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 27 May 2019 14:32:45 +0200 Subject: [PATCH 11/13] create folder --- x-pack/functionbeat/tests/system/test_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/functionbeat/tests/system/test_base.py b/x-pack/functionbeat/tests/system/test_base.py index 6c8895c25600..f238ba7c94b1 100644 --- a/x-pack/functionbeat/tests/system/test_base.py +++ b/x-pack/functionbeat/tests/system/test_base.py @@ -97,7 +97,8 @@ def test_export_function_template_with_invalid_configuration(self): assert exit_code != 0 def _generate_dummy_binary_for_template_checksum(self): - with open('pkg/functionbeat', 'wb') as f: + os.mkdir("pkg") + with open("pkg/functionbeat", "wb") as f: f.write("my dummy functionbeat binary") def _get_generated_function_template(self): From c67b842e558cbf9350a01907b55a52ceb5ecf127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 30 May 2019 09:16:07 +0200 Subject: [PATCH 12/13] do not create bin if exists --- x-pack/functionbeat/tests/system/test_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/functionbeat/tests/system/test_base.py b/x-pack/functionbeat/tests/system/test_base.py index f238ba7c94b1..d383e6bacfc4 100644 --- a/x-pack/functionbeat/tests/system/test_base.py +++ b/x-pack/functionbeat/tests/system/test_base.py @@ -97,7 +97,9 @@ def test_export_function_template_with_invalid_configuration(self): assert exit_code != 0 def _generate_dummy_binary_for_template_checksum(self): - os.mkdir("pkg") + if os.path.exists("pkg/functionbeat"): + return + with open("pkg/functionbeat", "wb") as f: f.write("my dummy functionbeat binary") From 9f19e93b2a69d4958bcc8fbc99062ce36d3db784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 30 May 2019 10:40:32 +0200 Subject: [PATCH 13/13] fix fix --- x-pack/functionbeat/tests/system/test_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/functionbeat/tests/system/test_base.py b/x-pack/functionbeat/tests/system/test_base.py index d383e6bacfc4..0717752c545c 100644 --- a/x-pack/functionbeat/tests/system/test_base.py +++ b/x-pack/functionbeat/tests/system/test_base.py @@ -100,6 +100,7 @@ def _generate_dummy_binary_for_template_checksum(self): if os.path.exists("pkg/functionbeat"): return + os.mkdir("pkg") with open("pkg/functionbeat", "wb") as f: f.write("my dummy functionbeat binary")