From 824c8c601cb8c0d6ccc42396535802042465bc8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Taveira=20Ara=C3=BAjo?= Date: Fri, 17 Nov 2023 14:02:20 -0800 Subject: [PATCH] feat(subscriber): introduce wildcard pattern Previously we were treating the absence of patterns or prefixes to signify "subscribe all". Semantically this is quite confusing once you try to wire it into cloudformation, where we would like to have the presence of inputs gate the installation of our app. This commit adds the ability to support wildcard on either `logGroupNamePatterns` or `logGroupNamePrefixes`. The subscriber cloudformation template is adjusted to still subscribe all log groups on install by default. --- apps/subscriber/template.yaml | 14 ++++++++------ docs/subscriber.md | 17 ++++++++++++++++- handler/subscriber/config.go | 28 +++++++++++++++++++--------- handler/subscriber/config_test.go | 13 +++++++++++++ handler/subscriber/discovery_test.go | 21 ++++++++++++++++++++- handler/subscriber/request.go | 22 +++++++++++++++------- integration/scripts/check_subscriber | 1 + 7 files changed, 92 insertions(+), 24 deletions(-) diff --git a/apps/subscriber/template.yaml b/apps/subscriber/template.yaml index 5663e9c9..e4333aa8 100644 --- a/apps/subscriber/template.yaml +++ b/apps/subscriber/template.yaml @@ -43,15 +43,17 @@ Parameters: LogGroupNamePatterns: Type: CommaDelimitedList Description: >- - Comma separated list of patterns. If not empty, the lambda function will - only apply to log groups that have names that match one of the provided - strings based on a case-sensitive substring search. - Default: '' + Comma separated list of patterns. + We will only subscribe to log groups that have names matching one of the + provided strings based on strings based on a case-sensitive substring + search. To subscribe to all log groups, use the wildcard operator *. + Default: '*' LogGroupNamePrefixes: Type: CommaDelimitedList Description: >- - Comma separated list of prefixes. If not empty, the lambda function will - only apply to log groups that start with a provided string. + Comma separated list of prefixes. The lambda function will only apply to + log groups that start with a provided string. To subscribe to all log + groups, use the wildcard operator *. Default: '' NumWorkers: Type: String diff --git a/docs/subscriber.md b/docs/subscriber.md index 31101b75..8844ea6b 100644 --- a/docs/subscriber.md +++ b/docs/subscriber.md @@ -90,7 +90,22 @@ aws logs describe-log-groups --log-group-name-pattern prod aws logs describe-log-groups --log-group-name-prefix /aws/lambda ``` -If no patterns or prefixes are provided, the lambda function will list all log groups. +To subscribe to all log groups, a wildcard can be provided to either `logGroupNamePatterns` or `logGroupNamePrefixes`. The following input: + +```json +{ + "discover": { + "logGroupNamePatterns": [ "*" ] + } +} +``` + +Will trigger a paginated request equivalent to the `awscli` command: + +```shell +aws logs describe-log-groups +``` + ### Response format diff --git a/handler/subscriber/config.go b/handler/subscriber/config.go index 38e64e1d..45aa4961 100644 --- a/handler/subscriber/config.go +++ b/handler/subscriber/config.go @@ -80,7 +80,7 @@ func (c *Config) Validate() error { } for _, s := range append(c.LogGroupNamePatterns, c.LogGroupNamePrefixes...) { - if !logGroupNameRe.MatchString(s) { + if !logGroupNameRe.MatchString(s) && s != "*" { errs = append(errs, fmt.Errorf("%w: %q", ErrInvalidLogGroupName, s)) } } @@ -89,23 +89,33 @@ func (c *Config) Validate() error { } func (c *Config) LogGroupFilter() FilterFunc { + var re *regexp.Regexp + filterFunc := func(logGroupName string) bool { + if re != nil { + return re.MatchString(logGroupName) + } + return true + } + var exprs []string - exprs = append(exprs, c.LogGroupNamePatterns...) + for _, pattern := range c.LogGroupNamePatterns { + if pattern == "*" { + return filterFunc + } + exprs = append(exprs, pattern) + } for _, prefix := range c.LogGroupNamePrefixes { + if prefix == "*" { + return filterFunc + } exprs = append(exprs, fmt.Sprintf("^%s.*", prefix)) } - var re *regexp.Regexp if len(exprs) != 0 { re = regexp.MustCompile(strings.Join(exprs, "|")) } - return func(logGroupName string) bool { - if re != nil { - return re.MatchString(logGroupName) - } - return true - } + return filterFunc } diff --git a/handler/subscriber/config_test.go b/handler/subscriber/config_test.go index b382466a..e1953bc2 100644 --- a/handler/subscriber/config_test.go +++ b/handler/subscriber/config_test.go @@ -57,6 +57,7 @@ func TestConfig(t *testing.T) { }, ExpectError: subscriber.ErrInvalidLogGroupName, }, + { Config: subscriber.Config{ CloudWatchLogsClient: &handlertest.CloudWatchLogsClient{}, @@ -69,6 +70,7 @@ func TestConfig(t *testing.T) { Config: subscriber.Config{ FilterName: "ok", CloudWatchLogsClient: &handlertest.CloudWatchLogsClient{}, + LogGroupNamePrefixes: []string{"*"}, }, }, } @@ -115,6 +117,17 @@ func TestLogFilter(t *testing.T) { "dev-local": false, }, }, + { + Config: subscriber.Config{ + LogGroupNamePatterns: []string{"prod", "*"}, + DestinationARN: "hello", + }, + Matches: map[string]bool{ + "prod-1": true, + "eu-prod": true, + "staging": true, + }, + }, } for i, tc := range testcases { diff --git a/handler/subscriber/discovery_test.go b/handler/subscriber/discovery_test.go index 3bf4ad19..a5f3e5d0 100644 --- a/handler/subscriber/discovery_test.go +++ b/handler/subscriber/discovery_test.go @@ -34,7 +34,9 @@ func TestHandleDiscovery(t *testing.T) { ExpectJSONResponse string }{ { - DiscoveryRequest: &subscriber.DiscoveryRequest{}, + DiscoveryRequest: &subscriber.DiscoveryRequest{ + LogGroupNamePatterns: []*string{aws.String("*")}, + }, /* matches: - /aws/hello - /aws/ello @@ -53,6 +55,23 @@ func TestHandleDiscovery(t *testing.T) { } }`, }, + { + DiscoveryRequest: &subscriber.DiscoveryRequest{}, + /* matches nothing + */ + ExpectJSONResponse: `{ + "discovery": { + "logGroupCount": 0, + "requestCount": 0, + "subscription": { + "deleted": 0, + "updated": 0, + "skipped": 0, + "processed": 0 + } + } + }`, + }, { DiscoveryRequest: &subscriber.DiscoveryRequest{ LogGroupNamePrefixes: []*string{ diff --git a/handler/subscriber/request.go b/handler/subscriber/request.go index d2b3bc23..08831044 100644 --- a/handler/subscriber/request.go +++ b/handler/subscriber/request.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" ) @@ -90,6 +91,13 @@ func (d *DiscoveryRequest) ToDescribeLogInputs() (inputs []*cloudwatchlogs.Descr } for _, pattern := range d.LogGroupNamePatterns { + if aws.ToString(pattern) == "*" { + return []*cloudwatchlogs.DescribeLogGroupsInput{ + { + Limit: d.Limit, + }, + } + } inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{ LogGroupNamePattern: pattern, Limit: d.Limit, @@ -97,18 +105,18 @@ func (d *DiscoveryRequest) ToDescribeLogInputs() (inputs []*cloudwatchlogs.Descr } for _, prefix := range d.LogGroupNamePrefixes { + if aws.ToString(prefix) == "*" { + return []*cloudwatchlogs.DescribeLogGroupsInput{ + { + Limit: d.Limit, + }, + } + } inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{ LogGroupNamePrefix: prefix, Limit: d.Limit, }) } - if len(inputs) == 0 { - // We should list all since we were provided with no log groups - // or filters. - inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{ - Limit: d.Limit, - }) - } return inputs } diff --git a/integration/scripts/check_subscriber b/integration/scripts/check_subscriber index c9150d20..273828bc 100755 --- a/integration/scripts/check_subscriber +++ b/integration/scripts/check_subscriber @@ -20,6 +20,7 @@ AWS_REGION=$(echo "$FUNCTION_ARN" | cut -d: -f4) check_result() { ERR=$(jq '.StatusCode != 200 or has("FunctionError")' <<<"$1") if [[ "$ERR" == true ]]; then + cat ${TMPFILE} echo "$1" return 1 fi