diff --git a/apps/subscriber/template.yaml b/apps/subscriber/template.yaml index 5663e9c9..e4333aa8 100644 --- a/apps/subscriber/template.yaml +++ b/apps/subscriber/template.yaml @@ -43,15 +43,17 @@ Parameters: LogGroupNamePatterns: Type: CommaDelimitedList Description: >- - Comma separated list of patterns. If not empty, the lambda function will - only apply to log groups that have names that match one of the provided - strings based on a case-sensitive substring search. - Default: '' + Comma separated list of patterns. + We will only subscribe to log groups that have names matching one of the + provided strings based on strings based on a case-sensitive substring + search. To subscribe to all log groups, use the wildcard operator *. + Default: '*' LogGroupNamePrefixes: Type: CommaDelimitedList Description: >- - Comma separated list of prefixes. If not empty, the lambda function will - only apply to log groups that start with a provided string. + Comma separated list of prefixes. The lambda function will only apply to + log groups that start with a provided string. To subscribe to all log + groups, use the wildcard operator *. Default: '' NumWorkers: Type: String diff --git a/docs/subscriber.md b/docs/subscriber.md index 31101b75..8844ea6b 100644 --- a/docs/subscriber.md +++ b/docs/subscriber.md @@ -90,7 +90,22 @@ aws logs describe-log-groups --log-group-name-pattern prod aws logs describe-log-groups --log-group-name-prefix /aws/lambda ``` -If no patterns or prefixes are provided, the lambda function will list all log groups. +To subscribe to all log groups, a wildcard can be provided to either `logGroupNamePatterns` or `logGroupNamePrefixes`. The following input: + +```json +{ + "discover": { + "logGroupNamePatterns": [ "*" ] + } +} +``` + +Will trigger a paginated request equivalent to the `awscli` command: + +```shell +aws logs describe-log-groups +``` + ### Response format diff --git a/handler/subscriber/config.go b/handler/subscriber/config.go index 38e64e1d..45aa4961 100644 --- a/handler/subscriber/config.go +++ b/handler/subscriber/config.go @@ -80,7 +80,7 @@ func (c *Config) Validate() error { } for _, s := range append(c.LogGroupNamePatterns, c.LogGroupNamePrefixes...) { - if !logGroupNameRe.MatchString(s) { + if !logGroupNameRe.MatchString(s) && s != "*" { errs = append(errs, fmt.Errorf("%w: %q", ErrInvalidLogGroupName, s)) } } @@ -89,23 +89,33 @@ func (c *Config) Validate() error { } func (c *Config) LogGroupFilter() FilterFunc { + var re *regexp.Regexp + filterFunc := func(logGroupName string) bool { + if re != nil { + return re.MatchString(logGroupName) + } + return true + } + var exprs []string - exprs = append(exprs, c.LogGroupNamePatterns...) + for _, pattern := range c.LogGroupNamePatterns { + if pattern == "*" { + return filterFunc + } + exprs = append(exprs, pattern) + } for _, prefix := range c.LogGroupNamePrefixes { + if prefix == "*" { + return filterFunc + } exprs = append(exprs, fmt.Sprintf("^%s.*", prefix)) } - var re *regexp.Regexp if len(exprs) != 0 { re = regexp.MustCompile(strings.Join(exprs, "|")) } - return func(logGroupName string) bool { - if re != nil { - return re.MatchString(logGroupName) - } - return true - } + return filterFunc } diff --git a/handler/subscriber/config_test.go b/handler/subscriber/config_test.go index b382466a..e1953bc2 100644 --- a/handler/subscriber/config_test.go +++ b/handler/subscriber/config_test.go @@ -57,6 +57,7 @@ func TestConfig(t *testing.T) { }, ExpectError: subscriber.ErrInvalidLogGroupName, }, + { Config: subscriber.Config{ CloudWatchLogsClient: &handlertest.CloudWatchLogsClient{}, @@ -69,6 +70,7 @@ func TestConfig(t *testing.T) { Config: subscriber.Config{ FilterName: "ok", CloudWatchLogsClient: &handlertest.CloudWatchLogsClient{}, + LogGroupNamePrefixes: []string{"*"}, }, }, } @@ -115,6 +117,17 @@ func TestLogFilter(t *testing.T) { "dev-local": false, }, }, + { + Config: subscriber.Config{ + LogGroupNamePatterns: []string{"prod", "*"}, + DestinationARN: "hello", + }, + Matches: map[string]bool{ + "prod-1": true, + "eu-prod": true, + "staging": true, + }, + }, } for i, tc := range testcases { diff --git a/handler/subscriber/discovery_test.go b/handler/subscriber/discovery_test.go index 3bf4ad19..a5f3e5d0 100644 --- a/handler/subscriber/discovery_test.go +++ b/handler/subscriber/discovery_test.go @@ -34,7 +34,9 @@ func TestHandleDiscovery(t *testing.T) { ExpectJSONResponse string }{ { - DiscoveryRequest: &subscriber.DiscoveryRequest{}, + DiscoveryRequest: &subscriber.DiscoveryRequest{ + LogGroupNamePatterns: []*string{aws.String("*")}, + }, /* matches: - /aws/hello - /aws/ello @@ -53,6 +55,23 @@ func TestHandleDiscovery(t *testing.T) { } }`, }, + { + DiscoveryRequest: &subscriber.DiscoveryRequest{}, + /* matches nothing + */ + ExpectJSONResponse: `{ + "discovery": { + "logGroupCount": 0, + "requestCount": 0, + "subscription": { + "deleted": 0, + "updated": 0, + "skipped": 0, + "processed": 0 + } + } + }`, + }, { DiscoveryRequest: &subscriber.DiscoveryRequest{ LogGroupNamePrefixes: []*string{ diff --git a/handler/subscriber/request.go b/handler/subscriber/request.go index d2b3bc23..08831044 100644 --- a/handler/subscriber/request.go +++ b/handler/subscriber/request.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" ) @@ -90,6 +91,13 @@ func (d *DiscoveryRequest) ToDescribeLogInputs() (inputs []*cloudwatchlogs.Descr } for _, pattern := range d.LogGroupNamePatterns { + if aws.ToString(pattern) == "*" { + return []*cloudwatchlogs.DescribeLogGroupsInput{ + { + Limit: d.Limit, + }, + } + } inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{ LogGroupNamePattern: pattern, Limit: d.Limit, @@ -97,18 +105,18 @@ func (d *DiscoveryRequest) ToDescribeLogInputs() (inputs []*cloudwatchlogs.Descr } for _, prefix := range d.LogGroupNamePrefixes { + if aws.ToString(prefix) == "*" { + return []*cloudwatchlogs.DescribeLogGroupsInput{ + { + Limit: d.Limit, + }, + } + } inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{ LogGroupNamePrefix: prefix, Limit: d.Limit, }) } - if len(inputs) == 0 { - // We should list all since we were provided with no log groups - // or filters. - inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{ - Limit: d.Limit, - }) - } return inputs } diff --git a/integration/scripts/check_subscriber b/integration/scripts/check_subscriber index c9150d20..273828bc 100755 --- a/integration/scripts/check_subscriber +++ b/integration/scripts/check_subscriber @@ -20,6 +20,7 @@ AWS_REGION=$(echo "$FUNCTION_ARN" | cut -d: -f4) check_result() { ERR=$(jq '.StatusCode != 200 or has("FunctionError")' <<<"$1") if [[ "$ERR" == true ]]; then + cat ${TMPFILE} echo "$1" return 1 fi