feat(subscriber): introduce wildcard pattern

apps/subscriber/template.yaml

@@ -43,15 +43,17 @@ Parameters:
   LogGroupNamePatterns:
     Type: CommaDelimitedList
     Description: >-
-      Comma separated list of patterns. If not empty, the lambda function will
-      only apply to log groups that have names that match one of the provided
-      strings based on a case-sensitive substring search.
-    Default: ''
+      Comma separated list of patterns. 
+      We will only subscribe to log groups that have names matching one of the
+      provided strings based on strings based on a case-sensitive substring
+      search. To subscribe to all log groups, use the wildcard operator *. 
+    Default: '*'
   LogGroupNamePrefixes:
     Type: CommaDelimitedList
     Description: >-
-      Comma separated list of prefixes. If not empty, the lambda function will
-      only apply to log groups that start with a provided string.
+      Comma separated list of prefixes. The lambda function will only apply to
+      log groups that start with a provided string. To subscribe to all log
+      groups, use the wildcard operator *.
     Default: ''
   NumWorkers:
     Type: String

docs/subscriber.md

@@ -90,7 +90,22 @@ aws logs describe-log-groups --log-group-name-pattern prod
 aws logs describe-log-groups --log-group-name-prefix /aws/lambda
 ```
 
-If no patterns or prefixes are provided, the lambda function will list all log groups.
+To subscribe to all log groups, a wildcard can be provided to either `logGroupNamePatterns` or `logGroupNamePrefixes`. The following input: 
+
+```json
+{
+    "discover": {
+        "logGroupNamePatterns": [ "*" ]
+    }
+}
+```
+
+Will trigger a paginated request equivalent to the `awscli` command:
+
+```shell
+aws logs describe-log-groups
+```
+
 
 ### Response format
 

handler/subscriber/config.go

@@ -80,7 +80,7 @@ func (c *Config) Validate() error {
 	}
 
 	for _, s := range append(c.LogGroupNamePatterns, c.LogGroupNamePrefixes...) {
-		if !logGroupNameRe.MatchString(s) {
+		if !logGroupNameRe.MatchString(s) && s != "*" {
 			errs = append(errs, fmt.Errorf("%w: %q", ErrInvalidLogGroupName, s))
 		}
 	}
@@ -89,23 +89,33 @@ func (c *Config) Validate() error {
 }
 
 func (c *Config) LogGroupFilter() FilterFunc {
+	var re *regexp.Regexp
+	filterFunc := func(logGroupName string) bool {
+		if re != nil {
+			return re.MatchString(logGroupName)
+		}
+		return true
+	}
+
 	var exprs []string
 
-	exprs = append(exprs, c.LogGroupNamePatterns...)
+	for _, pattern := range c.LogGroupNamePatterns {
+		if pattern == "*" {
+			return filterFunc
+		}
+		exprs = append(exprs, pattern)
+	}
 
 	for _, prefix := range c.LogGroupNamePrefixes {
+		if prefix == "*" {
+			return filterFunc
+		}
 		exprs = append(exprs, fmt.Sprintf("^%s.*", prefix))
 	}
 
-	var re *regexp.Regexp
 	if len(exprs) != 0 {
 		re = regexp.MustCompile(strings.Join(exprs, "|"))
 	}
 
-	return func(logGroupName string) bool {
-		if re != nil {
-			return re.MatchString(logGroupName)
-		}
-		return true
-	}
+	return filterFunc
 }

handler/subscriber/config_test.go

@@ -57,6 +57,7 @@ func TestConfig(t *testing.T) {
 			},
 			ExpectError: subscriber.ErrInvalidLogGroupName,
 		},
+
 		{
 			Config: subscriber.Config{
 				CloudWatchLogsClient: &handlertest.CloudWatchLogsClient{},
@@ -69,6 +70,7 @@ func TestConfig(t *testing.T) {
 			Config: subscriber.Config{
 				FilterName:           "ok",
 				CloudWatchLogsClient: &handlertest.CloudWatchLogsClient{},
+				LogGroupNamePrefixes: []string{"*"},
 			},
 		},
 	}
@@ -115,6 +117,17 @@ func TestLogFilter(t *testing.T) {
 				"dev-local":  false,
 			},
 		},
+		{
+			Config: subscriber.Config{
+				LogGroupNamePatterns: []string{"prod", "*"},
+				DestinationARN:       "hello",
+			},
+			Matches: map[string]bool{
+				"prod-1":  true,
+				"eu-prod": true,
+				"staging": true,
+			},
+		},
 	}
 
 	for i, tc := range testcases {

handler/subscriber/discovery_test.go

@@ -34,7 +34,9 @@ func TestHandleDiscovery(t *testing.T) {
 		ExpectJSONResponse string
 	}{
 		{
-			DiscoveryRequest: &subscriber.DiscoveryRequest{},
+			DiscoveryRequest: &subscriber.DiscoveryRequest{
+				LogGroupNamePatterns: []*string{aws.String("*")},
+			},
 			/* matches:
 			- /aws/hello
 			- /aws/ello
@@ -53,6 +55,23 @@ func TestHandleDiscovery(t *testing.T) {
 				}
 			}`,
 		},
+		{
+			DiscoveryRequest: &subscriber.DiscoveryRequest{},
+			/* matches nothing
+			 */
+			ExpectJSONResponse: `{
+				"discovery": {
+					"logGroupCount": 0,
+					"requestCount": 0,
+					"subscription": {
+						"deleted": 0,
+						"updated": 0,
+						"skipped": 0,
+						"processed": 0
+					}
+				}
+			}`,
+		},
 		{
 			DiscoveryRequest: &subscriber.DiscoveryRequest{
 				LogGroupNamePrefixes: []*string{

handler/subscriber/request.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs"
 )
 
@@ -90,25 +91,32 @@ func (d *DiscoveryRequest) ToDescribeLogInputs() (inputs []*cloudwatchlogs.Descr
 	}
 
 	for _, pattern := range d.LogGroupNamePatterns {
+		if aws.ToString(pattern) == "*" {
+			return []*cloudwatchlogs.DescribeLogGroupsInput{
+				{
+					Limit: d.Limit,
+				},
+			}
+		}
 		inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{
 			LogGroupNamePattern: pattern,
 			Limit:               d.Limit,
 		})
 	}
 
 	for _, prefix := range d.LogGroupNamePrefixes {
+		if aws.ToString(prefix) == "*" {
+			return []*cloudwatchlogs.DescribeLogGroupsInput{
+				{
+					Limit: d.Limit,
+				},
+			}
+		}
 		inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{
 			LogGroupNamePrefix: prefix,
 			Limit:              d.Limit,
 		})
 	}
 
-	if len(inputs) == 0 {
-		// We should list all since we were provided with no log groups
-		// or filters.
-		inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{
-			Limit: d.Limit,
-		})
-	}
 	return inputs
 }

integration/scripts/check_subscriber

@@ -20,6 +20,7 @@ AWS_REGION=$(echo "$FUNCTION_ARN" | cut -d: -f4)
 check_result() {
     ERR=$(jq '.StatusCode != 200 or has("FunctionError")' <<<"$1")
     if [[ "$ERR" == true ]]; then
+        cat ${TMPFILE}
         echo "$1"
         return 1
     fi