feat(subscriber): introduce wildcard pattern (#95)

Previously we were treating the absence of patterns or prefixes to
signify "subscribe all". Semantically this is quite confusing once you
try to wire it into cloudformation, where we would like to have the
presence of inputs gate the installation of our app.

This commit adds the ability to support wildcard on either
`logGroupNamePatterns` or `logGroupNamePrefixes`. The subscriber
cloudformation template is adjusted to still subscribe all log groups on
install by default.

apps/subscriber/template.yaml

@@ -43,15 +43,17 @@ Parameters:
   LogGroupNamePatterns:
     Type: CommaDelimitedList
     Description: >-
-      Comma separated list of patterns. If not empty, the lambda function will
-      only apply to log groups that have names that match one of the provided
-      strings based on a case-sensitive substring search.
-    Default: ''
+      Comma separated list of patterns. 
+      We will only subscribe to log groups that have names matching one of the
+      provided strings based on strings based on a case-sensitive substring
+      search. To subscribe to all log groups, use the wildcard operator *. 
+    Default: '*'
   LogGroupNamePrefixes:
     Type: CommaDelimitedList
     Description: >-
-      Comma separated list of prefixes. If not empty, the lambda function will
-      only apply to log groups that start with a provided string.
+      Comma separated list of prefixes. The lambda function will only apply to
+      log groups that start with a provided string. To subscribe to all log
+      groups, use the wildcard operator *.
     Default: ''
   NumWorkers:
     Type: String

docs/subscriber.md

@@ -90,7 +90,22 @@ aws logs describe-log-groups --log-group-name-pattern prod
 aws logs describe-log-groups --log-group-name-prefix /aws/lambda
 ```
 
-If no patterns or prefixes are provided, the lambda function will list all log groups.
+To subscribe to all log groups, a wildcard can be provided to either `logGroupNamePatterns` or `logGroupNamePrefixes`. The following input: 
+
+```json
+{
+    "discover": {
+        "logGroupNamePatterns": [ "*" ]
+    }
+}
+```
+
+Will trigger a paginated request equivalent to the `awscli` command:
+
+```shell
+aws logs describe-log-groups
+```
+
 
 ### Response format
 

handler/subscriber/config.go

@@ -80,7 +80,7 @@ func (c *Config) Validate() error {
 	}
 
 	for _, s := range append(c.LogGroupNamePatterns, c.LogGroupNamePrefixes...) {
-		if !logGroupNameRe.MatchString(s) {
+		if !logGroupNameRe.MatchString(s) && s != "*" {
 			errs = append(errs, fmt.Errorf("%w: %q", ErrInvalidLogGroupName, s))
 		}
 	}
@@ -89,23 +89,33 @@ func (c *Config) Validate() error {
 }
 
 func (c *Config) LogGroupFilter() FilterFunc {
+	var re *regexp.Regexp
+	filterFunc := func(logGroupName string) bool {
+		if re != nil {
+			return re.MatchString(logGroupName)
+		}
+		return true
+	}
+
 	var exprs []string
 
-	exprs = append(exprs, c.LogGroupNamePatterns...)
+	for _, pattern := range c.LogGroupNamePatterns {
+		if pattern == "*" {
+			return filterFunc
+		}
+		exprs = append(exprs, pattern)
+	}
 
 	for _, prefix := range c.LogGroupNamePrefixes {
+		if prefix == "*" {
+			return filterFunc
+		}
 		exprs = append(exprs, fmt.Sprintf("^%s.*", prefix))
 	}
 
-	var re *regexp.Regexp
 	if len(exprs) != 0 {
 		re = regexp.MustCompile(strings.Join(exprs, "|"))
 	}
 
-	return func(logGroupName string) bool {
-		if re != nil {
-			return re.MatchString(logGroupName)
-		}
-		return true
-	}
+	return filterFunc
 }

handler/subscriber/config_test.go

@@ -57,6 +57,7 @@ func TestConfig(t *testing.T) {
 			},
 			ExpectError: subscriber.ErrInvalidLogGroupName,
 		},
+
 		{
 			Config: subscriber.Config{
 				CloudWatchLogsClient: &handlertest.CloudWatchLogsClient{},
@@ -69,6 +70,7 @@ func TestConfig(t *testing.T) {
 			Config: subscriber.Config{
 				FilterName:           "ok",
 				CloudWatchLogsClient: &handlertest.CloudWatchLogsClient{},
+				LogGroupNamePrefixes: []string{"*"},
 			},
 		},
 	}
@@ -115,6 +117,17 @@ func TestLogFilter(t *testing.T) {
 				"dev-local":  false,
 			},
 		},
+		{
+			Config: subscriber.Config{
+				LogGroupNamePatterns: []string{"prod", "*"},
+				DestinationARN:       "hello",
+			},
+			Matches: map[string]bool{
+				"prod-1":  true,
+				"eu-prod": true,
+				"staging": true,
+			},
+		},
 	}
 
 	for i, tc := range testcases {

handler/subscriber/discovery_test.go

@@ -34,7 +34,9 @@ func TestHandleDiscovery(t *testing.T) {
 		ExpectJSONResponse string
 	}{
 		{
-			DiscoveryRequest: &subscriber.DiscoveryRequest{},
+			DiscoveryRequest: &subscriber.DiscoveryRequest{
+				LogGroupNamePatterns: []*string{aws.String("*")},
+			},
 			/* matches:
 			- /aws/hello
 			- /aws/ello
@@ -53,6 +55,23 @@ func TestHandleDiscovery(t *testing.T) {
 				}
 			}`,
 		},
+		{
+			DiscoveryRequest: &subscriber.DiscoveryRequest{},
+			/* matches nothing
+			 */
+			ExpectJSONResponse: `{
+				"discovery": {
+					"logGroupCount": 0,
+					"requestCount": 0,
+					"subscription": {
+						"deleted": 0,
+						"updated": 0,
+						"skipped": 0,
+						"processed": 0
+					}
+				}
+			}`,
+		},
 		{
 			DiscoveryRequest: &subscriber.DiscoveryRequest{
 				LogGroupNamePrefixes: []*string{

handler/subscriber/request.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs"
 )
 
@@ -90,25 +91,32 @@ func (d *DiscoveryRequest) ToDescribeLogInputs() (inputs []*cloudwatchlogs.Descr
 	}
 
 	for _, pattern := range d.LogGroupNamePatterns {
+		if aws.ToString(pattern) == "*" {
+			return []*cloudwatchlogs.DescribeLogGroupsInput{
+				{
+					Limit: d.Limit,
+				},
+			}
+		}
 		inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{
 			LogGroupNamePattern: pattern,
 			Limit:               d.Limit,
 		})
 	}
 
 	for _, prefix := range d.LogGroupNamePrefixes {
+		if aws.ToString(prefix) == "*" {
+			return []*cloudwatchlogs.DescribeLogGroupsInput{
+				{
+					Limit: d.Limit,
+				},
+			}
+		}
 		inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{
 			LogGroupNamePrefix: prefix,
 			Limit:              d.Limit,
 		})
 	}
 
-	if len(inputs) == 0 {
-		// We should list all since we were provided with no log groups
-		// or filters.
-		inputs = append(inputs, &cloudwatchlogs.DescribeLogGroupsInput{
-			Limit: d.Limit,
-		})
-	}
 	return inputs
 }

integration/scripts/check_subscriber

@@ -20,6 +20,7 @@ AWS_REGION=$(echo "$FUNCTION_ARN" | cut -d: -f4)
 check_result() {
     ERR=$(jq '.StatusCode != 200 or has("FunctionError")' <<<"$1")
     if [[ "$ERR" == true ]]; then
+        cat ${TMPFILE}
         echo "$1"
         return 1
     fi