From f507fc84903d89c53c471aacd1b248560b502460 Mon Sep 17 00:00:00 2001 From: obs-gh-colinhutchinson <131207535+obs-gh-colinhutchinson@users.noreply.github.com> Date: Fri, 24 Nov 2023 10:26:57 -0500 Subject: [PATCH] Revert "feat(subscriber): add eventbridge (#103)" (#107) This reverts commit 750a09459fb809d7ace836b90d14d0773ef3967e. --- apps/subscriber/template.yaml | 74 +------------------------ docs/subscriber.md | 15 +---- handler/subscriber/discovery.go | 4 -- integration/scripts/check_subscriber | 12 ---- integration/tests/subscriber.tftest.hcl | 6 +- 5 files changed, 4 insertions(+), 107 deletions(-) diff --git a/apps/subscriber/template.yaml b/apps/subscriber/template.yaml index df880d0f..e4333aa8 100644 --- a/apps/subscriber/template.yaml +++ b/apps/subscriber/template.yaml @@ -1,8 +1,6 @@ --- AWSTemplateFormatVersion: '2010-09-09' -Transform: - - AWS::Serverless-2016-10-31 - - AWS::LanguageExtensions +Transform: AWS::Serverless-2016-10-31 Description: 'Subscribe logs to Observe.' Metadata: AWS::ServerlessRepo::Application: @@ -62,12 +60,6 @@ Parameters: Description: Maximum number of concurrent workers when processing log groups. Default: '' AllowedPattern: '^[0-9]*$' - DiscoveryRate: - Type: String - Description: EventBridge rate expression for periodically triggering - discovery. If not set, no eventbridge rules are configured. - Default: '' - AllowedPattern: '^([1-9]\d* (minute|hour|day)s?)?$' NameOverride: Type: String Description: >- @@ -81,22 +73,6 @@ Conditions: UseStackName: !Equals - !Ref NameOverride - '' - HasDiscoveryRate: !Not - - !Equals - - !Ref DiscoveryRate - - '' - HasLogGroupNamePatterns: !Not - - !Equals - - !Join - - ',' - - !Ref LogGroupNamePatterns - - '' - HasLogGroupNamePrefixes: !Not - - !Equals - - !Join - - ',' - - !Ref LogGroupNamePrefixes - - '' Resources: DeadLetter: @@ -232,54 +208,6 @@ Resources: QUEUE_URL: !Ref Queue VERBOSITY: 9 NUM_WORKERS: !Ref NumWorkers - SubscriptionEvents: - Type: AWS::Events::Rule - Condition: HasDiscoveryRate - DependsOn: QueuePolicy - Properties: - Description: "Subscribe new log groups" - EventPattern: - source: - - "aws.logs" - detail-type: - - "AWS API Call via CloudTrail" - detail: - eventSource: - - "logs.amazonaws.com" - eventName: - - "CreateLogGroup" - Targets: - - Arn: !GetAtt Queue.Arn - Id: SubscriptionEvent - InputTransformer: - InputPathsMap: - logGroupName: "$.detail.requestParameters.logGroupName" - InputTemplate: >- - {"subscribe": {"logGroups": [{ "logGroupName": ""}]}} - DiscoveryEvents: - Type: 'AWS::Events::Rule' - Condition: HasDiscoveryRate - # We must have the appropriate permissions before attempting to write to - # queue. Removing this dependency will cause first trigger to be silently - # dropped. - DependsOn: QueuePolicy - Properties: - Description: Trigger log group discovery - ScheduleExpression: !Sub 'rate(${DiscoveryRate})' - Targets: - - Arn: !GetAtt Queue.Arn - Id: DiscoveryEvent - Input: !ToJsonString - discover: - logGroupNamePatterns: !If - - HasLogGroupNamePatterns - - !Ref LogGroupNamePatterns - - [] - logGroupNamePrefixes: !If - - HasLogGroupNamePrefixes - - !Ref LogGroupNamePrefixes - - [] - Outputs: Function: Description: "Lambda Function ARN" diff --git a/docs/subscriber.md b/docs/subscriber.md index bc66c2fb..8844ea6b 100644 --- a/docs/subscriber.md +++ b/docs/subscriber.md @@ -5,8 +5,6 @@ The subscriber stack subscribes CloudWatch Log Groups to a supported destination - subscription requests contain a list of log groups which we wish to subscribe to our destination. - discovery requests contain a list of filters which are used to generate subscription requests. -Additionally, the stack provides a method for automatically triggering subscription through Eventbridge rules. - ## Configuration The subscriber lambda is responsible for managing subscription filters for a set of log groups. @@ -17,7 +15,7 @@ The subscription filter will be configured according the following environment v | `FILTER_NAME` | **Required**. Subscription filter name. Existing filters that have this name as a prefix will be removed. | | `FILTER_PATTERN` | Subscription filter pattern. Refer to [AWS documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/SubscriptionFilters.html). | | `DESTINATION_ARN` | Destination ARN. If empty, any matching subscription filter named `FILTER_NAME` will be removed. | -| `ROLE_ARN` | Role ARN. Can only be set if `DESTINATION_ARN` is also set. | +| `ROLE_ARN` | Role ARN. Can only be set if `DESTINATION_ARN` is also set | Additionally, the set of log groups the lambda is applicable to is controlled through the following variables: @@ -156,14 +154,3 @@ The response for a successful invocation will embed the corresponding subscripti } } ``` - -## Automatic subscription through Eventbridge rules - -The stack optionally installs eventbridge rules which automatically subscribe log groups the the configured destination. To enable this feature, you must set the `DiscoveryRate` parameter to a valid [AWS EventBridge rate expression](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-rate-expressions.html) (e.g. `1 hour`). - -If this parameter is set, two EventBridge rules are installed: - -- a discovery request that will be fire at the desired rate, -- a subscription request will be fired on log group creation. This rule will only fire if CloudTrail is configured within the account and region our subscriber is running in. - -Both rules will send requests to the SQS queue, which in turn are consumed by the subscriber lambda. diff --git a/handler/subscriber/discovery.go b/handler/subscriber/discovery.go index e81da200..7d33a8e9 100644 --- a/handler/subscriber/discovery.go +++ b/handler/subscriber/discovery.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" - "github.com/go-logr/logr" ) var ErrNoQueue = errors.New("no queue defined") @@ -16,9 +15,6 @@ func (h *Handler) HandleDiscoveryRequest(ctx context.Context, discoveryReq *Disc Discovery: new(DiscoveryStats), } - logger := logr.FromContextOrDiscard(ctx) - logger.V(3).Info("handling discovery request", "request", discoveryReq) - var inline bool if discoveryReq.Inline == nil { inline = h.Queue == nil diff --git a/integration/scripts/check_subscriber b/integration/scripts/check_subscriber index cbcb0ee2..273828bc 100755 --- a/integration/scripts/check_subscriber +++ b/integration/scripts/check_subscriber @@ -17,17 +17,6 @@ TMPFILE=$(mktemp) FUNCTION_NAME=$(echo "$FUNCTION_ARN" | cut -d: -f7) AWS_REGION=$(echo "$FUNCTION_ARN" | cut -d: -f4) -LOG_EVENTS=$(aws logs filter-log-events \ - --region ${AWS_REGION} \ - --log-group-name /aws/lambda/${FUNCTION_NAME} | jq -r '.events[] | .message | fromjson?' ) - -[[ ! -z "${LOG_EVENTS}" ]] || DIE "subscriber lambda not invoked by eventbridge rule" - -[[ -z $(jq -r 'select(.level == "ERROR")' <<< ${LOG_EVENTS}) ]] || DIE "errors detected in lambda logs" - -# this requires verbosity to be set to at least 3 -[[ ! -z $(jq -r 'select(.msg == "handling discovery request")' <<< ${LOG_EVENTS}) ]] || DIE "no discovery request detected" - check_result() { ERR=$(jq '.StatusCode != 200 or has("FunctionError")' <<<"$1") if [[ "$ERR" == true ]]; then @@ -38,7 +27,6 @@ check_result() { } echo '{"subscribe": {"logGroups": [{"logGroupName": "does_not_exist"}]}}' > ${TMPFILE} - RESULT=$(aws lambda invoke \ --function-name ${FUNCTION_NAME} \ --payload fileb://${TMPFILE} ${TMPFILE} \ diff --git a/integration/tests/subscriber.tftest.hcl b/integration/tests/subscriber.tftest.hcl index 29bc3330..c282089b 100644 --- a/integration/tests/subscriber.tftest.hcl +++ b/integration/tests/subscriber.tftest.hcl @@ -9,8 +9,6 @@ run "install" { name = run.setup.id app = "subscriber" parameters = { - LogGroupNamePatterns = "*" - DiscoveryRate = "1 hour" } capabilities = [ "CAPABILITY_IAM", @@ -19,7 +17,7 @@ run "install" { } } -run "check_eventbridge_invoked" { +run "check_invoke" { module { source = "./modules/exec" } @@ -33,6 +31,6 @@ run "check_eventbridge_invoked" { assert { condition = output.error == "" - error_message = "Failed to verify subscriber invocation" + error_message = "Failed to invoke lambda function" } }