diff --git a/apps/logwriter/template.yaml b/apps/logwriter/template.yaml index ac81a7ab..3731909a 100644 --- a/apps/logwriter/template.yaml +++ b/apps/logwriter/template.yaml @@ -15,11 +15,35 @@ Metadata: SemanticVersion: '0.0.1' SourceCodeUrl: https://github.com/observeinc/aws-sam-apps -Globals: - Function: - Timeout: 120 - MemorySize: 128 - + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: Destination settings + Parameters: + - BucketARN + - Prefix + - Label: + default: Subscription settings + Parameters: + - LogGroupNamePatterns + - LogGroupNamePrefixes + - DiscoveryRate + - FilterName + - FilterPattern + - NameOverride + - Label: + default: Sizing + Parameters: + - BufferingInterval + - BufferingSize + - NumWorkers + - MemorySize + - Timeout + - Label: + default: Debugging + Parameters: + - DebugEndpoint + - Verbosity Parameters: BucketARN: Type: String @@ -46,6 +70,12 @@ Parameters: log groups that start with a provided string. To subscribe to all log groups, use the wildcard operator *. Default: '' + DiscoveryRate: + Type: String + Description: EventBridge rate expression for periodically triggering + discovery. If not set, no eventbridge rules are configured. + Default: '' + AllowedPattern: '^([1-9]\d* (minute|hour|day)s?)?$' FilterName: Type: String Description: >- @@ -56,6 +86,12 @@ Parameters: Description: >- Subscription filter pattern. Default: '' + NameOverride: + Type: String + Description: >- + Name of Lambda function. + Default: '' + MaxLength: 64 BufferingInterval: Type: Number Default: 60 @@ -77,27 +113,43 @@ Parameters: Description: Maximum number of concurrent workers when processing log groups. Default: '' AllowedPattern: '^[0-9]*$' - DiscoveryRate: + MemorySize: Type: String - Description: EventBridge rate expression for periodically triggering - discovery. If not set, no eventbridge rules are configured. + Description: >- + The amount of memory, in megabytes, that your function has access to. Default: '' - AllowedPattern: '^([1-9]\d* (minute|hour|day)s?)?$' - NameOverride: + AllowedValues: + - '' + - '64' + - '128' + - '256' + - '512' + - '1024' + - '2048' + - '4096' + - '8128' + AllowedPattern: "^[0-9]*$" + Timeout: Type: String Description: >- - Name of Lambda function. + The amount of time that Lambda allows a function to run before stopping + it. The maximum allowed value is 900 seconds. Default: '' - MaxLength: 64 + AllowedPattern: "^[0-9]*$" DebugEndpoint: Type: String Description: >- Endpoint to send additional debug telemetry to. Default: '' AllowedPattern: "^(http(s)?:\/\/.*)?$" - + Verbosity: + Type: String + Default: '' + Description: >- + Logging verbosity for Lambda. Highest log verbosity is 9. + AllowedPattern: "^[0-9]?$" Conditions: - SetDefaultFilterName: !Equals + UseDefaultFilterName: !Equals - !Ref FilterName - '' UseStackName: !Equals @@ -127,6 +179,15 @@ Conditions: DisableOTEL: !Equals - !Ref DebugEndpoint - '' + UseDefaultMemorySize: !Equals + - !Ref MemorySize + - '' + UseDefaultTimeout: !Equals + - !Ref Timeout + - '' + UseDefaultVerbosity: !Equals + - !Ref Verbosity + - '' Resources: FirehoseRole: Type: 'AWS::IAM::Role' @@ -251,7 +312,10 @@ Resources: RedrivePolicy: deadLetterTargetArn: !GetAtt DeadLetter.Arn maxReceiveCount: 4 - VisibilityTimeout: 120 + VisibilityTimeout: !If + - UseDefaultTimeout + - "120" + - !Ref Timeout QueuePolicy: Type: AWS::SQS::QueuePolicy Condition: EnableSubscription @@ -350,6 +414,14 @@ Resources: CodeUri: ../.. Handler: bootstrap Runtime: provided.al2 + MemorySize: !If + - UseDefaultMemorySize + - "128" + - !Ref MemorySize + Timeout: !If + - UseDefaultTimeout + - "120" + - !Ref Timeout Architectures: - arm64 Events: @@ -366,7 +438,7 @@ Resources: Environment: Variables: FILTER_NAME: !If - - SetDefaultFilterName + - UseDefaultFilterName - 'observe-logs-subscription' - !Ref FilterName FILTER_PATTERN: !Ref FilterPattern @@ -379,7 +451,10 @@ Resources: - !Ref LogGroupNamePatterns ROLE_ARN: !GetAtt DestinationRole.Arn QUEUE_URL: !Ref Queue - VERBOSITY: 9 + VERBOSITY: !If + - UseDefaultVerbosity + - "1" + - !Ref Verbosity NUM_WORKERS: !Ref NumWorkers OTEL_EXPORTER_OTLP_ENDPOINT: !Ref DebugEndpoint OTEL_TRACES_EXPORTER: !If [ DisableOTEL, "none", "otlp" ] diff --git a/docs/forwarder.md b/docs/forwarder.md index c3868ff5..9dd641ce 100644 --- a/docs/forwarder.md +++ b/docs/forwarder.md @@ -153,3 +153,9 @@ In order to grant the Forwarder lambda function permission to use the KMS key fo 1. **Update your Forwarder stack**: include your KMS Key ARN in `SourceKMSKeyArns` in your forwarder stack. 2. **Update your KMS key policy**: your key policy must grant the Forwarder Lambda function permission to call `kms:Decrypt`. The [default KMS key policy](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html) is sufficient to satisfy this constraint, since it will delegate access to the KMS key to IAM. + +## HTTP destination + +For backward compatability, the forwarder supports sending data to an HTTPS endpoint. Every `s3:CopyObject` triggers an `s3:GetObject` from the source. The source file is converted into newline delimited JSON and submitted over one or more HTTP POST requests. By default, a request body will not exceed 10MB when uncompressed. + +Submitting to an HTTP endpoint has multiple limitations when compared to using Filedrop. The forwarder must read, process and transmit the source file, which consumes both memory and time. The lambda function must therefore be sized according to the maximum file size it is expected to handle. Overall, HTTP mode supports smaller file sizes and less content types, and is provided only as a bridge towards Filedrop adoption. diff --git a/docs/logwriter.md b/docs/logwriter.md index 902fdee0..4c0ea21e 100644 --- a/docs/logwriter.md +++ b/docs/logwriter.md @@ -12,14 +12,17 @@ Additionally, the stack is capable of subscribing log groups and provides a meth | `Prefix` | String | Optional prefix to write log records to. | | `LogGroupNamePatterns` | CommaDelimitedList | Comma separated list of patterns. We will only subscribe to log groups that have names matching one of the provided strings based on strings based on a case-sensitive substring search. To subscribe to all log groups, use the wildcard operator *. | | `LogGroupNamePrefixes` | CommaDelimitedList | Comma separated list of prefixes. The lambda function will only apply to log groups that start with a provided string. To subscribe to all log groups, use the wildcard operator *. | +| `DiscoveryRate` | String | EventBridge rate expression for periodically triggering discovery. If not set, no eventbridge rules are configured. | | `FilterName` | String | Subscription filter name. Existing filters that have this name as a prefix will be removed. | | `FilterPattern` | String | Subscription filter pattern. | +| `NameOverride` | String | Name of Lambda function. | | `BufferingInterval` | Number | Buffer incoming data for the specified period of time, in seconds, before delivering it to S3. | | `BufferingSize` | Number | Buffer incoming data to the specified size, in MiBs, before delivering it to S3. | | `NumWorkers` | String | Maximum number of concurrent workers when processing log groups. | -| `DiscoveryRate` | String | EventBridge rate expression for periodically triggering discovery. If not set, no eventbridge rules are configured. | -| `NameOverride` | String | Name of Lambda function. | +| `MemorySize` | String | The amount of memory, in megabytes, that your function has access to. | +| `Timeout` | String | The amount of time that Lambda allows a function to run before stopping it. The maximum allowed value is 900 seconds. | | `DebugEndpoint` | String | Endpoint to send additional debug telemetry to. | +| `Verbosity` | String | Logging verbosity for Lambda. Highest log verbosity is 9. | **Note**: If neither `LogGroupNamePatterns` nor `LogGroupNamePrefixes` are provided, the Lambda function will not operate on any log groups. It requires explicit patterns or prefixes to define its scope of operation. diff --git a/handler/forwarder/override/set.go b/handler/forwarder/override/set.go index 3ec97320..62b8fffa 100644 --- a/handler/forwarder/override/set.go +++ b/handler/forwarder/override/set.go @@ -30,7 +30,7 @@ func (s *Set) Apply(ctx context.Context, input *s3.CopyObjectInput) (modified bo if id == "" { id = fmt.Sprintf("%d", i) } - s.Logger.Info("applied rule", "id", id) + s.Logger.V(3).Info("applied rule", "id", id) if !rule.Continue { return } diff --git a/integration/tests/logwriter.tftest.hcl b/integration/tests/logwriter.tftest.hcl index 08f9efb4..1881233f 100644 --- a/integration/tests/logwriter.tftest.hcl +++ b/integration/tests/logwriter.tftest.hcl @@ -103,6 +103,7 @@ run "install" { LogGroupNamePatterns = "*" DiscoveryRate = "24 hours" NameOverride = run.setup.id + Verbosity = 3 } capabilities = [ "CAPABILITY_IAM",