fix(logwriter): expose verbosity parameter (#261)

Same deal as with forwarder. Update the forwarder docs while we're at
it.

apps/logwriter/template.yaml

@@ -15,11 +15,35 @@ Metadata:
     SemanticVersion: '0.0.1'
     SourceCodeUrl: https://github.com/observeinc/aws-sam-apps
 
-Globals:
-  Function:
-    Timeout: 120
-    MemorySize: 128
-
+  AWS::CloudFormation::Interface:
+    ParameterGroups:
+      - Label:
+          default: Destination settings
+        Parameters:
+          - BucketARN
+          - Prefix
+      - Label:
+          default: Subscription settings
+        Parameters:
+          - LogGroupNamePatterns
+          - LogGroupNamePrefixes
+          - DiscoveryRate
+          - FilterName
+          - FilterPattern
+          - NameOverride
+      - Label:
+          default: Sizing
+        Parameters:
+          - BufferingInterval
+          - BufferingSize
+          - NumWorkers
+          - MemorySize
+          - Timeout
+      - Label:
+          default: Debugging
+        Parameters:
+          - DebugEndpoint
+          - Verbosity
 Parameters:
   BucketARN:
     Type: String
@@ -46,6 +70,12 @@ Parameters:
       log groups that start with a provided string. To subscribe to all log
       groups, use the wildcard operator *.
     Default: ''
+  DiscoveryRate:
+    Type: String
+    Description: EventBridge rate expression for periodically triggering
+      discovery. If not set, no eventbridge rules are configured.
+    Default: ''
+    AllowedPattern: '^([1-9]\d* (minute|hour|day)s?)?$'
   FilterName:
     Type: String
     Description: >-
@@ -56,6 +86,12 @@ Parameters:
     Description: >-
       Subscription filter pattern.
     Default: ''
+  NameOverride:
+    Type: String
+    Description: >-
+      Name of Lambda function.
+    Default: ''
+    MaxLength: 64
   BufferingInterval:
     Type: Number
     Default: 60
@@ -77,27 +113,43 @@ Parameters:
     Description: Maximum number of concurrent workers when processing log groups.
     Default: ''
     AllowedPattern: '^[0-9]*$'
-  DiscoveryRate:
+  MemorySize:
     Type: String
-    Description: EventBridge rate expression for periodically triggering
-      discovery. If not set, no eventbridge rules are configured.
+    Description: >-
+      The amount of memory, in megabytes, that your function has access to.
     Default: ''
-    AllowedPattern: '^([1-9]\d* (minute|hour|day)s?)?$'
-  NameOverride:
+    AllowedValues:
+      - ''
+      - '64'
+      - '128'
+      - '256'
+      - '512'
+      - '1024'
+      - '2048'
+      - '4096'
+      - '8128'
+    AllowedPattern: "^[0-9]*$"
+  Timeout:
     Type: String
     Description: >-
-      Name of Lambda function.
+      The amount of time that Lambda allows a function to run before stopping
+      it. The maximum allowed value is 900 seconds.
     Default: ''
-    MaxLength: 64
+    AllowedPattern: "^[0-9]*$"
   DebugEndpoint:
     Type: String
     Description: >-
       Endpoint to send additional debug telemetry to.
     Default: ''
     AllowedPattern: "^(http(s)?:\/\/.*)?$"
-
+  Verbosity:
+    Type: String
+    Default: ''
+    Description: >-
+      Logging verbosity for Lambda. Highest log verbosity is 9.
+    AllowedPattern: "^[0-9]?$"
 Conditions:
-  SetDefaultFilterName: !Equals
+  UseDefaultFilterName: !Equals
     - !Ref FilterName
     - ''
   UseStackName: !Equals
@@ -127,6 +179,15 @@ Conditions:
   DisableOTEL: !Equals
     - !Ref DebugEndpoint
     - ''
+  UseDefaultMemorySize: !Equals
+    - !Ref MemorySize
+    - ''
+  UseDefaultTimeout: !Equals
+    - !Ref Timeout
+    - ''
+  UseDefaultVerbosity: !Equals
+    - !Ref Verbosity
+    - ''
 Resources:
   FirehoseRole:
     Type: 'AWS::IAM::Role'
@@ -251,7 +312,10 @@ Resources:
       RedrivePolicy:
         deadLetterTargetArn: !GetAtt DeadLetter.Arn
         maxReceiveCount: 4
-      VisibilityTimeout: 120
+      VisibilityTimeout: !If
+        - UseDefaultTimeout
+        - "120"
+        - !Ref Timeout
   QueuePolicy:
     Type: AWS::SQS::QueuePolicy
     Condition: EnableSubscription
@@ -350,6 +414,14 @@ Resources:
       CodeUri: ../..
       Handler: bootstrap
       Runtime: provided.al2
+      MemorySize: !If
+        - UseDefaultMemorySize
+        - "128"
+        - !Ref MemorySize
+      Timeout: !If
+        - UseDefaultTimeout
+        - "120"
+        - !Ref Timeout
       Architectures:
         - arm64
       Events:
@@ -366,7 +438,7 @@ Resources:
       Environment:
         Variables:
           FILTER_NAME: !If
-           - SetDefaultFilterName
+           - UseDefaultFilterName
            - 'observe-logs-subscription'
            - !Ref FilterName
           FILTER_PATTERN: !Ref FilterPattern
@@ -379,7 +451,10 @@ Resources:
             - !Ref LogGroupNamePatterns
           ROLE_ARN: !GetAtt DestinationRole.Arn
           QUEUE_URL: !Ref Queue
-          VERBOSITY: 9
+          VERBOSITY: !If
+            - UseDefaultVerbosity
+            - "1"
+            - !Ref Verbosity
           NUM_WORKERS: !Ref NumWorkers
           OTEL_EXPORTER_OTLP_ENDPOINT: !Ref DebugEndpoint
           OTEL_TRACES_EXPORTER: !If [ DisableOTEL, "none", "otlp" ]

docs/forwarder.md

@@ -153,3 +153,9 @@ In order to grant the Forwarder lambda function permission to use the KMS key fo
 
 1. **Update your Forwarder stack**: include your KMS Key ARN in `SourceKMSKeyArns` in your forwarder stack.
 2. **Update your KMS key policy**: your key policy must grant the Forwarder Lambda function permission to call `kms:Decrypt`. The [default KMS key policy](https://docs.aws.amazon.com/kms/latest/developerguide/key-policy-default.html) is sufficient to satisfy this constraint, since it will delegate access to the KMS key to IAM.
+
+## HTTP destination
+
+For backward compatability, the forwarder supports sending data to an HTTPS endpoint. Every `s3:CopyObject` triggers an `s3:GetObject` from the source. The source file is converted into newline delimited JSON and submitted over one or more HTTP POST requests. By default, a request body will not exceed 10MB when uncompressed.
+
+Submitting to an HTTP endpoint has multiple limitations when compared to using Filedrop. The forwarder must read, process and transmit the source file, which consumes both memory and time. The lambda function must therefore be sized according to the maximum file size it is expected to handle. Overall, HTTP mode supports smaller file sizes and less content types, and is provided only as a bridge towards Filedrop adoption.

docs/logwriter.md

@@ -12,14 +12,17 @@ Additionally, the stack is capable of subscribing log groups and provides a meth
 | `Prefix` | String | Optional prefix to write log records to. |
 | `LogGroupNamePatterns` | CommaDelimitedList | Comma separated list of patterns. We will only subscribe to log groups that have names matching one of the provided strings based on strings based on a case-sensitive substring search. To subscribe to all log groups, use the wildcard operator *. |
 | `LogGroupNamePrefixes` | CommaDelimitedList | Comma separated list of prefixes. The lambda function will only apply to log groups that start with a provided string. To subscribe to all log groups, use the wildcard operator *. |
+| `DiscoveryRate` | String | EventBridge rate expression for periodically triggering discovery. If not set, no eventbridge rules are configured. |
 | `FilterName` | String | Subscription filter name. Existing filters that have this name as a prefix will be removed. |
 | `FilterPattern` | String | Subscription filter pattern. |
+| `NameOverride` | String | Name of Lambda function. |
 | `BufferingInterval` | Number | Buffer incoming data for the specified period of time, in seconds, before delivering it to S3.  |
 | `BufferingSize` | Number | Buffer incoming data to the specified size, in MiBs, before delivering it to S3.  |
 | `NumWorkers` | String | Maximum number of concurrent workers when processing log groups. |
-| `DiscoveryRate` | String | EventBridge rate expression for periodically triggering discovery. If not set, no eventbridge rules are configured. |
-| `NameOverride` | String | Name of Lambda function. |
+| `MemorySize` | String | The amount of memory, in megabytes, that your function has access to. |
+| `Timeout` | String | The amount of time that Lambda allows a function to run before stopping it. The maximum allowed value is 900 seconds. |
 | `DebugEndpoint` | String | Endpoint to send additional debug telemetry to. |
+| `Verbosity` | String | Logging verbosity for Lambda. Highest log verbosity is 9. |
 
 **Note**: If neither `LogGroupNamePatterns` nor `LogGroupNamePrefixes` are provided, the Lambda function will not operate on any log groups. It requires explicit patterns or prefixes to define its scope of operation.
 

handler/forwarder/override/set.go

@@ -30,7 +30,7 @@ func (s *Set) Apply(ctx context.Context, input *s3.CopyObjectInput) (modified bo
 			if id == "" {
 				id = fmt.Sprintf("%d", i)
 			}
-			s.Logger.Info("applied rule", "id", id)
+			s.Logger.V(3).Info("applied rule", "id", id)
 			if !rule.Continue {
 				return
 			}

integration/tests/logwriter.tftest.hcl

@@ -103,6 +103,7 @@ run "install" {
       LogGroupNamePatterns = "*"
       DiscoveryRate        = "24 hours"
       NameOverride         = run.setup.id
+      Verbosity            = 3
     }
     capabilities = [
       "CAPABILITY_IAM",