fix(forwarder): add verbosity parameter (#260)

Expose paramter to configure logging verbosity and dial down default value. Update documentation of forwarder parameters.
observeinc · May 17, 2024 · 298cf1c · 298cf1c
1 parent 7174e99
commit 298cf1c
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 22 deletions.
diff --git a/apps/forwarder/template.yaml b/apps/forwarder/template.yaml
@@ -26,16 +26,19 @@ Metadata:
         Parameters:
           - SourceBucketNames
           - SourceTopicArns
+          - SourceKMSKeyArns
+          - ContentTypeOverrides
       - Label:
-          default: Miscellaneous
+          default: Sizing
         Parameters:
           - MaxFileSize
-          - ContentTypeOverrides
-          - SourceKMSKeyArns
-          - DebugEndpoint
           - MemorySize
           - Timeout
-
+      - Label:
+          default: Debugging
+        Parameters:
+          - DebugEndpoint
+          - Verbosity
 Parameters:
   DestinationUri:
     Type: String
@@ -71,11 +74,12 @@ Parameters:
       A list of SNS topics the forwarder is allowed to be subscribed to.
     Default: ''
     AllowedPattern: "^(arn:.*)?$"
-  MaxFileSize:
-    Type: String
-    Description: Max file size for objects to process (in bytes), default is 1GB
+  SourceKMSKeyArns:
+    Type: CommaDelimitedList
+    Description: >-
+      A list of KMS Key ARNs the forwarder is allowed to use to decrypt objects in S3.
     Default: ''
-    AllowedPattern: '^[0-9]*$'
+    AllowedPattern: "^(arn:.*)?$"
   ContentTypeOverrides:
     Type: CommaDelimitedList
     Description: >-
@@ -86,18 +90,11 @@ Parameters:
       as newline delimited JSON
       files.
     Default: ''
-  SourceKMSKeyArns:
-    Type: CommaDelimitedList
-    Description: >-
-      A list of KMS Key ARNs the forwarder is allowed to use to decrypt objects in S3.
-    Default: ''
-    AllowedPattern: "^(arn:.*)?$"
-  DebugEndpoint:
+  MaxFileSize:
     Type: String
-    Description: >-
-      Endpoint to send additional debug telemetry to.
+    Description: Max file size for objects to process (in bytes), default is 1GB
     Default: ''
-    AllowedPattern: "^(http(s)?:\/\/.*)?$"
+    AllowedPattern: '^[0-9]*$'
   MemorySize:
     Type: String
     Description: >-
@@ -121,6 +118,18 @@ Parameters:
       it. The maximum allowed value is 900 seconds.
     Default: ''
     AllowedPattern: "^[0-9]*$"
+  DebugEndpoint:
+    Type: String
+    Description: >-
+      Endpoint to send additional debug telemetry to.
+    Default: ''
+    AllowedPattern: "^(http(s)?:\/\/.*)?$"
+  Verbosity:
+    Type: String
+    Default: ''
+    Description: >-
+      Logging verbosity for Lambda. Highest log verbosity is 9.
+    AllowedPattern: "^[0-9]?$"
 Conditions:
   NoDataAccessPointArn: !Equals
     - !Ref DataAccessPointArn
@@ -140,6 +149,9 @@ Conditions:
   UseDefaultTimeout: !Equals
     - !Ref Timeout
     - ''
+  UseDefaultVerbosity: !Equals
+    - !Ref Verbosity
+    - ''
   DisableSourceS3: !Equals
     - !Join
       - ''
@@ -395,7 +407,10 @@ Resources:
       Environment:
         Variables:
           DESTINATION_URI: !Ref DestinationUri
-          VERBOSITY: 9
+          VERBOSITY: !If
+            - UseDefaultVerbosity
+            - 1
+            - !Ref Verbosity
           MAX_FILE_SIZE: !If
             - UseDefaultMaxFileSize
             - "1073741824"

diff --git a/docs/forwarder.md b/docs/forwarder.md
@@ -41,10 +41,13 @@ The forwarder stack can be configured with the following parameters:
 | `NameOverride` | String | Name of IAM role expected by Filedrop. This name will also be applied to the SQS Queue and Lambda Function processing events. In the absence of a value, the stack name will be used. |
 | `SourceBucketNames` | CommaDelimitedList | A list of bucket names which the forwarder is allowed to read from.  This list only affects permissions, and supports wildcards. In order to have files copied to Filedrop, you must also subscribe S3 Bucket Notifications to the forwarder. |
 | `SourceTopicArns` | CommaDelimitedList | A list of SNS topics the forwarder is allowed to be subscribed to. |
-| `MaxFileSize` | String | Max file size for objects to process (in bytes), default is 1GB |
-| `ContentTypeOverrides` | CommaDelimitedList | A list of key value pairs. The key is a regular expression which is applied to the S3 source (<bucket>/<key>) of forwarded files. The value is the content type to set for matching files. For example, `\.json$=application/x-ndjson` would forward all files ending in `.json` as newline delimited JSON files. |
 | `SourceKMSKeyArns` | CommaDelimitedList | A list of KMS Key ARNs the forwarder is allowed to use to decrypt objects in S3. |
+| `ContentTypeOverrides` | CommaDelimitedList | A list of key value pairs. The key is a regular expression which is applied to the S3 source (<bucket>/<key>) of forwarded files. The value is the content type to set for matching files. For example, `\.json$=application/x-ndjson` would forward all files ending in `.json` as newline delimited JSON files. |
+| `MaxFileSize` | String | Max file size for objects to process (in bytes), default is 1GB |
+| `MemorySize` | String | The amount of memory, in megabytes, that your function has access to. |
+| `Timeout` | String | The amount of time that Lambda allows a function to run before stopping it. The maximum allowed value is 900 seconds. |
 | `DebugEndpoint` | String | Endpoint to send additional debug telemetry to. |
+| `Verbosity` | String | Logging verbosity for Lambda. Highest log verbosity is 9. |
 
 ## Installation