feat: OB-36691 add lambda and s3 bucket to record push metrics config

DEVELOPER.md

@@ -37,7 +37,7 @@ and the most important variables, run `make help`, e.g:
 VARIABLES:
   APPS          = config configsubscription firehose forwarder logwriter metricstream stack
   AWS_REGION    = us-west-2
-  GO_BINS       = forwarder subscriber
+  GO_BINS       = forwarder subscriber metricsconfigurator
   GO_BUILD_DIRS = bin/linux_arm64 .go/bin/linux_arm64 .go/cache .go/pkg
   TF_TESTS      = config configsubscription firehose forwarder forwarder_s3 logwriter metricstream simple stack
   VERSION       = v1.19.2-4-gb1238b5-dirty

apps/metricstream/template.yaml

@@ -1,5 +1,6 @@
 ---
 AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
 Description: 'Stream CloudWatch Metrics to S3.'
 Metadata:
   AWS::ServerlessRepo::Application:
@@ -16,7 +17,7 @@ Parameters:
   BucketArn:
     Type: String
     Description: >-
-      S3 Bucket ARN to write log records to.
+      S3 Bucket ARN to write metrics.
     AllowedPattern: "^arn:.*$"
   Prefix:
     Type: String
@@ -27,8 +28,8 @@ Parameters:
     Type: String
     Description: >-
       A file hosted in S3 containing list of metrics to stream.
-    Default: 's3://observeinc/cloudwatchmetrics/filters/recommended.yaml'
-    AllowedPattern: "^s3:\/\/.*"
+    Default: 's3://observeinc/cloudwatchmetrics/filters/empty.yaml'
+    AllowedPattern: "^(s3:\/\/.*)?$"
   OutputFormat:
     Type: String
     Description: >-
@@ -61,11 +62,41 @@ Parameters:
     Description: |
       Buffer incoming data to the specified size, in MiBs, before delivering it
       to the destination.
-
+  ObserveAccountID:
+    Type: String
+    Description: Observe Account ID
+    AllowedPattern: '\d*'
+    Default: ''
+  ObserveDomainName:
+    Type: String
+    Description: >-
+      The domain name we will retrieve metric configuration from.
+    Default: ''
+  UpdateTimestamp:
+    Type: String
+    Description: Timestamp when the metric stream was created or updated.
+    Default: ''
+    AllowedPattern: '^[0-9]*$'
+  DatasourceID:
+    Type: String
+    Description: >-
+      The datasource for this metric stream.
+    Default: ''
+    AllowedPattern: '\d*'
+  GQLToken:
+    Type: String
+    NoEcho: true
+    Description: >-
+      The token used to retrieve metric configuration.
+    Default: ''
 Conditions:
   UseStackName: !Equals
     - !Ref NameOverride
     - ''
+  DeployLambda: !Not
+    - !Equals
+      - !Ref DatasourceID
+      - ""
 
 Resources:
   DeliveryStreamRole:
@@ -106,6 +137,27 @@ Resources:
                 Resource:
                   - !Ref BucketArn
                   - !Sub '${BucketArn}/${Prefix}*'
+  LambdaLogGroup:
+    Type: 'AWS::Logs::LogGroup'
+    Properties:
+      LogGroupName: !Join
+        - ''
+        - - /aws/lambda/
+          - !If
+            - UseStackName
+            - !Ref AWS::StackName
+            - !Ref NameOverride
+          - '-'
+          - !Select [2, !Split ["/", !Ref "AWS::StackId"]]
+          # This grabs the unique id part of the stack id.
+          # This part is unique every time the stack is created.
+          # the reason we have this is to avoid an error
+          # if the stack is destroyed and created again.
+          # If the log group name is the same on recreation,
+          # an error will be thrown.
+      RetentionInDays: 365
+    DeletionPolicy: Retain
+    UpdateReplacePolicy: Retain
   LogGroup:
     Type: 'AWS::Logs::LogGroup'
     Properties:
@@ -186,9 +238,91 @@ Resources:
         Name: "AWS::Include"
         Parameters:
           Location: !Ref FilterUri
-      # end of processable content for include
+      # End of processable content for include.
+      # AWS::Include will be replaced with the file FilterUri references.
+      # This macro always runs, so we must always provide it with a valid
+      # S3 file, even if we overwrite it with a lambda.
+      # filterURI is decided in the parent stack becauseAWS does not
+      # allow an !If function inside an AWS::Include.
       OutputFormat: !Ref OutputFormat
-
+  MetricsConfiguratorRole:
+    Type: AWS::IAM::Role
+    Condition: DeployLambda
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action: sts:AssumeRole
+      Policies:
+        - PolicyName: MetricsConfiguratorPolicy
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - cloudwatch:PutMetricStream
+                Resource: !GetAtt MetricStream.Arn
+              - Effect: Allow
+                Action:
+                  - iam:PassRole
+                Resource: !GetAtt MetricStreamRole.Arn
+              - Effect: Allow
+                Action:
+                  - logs:CreateLogGroup
+                  - logs:CreateLogStream
+                  - logs:PutLogEvents
+                Resource: !GetAtt LambdaLogGroup.Arn
+              - Effect: Allow
+                Action:
+                  - secretsmanager:GetSecretValue
+                Resource: !Ref GQLTokenSecret
+  MetricsConfigurator:
+    Type: AWS::Serverless::Function
+    Condition: DeployLambda
+    Metadata:
+      BuildMethod: makefile
+    Properties:
+      FunctionName: !If
+        - UseStackName
+        - !Ref AWS::StackName
+        - !Ref NameOverride
+      Role: !GetAtt MetricsConfiguratorRole.Arn
+      CodeUri: ../../bin/linux_arm64
+      Handler: bootstrap
+      Runtime: provided.al2
+      Architectures:
+        - arm64
+      LoggingConfig:
+        LogGroup: !Ref LambdaLogGroup
+      Environment:
+        Variables:
+          VERBOSITY: 6
+          ACCOUNT_ID: !Ref ObserveAccountID
+          OBSERVE_DOMAIN_NAME: !Ref ObserveDomainName
+          DATASOURCE_ID: !Ref DatasourceID
+          SECRET_NAME: !Ref GQLTokenSecret
+          # fields are necessary to update the metric stream
+          METRIC_STREAM_NAME: !Ref MetricStream
+          FIREHOSE_ARN: !GetAtt DeliveryStream.Arn
+          ROLE_ARN: !GetAtt MetricStreamRole.Arn
+          OUTPUT_FORMAT: !Ref OutputFormat
+  GQLTokenSecret:
+    Type: AWS::SecretsManager::Secret
+    Condition: DeployLambda
+    Properties:
+      Description: GQL Token Secret
+      Name: !Sub "observe-gql-token-${AWS::StackName}"
+      SecretString: !Ref GQLToken
+  StackCreationUpdateCustomResource:
+    Type: Custom::StackCreationUpdateTrigger
+    Condition: DeployLambda
+    Properties:
+      ServiceToken: !GetAtt MetricsConfigurator.Arn
+      StackName: !Ref AWS::StackName
+      UpdateTimestamp: !Ref UpdateTimestamp
 Outputs:
   FirehoseArn:
     Description: >-

apps/stack/template.yaml

@@ -107,13 +107,53 @@ Parameters:
     Type: String
     Description: >-
       S3 URI containing filters for metrics to be collected by CloudWatch
-      Metrics Stream. If empty, no metrics will be collected.
-    Default: 's3://observeinc/cloudwatchmetrics/filters/recommended.yaml'
+      Metrics Stream. If neither this nor DatasourceID is provided, no metrics
+      will be collected.
+    Default: 's3://observeinc/cloudwatchmetrics/filters/empty.yaml'
+    # the empty.yaml file functions as a sentinel value: it lists a namespace
+    # that does not exist, and this causes no metrics to be collected.
     AllowedPattern: "^(s3:\/\/.*)?$"
+  ObserveAccountID:
+    # this and the field below are necessary for us to be able to make a
+    # request to Observe's GraphQL API - we use them to form the URL we
+    # make the request to.
+    Type: String
+    Description: Observe Account ID
+    AllowedPattern: '\d*'
+    Default: ''
+  ObserveDomainName:
+    Type: String
+    Description: >-
+      The domain name we will retrieve metric configuration from.
+    Default: ''
+  UpdateTimestamp:
+    # Used to trigger a stack update when the lambda is used to update the
+    # metric stream. The change in timestamp is used to trigger an update
+    # of the custom resource. Without a change, the custom resource will
+    # not be updated, and the lambda will not run.
+    Type: String
+    Description: Unix timestamp when metric stream was created or updated.
+    Default: ''
+    AllowedPattern: "^[0-9]*$"
+  DatasourceID:
+    Type: String
+    Description: >-
+      The datasource for this metric stream.
+    AllowedPattern: '\d*'
+    Default: ''
+  GQLToken:
+    # The token for making a request to Observe's
+    # GraphQL API. It is stored in Secrets Manager.
+    # noEcho prevents it from being displayed in the console.
+    Type: String
+    NoEcho: true
+    Description: >-
+      The token used to retrieve metric configuration.
+    Default: ''
   SourceBucketNames:
     Type: CommaDelimitedList
     Description: >-
-      A list of bucket names which the forwarder is allowed to read from.
+      A list of bucket names which the forwarder will read from.
     Default: ""
     AllowedPattern: "^[a-z0-9-.]*(\\*)?$"
   ContentTypeOverrides:
@@ -161,9 +201,18 @@ Conditions:
   UseStackName: !Equals
     - !Ref NameOverride
     - ""
-  EnableMetricStream: !Not
+  EnableMetricStream: !Or
+    - !Not
+      - !Equals
+        - !Ref MetricStreamFilterUri
+        - ""
+    - !Not
+      - !Equals
+        - !Ref DatasourceID
+        - ""
+  DeployLambda: !Not
     - !Equals
-      - !Ref MetricStreamFilterUri
+      - !Ref DatasourceID
       - ""
 
 Resources:
@@ -310,7 +359,21 @@ Resources:
         - !Ref Topic
       Parameters:
         BucketArn: !GetAtt Bucket.Arn
-        FilterUri: !Ref MetricStreamFilterUri
+        FilterUri: !If
+          - DeployLambda
+          - 's3://observeinc/cloudwatchmetrics/filters/default.yaml'
+          - !Ref MetricStreamFilterUri
+          # We provide a yaml file URI here where the contents are a sentinel
+          # value that the user would never enter. Then the lambda modifies
+          # the metricstream to reflect the actual config provided. We need
+          # this sentinel value here initially so that if we switch back to the
+          # yaml based approach, the cloudformation stack will recognize that
+          # the value from the yaml file has changed and will update.
+        ObserveAccountID: !Ref ObserveAccountID
+        ObserveDomainName: !Ref ObserveDomainName
+        UpdateTimestamp: !Ref UpdateTimestamp
+        GQLToken: !Ref GQLToken
+        DatasourceID: !Ref DatasourceID
         NameOverride: !If
           - UseStackName
           - !Sub "${AWS::StackName}-MetricStream"

cmd/metricsconfigurator/main.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	awslambda "github.com/aws/aws-lambda-go/lambda"
+
+	handler "github.com/observeinc/aws-sam-apps/pkg/handler/metricsconfigurator"
+	"github.com/observeinc/aws-sam-apps/pkg/lambda"
+	"github.com/observeinc/aws-sam-apps/pkg/lambda/metricsconfigurator"
+)
+
+var (
+	rec *metricsconfigurator.Lambda
+)
+
+func init() {
+	ctx := context.Background()
+
+	var config handler.Config
+	err := lambda.ProcessEnv(ctx, &config)
+	if err != nil {
+		panic(fmt.Errorf("failed to initialize config: %w", err))
+	}
+
+	rec, err = metricsconfigurator.New(ctx, &config)
+	if err != nil {
+		panic(fmt.Errorf("failed to configure entrypoint: %w", err))
+	}
+}
+
+func main() {
+	awslambda.StartWithOptions(rec.Entrypoint, awslambda.WithEnableSIGTERM(rec.Shutdown))
+}

go.mod

@@ -6,13 +6,15 @@ toolchain go1.22.2
 
 require (
 	github.com/aws/aws-lambda-go v1.47.0
-	github.com/aws/aws-sdk-go-v2 v1.30.3
+	github.com/aws/aws-sdk-go-v2 v1.31.0
 	github.com/aws/aws-sdk-go-v2/config v1.27.27
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10
+	github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.41.2
 	github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.37.3
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.4
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3
-	github.com/aws/smithy-go v1.20.3
+	github.com/aws/smithy-go v1.21.0
 	github.com/go-logr/logr v1.4.2
 	github.com/google/go-cmp v0.6.0
 	github.com/hashicorp/go-retryablehttp v0.7.7
@@ -38,8 +40,8 @@ require (
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.34.1 // indirect