feat: add pricing monitor dashboard

README.md

@@ -59,3 +59,17 @@ You will be prompted for:
 - The HTTPS URL of your website GitHub repository, e.g., `https://github.com/awsbites/aws-bites-site.git`
 
 Once deployment has completed, you can check the Step Function that orchestrates the whole process in the AWS Console. This state machine is automatically executed when transcripts are placed in the `processed-transcripts/` prefix.
+
+## Price Monitoring
+
+Bedrock pricing can be difficult to estimate. This repo comes with a pricing CloudWatch dashboard that helps to show the cost for a given period and the relationship between invocations, input tokens and output tokens. This is calculated based on published on-demand pricing for the ClaudeV2 model as of 28 October 2023. A CloudWatch alarm is also created for the total cost per hour, defaulting to breach when the cost exceeds $1 per hour for three consecutive hours.
+
+![Pricing Dashboard](./price-monitor/price-dashboard.png)
+
+The pricing dashboard can be deployed with CDK:
+
+```bash
+cd price-monitor
+npm install
+npx cdk deploy -c bedrockRegion=us-east-1
+```

price-monitor/.gitignore

@@ -0,0 +1,8 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

price-monitor/.npmignore

@@ -0,0 +1,6 @@
+*.ts
+!*.d.ts
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

price-monitor/README.md

@@ -0,0 +1,14 @@
+# Welcome to your CDK TypeScript project
+
+This is a blank project for CDK development with TypeScript.
+
+The `cdk.json` file tells the CDK Toolkit how to execute your app.
+
+## Useful commands
+
+* `npm run build`   compile typescript to js
+* `npm run watch`   watch for changes and compile
+* `npm run test`    perform the jest unit tests
+* `cdk deploy`      deploy this stack to your default AWS account/region
+* `cdk diff`        compare deployed stack with current state
+* `cdk synth`       emits the synthesized CloudFormation template

price-monitor/bin/price-monitor.ts

@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+import 'source-map-support/register'
+import * as cdk from 'aws-cdk-lib'
+import { PriceMonitorStack } from '../lib/price-monitor-stack'
+
+const app = new cdk.App()
+const bedrockRegion = app.node.tryGetContext('bedrockRegion') ?? 'us-east-1'
+
+// eslint-disable-next-line no-new
+new PriceMonitorStack(app, 'PriceMonitorStack', {
+  env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: bedrockRegion },
+  modelId: 'anthropic.claude-v2',
+  bedrockRegion: 'us-east-1'
+})

price-monitor/cdk.json

@@ -0,0 +1,63 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/price-monitor.ts",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ],
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
+    "@aws-cdk/aws-iam:standardizedServicePrincipals": true,
+    "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
+    "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
+    "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
+    "@aws-cdk/aws-route53-patters:useCertificate": true,
+    "@aws-cdk/customresources:installLatestAwsSdkDefault": false,
+    "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true,
+    "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true,
+    "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true,
+    "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true,
+    "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true,
+    "@aws-cdk/aws-redshift:columnId": true,
+    "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true,
+    "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true,
+    "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true,
+    "@aws-cdk/aws-kms:aliasNameRef": true,
+    "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true,
+    "@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
+    "@aws-cdk/aws-efs:denyAnonymousAccess": true,
+    "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true,
+    "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true,
+    "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true,
+    "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true,
+    "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true,
+    "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true
+  }
+}

price-monitor/jest.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  testEnvironment: 'node',
+  roots: ['<rootDir>/test'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest'
+  }
+};

price-monitor/lib/price-monitor-stack.ts

@@ -0,0 +1,146 @@
+/* eslint-disable no-new */
+import * as cdk from 'aws-cdk-lib'
+import { } from '@aws-sdk/client-pricing'
+import { Construct } from 'constructs'
+
+import * as cloudwatch from 'aws-cdk-lib/aws-cloudwatch'
+
+const BEDROCK_METRIC_NAMESPACE = 'AWS/Bedrock'
+
+const CLAUDE_V2_ON_DEMAND_COST_PER_1000_INPUT_TOKENS = '0.01102'
+const CLAUDE_V2_ON_DEMAND_COST_PER_1000_OUTPUT_TOKENS = '0.03268'
+const USD_HOURLY_COST_ALARM_THRESHOLD = 1.0
+
+interface PriceMonitorStackProps extends cdk.StackProps {
+  modelId: string
+  bedrockRegion: string
+}
+
+export class PriceMonitorStack extends cdk.Stack {
+  constructor (scope: Construct, id: string, props: PriceMonitorStackProps) {
+    super(scope, id, props)
+
+    const { modelId, bedrockRegion } = props
+
+    const dashboard = new cloudwatch.Dashboard(this, 'Dashboard', {
+      dashboardName: 'EpisoderPriceMonitor',
+      defaultInterval: cdk.Duration.days(7),
+      periodOverride: cloudwatch.PeriodOverride.AUTO
+    })
+
+    const invocationsMetric = new cloudwatch.Metric({
+      namespace: BEDROCK_METRIC_NAMESPACE,
+      metricName: 'Invocations',
+      region: bedrockRegion,
+      statistic: 'Sum',
+      label: 'Invocations',
+      dimensionsMap: {
+        ModelId: modelId
+      }
+    })
+
+    const inputTokensMetric = new cloudwatch.Metric({
+      namespace: BEDROCK_METRIC_NAMESPACE,
+      metricName: 'InputTokenCount',
+      region: bedrockRegion,
+      statistic: 'Sum',
+      label: 'Input Tokens',
+      dimensionsMap: {
+        ModelId: modelId
+      }
+    })
+
+    const outputTokensMetric = new cloudwatch.Metric({
+      namespace: BEDROCK_METRIC_NAMESPACE,
+      metricName: 'OutputTokenCount',
+      region: bedrockRegion,
+      statistic: 'Sum',
+      label: 'Input Tokens',
+      dimensionsMap: {
+        ModelId: modelId
+      }
+    })
+
+    const inputTokensCostMetric = new cloudwatch.MathExpression({
+      label: 'Input Token Cost ($)',
+      expression: `(inputTokens / 1000) * ${CLAUDE_V2_ON_DEMAND_COST_PER_1000_INPUT_TOKENS}`,
+      usingMetrics: {
+        inputTokens: inputTokensMetric
+      }
+    })
+
+    const outputTokensCostMetric = new cloudwatch.MathExpression({
+      label: 'Output Token Cost ($)',
+      expression: `(outputTokens / 1000) * ${CLAUDE_V2_ON_DEMAND_COST_PER_1000_OUTPUT_TOKENS}`,
+      usingMetrics: {
+        outputTokens: outputTokensMetric
+      }
+    })
+
+    const totalCostMetric = new cloudwatch.MathExpression({
+      label: 'Total Cost ($)',
+      expression: '(inputTokensCost + outputTokensCost)',
+      usingMetrics: {
+        inputTokensCost: inputTokensCostMetric,
+        outputTokensCost: outputTokensCostMetric
+      }
+    })
+
+    const costPerInvocationMetric = new cloudwatch.MathExpression({
+      label: 'Cost per Invocation ($)',
+      expression: 'totalCost / invocations',
+      usingMetrics: {
+        totalCost: totalCostMetric,
+        invocations: invocationsMetric
+      }
+    })
+
+    const priceWidget = new cloudwatch.SingleValueWidget({
+      title: `Bedrock ${modelId} Price Monitor`,
+      setPeriodToTimeRange: true,
+      period: cdk.Duration.days(1),
+      width: 24,
+      height: 6,
+      metrics: [
+        invocationsMetric,
+        inputTokensMetric,
+        outputTokensMetric,
+        inputTokensCostMetric,
+        outputTokensCostMetric,
+        totalCostMetric,
+        costPerInvocationMetric
+      ]
+    })
+
+    dashboard.addWidgets(priceWidget)
+
+    const hourlyCostMetric = new cloudwatch.MathExpression({
+      label: 'Total Cost ($)',
+      expression: '(inputTokensCost + outputTokensCost)',
+      period: cdk.Duration.hours(1),
+      usingMetrics: {
+        inputTokensCost: inputTokensCostMetric,
+        outputTokensCost: outputTokensCostMetric
+      }
+    })
+
+    const hourlyCostAlarm = hourlyCostMetric.createAlarm(this, 'TotalCostAlarm', {
+      threshold: USD_HOURLY_COST_ALARM_THRESHOLD,
+      evaluationPeriods: 3,
+      comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_THRESHOLD,
+      treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
+      datapointsToAlarm: 3,
+      alarmName: `Bedrock ${modelId} Hourly Cost`,
+      alarmDescription: `Total Cost per Hour exceeds $${USD_HOURLY_COST_ALARM_THRESHOLD}`
+    })
+
+    const hourlyCostAlarmWidget = new cloudwatch.AlarmWidget({
+      alarm: hourlyCostAlarm,
+      height: 6,
+      width: 24,
+      region: bedrockRegion,
+      title: `Bedrock ${modelId} Hourly Cost Alarm`
+    })
+    dashboard.addWidgets(hourlyCostAlarmWidget)
+  }
+}