revert: logging stack changes as its not ready yet.

packages/_infra/package.json

@@ -20,21 +20,22 @@
     "test": "ospec --globs 'build/**/*.test.js'"
   },
   "devDependencies": {
-    "@aws-cdk/aws-batch": ">=1.100.0",
-    "@aws-cdk/aws-certificatemanager": ">=1.100.0",
-    "@aws-cdk/aws-cloudfront": ">=1.100.0",
-    "@aws-cdk/aws-dynamodb": ">=1.100.0",
-    "@aws-cdk/aws-ecr": ">=1.100.0",
-    "@aws-cdk/aws-ecr-assets": ">=1.100.0",
-    "@aws-cdk/aws-elasticloadbalancingv2": ">=1.100.0",
-    "@aws-cdk/aws-elasticloadbalancingv2-targets": ">=1.100.0",
-    "@aws-cdk/aws-events-targets": " >=1.100.0",
-    "@aws-cdk/aws-kinesisfirehose": ">=1.100.0",
-    "@aws-cdk/aws-lambda-event-sources": ">=1.100.0",
-    "@aws-cdk/aws-s3": ">=1.100.0",
-    "@aws-cdk/core": ">=1.100.0",
+    "@aws-cdk/aws-batch": ">=1.111.0",
+    "@aws-cdk/aws-certificatemanager": ">=1.111.0",
+    "@aws-cdk/aws-cloudfront": ">=1.111.0",
+    "@aws-cdk/aws-dynamodb": ">=1.111.0",
+    "@aws-cdk/aws-ecr": ">=1.111.0",
+    "@aws-cdk/aws-ecr-assets": ">=1.111.0",
+    "@aws-cdk/aws-elasticloadbalancingv2": ">=1.111.0",
+    "@aws-cdk/aws-elasticloadbalancingv2-targets": ">=1.111.0",
+    "@aws-cdk/aws-events-targets": " >=1.111.0",
+    "@aws-cdk/aws-kinesisfirehose": ">=1.111.0",
+    "@aws-cdk/aws-lambda-event-sources": ">=1.111.0",
+    "@aws-cdk/aws-s3": ">=1.111.0",
+    "@aws-cdk/core": ">=1.111.0",
     "@basemaps/lambda-tiler": "^6.2.0",
     "@basemaps/shared": "^6.1.0",
-    "aws-cdk": "^1.100.0"
+    "@linzjs/cdk-elastic-shipper": "^0.6.2",
+    "aws-cdk": "^1.111.0"
   }
 }

packages/_infra/src/deploy.env.ts

@@ -7,4 +7,7 @@ export const DeployEnv = {
 
     /** TLS certificate to use with Cloudfront */
     CloudFrontTlsCertArn: 'CLOUDFRONT_CERTIFICATE_ARN',
+
+    /** Allow another account to write logs into our bucket */
+    LogAccountId: 'LOG_ACCOUNT_ID',
 };

packages/_infra/src/index.ts

@@ -5,6 +5,7 @@ import { CogBuilderStack } from './cogify';
 import { BaseMapsRegion } from './config';
 import { DeployEnv } from './deploy.env';
 import { EdgeStack } from './edge';
+import { LoggingStack } from './log';
 import { getEdgeParameters } from './parameters';
 import { ServeStack } from './serve';
 
@@ -14,6 +15,8 @@ async function main(): Promise<void> {
     /** Using VPC lookups requires a hard coded AWS "account" */
     const account = Env.get(DeployEnv.CdkAccount);
 
+    new LoggingStack(basemaps, 'Logging', { env: { region: BaseMapsRegion, account } });
+
     /**
      * Because we are using Lambda@Edge the edge stack has to be deployed into us-east-1,
      * The dynamoDb table needs to be close to our users that has to be deployed in ap-southeast-2

packages/_infra/src/log/__test__/log.filter.test.ts

@@ -0,0 +1,13 @@
+import o from 'ospec';
+import { onLog } from '..';
+
+o.spec('LogFilter', () => {
+    o('should drop lambda logs', () => {
+        o(onLog({ '@tags': ['Lambda log'] } as any)).equals(true);
+        o(onLog({ '@tags': ['Flow Log'] } as any)).equals(undefined);
+    });
+
+    o('should not die if "@tags" doesnt exist', () => {
+        o(onLog({} as any)).equals(undefined);
+    });
+});

packages/_infra/src/log/index.ts

@@ -0,0 +1,53 @@
+import { Vpc } from '@aws-cdk/aws-ec2';
+import * as iam from '@aws-cdk/aws-iam';
+import * as firehose from '@aws-cdk/aws-kinesisfirehose';
+import * as s3 from '@aws-cdk/aws-s3';
+import { StringParameter } from '@aws-cdk/aws-ssm';
+import * as cdk from '@aws-cdk/core';
+import { Env } from '@basemaps/shared';
+import { LambdaLogShipperFunction, LogObject } from '@linzjs/cdk-elastic-shipper';
+import { DeployEnv } from '../deploy.env';
+
+const ConfigName = `/es-shipper-config/accounts`;
+
+export function onLog(lo: LogObject): boolean | void {
+    if (lo['@tags']?.includes('Lambda log')) return true;
+}
+
+/**
+ * Basemap's logging
+ *
+ * Store all logs generated by basemaps into a s3 bucket for long term storage
+ * One logs are added to the bucket configure a lambda to ship them elastic search
+ * for monitoring and alerting
+ *
+ * General log flow:
+ *
+ * Lambda -> CloudWatch -> Kinesis -> Kinesis Firehose -> S3 -> Lambda -> ElasticSearch
+ */
+export class LoggingStack extends cdk.Stack {
+    fireHose: firehose.CfnDeliveryStream;
+    public shipper: LambdaLogShipperFunction;
+
+    public constructor(scope: cdk.Construct, id: string, props?: cdk.StackProps) {
+        super(scope, id, props);
+
+        const logBucket = new s3.Bucket(this, 'LogBucket', {
+            versioned: true,
+            blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
+        });
+
+        const accountId = Env.get(DeployEnv.LogAccountId);
+        if (accountId != null && accountId !== '') {
+            logBucket.grantReadWrite(new iam.AccountPrincipal(accountId));
+        }
+
+        new cdk.CfnOutput(this, 'LogBucketArn', { value: logBucket.bucketArn });
+
+        const vpc = Vpc.fromLookup(this, 'ShipperVpc', { tags: { default: 'true' } });
+
+        const configParameter = StringParameter.fromStringParameterName(this, 'ShipperConfig', ConfigName);
+        this.shipper = new LambdaLogShipperFunction(this, 'ShipperFunction', { configParameter, vpc, onLog });
+        this.shipper.addS3Source(logBucket);
+    }
+}