feat(lambda-tiler): Increase limit of total file size. (#2205)

* Fix the enviroment setup for cogBuilderStack

* Fix the output path for the imagery config.

* Increase the limit of total file number and size.

* Fix broken test

* Add import api limit ad environment varaiables.

packages/_infra/src/cogify/index.ts

@@ -148,7 +148,10 @@ export class CogBuilderStack extends cdk.Stack {
          * Eg a instance with 8192MB allocates 7953MB usable
          */
         memory: 3900,
-        environment: [{ name: Env.TempFolder, value: ScratchData.Folder, [Env.PublicUrlBase]: config.PublicUrlBase }],
+        environment: [
+          { name: Env.TempFolder, value: ScratchData.Folder },
+          { name: Env.PublicUrlBase, value: config.PublicUrlBase },
+        ],
         mountPoints: [{ containerPath: ScratchData.Folder, sourceVolume: 'scratch' }],
         volumes: [{ name: 'scratch', host: { sourcePath: ScratchData.Folder } }],
       },

packages/_infra/src/config.ts

@@ -19,8 +19,14 @@ export interface BaseMapsConfig {
   /** AWS role config bucket */
   AwsRoleConfigBucket: string;
 
-  /** ImportImageryBucket */
+  /** Import Imagery Bucket */
   ImportImageryBucket: string;
+
+  /** Import Imagery files number limit */
+  ImportFilesNumberLimit: string;
+
+  /** Import Imagery files total size limit in GB */
+  ImportFilesSizeLimit: string;
 }
 
 export const BaseMapsProdConfig: BaseMapsConfig = {
@@ -31,6 +37,8 @@ export const BaseMapsProdConfig: BaseMapsConfig = {
   PublicUrlBase: 'https://basemaps.linz.govt.nz',
   AwsRoleConfigBucket: 'linz-bucket-config',
   ImportImageryBucket: 'linz-basemaps-cache',
+  ImportFilesNumberLimit: '10000',
+  ImportFilesSizeLimit: '500',
 };
 
 export const BaseMapsDevConfig: BaseMapsConfig = {
@@ -41,6 +49,8 @@ export const BaseMapsDevConfig: BaseMapsConfig = {
   PublicUrlBase: 'https://dev.basemaps.linz.govt.nz',
   AwsRoleConfigBucket: 'linz-bucket-config',
   ImportImageryBucket: 'basemaps-cog-test',
+  ImportFilesNumberLimit: '10000',
+  ImportFilesSizeLimit: '500',
 };
 
 export function getConfig(): BaseMapsConfig {

packages/_infra/src/serve/lambda.tiler.ts

@@ -39,6 +39,8 @@ export class LambdaTiler extends Construct {
         [Env.PublicUrlBase]: config.PublicUrlBase,
         [Env.AwsRoleConfigBucket]: config.AwsRoleConfigBucket,
         [Env.ImportImageryBucket]: config.ImportImageryBucket,
+        [Env.ImportFilesNumberLimit]: config.ImportFilesNumberLimit,
+        [Env.ImportFilesSizeLimit]: config.ImportFilesSizeLimit,
         AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1',
       },
       logRetention: RetentionDays.ONE_MONTH,

packages/cli/src/cli/cogify/imagery.config.ts

@@ -17,7 +17,7 @@ export async function insertConfigImagery(job: CogStacJob, logger: LogType): Pro
     updatedAt: now,
     projection: job.tileMatrix.projection.code,
     tileMatrix: job.tileMatrix.identifier,
-    uri: job.output.location.path,
+    uri: job.getJobPath(),
     bounds: job.output.bounds,
     files: job.output.files,
   };

packages/lambda-tiler/src/__test__/tile.import.test.ts

@@ -15,16 +15,24 @@ o.spec('Import', () => {
   const sandbox = sinon.createSandbox();
   const outputBucket = 'testOutputBucket';
   const configBucket = 'testConfigBucket';
+  const fileNumberLimit = '5';
+  const fileSizeLimit = '2';
   const origConfigBucket = process.env[Env.AwsRoleConfigBucket];
   const origOutputBucket = process.env[Env.ImportImageryBucket];
+  const origFileNumberLimit = process.env[Env.ImportFilesNumberLimit];
+  const origFileSizeLimit = process.env[Env.ImportFilesSizeLimit];
   o.beforeEach(() => {
     process.env[Env.AwsRoleConfigBucket] = configBucket;
     process.env[Env.ImportImageryBucket] = outputBucket;
+    process.env[Env.ImportFilesNumberLimit] = fileNumberLimit;
+    process.env[Env.ImportFilesSizeLimit] = fileSizeLimit;
   });
 
   o.afterEach(() => {
     process.env[Env.AwsRoleConfigBucket] = origConfigBucket;
     process.env[Env.ImportImageryBucket] = origOutputBucket;
+    process.env[Env.ImportFilesNumberLimit] = origFileNumberLimit;
+    process.env[Env.ImportFilesSizeLimit] = origFileSizeLimit;
     sandbox.restore();
   });
 
@@ -38,8 +46,8 @@ o.spec('Import', () => {
   };
 
   const files = [`${path}/1.tiff`, `${path}/2.tiff`];
-  async function* listFiles(): AsyncGenerator<string, any, unknown> {
-    for (const key in files) yield files[key];
+  async function* listFiles(): AsyncGenerator<{ path: string; size: number }, any, unknown> {
+    for (const key in files) yield { path: files[key], size: 40_000_000 };
   }
 
   const ctx: JobCreationContext = {
@@ -106,8 +114,8 @@ o.spec('Import', () => {
   o('should return Imagery not found', async () => {
     // Given... none imagery find from bucket
     sandbox.stub(fsa, 'readJson').resolves({ buckets: [role] });
-    sandbox.stub(fsa, 'list').callsFake(async function* () {
-      yield `${path}1.json`;
+    sandbox.stub(fsa, 'details').callsFake(async function* () {
+      yield { path: `${path}1.json`, size: 4000000 };
     });
 
     const req = getRequest(path, '2193');
@@ -120,7 +128,7 @@ o.spec('Import', () => {
   o('should return 200 with existing import', async () => {
     // Given... different bucket have no access role
     sandbox.stub(fsa, 'readJson').resolves({ buckets: [role] });
-    sandbox.stub(fsa, 'list').callsFake(listFiles);
+    sandbox.stub(fsa, 'details').callsFake(listFiles);
     sandbox.stub(CogJobFactory, 'create').resolves(undefined);
 
     const jobConfig = {
@@ -137,4 +145,36 @@ o.spec('Import', () => {
     const body = Buffer.from(res.body ?? '', 'base64').toString();
     o(JSON.parse(body)).deepEquals(jobConfig);
   });
+
+  o('should return 400 with reach file number limit', async () => {
+    // Given... different bucket have no access role
+    async function* listTooManyFiles(): AsyncGenerator<{ path: string; size: number }, any, unknown> {
+      const files = [`${path}/1.tiff`, `${path}/2.tiff`, `${path}/3.tiff`, `${path}/4.tiff`, `${path}/5.tiff`];
+      for (const key in files) yield { path: files[key], size: 300_000_000 };
+    }
+    sandbox.stub(fsa, 'readJson').resolves({ buckets: [role] });
+    sandbox.stub(fsa, 'details').callsFake(listTooManyFiles);
+    sandbox.stub(CogJobFactory, 'create').resolves(undefined);
+    const req = getRequest(path, '2193');
+
+    // When ...Then ...
+    const res = await Import(req);
+    o(res.body).equals('{"status":400,"message":"Too many files to process. Files: 5. TotalSize: 1.4GB"}');
+  });
+
+  o('should return 400 with reach file size limit', async () => {
+    // Given... different bucket have no access role
+    async function* listTooLargeFiles(): AsyncGenerator<{ path: string; size: number }, any, unknown> {
+      const files = [`${path}/1.tiff`];
+      for (const key in files) yield { path: files[key], size: 3_000_000_000 };
+    }
+    sandbox.stub(fsa, 'readJson').resolves({ buckets: [role] });
+    sandbox.stub(fsa, 'details').callsFake(listTooLargeFiles);
+    sandbox.stub(CogJobFactory, 'create').resolves(undefined);
+    const req = getRequest(path, '2193');
+
+    // When ...Then ...
+    const res = await Import(req);
+    o(res.body).equals(`{"status":400,"message":"Too many files to process. Files: 1. TotalSize: 2.79GB"}`);
+  });
 });

packages/lambda-tiler/src/import/imagery.find.ts

@@ -50,11 +50,15 @@ export class RoleRegister {
 }
 
 /** Search for the imagery across all of our buckets */
-export async function findImagery(path: string): Promise<string[]> {
+export async function findImagery(path: string): Promise<{ files: string[]; totalSize: number }> {
   const files: string[] = [];
-  for await (const key of fsa.list(path)) {
-    const searchKey = key.toLowerCase();
-    if (searchKey.endsWith('.tif') || searchKey.endsWith('.tiff')) files.push(key);
+  let totalSize = 0;
+  for await (const key of fsa.details(path)) {
+    const searchKey = key.path.toLowerCase();
+    if (searchKey.endsWith('.tif') || searchKey.endsWith('.tiff')) {
+      files.push(key.path);
+      if (key.size != null) totalSize += key.size;
+    }
   }
-  return files;
+  return { files, totalSize };
 }

packages/lambda-tiler/src/routes/import.ts

@@ -1,5 +1,5 @@
 import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda';
-import { Config, extractYearRangeFromName, fsa } from '@basemaps/shared';
+import { Config, Env, extractYearRangeFromName, fsa } from '@basemaps/shared';
 import { createHash } from 'crypto';
 import { findImagery, RoleRegister } from '../import/imagery.find.js';
 import { Nztm2000Tms, TileMatrixSets } from '@basemaps/geo';
@@ -42,9 +42,17 @@ export async function Import(req: LambdaHttpRequest): Promise<LambdaHttpResponse
   // Find the imagery from s3
   const role = await RoleRegister.findRole(path);
   if (role == null) return new LambdaHttpResponse(403, 'Unable to Access the s3 bucket');
-  const files = await findImagery(path);
-  if (files.length === 0) return new LambdaHttpResponse(404, 'Imagery Not Found');
-  if (files.length >= 5_000) return new LambdaHttpResponse(400, `Too many files to process. Files: ${files.length}`);
+  const fileInfo = await findImagery(path);
+  if (fileInfo.files.length === 0) return new LambdaHttpResponse(404, 'Imagery Not Found');
+  const files = fileInfo.files;
+  const numberLimit = Number(Env.get(Env.ImportFilesNumberLimit));
+  const sizeLimit = Number(Env.get(Env.ImportFilesSizeLimit));
+  const totalSizeInGB = Math.round((fileInfo.totalSize / Math.pow(1024, 3)) * 100) / 100;
+  if (files.length >= numberLimit || totalSizeInGB >= sizeLimit)
+    return new LambdaHttpResponse(
+      400,
+      `Too many files to process. Files: ${files.length}. TotalSize: ${totalSizeInGB}GB`,
+    );
 
   // Prepare Cog jobs
   const ctx = await getJobCreationContext(path, targetTms, role, files);

packages/shared/src/const.ts

@@ -36,6 +36,12 @@ export const Env = {
   /** Import Imagery bucket */
   ImportImageryBucket: 'IMPORT_IMAGERY_BUCKET',
 
+  /** Import Imagery files number limit */
+  ImportFilesNumberLimit: 'FILES_NUMBER_LIMIT',
+
+  /** Import Imagery files total size limit in GB */
+  ImportFilesSizeLimit: 'FILES_SIZE_LIMIT',
+
   Gdal: {
     /** Should the gdal docker container be used? */
     UseDocker: 'GDAL_DOCKER',