From 680e7dc3b53d7d0cfc614bb06af5130231559bb5 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Fri, 29 Apr 2022 12:14:23 +1200 Subject: [PATCH 1/9] feat(cli): allow overriding imagery names --- packages/cli/src/cli/cogify/action.job.ts | 2 ++ packages/cli/src/cog/cog.stac.job.ts | 2 ++ packages/cli/src/cog/job.factory.ts | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/cli/cogify/action.job.ts b/packages/cli/src/cli/cogify/action.job.ts index 6e9cda1d9..1a10b0e88 100644 --- a/packages/cli/src/cli/cogify/action.job.ts +++ b/packages/cli/src/cli/cogify/action.job.ts @@ -6,6 +6,7 @@ import { CommandLineIntegerParameter, CommandLineStringParameter, } from '@rushstack/ts-command-line'; +import { basename } from 'path'; import { JobCreationContext } from '../../cog/cog.stac.job.js'; import { CogJobFactory, MaxConcurrencyDefault } from '../../cog/job.factory.js'; import { GdalCogBuilderDefaults, GdalCogBuilderResampling, GdalResamplingOptions } from '../../gdal/gdal.config.js'; @@ -120,6 +121,7 @@ export class ActionJobCreate extends CommandLineAction { } const ctx: JobCreationContext = { + imageryName: basename(sourceLocation.path), sourceLocation, outputLocation, cutline, diff --git a/packages/cli/src/cog/cog.stac.job.ts b/packages/cli/src/cog/cog.stac.job.ts index fc6639c00..561bc3a45 100644 --- a/packages/cli/src/cog/cog.stac.job.ts +++ b/packages/cli/src/cog/cog.stac.job.ts @@ -34,6 +34,8 @@ import { export const MaxConcurrencyDefault = 50; export interface JobCreationContext { + imageryName: string; + /** Source config */ sourceLocation: FileConfig | FileConfigPath; diff --git a/packages/cli/src/cog/job.factory.ts b/packages/cli/src/cog/job.factory.ts index e2112d173..3218718b2 100644 --- a/packages/cli/src/cog/job.factory.ts +++ b/packages/cli/src/cog/job.factory.ts @@ -22,7 +22,7 @@ export const CogJobFactory = { */ async create(ctx: JobCreationContext): Promise { const id = ctx.override?.id ?? ulid.ulid(); - const imageryName = basename(ctx.sourceLocation.path).replace(/\./g, '-'); // batch does not allow '.' in names + const imageryName = ctx.imageryName; // batch does not allow '.' in names const logger = LogConfig.get().child({ id, imageryName }); const gdalVersion = await Gdal.version(logger); From fe6e2c02a95d4c1f446ef1ff9c5c75a762b45de5 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Wed, 11 May 2022 14:22:08 +1200 Subject: [PATCH 2/9] refactor: cleanup lint --- packages/cli/src/cog/job.factory.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/cog/job.factory.ts b/packages/cli/src/cog/job.factory.ts index 3218718b2..c35993544 100644 --- a/packages/cli/src/cog/job.factory.ts +++ b/packages/cli/src/cog/job.factory.ts @@ -1,6 +1,5 @@ import { Bounds } from '@basemaps/geo'; import { fsa, isConfigS3Role, isFileConfigPath, LogConfig } from '@basemaps/shared'; -import { basename } from 'path'; import * as ulid from 'ulid'; import { CogBuilder } from '../index.js'; import { ActionBatchJob } from '../cli/cogify/action.batch.js'; @@ -22,8 +21,7 @@ export const CogJobFactory = { */ async create(ctx: JobCreationContext): Promise { const id = ctx.override?.id ?? ulid.ulid(); - const imageryName = ctx.imageryName; // batch does not allow '.' in names - const logger = LogConfig.get().child({ id, imageryName }); + const logger = LogConfig.get().child({ id, imageryName: ctx.imageryName }); const gdalVersion = await Gdal.version(logger); logger.info({ version: gdalVersion }, 'GdalVersion'); @@ -101,7 +99,7 @@ export const CogJobFactory = { const job = await CogStacJob.create({ id, - imageryName, + imageryName: ctx.imageryName, metadata, ctx, addAlpha, From 8585d7528e05d0eee7502b40765bd1b620f416ad Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Mon, 16 May 2022 13:25:44 +1200 Subject: [PATCH 3/9] Allow custom imagery name for import api. --- packages/lambda-tiler/src/__test__/tile.import.test.ts | 1 + packages/lambda-tiler/src/import/make.cog.ts | 3 +++ packages/lambda-tiler/src/routes/import.ts | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/lambda-tiler/src/__test__/tile.import.test.ts b/packages/lambda-tiler/src/__test__/tile.import.test.ts index dc1246ba4..b3a2842c3 100644 --- a/packages/lambda-tiler/src/__test__/tile.import.test.ts +++ b/packages/lambda-tiler/src/__test__/tile.import.test.ts @@ -43,6 +43,7 @@ o.spec('Import', () => { } const ctx: JobCreationContext = { + imageryName: 'testImagery', override: { projection: tileMatrix.projection, resampling: { diff --git a/packages/lambda-tiler/src/import/make.cog.ts b/packages/lambda-tiler/src/import/make.cog.ts index d28e1f638..9fe24f299 100644 --- a/packages/lambda-tiler/src/import/make.cog.ts +++ b/packages/lambda-tiler/src/import/make.cog.ts @@ -2,16 +2,19 @@ import { JobCreationContext } from '@basemaps/cli/build/cog/cog.stac.job'; import { TileMatrixSet } from '@basemaps/geo'; import { Env } from '@basemaps/shared'; import { RoleConfig } from './imagery.find.js'; +import { basename } from 'path'; export async function getJobCreationContext( path: string, tileMatrix: TileMatrixSet, + name: string | null, role: RoleConfig, files: string[], ): Promise { const bucket = Env.get(Env.ImportImageryBucket); if (bucket == null) throw new Error('Output AWS s3 bucket Not Found.'); const ctx: JobCreationContext = { + imageryName: name != null ? name : basename(path), override: { projection: tileMatrix.projection, resampling: { diff --git a/packages/lambda-tiler/src/routes/import.ts b/packages/lambda-tiler/src/routes/import.ts index 37baafa00..b3a302bc5 100644 --- a/packages/lambda-tiler/src/routes/import.ts +++ b/packages/lambda-tiler/src/routes/import.ts @@ -16,6 +16,7 @@ import { CogJobFactory } from '@basemaps/cli'; export async function Import(req: LambdaHttpRequest): Promise { const path = req.query.get('path'); const projection = req.query.get('p'); + const imageryName = req.query.get('name'); // Parse projection as target, default to process both NZTM2000Quad let targetTms = Nztm2000Tms; @@ -33,7 +34,7 @@ export async function Import(req: LambdaHttpRequest): Promise Date: Mon, 16 May 2022 14:31:49 +1200 Subject: [PATCH 4/9] Validate the imagery name in before start processing. --- .../src/__test__/tile.import.test.ts | 6 +++--- packages/lambda-tiler/src/import/make.cog.ts | 5 ++--- packages/lambda-tiler/src/routes/import.ts | 16 +++++++++++++--- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/lambda-tiler/src/__test__/tile.import.test.ts b/packages/lambda-tiler/src/__test__/tile.import.test.ts index 9e86412c3..2c89ffc7a 100644 --- a/packages/lambda-tiler/src/__test__/tile.import.test.ts +++ b/packages/lambda-tiler/src/__test__/tile.import.test.ts @@ -30,7 +30,7 @@ o.spec('Import', () => { const tileMatrix = Nztm2000Tms; const bucket = 'testSourceBucket'; - const path = `s3://${bucket}/imagery/`; + const path = `s3://${bucket}/imagery_2022/`; const role: RoleConfig = { bucket, accountId: '123456', @@ -43,7 +43,7 @@ o.spec('Import', () => { } const ctx: JobCreationContext = { - imageryName: 'testImagery', + imageryName: 'imagery_2022', override: { projection: tileMatrix.projection, resampling: { @@ -97,7 +97,7 @@ o.spec('Import', () => { o('should return Unable to access bucket', async () => { // Given... different bucket have no access role sandbox.stub(fsa, 'readJson').resolves({ buckets: [role] }); - const req = getRequest(`s3://wrong-bucket/imagery/`, '2193'); + const req = getRequest(`s3://wrong-bucket/imagery_2022/`, '2193'); // When ...Then ... const res = await Import(req); diff --git a/packages/lambda-tiler/src/import/make.cog.ts b/packages/lambda-tiler/src/import/make.cog.ts index 9fe24f299..bf104879b 100644 --- a/packages/lambda-tiler/src/import/make.cog.ts +++ b/packages/lambda-tiler/src/import/make.cog.ts @@ -2,19 +2,18 @@ import { JobCreationContext } from '@basemaps/cli/build/cog/cog.stac.job'; import { TileMatrixSet } from '@basemaps/geo'; import { Env } from '@basemaps/shared'; import { RoleConfig } from './imagery.find.js'; -import { basename } from 'path'; export async function getJobCreationContext( path: string, tileMatrix: TileMatrixSet, - name: string | null, + name: string, role: RoleConfig, files: string[], ): Promise { const bucket = Env.get(Env.ImportImageryBucket); if (bucket == null) throw new Error('Output AWS s3 bucket Not Found.'); const ctx: JobCreationContext = { - imageryName: name != null ? name : basename(path), + imageryName: name, override: { projection: tileMatrix.projection, resampling: { diff --git a/packages/lambda-tiler/src/routes/import.ts b/packages/lambda-tiler/src/routes/import.ts index 00f720e43..40cdfe4dc 100644 --- a/packages/lambda-tiler/src/routes/import.ts +++ b/packages/lambda-tiler/src/routes/import.ts @@ -1,5 +1,5 @@ import { HttpHeader, LambdaHttpRequest, LambdaHttpResponse } from '@linzjs/lambda'; -import { Config, fsa } from '@basemaps/shared'; +import { Config, extractYearRangeFromName, fsa } from '@basemaps/shared'; import { createHash } from 'crypto'; import { findImagery, RoleRegister } from '../import/imagery.find.js'; import { Nztm2000Tms, TileMatrixSets } from '@basemaps/geo'; @@ -7,6 +7,7 @@ import { getJobCreationContext } from '../import/make.cog.js'; import { ConfigProcessingJob, JobStatus } from '@basemaps/config'; import { CogJobFactory } from '@basemaps/cli'; import * as ulid from 'ulid'; +import { basename } from 'path'; /** * Trigger import imagery job by this endpoint @@ -17,7 +18,7 @@ import * as ulid from 'ulid'; export async function Import(req: LambdaHttpRequest): Promise { const path = req.query.get('path'); const projection = req.query.get('p'); - const imageryName = req.query.get('name'); + const name = req.query.get('name'); // Parse projection as target, default to process both NZTM2000Quad let targetTms = Nztm2000Tms; @@ -27,8 +28,17 @@ export async function Import(req: LambdaHttpRequest): Promise Date: Mon, 16 May 2022 14:53:58 +1200 Subject: [PATCH 5/9] Add default year when not exists. --- packages/lambda-tiler/src/routes/import.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/lambda-tiler/src/routes/import.ts b/packages/lambda-tiler/src/routes/import.ts index 40cdfe4dc..ebf6ec3bc 100644 --- a/packages/lambda-tiler/src/routes/import.ts +++ b/packages/lambda-tiler/src/routes/import.ts @@ -18,7 +18,6 @@ import { basename } from 'path'; export async function Import(req: LambdaHttpRequest): Promise { const path = req.query.get('path'); const projection = req.query.get('p'); - const name = req.query.get('name'); // Parse projection as target, default to process both NZTM2000Quad let targetTms = Nztm2000Tms; @@ -32,10 +31,11 @@ export async function Import(req: LambdaHttpRequest): Promise Date: Mon, 16 May 2022 15:05:12 +1200 Subject: [PATCH 6/9] Use ulid_year as imagery name for invalid names. --- packages/lambda-tiler/src/routes/import.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/lambda-tiler/src/routes/import.ts b/packages/lambda-tiler/src/routes/import.ts index ebf6ec3bc..299dcdaba 100644 --- a/packages/lambda-tiler/src/routes/import.ts +++ b/packages/lambda-tiler/src/routes/import.ts @@ -18,6 +18,7 @@ import { basename } from 'path'; export async function Import(req: LambdaHttpRequest): Promise { const path = req.query.get('path'); const projection = req.query.get('p'); + const id = ulid.ulid(); // Parse projection as target, default to process both NZTM2000Quad let targetTms = Nztm2000Tms; @@ -35,7 +36,7 @@ export async function Import(req: LambdaHttpRequest): Promise Date: Mon, 16 May 2022 15:37:10 +1200 Subject: [PATCH 7/9] Make custom imagery name as option attribute for JobCreationContext. --- packages/cli/src/cli/cogify/action.job.ts | 2 -- packages/cli/src/cog/cog.stac.job.ts | 3 ++- packages/cli/src/cog/job.factory.ts | 8 ++++++-- packages/lambda-tiler/src/__test__/tile.import.test.ts | 4 ++-- packages/lambda-tiler/src/import/make.cog.ts | 2 -- packages/lambda-tiler/src/routes/import.ts | 3 ++- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/cli/cogify/action.job.ts b/packages/cli/src/cli/cogify/action.job.ts index 1a10b0e88..6e9cda1d9 100644 --- a/packages/cli/src/cli/cogify/action.job.ts +++ b/packages/cli/src/cli/cogify/action.job.ts @@ -6,7 +6,6 @@ import { CommandLineIntegerParameter, CommandLineStringParameter, } from '@rushstack/ts-command-line'; -import { basename } from 'path'; import { JobCreationContext } from '../../cog/cog.stac.job.js'; import { CogJobFactory, MaxConcurrencyDefault } from '../../cog/job.factory.js'; import { GdalCogBuilderDefaults, GdalCogBuilderResampling, GdalResamplingOptions } from '../../gdal/gdal.config.js'; @@ -121,7 +120,6 @@ export class ActionJobCreate extends CommandLineAction { } const ctx: JobCreationContext = { - imageryName: basename(sourceLocation.path), sourceLocation, outputLocation, cutline, diff --git a/packages/cli/src/cog/cog.stac.job.ts b/packages/cli/src/cog/cog.stac.job.ts index 44fed2c5c..4dffc28b6 100644 --- a/packages/cli/src/cog/cog.stac.job.ts +++ b/packages/cli/src/cog/cog.stac.job.ts @@ -34,7 +34,8 @@ import { export const MaxConcurrencyDefault = 50; export interface JobCreationContext { - imageryName: string; + /** Custom Imagery Name if defined */ + imageryName?: string; /** Source config */ sourceLocation: FileConfig | FileConfigPath; diff --git a/packages/cli/src/cog/job.factory.ts b/packages/cli/src/cog/job.factory.ts index 2fcce92b7..6211ec95c 100644 --- a/packages/cli/src/cog/job.factory.ts +++ b/packages/cli/src/cog/job.factory.ts @@ -7,6 +7,7 @@ import { Gdal } from '../gdal/gdal.js'; import { CogStacJob, JobCreationContext } from './cog.stac.job.js'; import { Cutline } from './cutline.js'; import { CogJob } from './types.js'; +import { basename } from 'path'; export const MaxConcurrencyDefault = 50; @@ -21,7 +22,10 @@ export const CogJobFactory = { */ async create(ctx: JobCreationContext): Promise { const id = ctx.override?.id ?? ulid.ulid(); - const logger = LogConfig.get().child({ id, imageryName: ctx.imageryName }); + let imageryName = ctx.imageryName; + if (imageryName == null) imageryName = basename(ctx.sourceLocation.path).replace(/\./g, '-'); // batch does not allow '.' in names + + const logger = LogConfig.get().child({ id, imageryName }); const gdalVersion = await Gdal.version(logger); logger.info({ version: gdalVersion }, 'GdalVersion'); @@ -99,7 +103,7 @@ export const CogJobFactory = { const job = await CogStacJob.create({ id, - imageryName: ctx.imageryName, + imageryName, metadata, ctx, addAlpha, diff --git a/packages/lambda-tiler/src/__test__/tile.import.test.ts b/packages/lambda-tiler/src/__test__/tile.import.test.ts index 2c89ffc7a..849063a2f 100644 --- a/packages/lambda-tiler/src/__test__/tile.import.test.ts +++ b/packages/lambda-tiler/src/__test__/tile.import.test.ts @@ -43,7 +43,7 @@ o.spec('Import', () => { } const ctx: JobCreationContext = { - imageryName: 'imagery_2022', + imageryName: 'imagery', override: { projection: tileMatrix.projection, resampling: { @@ -97,7 +97,7 @@ o.spec('Import', () => { o('should return Unable to access bucket', async () => { // Given... different bucket have no access role sandbox.stub(fsa, 'readJson').resolves({ buckets: [role] }); - const req = getRequest(`s3://wrong-bucket/imagery_2022/`, '2193'); + const req = getRequest(`s3://wrong-bucket/imagery/`, '2193'); // When ...Then ... const res = await Import(req); diff --git a/packages/lambda-tiler/src/import/make.cog.ts b/packages/lambda-tiler/src/import/make.cog.ts index bf104879b..d28e1f638 100644 --- a/packages/lambda-tiler/src/import/make.cog.ts +++ b/packages/lambda-tiler/src/import/make.cog.ts @@ -6,14 +6,12 @@ import { RoleConfig } from './imagery.find.js'; export async function getJobCreationContext( path: string, tileMatrix: TileMatrixSet, - name: string, role: RoleConfig, files: string[], ): Promise { const bucket = Env.get(Env.ImportImageryBucket); if (bucket == null) throw new Error('Output AWS s3 bucket Not Found.'); const ctx: JobCreationContext = { - imageryName: name, override: { projection: tileMatrix.projection, resampling: { diff --git a/packages/lambda-tiler/src/routes/import.ts b/packages/lambda-tiler/src/routes/import.ts index 299dcdaba..adcd6762e 100644 --- a/packages/lambda-tiler/src/routes/import.ts +++ b/packages/lambda-tiler/src/routes/import.ts @@ -46,13 +46,14 @@ export async function Import(req: LambdaHttpRequest): Promise Date: Mon, 16 May 2022 15:38:32 +1200 Subject: [PATCH 8/9] Revert the test. --- packages/lambda-tiler/src/__test__/tile.import.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/lambda-tiler/src/__test__/tile.import.test.ts b/packages/lambda-tiler/src/__test__/tile.import.test.ts index 849063a2f..95b9adceb 100644 --- a/packages/lambda-tiler/src/__test__/tile.import.test.ts +++ b/packages/lambda-tiler/src/__test__/tile.import.test.ts @@ -30,7 +30,7 @@ o.spec('Import', () => { const tileMatrix = Nztm2000Tms; const bucket = 'testSourceBucket'; - const path = `s3://${bucket}/imagery_2022/`; + const path = `s3://${bucket}/imagery/`; const role: RoleConfig = { bucket, accountId: '123456', @@ -43,7 +43,6 @@ o.spec('Import', () => { } const ctx: JobCreationContext = { - imageryName: 'imagery', override: { projection: tileMatrix.projection, resampling: { From d1688f72283d1dd62d3e95d2f15d19efcc75b96f Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Tue, 17 May 2022 09:10:35 +1200 Subject: [PATCH 9/9] Check job status after found file eixsts. --- packages/cli/src/cli/cogify/action.cog.ts | 1 + packages/cli/src/cog/job.factory.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/cli/cogify/action.cog.ts b/packages/cli/src/cli/cogify/action.cog.ts index ca3be95b4..9dacc3ee6 100644 --- a/packages/cli/src/cli/cogify/action.cog.ts +++ b/packages/cli/src/cli/cogify/action.cog.ts @@ -96,6 +96,7 @@ export class ActionCogCreate extends CommandLineAction { // Output file exists don't try and overwrite it if (outputExists) { logger.warn({ targetPath }, 'CogCreate:OutputExists'); + await this.checkJobStatus(job, logger); return; } diff --git a/packages/cli/src/cog/job.factory.ts b/packages/cli/src/cog/job.factory.ts index 6211ec95c..3280734d4 100644 --- a/packages/cli/src/cog/job.factory.ts +++ b/packages/cli/src/cog/job.factory.ts @@ -23,7 +23,7 @@ export const CogJobFactory = { async create(ctx: JobCreationContext): Promise { const id = ctx.override?.id ?? ulid.ulid(); let imageryName = ctx.imageryName; - if (imageryName == null) imageryName = basename(ctx.sourceLocation.path).replace(/\./g, '-'); // batch does not allow '.' in names + if (imageryName == null) imageryName = basename(ctx.sourceLocation.path); const logger = LogConfig.get().child({ id, imageryName });