From 41dda90278e7ef1fbd1f7ed3a92e14f4196d1935 Mon Sep 17 00:00:00 2001 From: Joseph Ramsay Date: Fri, 8 May 2020 15:42:41 +1200 Subject: [PATCH] fix: cut blend merge build: production builds need to be tagged as production (#574) * build: production builds need to be tagged as production * build: dont fail to deploy if old landing page does not exist v1.4.2 feat(landing): support different imagery sets other than aerial with ?i=:imageId (#575) fix(cli): root quadkey causes issues with dynamodb so never use it (#576) * fix(cli): root quadkey causes issues with dynamodb so never use it * feat(cli): support unprefixed imagery ids Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> fix(cli): do not error when --replace-with is not supplied (#577) v1.5.0 fix(cli): aws assume role needs to be able to assume any role provided via the cli (#578) v1.5.1 fix(serve): allow any tile set name to be used (#579) This allows any tile set in the database to be served, rather than 404ing on anything that is not "aerial" fix(tiler): position non square COGs correctly (#580) Non square COGs were being shifted up from their intended position fix: add blend defaults build(deps): cogeotiff 1.0.6 (#582) build(deps): aws-cdk 1.37.0 (#581) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> feat(cli): submit jobs automatically to aws batch with --batch (#583) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> v1.6.0 fix(cli): role assumptions must have role names shorter than 64 chars (#585) feat(cli): support webp quality setting (#586) build(deps-dev): bump mime-types from 2.1.26 to 2.1.27 (#523) Bumps [mime-types](https://github.com/jshttp/mime-types) from 2.1.26 to 2.1.27. - [Release notes](https://github.com/jshttp/mime-types/releases) - [Changelog](https://github.com/jshttp/mime-types/blob/master/HISTORY.md) - [Commits](https://github.com/jshttp/mime-types/compare/2.1.26...2.1.27) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> build(deps-dev): bump @types/sharp from 0.24.0 to 0.25.0 (#587) Bumps [@types/sharp](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/sharp) from 0.24.0 to 0.25.0. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/sharp) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> build(deps-dev): bump eslint-config-prettier from 6.10.1 to 6.11.0 (#520) Bumps [eslint-config-prettier](https://github.com/prettier/eslint-config-prettier) from 6.10.1 to 6.11.0. - [Release notes](https://github.com/prettier/eslint-config-prettier/releases) - [Changelog](https://github.com/prettier/eslint-config-prettier/blob/master/CHANGELOG.md) - [Commits](https://github.com/prettier/eslint-config-prettier/compare/v6.10.1...v6.11.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> build(deps-dev): bump @typescript-eslint/parser from 2.29.0 to 2.31.0 (#588) Bumps [@typescript-eslint/parser](https://github.com/typescript-eslint/typescript-eslint/tree/HEAD/packages/parser) from 2.29.0 to 2.31.0. - [Release notes](https://github.com/typescript-eslint/typescript-eslint/releases) - [Changelog](https://github.com/typescript-eslint/typescript-eslint/blob/master/packages/parser/CHANGELOG.md) - [Commits](https://github.com/typescript-eslint/typescript-eslint/commits/v2.31.0/packages/parser) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> v1.7.0 feat: support rendering different backgrounds for tiles (#591) * feat: support rendering different backgrounds for tiles * feat(cli): support changing background of tile sets build(deps): cogeotiff 1.0.8 (#592) fixes a issue where some s3 reads would fail fix(cog): add padding to projwin' (#594) Was sometimes one pixel short on lower right sides Now add 1% padding to lower right sides v1.8.0 --- .github/workflows/push.yml | 1 + CHANGELOG.md | 85 ++ lerna.json | 2 +- packages/_infra/CHANGELOG.md | 51 + packages/_infra/package.json | 30 +- packages/_infra/src/cogify/index.ts | 6 + packages/cog/CHANGELOG.md | 58 ++ packages/cog/package.json | 14 +- .../cli/basemaps/__test__/test.update.test.ts | 95 +- .../src/cli/basemaps/action.tileset.update.ts | 81 +- packages/cog/src/cli/basemaps/tileset.util.ts | 3 + packages/cog/src/cli/cogify/action.batch.ts | 36 +- packages/cog/src/cli/cogify/action.job.ts | 48 +- packages/cog/src/cog/__test__/cog.test.ts | 9 +- packages/cog/src/cog/__test__/cutline.test.ts | 8 + packages/cog/src/cog/cog.ts | 5 +- packages/cog/src/cog/cutline.ts | 15 +- packages/cog/src/cog/types.ts | 5 + packages/cog/src/gdal/gdal.config.ts | 6 + packages/cog/src/gdal/gdal.ts | 3 + packages/geo/CHANGELOG.md | 8 + packages/geo/package.json | 2 +- packages/lambda-api-tracker/CHANGELOG.md | 40 + packages/lambda-api-tracker/package.json | 6 +- packages/lambda-shared/CHANGELOG.md | 52 + packages/lambda-shared/package.json | 8 +- .../src/__test__/api.path.test.ts | 4 +- packages/lambda-shared/src/api.path.ts | 18 +- packages/lambda-shared/src/aws/credentials.ts | 2 +- .../lambda-shared/src/aws/tile.metadata.ts | 4 + packages/lambda-xyz/CHANGELOG.md | 46 + packages/lambda-xyz/package.json | 14 +- packages/lambda-xyz/src/__test__/xyz.test.ts | 8 +- packages/lambda-xyz/src/routes/tile.ts | 15 +- packages/lambda-xyz/src/tile.set.ts | 4 + packages/landing/CHANGELOG.md | 51 + packages/landing/deploy.js | 23 +- packages/landing/package.json | 4 +- packages/landing/static/index.html | 4 +- packages/metrics/CHANGELOG.md | 8 + packages/metrics/package.json | 2 +- packages/tiler-sharp/CHANGELOG.md | 35 + packages/tiler-sharp/package.json | 10 +- .../src/__test__/tile.benchmark.ts | 3 +- .../src/__test__/tile.creation.test.ts | 12 +- packages/tiler-sharp/src/index.ts | 12 +- packages/tiler/CHANGELOG.md | 30 + packages/tiler/package.json | 14 +- packages/tiler/src/__test__/tiler.test.ts | 40 + packages/tiler/src/raster.ts | 1 + packages/tiler/src/tiler.ts | 4 +- yarn.lock | 946 +++++++++--------- 52 files changed, 1360 insertions(+), 631 deletions(-) create mode 100644 packages/tiler/src/__test__/tiler.test.ts diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 8d22dbdbcb..a6fab62fc7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -81,3 +81,4 @@ jobs: env: ALB_CERTIFICATE_ARN: ${{secrets.ALB_CERTIFICATE_ARN_PROD}} CLOUDFRONT_CERTIFICATE_ARN: ${{secrets.CLOUDFRONT_CERTIFICATE_ARN_PROD}} + NODE_ENV: 'production' diff --git a/CHANGELOG.md b/CHANGELOG.md index bc6ba3ef9c..9b14d305de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,91 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +# [1.8.0](https://github.com/linz/basemaps/compare/v1.7.0...v1.8.0) (2020-05-11) + + +### Bug Fixes + +* **cog:** add padding to projwin' ([#594](https://github.com/linz/basemaps/issues/594)) ([72a324a](https://github.com/linz/basemaps/commit/72a324a97ff470e436c3d5360954b9338ff36e59)) + + +### Features + +* support rendering different backgrounds for tiles ([#591](https://github.com/linz/basemaps/issues/591)) ([22f38f5](https://github.com/linz/basemaps/commit/22f38f555a678e6968206351d8fbb62a604da39e)) + + + + + +# [1.7.0](https://github.com/linz/basemaps/compare/v1.6.0...v1.7.0) (2020-05-10) + + +### Bug Fixes + +* **cli:** role assumptions must have role names shorter than 64 chars ([#585](https://github.com/linz/basemaps/issues/585)) ([d889cb7](https://github.com/linz/basemaps/commit/d889cb7666685a8c3a4c7a0816c92fe62626e2e4)) + + +### Features + +* **cli:** support webp quality setting ([#586](https://github.com/linz/basemaps/issues/586)) ([a456404](https://github.com/linz/basemaps/commit/a456404e2774c7a7adeffd8d114c192b073106b7)) + + + + + +# [1.6.0](https://github.com/linz/basemaps/compare/v1.5.1...v1.6.0) (2020-05-08) + + +### Bug Fixes + +* **serve:** allow any tile set name to be used ([#579](https://github.com/linz/basemaps/issues/579)) ([e3e6a03](https://github.com/linz/basemaps/commit/e3e6a03e66b496ae6f9247dc9cbbb0110f5993c5)) +* **tiler:** position non square COGs correctly ([#580](https://github.com/linz/basemaps/issues/580)) ([3eb267a](https://github.com/linz/basemaps/commit/3eb267a1cfceefcdc9fa9872183a71d8da5818f7)) + + +### Features + +* **cli:** submit jobs automatically to aws batch with --batch ([#583](https://github.com/linz/basemaps/issues/583)) ([6b35696](https://github.com/linz/basemaps/commit/6b356961a2f7d1497f51f69199aa038e64fbdca9)) + + + + + +## [1.5.1](https://github.com/linz/basemaps/compare/v1.5.0...v1.5.1) (2020-05-07) + + +### Bug Fixes + +* **cli:** aws assume role needs to be able to assume any role provided via the cli ([#578](https://github.com/linz/basemaps/issues/578)) ([d432c89](https://github.com/linz/basemaps/commit/d432c891280bbf312d6a547c4ccb3a766eca3670)) + + + + + +# [1.5.0](https://github.com/linz/basemaps/compare/v1.4.2...v1.5.0) (2020-05-07) + + +### Bug Fixes + +* **cli:** do not error when --replace-with is not supplied ([#577](https://github.com/linz/basemaps/issues/577)) ([2c4f5dc](https://github.com/linz/basemaps/commit/2c4f5dc5f46823ce4e6f03420b9ec9fc233505ea)) +* **cli:** root quadkey causes issues with dynamodb so never use it ([#576](https://github.com/linz/basemaps/issues/576)) ([4dfa860](https://github.com/linz/basemaps/commit/4dfa86027980231514ae417ce59e94f02e78c3f6)) + + +### Features + +* **landing:** support different imagery sets other than aerial with ?i=:imageId ([#575](https://github.com/linz/basemaps/issues/575)) ([f1b730e](https://github.com/linz/basemaps/commit/f1b730ea8fd61bd907e54be20abe18cd1146e3a9)) + + + + + +## [1.4.2](https://github.com/linz/basemaps/compare/v1.4.1...v1.4.2) (2020-05-06) + +**Note:** Version bump only for package @basemaps/core + + + + + ## [1.4.1](https://github.com/linz/basemaps/compare/v1.4.0...v1.4.1) (2020-05-06) **Note:** Version bump only for package @basemaps/core diff --git a/lerna.json b/lerna.json index 5c78a4c42b..4e328786d3 100644 --- a/lerna.json +++ b/lerna.json @@ -3,5 +3,5 @@ "packages": [ "packages/*" ], - "version": "1.4.1" + "version": "1.8.0" } diff --git a/packages/_infra/CHANGELOG.md b/packages/_infra/CHANGELOG.md index e8dc8b2f4b..d0b45ef481 100644 --- a/packages/_infra/CHANGELOG.md +++ b/packages/_infra/CHANGELOG.md @@ -3,6 +3,57 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +# [1.8.0](https://github.com/linz/basemaps/compare/v1.7.0...v1.8.0) (2020-05-11) + +**Note:** Version bump only for package @basemaps/infra + + + + + +# [1.7.0](https://github.com/linz/basemaps/compare/v1.6.0...v1.7.0) (2020-05-10) + +**Note:** Version bump only for package @basemaps/infra + + + + + +# [1.6.0](https://github.com/linz/basemaps/compare/v1.5.1...v1.6.0) (2020-05-08) + +**Note:** Version bump only for package @basemaps/infra + + + + + +## [1.5.1](https://github.com/linz/basemaps/compare/v1.5.0...v1.5.1) (2020-05-07) + + +### Bug Fixes + +* **cli:** aws assume role needs to be able to assume any role provided via the cli ([#578](https://github.com/linz/basemaps/issues/578)) ([d432c89](https://github.com/linz/basemaps/commit/d432c891280bbf312d6a547c4ccb3a766eca3670)) + + + + + +# [1.5.0](https://github.com/linz/basemaps/compare/v1.4.2...v1.5.0) (2020-05-07) + +**Note:** Version bump only for package @basemaps/infra + + + + + +## [1.4.2](https://github.com/linz/basemaps/compare/v1.4.1...v1.4.2) (2020-05-06) + +**Note:** Version bump only for package @basemaps/infra + + + + + ## [1.4.1](https://github.com/linz/basemaps/compare/v1.4.0...v1.4.1) (2020-05-06) **Note:** Version bump only for package @basemaps/infra diff --git a/packages/_infra/package.json b/packages/_infra/package.json index 909d0dc7f3..c98b247584 100644 --- a/packages/_infra/package.json +++ b/packages/_infra/package.json @@ -1,6 +1,6 @@ { "name": "@basemaps/infra", - "version": "1.4.1", + "version": "1.8.0", "private": true, "repository": "git@github.com:linz/basemaps.git", "author": "", @@ -14,20 +14,20 @@ "git-rev-sync": "^2.0.0" }, "devDependencies": { - "@aws-cdk/aws-batch": "^1.34.1", - "@aws-cdk/aws-certificatemanager": "^1.34.1", - "@aws-cdk/aws-cloudfront": "^1.34.1", - "@aws-cdk/aws-dynamodb": "^1.34.1", - "@aws-cdk/aws-ecr": "^1.34.1", - "@aws-cdk/aws-ecr-assets": "^1.34.1", - "@aws-cdk/aws-elasticloadbalancingv2": "^1.34.1", - "@aws-cdk/aws-elasticloadbalancingv2-targets": "^1.34.1", - "@aws-cdk/aws-s3": "^1.34.1", - "@aws-cdk/core": "^1.34.1", - "@basemaps/lambda-api-tracker": "^1.4.1", - "@basemaps/lambda-shared": "^1.4.1", - "@basemaps/lambda-xyz": "^1.4.1", + "@aws-cdk/aws-batch": "^1.37.0", + "@aws-cdk/aws-certificatemanager": "^1.37.0", + "@aws-cdk/aws-cloudfront": "^1.37.0", + "@aws-cdk/aws-dynamodb": "^1.37.0", + "@aws-cdk/aws-ecr": "^1.37.0", + "@aws-cdk/aws-ecr-assets": "^1.37.0", + "@aws-cdk/aws-elasticloadbalancingv2": "^1.37.0", + "@aws-cdk/aws-elasticloadbalancingv2-targets": "^1.37.0", + "@aws-cdk/aws-s3": "^1.37.0", + "@aws-cdk/core": "^1.37.0", + "@basemaps/lambda-api-tracker": "^1.8.0", + "@basemaps/lambda-shared": "^1.8.0", + "@basemaps/lambda-xyz": "^1.8.0", "@types/git-rev-sync": "^1.12.0", - "aws-cdk": "^1.34.1" + "aws-cdk": "^1.37.0" } } diff --git a/packages/_infra/src/cogify/index.ts b/packages/_infra/src/cogify/index.ts index e3099223ec..6b3b18b275 100644 --- a/packages/_infra/src/cogify/index.ts +++ b/packages/_infra/src/cogify/index.ts @@ -49,6 +49,12 @@ export class CogBuilderStack extends cdk.Stack { dynamoPolicy.addResources(TileMetadataTableArn.getArn(this)); batchInstanceRole.addToPolicy(dynamoPolicy); + // Since roles are passed in via the CLI we ned to assume all the roles + const stsPolicy = new iam.PolicyStatement(); + stsPolicy.addActions('sts:AssumeRole'); + stsPolicy.addAllResources(); // literally all of the roles! + batchInstanceRole.addToPolicy(stsPolicy); + new iam.CfnInstanceProfile(this, 'CogBatchInstanceProfile', { instanceProfileName: batchInstanceRole.roleName, roles: [batchInstanceRole.roleName], diff --git a/packages/cog/CHANGELOG.md b/packages/cog/CHANGELOG.md index 6fbd59fd69..ae5110ed2f 100644 --- a/packages/cog/CHANGELOG.md +++ b/packages/cog/CHANGELOG.md @@ -3,6 +3,64 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +# [1.8.0](https://github.com/linz/basemaps/compare/v1.7.0...v1.8.0) (2020-05-11) + + +### Bug Fixes + +* **cog:** add padding to projwin' ([#594](https://github.com/linz/basemaps/issues/594)) ([72a324a](https://github.com/linz/basemaps/commit/72a324a97ff470e436c3d5360954b9338ff36e59)) + + +### Features + +* support rendering different backgrounds for tiles ([#591](https://github.com/linz/basemaps/issues/591)) ([22f38f5](https://github.com/linz/basemaps/commit/22f38f555a678e6968206351d8fbb62a604da39e)) + + + + + +# [1.7.0](https://github.com/linz/basemaps/compare/v1.6.0...v1.7.0) (2020-05-10) + + +### Features + +* **cli:** support webp quality setting ([#586](https://github.com/linz/basemaps/issues/586)) ([a456404](https://github.com/linz/basemaps/commit/a456404e2774c7a7adeffd8d114c192b073106b7)) + + + + + +# [1.6.0](https://github.com/linz/basemaps/compare/v1.5.1...v1.6.0) (2020-05-08) + + +### Features + +* **cli:** submit jobs automatically to aws batch with --batch ([#583](https://github.com/linz/basemaps/issues/583)) ([6b35696](https://github.com/linz/basemaps/commit/6b356961a2f7d1497f51f69199aa038e64fbdca9)) + + + + + +# [1.5.0](https://github.com/linz/basemaps/compare/v1.4.2...v1.5.0) (2020-05-07) + + +### Bug Fixes + +* **cli:** do not error when --replace-with is not supplied ([#577](https://github.com/linz/basemaps/issues/577)) ([2c4f5dc](https://github.com/linz/basemaps/commit/2c4f5dc5f46823ce4e6f03420b9ec9fc233505ea)) +* **cli:** root quadkey causes issues with dynamodb so never use it ([#576](https://github.com/linz/basemaps/issues/576)) ([4dfa860](https://github.com/linz/basemaps/commit/4dfa86027980231514ae417ce59e94f02e78c3f6)) + + + + + +## [1.4.2](https://github.com/linz/basemaps/compare/v1.4.1...v1.4.2) (2020-05-06) + +**Note:** Version bump only for package @basemaps/cog + + + + + ## [1.4.1](https://github.com/linz/basemaps/compare/v1.4.0...v1.4.1) (2020-05-06) **Note:** Version bump only for package @basemaps/cog diff --git a/packages/cog/package.json b/packages/cog/package.json index 09c42e87c4..1b4317a3d1 100644 --- a/packages/cog/package.json +++ b/packages/cog/package.json @@ -1,6 +1,6 @@ { "name": "@basemaps/cog", - "version": "1.4.1", + "version": "1.8.0", "private": true, "repository": "git@github.com:linz/basemaps.git", "author": "", @@ -17,12 +17,12 @@ "test": "ospec --globs 'build/**/*.test.js' --preload ../../scripts/test.before.js" }, "dependencies": { - "@basemaps/geo": "^1.4.1", - "@basemaps/lambda-shared": "^1.4.1", - "@cogeotiff/core": "^1.0.3", - "@cogeotiff/source-aws": "^1.0.3", - "@cogeotiff/source-file": "^1.0.3", - "@cogeotiff/source-url": "^1.0.3", + "@basemaps/geo": "^1.4.2", + "@basemaps/lambda-shared": "^1.8.0", + "@cogeotiff/core": "^1.0.8", + "@cogeotiff/source-aws": "^1.0.8", + "@cogeotiff/source-file": "^1.0.8", + "@cogeotiff/source-url": "^1.0.8", "@turf/turf": "^5.1.6", "chalk": "^4.0.0", "p-limit": "^2.2.1", diff --git a/packages/cog/src/cli/basemaps/__test__/test.update.test.ts b/packages/cog/src/cli/basemaps/__test__/test.update.test.ts index 250606edb5..a732d7e7c8 100644 --- a/packages/cog/src/cli/basemaps/__test__/test.update.test.ts +++ b/packages/cog/src/cli/basemaps/__test__/test.update.test.ts @@ -1,17 +1,18 @@ +/* eslint-disable @typescript-eslint/camelcase */ import * as o from 'ospec'; -import { TileSetUpdateAction } from '../action.tileset.update'; +import { TileSetUpdateAction, parseRgba } from '../action.tileset.update'; import { TileMetadataSetRecord, LogConfig, Aws } from '@basemaps/lambda-shared'; function fakeTileSet(): TileMetadataSetRecord { - Aws.tileMetadata.Imagery.imagery.set('0', { name: '0' } as any); - Aws.tileMetadata.Imagery.imagery.set('1', { name: '1' } as any); - Aws.tileMetadata.Imagery.imagery.set('2', { name: '2' } as any); - Aws.tileMetadata.Imagery.imagery.set('3', { name: '3' } as any); + Aws.tileMetadata.Imagery.imagery.set('im_0', { name: '0' } as any); + Aws.tileMetadata.Imagery.imagery.set('im_1', { name: '1' } as any); + Aws.tileMetadata.Imagery.imagery.set('im_2', { name: '2' } as any); + Aws.tileMetadata.Imagery.imagery.set('im_3', { name: '3' } as any); return { imagery: { - '0': { id: '0', maxZoom: 32, minZoom: 0, priority: 10 }, - '1': { id: '1', maxZoom: 32, minZoom: 0, priority: 10 }, - '2': { id: '2', maxZoom: 32, minZoom: 0, priority: 100 }, + im_0: { id: 'im_0', maxZoom: 32, minZoom: 0, priority: 10 }, + im_1: { id: 'im_1', maxZoom: 32, minZoom: 0, priority: 10 }, + im_2: { id: 'im_2', maxZoom: 32, minZoom: 0, priority: 100 }, }, } as any; } @@ -43,29 +44,29 @@ o.spec('TileSetUpdateAction', () => { cmd.minZoom = { value: 0 } as any; cmd.maxZoom = { value: 30 } as any; - const hasChanges = await cmd.updateZoom(tileSet, '0'); + const hasChanges = await cmd.updateZoom(tileSet, 'im_0'); o(hasChanges).equals(true); - o(tileSet.imagery[0].maxZoom).equals(30); + o(tileSet.imagery['im_0'].maxZoom).equals(30); }); o('should not have changes when nothing changed', async () => { // No values to change - const hasChangesA = await cmd.updateZoom(tileSet, '0'); + const hasChangesA = await cmd.updateZoom(tileSet, 'im_0'); o(hasChangesA).equals(false); // Missing maxZoom cmd.minZoom = { value: 0 } as any; - const hasChangesB = await cmd.updateZoom(tileSet, '0'); + const hasChangesB = await cmd.updateZoom(tileSet, 'im_0'); o(hasChangesB).equals(false); // Valid but missing id cmd.maxZoom = { value: 0 } as any; - const hasChangesC = await cmd.updateZoom(tileSet, '-1'); + const hasChangesC = await cmd.updateZoom(tileSet, 'im_A'); o(hasChangesC).equals(false); // No changes cmd.maxZoom = { value: 32 } as any; - const hasChangesD = await cmd.updateZoom(tileSet, '0'); + const hasChangesD = await cmd.updateZoom(tileSet, 'im_0'); o(hasChangesD).equals(false); }); }); @@ -73,14 +74,14 @@ o.spec('TileSetUpdateAction', () => { o.spec('Replace', () => { o('should replace imagery', async () => { cmd.replaceImageryId = { value: '3' } as any; - const hasChanges = await cmd.replaceUpdate(tileSet, '1'); + const hasChanges = await cmd.replaceUpdate(tileSet, 'im_1'); o(hasChanges).equals(true); - o(tileSetId(tileSet)).deepEquals(['0', '3', '2']); + o(tileSetId(tileSet)).deepEquals(['im_0', 'im_3', 'im_2']); }); o('should not replace imagery if already exists', async () => { cmd.replaceImageryId = { value: '0' } as any; - const hasChanges = await cmd.replaceUpdate(tileSet, '1'); + const hasChanges = await cmd.replaceUpdate(tileSet, 'im_1'); o(hasChanges).equals(false); }); }); @@ -88,44 +89,82 @@ o.spec('TileSetUpdateAction', () => { o.spec('UpdatePriority', () => { o('should remove when priority -1', async () => { cmd.priority = { value: -1 } as any; - const hasChanges = await cmd.updatePriority(tileSet, '0'); + const hasChanges = await cmd.updatePriority(tileSet, 'im_0'); o(hasChanges).equals(true); o(Object.keys(tileSet.imagery).length).equals(2); - o(tileSetId(tileSet)).deepEquals(['1', '2']); + o(tileSetId(tileSet)).deepEquals(['im_1', 'im_2']); }); o('should insert at priority 0', async () => { cmd.priority = { value: 0 } as any; - const hasChanges = await cmd.updatePriority(tileSet, '3'); + const hasChanges = await cmd.updatePriority(tileSet, 'im_3'); o(hasChanges).equals(true); - o(tileSetId(tileSet)).deepEquals(['3', '0', '1', '2']); + o(tileSetId(tileSet)).deepEquals(['im_3', 'im_0', 'im_1', 'im_2']); }); o('should insert at priority 999', async () => { cmd.priority = { value: 999 } as any; - const hasChanges = await cmd.updatePriority(tileSet, '3'); + const hasChanges = await cmd.updatePriority(tileSet, 'im_3'); o(hasChanges).equals(true); - o(tileSetId(tileSet)).deepEquals(['0', '1', '2', '3']); + o(tileSetId(tileSet)).deepEquals(['im_0', 'im_1', 'im_2', 'im_3']); }); o('should insert at priority 10', async () => { cmd.priority = { value: 10 } as any; - const hasChanges = await cmd.updatePriority(tileSet, '3'); + const hasChanges = await cmd.updatePriority(tileSet, 'im_3'); o(hasChanges).equals(true); - o(tileSetId(tileSet)).deepEquals(['0', '1', '3', '2']); + o(tileSetId(tileSet)).deepEquals(['im_0', 'im_1', 'im_3', 'im_2']); }); o('should reorder', async () => { cmd.priority = { value: 50 } as any; - const hasChanges = await cmd.updatePriority(tileSet, '0'); + const hasChanges = await cmd.updatePriority(tileSet, 'im_0'); o(hasChanges).equals(true); - o(tileSetId(tileSet)).deepEquals(['1', '0', '2']); + o(tileSetId(tileSet)).deepEquals(['im_1', 'im_0', 'im_2']); }); o('should have no changes if not reordering', async () => { cmd.priority = { value: 10 } as any; - const hasChanges = await cmd.updatePriority(tileSet, '0'); + const hasChanges = await cmd.updatePriority(tileSet, 'im_0'); o(hasChanges).equals(false); }); }); + + o.spec('background', () => { + o('should support 0x', () => { + const colors = parseRgba('0xff00ff00'); + o(colors).deepEquals({ r: 255, g: 0, b: 255, alpha: 0 }); + }); + + o('should support smaller hex strings', () => { + o(parseRgba('0x')).deepEquals({ r: 0, g: 0, b: 0, alpha: 0 }); + o(parseRgba('0xff')).deepEquals({ r: 255, g: 0, b: 0, alpha: 0 }); + o(parseRgba('0xffff')).deepEquals({ r: 255, g: 255, b: 0, alpha: 0 }); + o(parseRgba('0xffffff')).deepEquals({ r: 255, g: 255, b: 255, alpha: 0 }); + o(parseRgba('0xffffffff')).deepEquals({ r: 255, g: 255, b: 255, alpha: 255 }); + }); + + o('should support all hex', () => { + for (let i = 0x00; i <= 0xff; i++) { + const hex = i.toString(16).padStart(2, '0'); + const colors = parseRgba(`${hex}${hex}${hex}${hex}`); + o(colors).deepEquals({ r: i, g: i, b: i, alpha: i }); + } + }); + + o('should update background', async () => { + cmd.background = { value: '0xff00ff00' } as any; + const hasChanges = await cmd.updateBackground(tileSet); + o(hasChanges).equals(true); + o(tileSet.background).deepEquals({ r: 255, g: 0, b: 255, alpha: 0 }); + }); + + o('should only update if changes background', async () => { + cmd.background = { value: '0xff00ff00' } as any; + tileSet.background = { r: 255, g: 0, b: 255, alpha: 0 }; + const hasChanges = await cmd.updateBackground(tileSet); + o(hasChanges).equals(false); + o(tileSet.background).deepEquals({ r: 255, g: 0, b: 255, alpha: 0 }); + }); + }); }); diff --git a/packages/cog/src/cli/basemaps/action.tileset.update.ts b/packages/cog/src/cli/basemaps/action.tileset.update.ts index 2aaf41c860..df9661bf5e 100644 --- a/packages/cog/src/cli/basemaps/action.tileset.update.ts +++ b/packages/cog/src/cli/basemaps/action.tileset.update.ts @@ -1,12 +1,45 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ -import { Aws, LogConfig, TileMetadataSetRecord, TileSetTag } from '@basemaps/lambda-shared'; +import { + Aws, + LogConfig, + RecordPrefix, + TileMetadataSetRecord, + TileMetadataTable, + TileSetTag, +} from '@basemaps/lambda-shared'; import { CommandLineFlagParameter, CommandLineIntegerParameter, CommandLineStringParameter, } from '@rushstack/ts-command-line'; import { TileSetBaseAction } from './tileset.action'; -import { printTileSet, invalidateCache } from './tileset.util'; +import { invalidateCache, printTileSet } from './tileset.util'; + +/** + * Parse a string as hex, return 0 on failure + * @param str string to parse + */ +function parseHex(str: string): number { + if (str == '') return 0; + const val = parseInt(str, 16); + if (isNaN(val)) return 0; + return val; +} +/** + * Parse a hexstring into RGBA + * + * Defaults to 0 if missing values + * @param str string to parse + */ +export function parseRgba(str: string): { r: number; g: number; b: number; alpha: number } { + if (str.startsWith('0x')) str = str.slice(2); + return { + r: parseHex(str.substr(0, 2)), + g: parseHex(str.substr(2, 2)), + b: parseHex(str.substr(4, 2)), + alpha: parseHex(str.substr(6, 2)), + }; +} export class TileSetUpdateAction extends TileSetBaseAction { priority: CommandLineIntegerParameter; @@ -14,6 +47,7 @@ export class TileSetUpdateAction extends TileSetBaseAction { commit: CommandLineFlagParameter; replaceImageryId: CommandLineStringParameter; + background: CommandLineStringParameter; minZoom: CommandLineIntegerParameter; maxZoom: CommandLineIntegerParameter; @@ -33,7 +67,7 @@ export class TileSetUpdateAction extends TileSetBaseAction { parameterLongName: '--imagery', parameterShortName: '-i', description: 'Imagery ID', - required: true, + required: false, }); this.priority = this.defineIntegerParameter({ @@ -43,6 +77,13 @@ export class TileSetUpdateAction extends TileSetBaseAction { required: false, }); + this.background = this.defineStringParameter({ + argumentName: 'BACKGROUND', + parameterLongName: '--background', + description: 'background color', + required: false, + }); + this.replaceImageryId = this.defineStringParameter({ argumentName: 'REPLACE_WITH', parameterLongName: '--replace-with', @@ -73,7 +114,7 @@ export class TileSetUpdateAction extends TileSetBaseAction { protected async onExecute(): Promise { const name = this.tileSet.value!; const projection = this.projection.value!; - const imgId = this.imageryId.value!; + const imgId = TileMetadataTable.prefix(RecordPrefix.Imagery, this.imageryId.value ?? ''); const tsData = await Aws.tileMetadata.TileSet.get(name, projection, TileSetTag.Head); @@ -81,16 +122,21 @@ export class TileSetUpdateAction extends TileSetBaseAction { LogConfig.get().fatal({ tileSet: name, projection }, 'Failed to find tile set'); process.exit(1); } - + const before = JSON.stringify(tsData); await Aws.tileMetadata.Imagery.getAll(tsData); - const priorityUpdate = await this.updatePriority(tsData, imgId); - const zoomUpdate = await this.updateZoom(tsData, imgId); - const replaceUpdate = await this.replaceUpdate(tsData, imgId); + if (imgId) { + await this.updatePriority(tsData, imgId); + await this.updateZoom(tsData, imgId); + await this.replaceUpdate(tsData, imgId); + } + + await this.updateBackground(tsData); + const after = JSON.stringify(tsData); await printTileSet(tsData); - if (priorityUpdate || zoomUpdate || replaceUpdate) { + if (before != after) { if (this.commit.value) { await Aws.tileMetadata.TileSet.create(tsData); await invalidateCache(name, projection, TileSetTag.Head, this.commit.value); @@ -102,12 +148,25 @@ export class TileSetUpdateAction extends TileSetBaseAction { } } + async updateBackground(tsData: TileMetadataSetRecord): Promise { + const existing = tsData.background; + const background = this.background.value; + if (background == null) return false; + const rgba = parseRgba(background); + if (rgba.r != existing?.r || rgba.g != existing?.g || rgba.b != existing?.b || rgba.alpha != existing?.alpha) { + tsData.background = rgba; + return true; + } + + return false; + } + async replaceUpdate(tsData: TileMetadataSetRecord, imgId: string): Promise { const existing = tsData.imagery[imgId]; if (existing == null) return false; - const replaceId = this.replaceImageryId.value; - if (replaceId == null) return false; + const replaceId = TileMetadataTable.prefix(RecordPrefix.Imagery, this.replaceImageryId.value ?? ''); + if (replaceId == '') return false; if (tsData.imagery[replaceId] != null) { LogConfig.get().warn({ replaceId }, 'Replacement already exists'); return false; diff --git a/packages/cog/src/cli/basemaps/tileset.util.ts b/packages/cog/src/cli/basemaps/tileset.util.ts index 8205b92732..9c0c9434a9 100644 --- a/packages/cog/src/cli/basemaps/tileset.util.ts +++ b/packages/cog/src/cli/basemaps/tileset.util.ts @@ -34,6 +34,9 @@ export async function printTileSet(tsData: TileMetadataSetRecord, printImagery = console.log(c.bold('CreatedAt:'), new Date(tsData.createdAt).toISOString()); console.log(c.bold('UpdatedAt:'), new Date(tsData.updatedAt).toISOString()); console.log(c.bold('Version:'), `v${tsData.version}`); + if (tsData.background) { + console.log(c.bold('Background'), tsData.background); + } if (printImagery) await printTileSetImagery(tsData); } diff --git a/packages/cog/src/cli/cogify/action.batch.ts b/packages/cog/src/cli/cogify/action.batch.ts index bec7fcf696..60e232a7e5 100644 --- a/packages/cog/src/cli/cogify/action.batch.ts +++ b/packages/cog/src/cli/cogify/action.batch.ts @@ -7,6 +7,7 @@ import { Aws, TileMetadataImageryRecord, TileMetadataSetRecord, + LogType, } from '@basemaps/lambda-shared'; import { CommandLineAction, CommandLineFlagParameter, CommandLineStringParameter } from '@rushstack/ts-command-line'; import * as aws from 'aws-sdk'; @@ -84,7 +85,8 @@ export class ActionBatchJob extends CommandLineAction { return `${job.id}-${job.name}-${quadKey}`; } - async batchOne( + static async batchOne( + jobPath: string, job: CogJob, batch: AWS.Batch, quadKey: string, @@ -96,11 +98,11 @@ export class ActionBatchJob extends CommandLineAction { const resDiff = 1 + Math.max(alignmentLevels - MagicAlignmentLevel, 0) * 0.25; const memory = 3900 * resDiff; - if (!isCommit || this.job?.value == null) { + if (!isCommit) { return { jobName, jobId: '', memory }; } - const commandStr = ['-V', 'cog', '--job', this.job.value, '--commit', '--quadkey', quadKey]; + const commandStr = ['-V', 'cog', '--job', jobPath, '--commit', '--quadkey', quadKey]; const batchJob = await batch .submitJob({ @@ -121,7 +123,7 @@ export class ActionBatchJob extends CommandLineAction { * List all the current jobs in batch and their statuses * @returns a map of JobName to if their status is "ok" (not failed) */ - async getCurrentJobList(batch: AWS.Batch): Promise> { + static async getCurrentJobList(batch: AWS.Batch): Promise> { // For some reason AWS only lets us query one status at a time. const allStatuses = ['SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING', 'RUNNING', 'SUCCEEDED', 'FAILED']; const allJobs = await Promise.all( @@ -146,18 +148,22 @@ export class ActionBatchJob extends CommandLineAction { if (this.job?.value == null) { throw new Error('Failed to read parameters'); } - const region = Env.get('AWS_DEFAULT_REGION', 'ap-southeast-2'); - const batch = new aws.Batch({ region }); - - const job = (await FileOperator.create(this.job.value).readJson(this.job.value)) as CogJob; const processId = ulid.ulid(); - const logger = LogConfig.get().child({ id: processId, correlationId: job.id, imageryName: job.name }); - LogConfig.set(logger); + const logger = LogConfig.get().child({ id: processId }); + + await ActionBatchJob.batchJob(this.job.value, this.commit?.value, logger); + } - const isCommit = this.commit?.value ?? false; + static async batchJob(jobPath: string, commit = false, logger: LogType): Promise { + if (!FileOperator.isS3(jobPath)) throw new Error(`AWS Batch job.json have to be in S3, jobPath:${jobPath}`); + const job = (await FileOperator.create(jobPath).readJson(jobPath)) as CogJob; + LogConfig.set(logger.child({ correlationId: job.id, imageryName: job.name })); + + const region = Env.get('AWS_DEFAULT_REGION', 'ap-southeast-2'); + const batch = new aws.Batch({ region }); const outputFs = FileOperator.create(job.output); - const runningJobs = await this.getCurrentJobList(batch); + const runningJobs = await ActionBatchJob.getCurrentJobList(batch); const stats = await Promise.all( job.quadkeys.map(async (quadKey) => { @@ -193,7 +199,7 @@ export class ActionBatchJob extends CommandLineAction { 'JobSubmit', ); - if (isCommit) { + if (commit) { const img = createImageryRecordFromJob(job); await Aws.tileMetadata.put(img); const tileMetadata: TileMetadataSetRecord = { @@ -210,11 +216,11 @@ export class ActionBatchJob extends CommandLineAction { } for (const quadKey of toSubmit) { - const jobStatus = await this.batchOne(job, batch, quadKey, isCommit); + const jobStatus = await ActionBatchJob.batchOne(jobPath, job, batch, quadKey, commit); logger.info(jobStatus, 'JobSubmitted'); } - if (!isCommit) { + if (!commit) { logger.warn('DryRun:Done'); return; } diff --git a/packages/cog/src/cli/cogify/action.job.ts b/packages/cog/src/cli/cogify/action.job.ts index 7fd97359a0..084137bbc8 100644 --- a/packages/cog/src/cli/cogify/action.job.ts +++ b/packages/cog/src/cli/cogify/action.job.ts @@ -19,6 +19,7 @@ import { Cutline } from '../../cog/cutline'; import { CogJob } from '../../cog/types'; import { GdalCogBuilderDefaults, GdalResamplingOptions } from '../../gdal/gdal.config'; import { getJobPath, makeTempFolder } from '../folder'; +import { ActionBatchJob } from './action.batch'; function filterTiff(a: string): boolean { const lowerA = a.toLowerCase(); @@ -55,14 +56,16 @@ export class CLiInputData { } export class ActionJobCreate extends CommandLineAction { - private source?: CLiInputData; - private output?: CLiInputData; - private maxConcurrency?: CommandLineIntegerParameter; - private generateVrt?: CommandLineFlagParameter; - private resampling?: CommandLineStringParameter; - private cutline?: CommandLineStringParameter; - private cutlineBlend?: CommandLineIntegerParameter; - private overrideId?: CommandLineStringParameter; + private source: CLiInputData; + private output: CLiInputData; + private maxConcurrency: CommandLineIntegerParameter; + private generateVrt: CommandLineFlagParameter; + private resampling: CommandLineStringParameter; + private cutline: CommandLineStringParameter; + private cutlineBlend: CommandLineIntegerParameter; + private overrideId: CommandLineStringParameter; + private submitBatch: CommandLineFlagParameter; + private quality: CommandLineIntegerParameter; MaxCogsDefault = 50; MaxConcurrencyDefault = 5; @@ -131,10 +134,13 @@ export class ActionJobCreate extends CommandLineAction { logger.info({ source: this.source.path.value, tiffCount: tiffList.length }, 'LoadingTiffs'); + const cutlineDefaults = Cutline.defaultCutline(imageryName); const cutlinePath = - this.cutline?.value == null ? Cutline.defaultCutline(imageryName)['path'] : this.cutline?.value; + this.cutline?.value == null ? outputConfig.path + cutlineDefaults['path'] : this.cutline.value; const cutline = cutlinePath == null ? new Cutline() : await Cutline.loadCutline(cutlinePath); + const blend = this.cutlineBlend == null ? cutlineDefaults['blend'] : this.cutlineBlend?.value; + const builder = new CogBuilder(maxConcurrency, logger); const metadata = await builder.build(tiffSource, cutline); @@ -194,7 +200,8 @@ export class ActionJobCreate extends CommandLineAction { output: { ...outputConfig, resampling, - cutlineBlend: cutline != null ? this.cutlineBlend?.value ?? 0 : undefined, + quality: this.quality.value ?? 90, + cutlineBlend: cutline != null ? blend ?? 0 : undefined, nodata: metadata.nodata, vrt: { options: vrtOptions, @@ -209,6 +216,7 @@ export class ActionJobCreate extends CommandLineAction { quadkeys, }; + const isVrtGenerated = this.generateVrt?.value == true; const tmpFolder = await makeTempFolder(`basemaps-${job.id}`); try { // Local file systems need directories to be created before writing to them @@ -217,7 +225,7 @@ export class ActionJobCreate extends CommandLineAction { } // TODO should this be done here, it could be done for each COG builder - if (this.generateVrt?.value) { + if (isVrtGenerated) { const vrtTmp = await buildVrtForTiffs(job, vrtOptions, tmpFolder, logger); const readStream = createReadStream(vrtTmp); await outputFs.write(getJobPath(job, '.vrt'), readStream, logger); @@ -237,6 +245,11 @@ export class ActionJobCreate extends CommandLineAction { const geoJsonCoveringOutput = getJobPath(job, `covering.geojson`); await outputFs.writeJson(geoJsonCoveringOutput, TileCover.toGeoJson(quadkeys), logger); + if (this.submitBatch.value) { + if (!isVrtGenerated) throw new Error('Unable to submit, no VRT generated'); + await ActionBatchJob.batchJob(jobFile, true, logger); + } + logger.info({ job: jobFile }, 'Done'); } finally { // Cleanup @@ -290,5 +303,18 @@ export class ActionJobCreate extends CommandLineAction { description: 'used mainly for debugging to create with a pre determined job id', required: false, }); + + this.submitBatch = this.defineFlagParameter({ + parameterLongName: `--batch`, + description: 'Submit the job to AWS Batch', + required: false, + }); + + this.quality = this.defineIntegerParameter({ + argumentName: 'QUALITY', + parameterLongName: '--quality', + description: 'Compression quality (0-100)', + required: false, + }); } } diff --git a/packages/cog/src/cog/__test__/cog.test.ts b/packages/cog/src/cog/__test__/cog.test.ts index 11b67d841b..96ec92e508 100644 --- a/packages/cog/src/cog/__test__/cog.test.ts +++ b/packages/cog/src/cog/__test__/cog.test.ts @@ -37,11 +37,12 @@ o.spec('cog', () => { // -projwin 18472078.003508832 -5948635.289265559 18785164.071364917 -6261721.357121641 // -projwin_srs EPSG:3857 o(gdalCogBuilder!.config).deepEquals({ - bbox: [18472078.003508832, -5948635.289265559, 18785164.071364917, -6261721.357121641], + bbox: [18472078.003508832, -5948635.289265559, 18788294.932043478, -6264852.217800202], alignmentLevels: 6, compression: 'webp', resampling: 'bilinear', blockSize: 512, + quality: 90, }); o(gdalCogBuilder!.source).equals('/tmp/test.vrt'); o(gdalCogBuilder!.target).equals('/tmp/out-tiff'); @@ -65,11 +66,13 @@ o.spec('cog', () => { 'COMPRESS=webp', '-co', 'ALIGNED_LEVELS=6', + '-co', + 'QUALITY=90', '-projwin', '18472078.003508832', '-5948635.289265559', - '18785164.071364917', - '-6261721.357121641', + '18788294.932043478', + '-6264852.217800202', '-projwin_srs', 'EPSG:3857', '/tmp/test.vrt', diff --git a/packages/cog/src/cog/__test__/cutline.test.ts b/packages/cog/src/cog/__test__/cutline.test.ts index c715fc72f4..8efcefaca9 100644 --- a/packages/cog/src/cog/__test__/cutline.test.ts +++ b/packages/cog/src/cog/__test__/cutline.test.ts @@ -1,3 +1,4 @@ +import { TileCover } from '@basemaps/geo'; import { MultiPolygon } from 'geojson'; import * as o from 'ospec'; import { Cutline, CutlineMap } from '../cutline'; @@ -82,4 +83,11 @@ o.spec('covering', () => { '313111001020', ]); }); + + o('optimize should not cover the world', () => { + const bounds = TileCover.toGeoJson(['']); + const cutline = new Cutline(); + const covering = cutline.optimizeCovering({ bounds, resolution: 0 } as SourceMetadata); + o(Array.from(covering)).deepEquals(['0', '1', '2', '3']); + }); }); diff --git a/packages/cog/src/cog/cog.ts b/packages/cog/src/cog/cog.ts index 10b9a6fb24..1a562fd7ac 100644 --- a/packages/cog/src/cog/cog.ts +++ b/packages/cog/src/cog/cog.ts @@ -54,14 +54,17 @@ export async function buildCogForQuadKey( const [ulX, ulY] = forward([left, upper]); const [lrX, lrY] = forward([right, lower]); + const padding = Math.max(Math.abs(lrY - ulY), Math.abs(lrX - ulX)) * 0.01; + const [x, y, z] = QuadKey.toXYZ(quadKey); const alignmentLevels = job.source.resolution - z; const cogBuild = new GdalCogBuilder(vrtLocation, outputTiffPath, { - bbox: [ulX, ulY, lrX, lrY], + bbox: [ulX, ulY, lrX + padding, lrY - padding], alignmentLevels, resampling: job.output.resampling, + quality: job.output.quality, }); if (cogBuild.gdal.mount) { job.source.files.forEach((f) => cogBuild.gdal.mount?.(f)); diff --git a/packages/cog/src/cog/cutline.ts b/packages/cog/src/cog/cutline.ts index 71ba3a79de..ace8ca9bbe 100644 --- a/packages/cog/src/cog/cutline.ts +++ b/packages/cog/src/cog/cutline.ts @@ -1,4 +1,4 @@ -import { Bounds, EPSG, GeoJson, Projection, QuadKeyTrie, TileCover } from '@basemaps/geo'; +import { Bounds, EPSG, GeoJson, Projection, QuadKeyTrie, TileCover, QuadKey } from '@basemaps/geo'; import { FileOperator } from '@basemaps/lambda-shared'; import bbox from '@turf/bbox'; import intersect from '@turf/intersect'; @@ -10,9 +10,9 @@ import { CogJob, SourceMetadata } from './types'; const PaddingFactor = 1.125; type CutlineItem = { path: string; blend: number }; export const CutlineMap: { [key: string]: CutlineItem } = { - sentinel: { path: 's3://basemaps-cog-test/NZCoastCutline.sentinel.geojson', blend: 5 }, - urban: { path: 's3://basemaps-cog-test/NZCoastCutline.urban.geojson', blend: 20 }, - rural: { path: 's3://basemaps-cog-test/NZCoastCutline.rural.geojson', blend: 20 }, + sentinel: { path: 'cutline/sentinel.geojson', blend: 5 }, + urban: { path: 'cutline/urban.geojson', blend: 20 }, + rural: { path: 'cutline/rural.geojson', blend: 20 }, }; function findGeoJsonProjection(geojson: any | null): EPSG { @@ -186,6 +186,9 @@ export class Cutline { covering.mergeQuadKeys(CoveringPercentage, minZ, minZ + 2); + /** We should never return a full cover, using '' as a index causes problems */ + if (covering.has('')) return QuadKeyTrie.fromList(QuadKey.children('')); + return covering; } @@ -217,9 +220,7 @@ export class Cutline { * @param imageryName the name of the dataset being cut, should contain a substring denoting dataset type; sentinel, urban, rural. */ static defaultCutline(imageryName: string): CutlineItem { - const foundNames = Object.keys(CutlineMap) - .map((cutName) => (imageryName.toLowerCase().search(cutName) > -1 ? cutName : '')) - .filter((matchedName) => !!matchedName); + const foundNames = Object.keys(CutlineMap).filter((f) => imageryName.toLowerCase().includes(f)); if (foundNames.length != 1) { throw new Error('Matched ' + foundNames.length + ' cutline names'); } diff --git a/packages/cog/src/cog/types.ts b/packages/cog/src/cog/types.ts index 3afa833b3b..24e42d99b1 100644 --- a/packages/cog/src/cog/types.ts +++ b/packages/cog/src/cog/types.ts @@ -31,6 +31,11 @@ export interface CogJob { output: { resampling: GdalCogBuilderOptionsResampling; nodata?: number; + /** + * Quality level to use + * @default 90 + */ + quality: number; cutlineBlend?: number; vrt: { options: VrtOptions; diff --git a/packages/cog/src/gdal/gdal.config.ts b/packages/cog/src/gdal/gdal.config.ts index 371bc8a94d..47c77a935d 100644 --- a/packages/cog/src/gdal/gdal.config.ts +++ b/packages/cog/src/gdal/gdal.config.ts @@ -34,6 +34,11 @@ export interface GdalCogBuilderOptions { * @default 512 */ blockSize: 256 | 512 | 1024 | 2048 | 4096; + + /** + * Compression quality + */ + quality: number; } export const GdalCogBuilderDefaults: GdalCogBuilderOptions = { @@ -41,6 +46,7 @@ export const GdalCogBuilderDefaults: GdalCogBuilderOptions = { compression: 'webp', alignmentLevels: 1, blockSize: 512, + quality: 90, }; export const GdalResamplingOptions: Record = { diff --git a/packages/cog/src/gdal/gdal.ts b/packages/cog/src/gdal/gdal.ts index db2663adbc..863844070d 100644 --- a/packages/cog/src/gdal/gdal.ts +++ b/packages/cog/src/gdal/gdal.ts @@ -58,6 +58,7 @@ export class GdalCogBuilder { compression: config.compression ?? GdalCogBuilderDefaults.compression, resampling: config.resampling ?? GdalCogBuilderDefaults.resampling, blockSize: config.blockSize ?? GdalCogBuilderDefaults.blockSize, + quality: config.quality ?? GdalCogBuilderDefaults.quality, }; this.gdal = GdalCogBuilder.getGdal(); @@ -104,6 +105,8 @@ export class GdalCogBuilder { // Number of levels to align to web mercator '-co', `ALIGNED_LEVELS=${this.config.alignmentLevels}`, + '-co', + `QUALITY=${this.config.quality}`, ...this.getBounds(), this.source, diff --git a/packages/geo/CHANGELOG.md b/packages/geo/CHANGELOG.md index de9b3587e8..95ece3ad8a 100644 --- a/packages/geo/CHANGELOG.md +++ b/packages/geo/CHANGELOG.md @@ -3,6 +3,14 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.4.2](https://github.com/linz/basemaps/compare/v1.4.1...v1.4.2) (2020-05-06) + +**Note:** Version bump only for package @basemaps/geo + + + + + ## [1.4.1](https://github.com/linz/basemaps/compare/v1.4.0...v1.4.1) (2020-05-06) **Note:** Version bump only for package @basemaps/geo diff --git a/packages/geo/package.json b/packages/geo/package.json index 79500f45e3..a8edd10c8c 100644 --- a/packages/geo/package.json +++ b/packages/geo/package.json @@ -1,6 +1,6 @@ { "name": "@basemaps/geo", - "version": "1.4.1", + "version": "1.4.2", "repository": "git@github.com:linz/basemaps.git", "author": "", "license": "MIT", diff --git a/packages/lambda-api-tracker/CHANGELOG.md b/packages/lambda-api-tracker/CHANGELOG.md index 35755f35de..7f5e62b678 100644 --- a/packages/lambda-api-tracker/CHANGELOG.md +++ b/packages/lambda-api-tracker/CHANGELOG.md @@ -3,6 +3,46 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +# [1.8.0](https://github.com/linz/basemaps/compare/v1.7.0...v1.8.0) (2020-05-11) + +**Note:** Version bump only for package @basemaps/lambda-api-tracker + + + + + +# [1.7.0](https://github.com/linz/basemaps/compare/v1.6.0...v1.7.0) (2020-05-10) + +**Note:** Version bump only for package @basemaps/lambda-api-tracker + + + + + +# [1.6.0](https://github.com/linz/basemaps/compare/v1.5.1...v1.6.0) (2020-05-08) + +**Note:** Version bump only for package @basemaps/lambda-api-tracker + + + + + +# [1.5.0](https://github.com/linz/basemaps/compare/v1.4.2...v1.5.0) (2020-05-07) + +**Note:** Version bump only for package @basemaps/lambda-api-tracker + + + + + +## [1.4.2](https://github.com/linz/basemaps/compare/v1.4.1...v1.4.2) (2020-05-06) + +**Note:** Version bump only for package @basemaps/lambda-api-tracker + + + + + ## [1.4.1](https://github.com/linz/basemaps/compare/v1.4.0...v1.4.1) (2020-05-06) **Note:** Version bump only for package @basemaps/lambda-api-tracker diff --git a/packages/lambda-api-tracker/package.json b/packages/lambda-api-tracker/package.json index 2bed958fa4..0b057a3f52 100644 --- a/packages/lambda-api-tracker/package.json +++ b/packages/lambda-api-tracker/package.json @@ -1,13 +1,13 @@ { "name": "@basemaps/lambda-api-tracker", - "version": "1.4.1", + "version": "1.8.0", "private": true, "repository": "git@github.com:linz/basemaps.git", "author": "", "license": "MIT", "dependencies": { - "@basemaps/geo": "^1.4.1", - "@basemaps/lambda-shared": "^1.4.1" + "@basemaps/geo": "^1.4.2", + "@basemaps/lambda-shared": "^1.8.0" }, "scripts": { "test": "ospec --globs 'build/**/*.test.js' --preload ../../scripts/test.before.js", diff --git a/packages/lambda-shared/CHANGELOG.md b/packages/lambda-shared/CHANGELOG.md index 02eceb9933..37de11336b 100644 --- a/packages/lambda-shared/CHANGELOG.md +++ b/packages/lambda-shared/CHANGELOG.md @@ -3,6 +3,58 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +# [1.8.0](https://github.com/linz/basemaps/compare/v1.7.0...v1.8.0) (2020-05-11) + + +### Features + +* support rendering different backgrounds for tiles ([#591](https://github.com/linz/basemaps/issues/591)) ([22f38f5](https://github.com/linz/basemaps/commit/22f38f555a678e6968206351d8fbb62a604da39e)) + + + + + +# [1.7.0](https://github.com/linz/basemaps/compare/v1.6.0...v1.7.0) (2020-05-10) + + +### Bug Fixes + +* **cli:** role assumptions must have role names shorter than 64 chars ([#585](https://github.com/linz/basemaps/issues/585)) ([d889cb7](https://github.com/linz/basemaps/commit/d889cb7666685a8c3a4c7a0816c92fe62626e2e4)) + + + + + +# [1.6.0](https://github.com/linz/basemaps/compare/v1.5.1...v1.6.0) (2020-05-08) + + +### Bug Fixes + +* **serve:** allow any tile set name to be used ([#579](https://github.com/linz/basemaps/issues/579)) ([e3e6a03](https://github.com/linz/basemaps/commit/e3e6a03e66b496ae6f9247dc9cbbb0110f5993c5)) + + + + + +# [1.5.0](https://github.com/linz/basemaps/compare/v1.4.2...v1.5.0) (2020-05-07) + + +### Bug Fixes + +* **cli:** root quadkey causes issues with dynamodb so never use it ([#576](https://github.com/linz/basemaps/issues/576)) ([4dfa860](https://github.com/linz/basemaps/commit/4dfa86027980231514ae417ce59e94f02e78c3f6)) + + + + + +## [1.4.2](https://github.com/linz/basemaps/compare/v1.4.1...v1.4.2) (2020-05-06) + +**Note:** Version bump only for package @basemaps/lambda-shared + + + + + ## [1.4.1](https://github.com/linz/basemaps/compare/v1.4.0...v1.4.1) (2020-05-06) **Note:** Version bump only for package @basemaps/lambda-shared diff --git a/packages/lambda-shared/package.json b/packages/lambda-shared/package.json index bce52d0669..ce42678d27 100644 --- a/packages/lambda-shared/package.json +++ b/packages/lambda-shared/package.json @@ -1,6 +1,6 @@ { "name": "@basemaps/lambda-shared", - "version": "1.4.1", + "version": "1.8.0", "private": true, "repository": "git@github.com:linz/basemaps.git", "author": "", @@ -11,9 +11,9 @@ "test": "ospec --globs 'build/**/*.test.js' --preload ../../scripts/test.before.js" }, "dependencies": { - "@basemaps/geo": "^1.4.1", - "@basemaps/metrics": "^1.4.1", - "@basemaps/tiler": "^1.4.1", + "@basemaps/geo": "^1.4.2", + "@basemaps/metrics": "^1.4.2", + "@basemaps/tiler": "^1.8.0", "@types/sax": "^1.2.1", "aws-sdk": "^2.508.0", "base-x": "^3.0.7", diff --git a/packages/lambda-shared/src/__test__/api.path.test.ts b/packages/lambda-shared/src/__test__/api.path.test.ts index e8e2691626..95be7968f6 100644 --- a/packages/lambda-shared/src/__test__/api.path.test.ts +++ b/packages/lambda-shared/src/__test__/api.path.test.ts @@ -40,7 +40,7 @@ o.spec('api.path', () => { o(tileFromPath(ctx.action.rest)).deepEquals({ type: TileType.Image, - tileSet: TileSetType.aerial, + name: TileSetType.aerial, projection: EPSG.Google, x: 2, y: 3, @@ -54,7 +54,7 @@ o.spec('api.path', () => { o(tileFromPath(ctx.action.rest)).deepEquals({ type: TileType.Image, - tileSet: TileSetType.aerial, + name: TileSetType.aerial, projection: EPSG.Google, x: 5, y: 6, diff --git a/packages/lambda-shared/src/api.path.ts b/packages/lambda-shared/src/api.path.ts index 98da896c3d..4519b2e3b0 100644 --- a/packages/lambda-shared/src/api.path.ts +++ b/packages/lambda-shared/src/api.path.ts @@ -10,8 +10,6 @@ export interface ActionData { export enum TileSetType { aerial = 'aerial', - aerialHead = 'aerial@head', - aerialBeta = 'aerial@beta', } export enum TileType { @@ -23,7 +21,7 @@ export type TileData = TileDataXyz | TileDataWmts; export interface TileDataXyz { type: TileType.Image; - tileSet: TileSetType; + name: string; projection: EPSG; x: number; y: number; @@ -39,11 +37,10 @@ export interface TileDataWmts { const TileSets: Record = { aerial: TileSetType.aerial, - 'aerial@head': TileSetType.aerialHead, - 'aerial@beta': TileSetType.aerialBeta, }; -function tileXyzFromPath(path: string[], tileSet: TileSetType): TileData | null { +function tileXyzFromPath(path: string[]): TileData | null { + const name = path[0]; const projection = Projection.parseEpsgString(path[1]); if (projection == null) return null; const z = parseInt(path[2], 10); @@ -56,7 +53,7 @@ function tileXyzFromPath(path: string[], tileSet: TileSetType): TileData | null const ext = getImageFormat(extStr); if (ext == null) return null; - return { type: TileType.Image, tileSet, projection, x, y, z, ext }; + return { type: TileType.Image, name, projection, x, y, z, ext }; } /** @@ -91,12 +88,11 @@ function tileWmtsFromPath(path: string[], tileSet: TileSetType): TileData | null export function tileFromPath(path: string[]): TileData | null { if (path.length < 1) return null; - const tileSet = TileSets[path[0]]; - if (tileSet == null) return null; - if (path.length == 5) { - return tileXyzFromPath(path, tileSet); + return tileXyzFromPath(path); } + const tileSet = TileSets[path[0]]; + if (tileSet == null) return null; return tileWmtsFromPath(path, tileSet); } diff --git a/packages/lambda-shared/src/aws/credentials.ts b/packages/lambda-shared/src/aws/credentials.ts index 61aa889a80..c981b52b40 100644 --- a/packages/lambda-shared/src/aws/credentials.ts +++ b/packages/lambda-shared/src/aws/credentials.ts @@ -21,7 +21,7 @@ class CredentialObjectCache extends ObjectCache { /** Generate mock ALBEvent */ @@ -95,7 +95,7 @@ o.spec('LambdaXyz', () => { const res = await handleRequest(request); o(res.status).equals(200); o(res.header('content-type')).equals('image/webp'); - o(res.header('eTaG')).equals('kOkbgX07nGYNVt4RO5HxkKxfL2/uM4UJpf1IJl9ySTk='); + o(res.header('eTaG')).equals('9Iiu/i3ZzjiLKroRycpaD5eLk0BHUHX1hUsy0CCSoIM='); o(res.getBody()).equals(rasterMockBuffer.toString('base64')); o(tileMock.calls.length).equals(1); @@ -120,12 +120,14 @@ o.spec('LambdaXyz', () => { }); o('should 304 if a tile is not modified', async () => { - const key = 'Je+AcRSzbjT8XIAe/VK/Sfh9KlDHPAmq3BkBbpnN3/Q='; + const key = 'J6AksQQEhXqW/wywDDsAGtd0OVVqOlKs6M8ViZlOU1g='; const request = req('/v1/tiles/aerial/global-mercator/0/0/0.png', 'get', { 'if-none-match': key, }); const res = await handleRequest(request); o(res.status).equals(304); + o(res.header('eTaG')).equals(undefined); + o(tileMock.calls.length).equals(1); o(rasterMock.calls.length).equals(0); diff --git a/packages/lambda-xyz/src/routes/tile.ts b/packages/lambda-xyz/src/routes/tile.ts index d9384355f2..624e008f26 100644 --- a/packages/lambda-xyz/src/routes/tile.ts +++ b/packages/lambda-xyz/src/routes/tile.ts @@ -35,6 +35,9 @@ function emptyPng(req: LambdaContext, cacheKey: string): LambdaHttpResponse { } const LoadingQueue = pLimit(Env.getNumber(Env.TiffConcurrency, 5)); +/** Background color of tiles where the tileset does not define a color */ +const DefaultBackground = { r: 0, g: 0, b: 0, alpha: 0 }; + /** Initialize the tiffs before reading */ async function initTiffs(tileSet: TileSet, qk: string, zoom: number, logger: LogType): Promise { const tiffs = await tileSet.getTiffsForQuadKey(qk, zoom); @@ -82,10 +85,10 @@ export async function Tile(req: LambdaContext, xyzData: TileDataXyz): Promise f.OutputKey == 'CloudFrontBucket'); + const bucket = stackInfo.Stacks[0].Outputs.find((f) => f.OutputKey == 'CloudFrontBucket'); if (bucket == null) throw new Error('Failed to find EdgeBucket'); const s3BucketName = bucket.OutputValue; const allBuckets = await s3.listBuckets().promise(); - const s3Bucket = allBuckets.Buckets.find(f => f.Name == s3BucketName); + const s3Bucket = allBuckets.Buckets.find((f) => f.Name == s3BucketName); if (s3Bucket == null) throw new Error('Failed to locate edge bucket in current account'); const files = await fs.readdir(DistDir); for (const fileName of files) { const fileData = await fs.readFile(`${DistDir}/${fileName}`); - const hash = crypto - .createHash('sha512') - .update(fileData) - .digest('base64'); + const hash = crypto.createHash('sha512').update(fileData).digest('base64'); - const obj = await s3.getObject({ Bucket: s3BucketName, Key: fileName }).promise(); + const oldHash = await getHash(s3BucketName, fileName); // Only upload files if they have changed - if (obj.Metadata[HashKey] == hash) { + if (oldHash == hash) { console.log('Skipped', fileName, `${(fileData.byteLength / 1024).toFixed(2)}Kb`, hash); continue; } diff --git a/packages/landing/package.json b/packages/landing/package.json index 1f75320093..fc3f43e808 100644 --- a/packages/landing/package.json +++ b/packages/landing/package.json @@ -1,6 +1,6 @@ { "name": "@basemaps/landing", - "version": "1.4.1", + "version": "1.8.0", "private": true, "repository": "git@github.com:linz/basemaps.git", "author": "", @@ -12,7 +12,7 @@ "deploy:deploy": "node deploy.js" }, "dependencies": { - "@basemaps/infra": "^1.4.1" + "@basemaps/infra": "^1.8.0" }, "devDependencies": { "mime-types": "^2.1.26" diff --git a/packages/landing/static/index.html b/packages/landing/static/index.html index ddcfb9c8d2..398d831e25 100644 --- a/packages/landing/static/index.html +++ b/packages/landing/static/index.html @@ -37,7 +37,9 @@