From 5befc6f7ad8b03254a645778c72732a24c21b0d9 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 21 Feb 2024 10:56:38 +1300 Subject: [PATCH 1/2] Update the chunkd verison for the fix, and allow trailing slash uri --- package-lock.json | 42 +++++++++---------- .../src/json/__tests__/config.loader.test.ts | 16 +++++++ .../config-loader/src/json/json.config.ts | 9 +++- packages/lambda-tiler/package.json | 2 +- packages/shared/package.json | 4 +- 5 files changed, 48 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index 317e2d47c..ae5ea33e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1457,36 +1457,24 @@ } }, "node_modules/@chunkd/fs-aws": { - "version": "11.2.1", - "resolved": "https://registry.npmjs.org/@chunkd/fs-aws/-/fs-aws-11.2.1.tgz", - "integrity": "sha512-dGoyQlq1Hox+NAwenZmXcnMew1wkKwJDNZ/FUJnywKRWgR3aAOpFmwtJ5hrcJGtaRbq89NJseigncUvyBOkolA==", + "version": "11.2.2", + "resolved": "https://registry.npmjs.org/@chunkd/fs-aws/-/fs-aws-11.2.2.tgz", + "integrity": "sha512-zTF0sVXHkzHIOZWwav4Qe1AcdTCCHgY5fyd9IWd7XjZtOjgYwlakwKV6GnkRkhD5QqAjTW/SVJRLca8Z7T1snw==", "dependencies": { "@aws-sdk/client-s3": "*", "@aws-sdk/credential-providers": "*", "@aws-sdk/lib-storage": "*", "@chunkd/fs": "11.2.0", - "@chunkd/source-aws": "11.0.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@chunkd/fs-aws/node_modules/@chunkd/source-aws": { - "version": "11.0.2", - "resolved": "https://registry.npmjs.org/@chunkd/source-aws/-/source-aws-11.0.2.tgz", - "integrity": "sha512-Z+yJbJg7GISTD/vHsfyvDa0rAdfCd2xbV7RNm9hrgR0drmNgBoFxl1WBobtdOKFh4/Du3w0veJBgVRmSrR5tMA==", - "dependencies": { - "@aws-sdk/client-s3": "*", - "@chunkd/source": "^11.1.0" + "@chunkd/source-aws": "11.0.3" }, "engines": { "node": ">=16.0.0" } }, "node_modules/@chunkd/middleware": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/@chunkd/middleware/-/middleware-11.0.1.tgz", - "integrity": "sha512-9Q0WyHH1UvUmAP4ly9lTvXgaRZwITFb4yZh8OV9wC7nrhS7RqhpwQExiz2SOos7vBpLEvRY+EEK4SEGuq7sgdA==", + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@chunkd/middleware/-/middleware-11.1.0.tgz", + "integrity": "sha512-xLEoDWDG3djf/plcsEsm0q6eAD3fFC/nJ6QV9234a+DE4iN8xuaqdHqtk9CRC877YfCpWZ1RDw3T62OUeqyWqQ==", "dependencies": { "@chunkd/source": "^11.1.0" }, @@ -1502,6 +1490,18 @@ "node": ">=16.0.0" } }, + "node_modules/@chunkd/source-aws": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/@chunkd/source-aws/-/source-aws-11.0.3.tgz", + "integrity": "sha512-zWWV39cOKdTmgiSW1PiNJLRM9PoxpaBkMD/z2Wn+8c8JnLt4NNILsdne0qrl722xc4vcFar0xXFdAxy9Stlolg==", + "dependencies": { + "@aws-sdk/client-s3": "*", + "@chunkd/source": "^11.1.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/@chunkd/source-file": { "version": "11.0.1", "resolved": "https://registry.npmjs.org/@chunkd/source-file/-/source-file-11.0.1.tgz", @@ -20023,8 +20023,8 @@ "@basemaps/geo": "^7.0.0", "@basemaps/tiler": "^7.0.0", "@chunkd/fs": "^11.2.0", - "@chunkd/fs-aws": "^11.2.1", - "@chunkd/middleware": "^11.0.1", + "@chunkd/fs-aws": "^11.2.2", + "@chunkd/middleware": "^11.1.0", "@cogeotiff/core": "^9.0.3", "@cotar/core": "^6.0.1", "@linzjs/geojson": "^7.0.0", diff --git a/packages/config-loader/src/json/__tests__/config.loader.test.ts b/packages/config-loader/src/json/__tests__/config.loader.test.ts index 4b0ada555..e5320a8a4 100644 --- a/packages/config-loader/src/json/__tests__/config.loader.test.ts +++ b/packages/config-loader/src/json/__tests__/config.loader.test.ts @@ -6,6 +6,7 @@ import { fsa, FsMemory, SourceMemory, Tiff } from '@basemaps/shared'; import { getTiffTagSize, TiffTag } from '@cogeotiff/core'; import { getImageryName, initConfigFromUrls } from '../tiff.config.js'; +import { matchUri } from '../json.config.js'; const simpleTiff = new URL('../../../../__tests__/static/rgba8_tiled.tiff', import.meta.url); @@ -152,4 +153,19 @@ describe('config import', () => { '05-ecan-banks-peninsula-original-9mjdj', ); }); + + it('should match uris with trailing slashes or macros', () => { + assert( + matchUri( + 's3://linz-basemaps/3857/ōtorohanga_urban_2021_0-1m_RGB/01FYWKATAEK2ZTJQ2PX44Y0XNT/', + 's3://linz-basemaps/3857/%C5%8Dtorohanga_urban_2021_0-1m_RGB/01FYWKATAEK2ZTJQ2PX44Y0XNT/', + ), + ); + assert( + matchUri( + 's3://linz-basemaps/3857/ōtorohanga_urban_2021_0-1m_RGB/01FYWKATAEK2ZTJQ2PX44Y0XNT/', + 's3://linz-basemaps/3857/ōtorohanga_urban_2021_0-1m_RGB/01FYWKATAEK2ZTJQ2PX44Y0XNT', + ), + ); + }); }); diff --git a/packages/config-loader/src/json/json.config.ts b/packages/config-loader/src/json/json.config.ts index 0a3cff8d9..2d34c4ee0 100644 --- a/packages/config-loader/src/json/json.config.ts +++ b/packages/config-loader/src/json/json.config.ts @@ -34,6 +34,13 @@ function isTiff(u: URL): boolean { return filePath.endsWith('.tiff') || filePath.endsWith('.tif'); } +export function matchUri(a: string, b: string): boolean { + const UrlA = new URL(a.endsWith('/') ? a : a + '/'); + const UrlB = new URL(b.endsWith('/') ? b : b + '/'); + if (UrlA.href === UrlB.href) return true; + return false; +} + export function guessIdFromUri(uri: string): string | null { const parts = uri.split('/'); const id = uri.endsWith('/') ? parts.at(-2) : parts.at(-1); @@ -199,7 +206,7 @@ export class ConfigJson { ): string | undefined { if (uri == null) return uri; if (uri.startsWith(ConfigPrefix.Imagery)) return uri; - const record = imagery.find((f) => f.uri === uri); /// + const record = imagery.find((f) => matchUri(f.uri, uri)); /// if (record == null) throw new Error('Unable to find imagery id for uri:' + uri); if (record.title && record.title !== layer.title) { diff --git a/packages/lambda-tiler/package.json b/packages/lambda-tiler/package.json index 775e6ce58..9644a10f7 100644 --- a/packages/lambda-tiler/package.json +++ b/packages/lambda-tiler/package.json @@ -48,8 +48,8 @@ "bundle": "./bundle.sh" }, "devDependencies": { - "@chunkd/fs": "^11.2.0", "@basemaps/attribution": "^7.0.0", + "@chunkd/fs": "^11.2.0", "@types/aws-lambda": "^8.10.75", "@types/pixelmatch": "^5.0.0", "pretty-json-log": "^1.0.0" diff --git a/packages/shared/package.json b/packages/shared/package.json index 1df876e1f..68c017ea5 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -30,8 +30,8 @@ "@basemaps/geo": "^7.0.0", "@basemaps/tiler": "^7.0.0", "@chunkd/fs": "^11.2.0", - "@chunkd/fs-aws": "^11.2.1", - "@chunkd/middleware": "^11.0.1", + "@chunkd/fs-aws": "^11.2.2", + "@chunkd/middleware": "^11.1.0", "@cogeotiff/core": "^9.0.3", "@cotar/core": "^6.0.1", "@linzjs/geojson": "^7.0.0", From 2662c49abbaae43acd8564920703e87aea354e8d Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 21 Feb 2024 11:03:03 +1300 Subject: [PATCH 2/2] Fix format --- packages/config-loader/src/json/__tests__/config.loader.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/config-loader/src/json/__tests__/config.loader.test.ts b/packages/config-loader/src/json/__tests__/config.loader.test.ts index e5320a8a4..25a0f3209 100644 --- a/packages/config-loader/src/json/__tests__/config.loader.test.ts +++ b/packages/config-loader/src/json/__tests__/config.loader.test.ts @@ -5,8 +5,8 @@ import { ConfigProviderMemory } from '@basemaps/config'; import { fsa, FsMemory, SourceMemory, Tiff } from '@basemaps/shared'; import { getTiffTagSize, TiffTag } from '@cogeotiff/core'; -import { getImageryName, initConfigFromUrls } from '../tiff.config.js'; import { matchUri } from '../json.config.js'; +import { getImageryName, initConfigFromUrls } from '../tiff.config.js'; const simpleTiff = new URL('../../../../__tests__/static/rgba8_tiled.tiff', import.meta.url);