From b9790bbf432371d86904fd5debe0a98879d14fb9 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Sun, 21 May 2023 10:53:43 +1200 Subject: [PATCH 01/14] Create a all tileset from imagery configs. --- .../memory/__tests__/memory.config.test.ts | 16 +++++++++-- packages/config/src/memory/memory.config.ts | 28 ++++++++++++++++++- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/packages/config/src/memory/__tests__/memory.config.test.ts b/packages/config/src/memory/__tests__/memory.config.test.ts index e07d54389..e93a952fd 100644 --- a/packages/config/src/memory/__tests__/memory.config.test.ts +++ b/packages/config/src/memory/__tests__/memory.config.test.ts @@ -64,9 +64,13 @@ o.spec('MemoryConfig', () => { config.createVirtualTileSets(); const cfg = config.toJson(); - o(cfg.tileSet.length).equals(2); + o(cfg.tileSet.length).equals(3); o(cfg.tileSet[0].id).equals('ts_Image123'); o(cfg.tileSet[1].id).equals('ts_ōtorohanga-urban-2021-0.1m'); + o(cfg.tileSet[2].id).equals('ts_all'); + const allTileSet = cfg.tileSet[2]; + o(allTileSet.layers.length).equals(1); + o(allTileSet.layers[0].name).equals('ōtorohanga-urban-2021-0.1m'); }); o('should create virtual tilesets by name', async () => { @@ -94,10 +98,16 @@ o.spec('MemoryConfig', () => { const cfg = config.toJson(); // 1 tileset per imagery id (2x) // 1 tileset per imagery name (1x) - o(cfg.tileSet.length).equals(3); + o(cfg.tileSet.length).equals(4); o(cfg.tileSet[0].id).equals('ts_Image123'); o(cfg.tileSet[1].id).equals('ts_ōtorohanga-urban-2021-0.1m'); o(cfg.tileSet[2].id).equals('ts_Image234'); + o(cfg.tileSet[3].id).equals('ts_all'); + o(cfg.tileSet[3].layers.length).equals(1); + o(cfg.tileSet[3].layers[0][2193]).equals('im_Image234'); + o(cfg.tileSet[3].layers[0][3857]).equals('im_Image123'); + o(cfg.tileSet[3].layers[0].maxZoom).equals(undefined); + o(cfg.tileSet[3].layers[0].minZoom).equals(32); }); o('virtual tilesets should overwrite existing projections', async () => { @@ -135,7 +145,6 @@ o.spec('MemoryConfig', () => { config.put({ ...baseImg, id: 'im_image-3857', projection: 3857 } as ConfigImagery); o(config.toJson().tileSet.length).equals(1); - config.createVirtualTileSets(); const tileSets = config.toJson().tileSet.map((c) => c.id); @@ -146,6 +155,7 @@ o.spec('MemoryConfig', () => { 'ts_image-2193', // By image id 'ts_ōtorohanga-urban-2021-0.1m', // By name 'ts_image-3857', // By image id + 'ts_all', // By image id ]); const target = await config.TileSet.get('ts_aerial:ōtorohanga_urban_2021_0-1m_RGB'); diff --git a/packages/config/src/memory/memory.config.ts b/packages/config/src/memory/memory.config.ts index a38c34d0a..edcdc61bb 100644 --- a/packages/config/src/memory/memory.config.ts +++ b/packages/config/src/memory/memory.config.ts @@ -105,6 +105,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { /** Find all imagery inside this configuration and create a virtual tile set for it */ createVirtualTileSets(): void { + const allLayers: ConfigLayer[] = []; for (const obj of this.objects.values()) { // Limit child tileset generation to `aerial` layers only if (isConfigTileSet(obj) && obj.name === 'aerial') { @@ -115,9 +116,34 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { } else if (isConfigImagery(obj)) { // TODO should this really overwrite existing tilesets this.put(ConfigProviderMemory.imageryToTileSet(obj)); - this.imageryToTileSetByName(obj); + const tileSet = this.imageryToTileSetByName(obj); + allLayers.push(tileSet.layers[0]); } } + // Create an all tileset contains all raster layers + if (allLayers.length) this.createVirtualAllTileSet(allLayers); + } + + createVirtualAllTileSet(layers: ConfigLayer[]): void { + const layerByName = new Map(); + // Set all layers as minZoom:32 + for (const l of layers) { + delete l.maxZoom; + l.minZoom = 32; + // TODO: This might overwrite the layer id for duplicated configImagery + if (layerByName.has(l.name)) layerByName.set(l.name, { ...layerByName.get(l.name), ...l }); + else layerByName.set(l.name, l); + } + const allTileset: ConfigTileSet = { + type: TileSetType.Raster, + id: 'ts_all', + name: 'all_imagery', + title: 'All Imagery Basemaps', + category: 'Basemaps', + format: ImageFormat.Webp, + layers: Array.from(layerByName.values()), + }; + this.put(allTileset); } /** Create a tileset by the standardized name */ From 57e406b061aa85c714d7ca837af6b093ed3c9652 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Sun, 21 May 2023 10:54:47 +1200 Subject: [PATCH 02/14] Remove unused comment --- packages/config/src/memory/__tests__/memory.config.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/config/src/memory/__tests__/memory.config.test.ts b/packages/config/src/memory/__tests__/memory.config.test.ts index e93a952fd..b1bf70890 100644 --- a/packages/config/src/memory/__tests__/memory.config.test.ts +++ b/packages/config/src/memory/__tests__/memory.config.test.ts @@ -155,7 +155,7 @@ o.spec('MemoryConfig', () => { 'ts_image-2193', // By image id 'ts_ōtorohanga-urban-2021-0.1m', // By name 'ts_image-3857', // By image id - 'ts_all', // By image id + 'ts_all', ]); const target = await config.TileSet.get('ts_aerial:ōtorohanga_urban_2021_0-1m_RGB'); From 1984f71a82f1a6d6cc39e549d7c4be7ed067a36b Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Tue, 30 May 2023 09:16:05 +1200 Subject: [PATCH 03/14] Clone the layer before modify --- packages/config/src/memory/memory.config.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/config/src/memory/memory.config.ts b/packages/config/src/memory/memory.config.ts index edcdc61bb..809e208b1 100644 --- a/packages/config/src/memory/memory.config.ts +++ b/packages/config/src/memory/memory.config.ts @@ -128,11 +128,13 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { const layerByName = new Map(); // Set all layers as minZoom:32 for (const l of layers) { - delete l.maxZoom; - l.minZoom = 32; + const newLayer = { ...l }; + delete newLayer.maxZoom; + newLayer.minZoom = 32; // TODO: This might overwrite the layer id for duplicated configImagery - if (layerByName.has(l.name)) layerByName.set(l.name, { ...layerByName.get(l.name), ...l }); - else layerByName.set(l.name, l); + if (layerByName.has(newLayer.name)) + layerByName.set(newLayer.name, { ...layerByName.get(newLayer.name), ...newLayer }); + else layerByName.set(newLayer.name, newLayer); } const allTileset: ConfigTileSet = { type: TileSetType.Raster, From b99f200765f9901f1aecebff240ea50eb8567d2d Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 31 May 2023 14:09:16 +1200 Subject: [PATCH 04/14] Stop using delete in js. --- packages/config/src/memory/memory.config.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/config/src/memory/memory.config.ts b/packages/config/src/memory/memory.config.ts index 809e208b1..eba63f28f 100644 --- a/packages/config/src/memory/memory.config.ts +++ b/packages/config/src/memory/memory.config.ts @@ -128,9 +128,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { const layerByName = new Map(); // Set all layers as minZoom:32 for (const l of layers) { - const newLayer = { ...l }; - delete newLayer.maxZoom; - newLayer.minZoom = 32; + const newLayer = { ...l, maxZoom: undefined, minZoom: 32 }; // TODO: This might overwrite the layer id for duplicated configImagery if (layerByName.has(newLayer.name)) layerByName.set(newLayer.name, { ...layerByName.get(newLayer.name), ...newLayer }); From 8667de26de8144746d34ff4b203f3189f9a18d18 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 31 May 2023 15:01:43 +1200 Subject: [PATCH 05/14] Compare the timestamps and set the newer to overwriting it. --- packages/config/src/memory/memory.config.ts | 25 ++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/config/src/memory/memory.config.ts b/packages/config/src/memory/memory.config.ts index eba63f28f..42f713517 100644 --- a/packages/config/src/memory/memory.config.ts +++ b/packages/config/src/memory/memory.config.ts @@ -124,15 +124,34 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { if (allLayers.length) this.createVirtualAllTileSet(allLayers); } + /** Get the last id from the s3 path and compare to get the latest id based on the timestamp */ + findLatestId(pathA: string, pathB: string): string { + const idA = pathA.split('/')[-1]; + const idB = pathB.split('/')[-1]; + const timeA = decodeTime(idA); + const timeB = decodeTime(idB); + if (timeA >= timeB) return idA; + else return idB; + } + createVirtualAllTileSet(layers: ConfigLayer[]): void { const layerByName = new Map(); // Set all layers as minZoom:32 for (const l of layers) { const newLayer = { ...l, maxZoom: undefined, minZoom: 32 }; // TODO: This might overwrite the layer id for duplicated configImagery - if (layerByName.has(newLayer.name)) - layerByName.set(newLayer.name, { ...layerByName.get(newLayer.name), ...newLayer }); - else layerByName.set(newLayer.name, newLayer); + const existing = layerByName.get(newLayer.name); + if (existing) { + if (existing[2193] && newLayer[2193]) { + layerByName.set(newLayer.name, { ...existing, '2193': this.findLatestId(existing[2193], newLayer[2193]) }); + } else if (existing[3857] && newLayer[3857]) { + layerByName.set(newLayer.name, { ...existing, '3857': this.findLatestId(existing[3857], newLayer[3857]) }); + } else { + layerByName.set(newLayer.name, { ...existing, ...newLayer }); + } + } else { + layerByName.set(newLayer.name, newLayer); + } } const allTileset: ConfigTileSet = { type: TileSetType.Raster, From 15eb4d6e2d7956995d5f5ccca2c53432199b447b Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 31 May 2023 17:49:44 +1200 Subject: [PATCH 06/14] Later imagery always overwrite earlier ones. --- .../memory/__tests__/memory.config.test.ts | 77 ++++++++++++------- packages/config/src/memory/memory.config.ts | 44 +++++------ 2 files changed, 72 insertions(+), 49 deletions(-) diff --git a/packages/config/src/memory/__tests__/memory.config.test.ts b/packages/config/src/memory/__tests__/memory.config.test.ts index b1bf70890..884e4795e 100644 --- a/packages/config/src/memory/__tests__/memory.config.test.ts +++ b/packages/config/src/memory/__tests__/memory.config.test.ts @@ -3,31 +3,35 @@ import { BaseConfig } from '../../config/base.js'; import { ConfigImagery } from '../../config/imagery.js'; import { ConfigTileSetRaster } from '../../config/tile.set.js'; import { ConfigProviderMemory } from '../memory.config.js'; +import { ulid } from 'ulid'; o.spec('MemoryConfig', () => { const config = new ConfigProviderMemory(); o.beforeEach(() => config.objects.clear()); + const id = ulid(); + const imId = `im_${id}`; + const tsId = `ts_${id}`; - const baseImg = { id: 'im_Image123', name: 'ōtorohanga_urban_2021_0-1m_RGB', projection: 3857 } as ConfigImagery; - const baseTs = { id: 'ts_TileSet123', description: 'tileset' } as ConfigTileSetRaster; + const baseImg = { id: imId, name: 'ōtorohanga_urban_2021_0-1m_RGB', projection: 3857 } as ConfigImagery; + const baseTs = { id: tsId, description: 'tileset' } as ConfigTileSetRaster; o('should load correct objects from memory', async () => { config.put(baseTs); config.put(baseImg); - const img = await config.Imagery.get('Image123'); - o(img?.id).equals('im_Image123'); + const img = await config.Imagery.get(imId); + o(img?.id).equals(imId); - const ts = await config.TileSet.get('TileSet123'); - o(ts?.id).equals('ts_TileSet123'); + const ts = await config.TileSet.get(tsId); + o(ts?.id).equals(tsId); o(ts?.description).equals('tileset'); }); o('should support prefixed keys', async () => { config.put(baseImg); - const img = await config.Imagery.get('im_Image123'); - o(img?.id).equals('im_Image123'); + const img = await config.Imagery.get(imId); + o(img?.id).equals(imId); }); o('should not find objects', async () => { @@ -65,7 +69,7 @@ o.spec('MemoryConfig', () => { const cfg = config.toJson(); o(cfg.tileSet.length).equals(3); - o(cfg.tileSet[0].id).equals('ts_Image123'); + o(cfg.tileSet[0].id).equals(tsId); o(cfg.tileSet[1].id).equals('ts_ōtorohanga-urban-2021-0.1m'); o(cfg.tileSet[2].id).equals('ts_all'); const allTileSet = cfg.tileSet[2]; @@ -73,9 +77,11 @@ o.spec('MemoryConfig', () => { o(allTileSet.layers[0].name).equals('ōtorohanga-urban-2021-0.1m'); }); + const newId = ulid(); + const newImId = `im_${newId}`; o('should create virtual tilesets by name', async () => { config.put(baseImg); - config.put({ ...baseImg, projection: 2193, id: 'im_Image234' } as ConfigImagery); + config.put({ ...baseImg, projection: 2193, id: newImId } as ConfigImagery); o(config.toJson().tileSet.length).equals(0); config.createVirtualTileSets(); @@ -83,13 +89,13 @@ o.spec('MemoryConfig', () => { const target = await config.TileSet.get('ōtorohanga-urban-2021-0.1m'); o(target?.layers.length).equals(1); o(target?.layers[0][3857]).equals(baseImg.id); - o(target?.layers[0][2193]).equals('im_Image234'); + o(target?.layers[0][2193]).equals(newImId); o(target?.name).equals('ōtorohanga-urban-2021-0.1m'); }); o('virtual tilesets can be called multiple times', () => { config.put(baseImg); - config.put({ ...baseImg, projection: 2193, id: 'im_Image234' } as ConfigImagery); + config.put({ ...baseImg, projection: 2193, id: newImId } as ConfigImagery); config.createVirtualTileSets(); config.createVirtualTileSets(); @@ -99,20 +105,20 @@ o.spec('MemoryConfig', () => { // 1 tileset per imagery id (2x) // 1 tileset per imagery name (1x) o(cfg.tileSet.length).equals(4); - o(cfg.tileSet[0].id).equals('ts_Image123'); + o(cfg.tileSet[0].id).equals(tsId); o(cfg.tileSet[1].id).equals('ts_ōtorohanga-urban-2021-0.1m'); - o(cfg.tileSet[2].id).equals('ts_Image234'); + o(cfg.tileSet[2].id).equals(`ts_${newId}`); o(cfg.tileSet[3].id).equals('ts_all'); o(cfg.tileSet[3].layers.length).equals(1); - o(cfg.tileSet[3].layers[0][2193]).equals('im_Image234'); - o(cfg.tileSet[3].layers[0][3857]).equals('im_Image123'); + o(cfg.tileSet[3].layers[0][2193]).equals(newImId); + o(cfg.tileSet[3].layers[0][3857]).equals(imId); o(cfg.tileSet[3].layers[0].maxZoom).equals(undefined); o(cfg.tileSet[3].layers[0].minZoom).equals(32); }); o('virtual tilesets should overwrite existing projections', async () => { config.put(baseImg); - config.put({ ...baseImg, id: 'im_Image234' } as ConfigImagery); + config.put({ ...baseImg, id: newImId } as ConfigImagery); o(config.toJson().tileSet.length).equals(0); @@ -120,12 +126,14 @@ o.spec('MemoryConfig', () => { const target = await config.TileSet.get('ts_ōtorohanga-urban-2021-0.1m'); o(target?.layers.length).equals(1); - o(target?.layers[0][3857]).equals('im_Image234'); + o(target?.layers[0][3857]).equals(newImId); o(target?.layers[0][2193]).equals(undefined); o(target?.name).equals('ōtorohanga-urban-2021-0.1m'); }); o('virtual tilesets should be created with `:`', async () => { + const idA = ulid(); + const idB = ulid(); config.objects.clear(); config.put({ ...baseTs, @@ -136,13 +144,13 @@ o.spec('MemoryConfig', () => { name: baseImg.name, title: '', category: '', - 2193: 'im_image-2193', - 3857: 'im_image-3857', + 2193: `im_${idA}`, + 3857: `im_${idB}`, }, ], } as BaseConfig); - config.put({ ...baseImg, id: 'im_image-2193', projection: 2193 } as ConfigImagery); - config.put({ ...baseImg, id: 'im_image-3857', projection: 3857 } as ConfigImagery); + config.put({ ...baseImg, id: `im_${idA}`, projection: 2193 } as ConfigImagery); + config.put({ ...baseImg, id: `im_${idB}`, projection: 3857 } as ConfigImagery); o(config.toJson().tileSet.length).equals(1); config.createVirtualTileSets(); @@ -152,17 +160,34 @@ o.spec('MemoryConfig', () => { o(tileSets).deepEquals([ 'ts_aerial', 'ts_aerial:ōtorohanga_urban_2021_0-1m_RGB', // deprecated by child `:` - 'ts_image-2193', // By image id + `ts_${idA}`, // By image id 'ts_ōtorohanga-urban-2021-0.1m', // By name - 'ts_image-3857', // By image id + `ts_${idB}`, // By image id 'ts_all', ]); const target = await config.TileSet.get('ts_aerial:ōtorohanga_urban_2021_0-1m_RGB'); o(target?.layers.length).equals(1); - o(target?.layers[0][3857]).equals('im_image-3857'); - o(target?.layers[0][2193]).equals('im_image-2193'); + o(target?.layers[0][3857]).equals(`im_${idB}`); + o(target?.layers[0][2193]).equals(`im_${idA}`); // the name should be mapped back to the expected name so tiles will be served via the same endpoints as by name o(target?.name).equals('ōtorohanga-urban-2021-0.1m'); }); + + o('The latest imagery should overwrite the old ones', async () => { + const idLater = ulid(); + const idLatest = ulid(); + config.put(baseImg); + config.put({ ...baseImg, id: `im_${idLater}` } as ConfigImagery); + config.put({ ...baseImg, id: `im_${idLatest}` } as ConfigImagery); + + o(config.toJson().imagery.length).equals(3); + + config.createVirtualTileSets(); + const target = await config.TileSet.get('ts_ōtorohanga-urban-2021-0.1m'); + o(target?.layers.length).equals(1); + o(target?.layers[0][3857]).equals(`im_${idLatest}`); + o(target?.layers[0][2193]).equals(undefined); + o(target?.name).equals('ōtorohanga-urban-2021-0.1m'); + }); }); diff --git a/packages/config/src/memory/memory.config.ts b/packages/config/src/memory/memory.config.ts index 42f713517..05863dd5d 100644 --- a/packages/config/src/memory/memory.config.ts +++ b/packages/config/src/memory/memory.config.ts @@ -114,7 +114,6 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { this.imageryToChildTileSet(obj, layer, EpsgCode.Google); } } else if (isConfigImagery(obj)) { - // TODO should this really overwrite existing tilesets this.put(ConfigProviderMemory.imageryToTileSet(obj)); const tileSet = this.imageryToTileSetByName(obj); allLayers.push(tileSet.layers[0]); @@ -125,13 +124,18 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { } /** Get the last id from the s3 path and compare to get the latest id based on the timestamp */ - findLatestId(pathA: string, pathB: string): string { - const idA = pathA.split('/')[-1]; - const idB = pathB.split('/')[-1]; - const timeA = decodeTime(idA); - const timeB = decodeTime(idB); - if (timeA >= timeB) return idA; - else return idB; + findLatestId(idA: string, idB: string): string { + const ulidA = ConfigId.unprefix(ConfigPrefix.Imagery, idA); + const ulidB = ConfigId.unprefix(ConfigPrefix.Imagery, idB); + try { + const timeA = decodeTime(ulidA); + const timeB = decodeTime(ulidB); + if (timeA >= timeB) return idA; + else return idB; + } finally { + //If not ulid return the idB + return idB; + } } createVirtualAllTileSet(layers: ConfigLayer[]): void { @@ -139,19 +143,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { // Set all layers as minZoom:32 for (const l of layers) { const newLayer = { ...l, maxZoom: undefined, minZoom: 32 }; - // TODO: This might overwrite the layer id for duplicated configImagery - const existing = layerByName.get(newLayer.name); - if (existing) { - if (existing[2193] && newLayer[2193]) { - layerByName.set(newLayer.name, { ...existing, '2193': this.findLatestId(existing[2193], newLayer[2193]) }); - } else if (existing[3857] && newLayer[3857]) { - layerByName.set(newLayer.name, { ...existing, '3857': this.findLatestId(existing[3857], newLayer[3857]) }); - } else { - layerByName.set(newLayer.name, { ...existing, ...newLayer }); - } - } else { - layerByName.set(newLayer.name, newLayer); - } + layerByName.set(newLayer.name, { ...layerByName.get(l.name), ...newLayer }); } const allTileset: ConfigTileSet = { type: TileSetType.Raster, @@ -185,8 +177,14 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { removeUndefined(existing); this.put(existing); } - // TODO this overwrites existing layers - existing.layers[0][i.projection] = i.id; + // The latest imagery overwrite the earlier ones. + const existingImageryId = existing.layers[0][i.projection]; + if (existingImageryId) { + existing.layers[0][i.projection] = this.findLatestId(existingImageryId, i.id); + } else { + existing.layers[0][i.projection] = i.id; + } + return existing; } From 54f58c23adda42741a972423ea7fafd377a681e4 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Tue, 8 Aug 2023 13:46:00 +1200 Subject: [PATCH 07/14] Capture duplicated imagery in the config --- packages/config/src/memory/memory.config.ts | 39 ++++++++++++--------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/packages/config/src/memory/memory.config.ts b/packages/config/src/memory/memory.config.ts index 05863dd5d..d18ff2986 100644 --- a/packages/config/src/memory/memory.config.ts +++ b/packages/config/src/memory/memory.config.ts @@ -22,6 +22,7 @@ export interface ConfigBundled { style: ConfigVectorStyle[]; provider: ConfigProvider[]; tileSet: ConfigTileSet[]; + duplicateImagery: ConfigTileSet[]; } function isConfigImagery(i: BaseConfig): i is ConfigImagery { @@ -31,6 +32,21 @@ function isConfigTileSet(i: BaseConfig): i is ConfigTileSet { return ConfigId.getPrefix(i.id) === ConfigPrefix.TileSet; } +/** Get the last id from the s3 path and compare to get the latest id based on the timestamp */ +function findLatestId(idA: string, idB: string): string { + const ulidA = ConfigId.unprefix(ConfigPrefix.Imagery, idA); + const ulidB = ConfigId.unprefix(ConfigPrefix.Imagery, idB); + try { + const timeA = decodeTime(ulidA); + const timeB = decodeTime(ulidB); + if (timeA >= timeB) return idA; + else return idB; + } finally { + //If not ulid return the return id alphabetically. + return idA.localeCompare(idB) ? idA : idB; + } +} + /** Force a unknown object into a Record type */ export function isObject(obj: unknown): obj is Record { if (typeof obj !== 'object') return false; @@ -61,6 +77,9 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { /** Asset path from the config bundle */ assets: string; + /** Catch configs with the same imagery that using the different imagery ids. */ + duplicateImagery: ConfigTileSet[]; + put(obj: BaseConfig): void { this.objects.set(obj.id, obj); } @@ -74,6 +93,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { style: [], provider: [], tileSet: [], + duplicateImagery: this.duplicateImagery, }; for (const val of this.objects.values()) { @@ -123,21 +143,6 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { if (allLayers.length) this.createVirtualAllTileSet(allLayers); } - /** Get the last id from the s3 path and compare to get the latest id based on the timestamp */ - findLatestId(idA: string, idB: string): string { - const ulidA = ConfigId.unprefix(ConfigPrefix.Imagery, idA); - const ulidB = ConfigId.unprefix(ConfigPrefix.Imagery, idB); - try { - const timeA = decodeTime(ulidA); - const timeB = decodeTime(ulidB); - if (timeA >= timeB) return idA; - else return idB; - } finally { - //If not ulid return the idB - return idB; - } - } - createVirtualAllTileSet(layers: ConfigLayer[]): void { const layerByName = new Map(); // Set all layers as minZoom:32 @@ -176,11 +181,13 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { } as ConfigTileSet; removeUndefined(existing); this.put(existing); + } else { + this.duplicateImagery.push(existing); } // The latest imagery overwrite the earlier ones. const existingImageryId = existing.layers[0][i.projection]; if (existingImageryId) { - existing.layers[0][i.projection] = this.findLatestId(existingImageryId, i.id); + existing.layers[0][i.projection] = findLatestId(existingImageryId, i.id); } else { existing.layers[0][i.projection] = i.id; } From c7f9b5f9323bcc23367b84fa9d0dfe89bd075e0c Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Tue, 8 Aug 2023 13:59:17 +1200 Subject: [PATCH 08/14] Fix broken test --- packages/config/src/memory/memory.config.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/config/src/memory/memory.config.ts b/packages/config/src/memory/memory.config.ts index d18ff2986..1e1831b61 100644 --- a/packages/config/src/memory/memory.config.ts +++ b/packages/config/src/memory/memory.config.ts @@ -39,8 +39,9 @@ function findLatestId(idA: string, idB: string): string { try { const timeA = decodeTime(ulidA); const timeB = decodeTime(ulidB); - if (timeA >= timeB) return idA; - else return idB; + if (timeA >= timeB) { + return idA; + } else return idB; } finally { //If not ulid return the return id alphabetically. return idA.localeCompare(idB) ? idA : idB; @@ -78,7 +79,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { assets: string; /** Catch configs with the same imagery that using the different imagery ids. */ - duplicateImagery: ConfigTileSet[]; + duplicateImagery: ConfigTileSet[] = []; put(obj: BaseConfig): void { this.objects.set(obj.id, obj); @@ -187,7 +188,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { // The latest imagery overwrite the earlier ones. const existingImageryId = existing.layers[0][i.projection]; if (existingImageryId) { - existing.layers[0][i.projection] = findLatestId(existingImageryId, i.id); + existing.layers[0][i.projection] = findLatestId(i.id, existingImageryId);; } else { existing.layers[0][i.projection] = i.id; } From 3a50d13a94238b8139a901a2e1de3ea950153127 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Tue, 8 Aug 2023 13:59:32 +1200 Subject: [PATCH 09/14] FFix broken test --- packages/config/src/memory/memory.config.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/config/src/memory/memory.config.ts b/packages/config/src/memory/memory.config.ts index 1e1831b61..57662e952 100644 --- a/packages/config/src/memory/memory.config.ts +++ b/packages/config/src/memory/memory.config.ts @@ -39,9 +39,7 @@ function findLatestId(idA: string, idB: string): string { try { const timeA = decodeTime(ulidA); const timeB = decodeTime(ulidB); - if (timeA >= timeB) { - return idA; - } else return idB; + if (timeA >= timeB) return idA; } finally { //If not ulid return the return id alphabetically. return idA.localeCompare(idB) ? idA : idB; @@ -188,7 +186,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { // The latest imagery overwrite the earlier ones. const existingImageryId = existing.layers[0][i.projection]; if (existingImageryId) { - existing.layers[0][i.projection] = findLatestId(i.id, existingImageryId);; + existing.layers[0][i.projection] = findLatestId(i.id, existingImageryId); } else { existing.layers[0][i.projection] = i.id; } From 71d45a61d40b30a99f03870f4b703bca9905a368 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 9 Aug 2023 14:51:25 +1200 Subject: [PATCH 10/14] Fix the place to push duplicated imagery --- packages/config/src/memory/memory.config.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/config/src/memory/memory.config.ts b/packages/config/src/memory/memory.config.ts index 57662e952..e791b7434 100644 --- a/packages/config/src/memory/memory.config.ts +++ b/packages/config/src/memory/memory.config.ts @@ -42,7 +42,7 @@ function findLatestId(idA: string, idB: string): string { if (timeA >= timeB) return idA; } finally { //If not ulid return the return id alphabetically. - return idA.localeCompare(idB) ? idA : idB; + return idA.localeCompare(idB) > 0 ? idA : idB; } } @@ -180,13 +180,12 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { } as ConfigTileSet; removeUndefined(existing); this.put(existing); - } else { - this.duplicateImagery.push(existing); } // The latest imagery overwrite the earlier ones. const existingImageryId = existing.layers[0][i.projection]; if (existingImageryId) { existing.layers[0][i.projection] = findLatestId(i.id, existingImageryId); + this.duplicateImagery.push(existing); } else { existing.layers[0][i.projection] = i.id; } From cc525880f126376224915a295468bc933c160792 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 9 Aug 2023 15:25:24 +1200 Subject: [PATCH 11/14] Add timeout between two test ulids --- packages/config/src/memory/__tests__/memory.config.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/config/src/memory/__tests__/memory.config.test.ts b/packages/config/src/memory/__tests__/memory.config.test.ts index 884e4795e..9529fb709 100644 --- a/packages/config/src/memory/__tests__/memory.config.test.ts +++ b/packages/config/src/memory/__tests__/memory.config.test.ts @@ -133,6 +133,7 @@ o.spec('MemoryConfig', () => { o('virtual tilesets should be created with `:`', async () => { const idA = ulid(); + await new Promise((resolve) => setTimeout(resolve, 5)); const idB = ulid(); config.objects.clear(); config.put({ From e0e25481e05f4baec42696618779d0d7957d0f4e Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 9 Aug 2023 15:32:19 +1200 Subject: [PATCH 12/14] correct await location. --- packages/config/src/memory/__tests__/memory.config.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/config/src/memory/__tests__/memory.config.test.ts b/packages/config/src/memory/__tests__/memory.config.test.ts index 9529fb709..8b194078e 100644 --- a/packages/config/src/memory/__tests__/memory.config.test.ts +++ b/packages/config/src/memory/__tests__/memory.config.test.ts @@ -133,7 +133,6 @@ o.spec('MemoryConfig', () => { o('virtual tilesets should be created with `:`', async () => { const idA = ulid(); - await new Promise((resolve) => setTimeout(resolve, 5)); const idB = ulid(); config.objects.clear(); config.put({ @@ -176,7 +175,9 @@ o.spec('MemoryConfig', () => { }); o('The latest imagery should overwrite the old ones', async () => { + await new Promise((resolve) => setTimeout(resolve, 5)); const idLater = ulid(); + await new Promise((resolve) => setTimeout(resolve, 5)); const idLatest = ulid(); config.put(baseImg); config.put({ ...baseImg, id: `im_${idLater}` } as ConfigImagery); From 7354175487f6509d2446485c8fe9a2ea3ab995bc Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 9 Aug 2023 15:43:21 +1200 Subject: [PATCH 13/14] Remove the unused timeout --- packages/config/src/memory/__tests__/memory.config.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/config/src/memory/__tests__/memory.config.test.ts b/packages/config/src/memory/__tests__/memory.config.test.ts index 8b194078e..9232da635 100644 --- a/packages/config/src/memory/__tests__/memory.config.test.ts +++ b/packages/config/src/memory/__tests__/memory.config.test.ts @@ -175,7 +175,6 @@ o.spec('MemoryConfig', () => { }); o('The latest imagery should overwrite the old ones', async () => { - await new Promise((resolve) => setTimeout(resolve, 5)); const idLater = ulid(); await new Promise((resolve) => setTimeout(resolve, 5)); const idLatest = ulid(); From 6a1a7b37cf7a0b4cf3b0204106b914296f44a646 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 9 Aug 2023 15:44:23 +1200 Subject: [PATCH 14/14] Use timers.setTimeout. --- packages/config/src/memory/__tests__/memory.config.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/config/src/memory/__tests__/memory.config.test.ts b/packages/config/src/memory/__tests__/memory.config.test.ts index 9232da635..06f4b9711 100644 --- a/packages/config/src/memory/__tests__/memory.config.test.ts +++ b/packages/config/src/memory/__tests__/memory.config.test.ts @@ -4,6 +4,7 @@ import { ConfigImagery } from '../../config/imagery.js'; import { ConfigTileSetRaster } from '../../config/tile.set.js'; import { ConfigProviderMemory } from '../memory.config.js'; import { ulid } from 'ulid'; +import timers from 'node:timers/promises'; o.spec('MemoryConfig', () => { const config = new ConfigProviderMemory(); @@ -176,7 +177,7 @@ o.spec('MemoryConfig', () => { o('The latest imagery should overwrite the old ones', async () => { const idLater = ulid(); - await new Promise((resolve) => setTimeout(resolve, 5)); + await timers.setTimeout(5); const idLatest = ulid(); config.put(baseImg); config.put({ ...baseImg, id: `im_${idLater}` } as ConfigImagery);