fix(lambda-tiler): allow reading config from memory (#2443)

linz · Aug 18, 2022 · 9f98719 · 9f98719
1 parent 8f946d8
commit 9f98719
Show file tree

Hide file tree

Showing 11 changed files with 69 additions and 33 deletions.
diff --git a/packages/lambda-tiler/src/arcgis/__tests__/arcgis.style.json.test.ts b/packages/lambda-tiler/src/arcgis/__tests__/arcgis.style.json.test.ts
@@ -17,7 +17,7 @@ o.spec('arcgis/stylejson', () => {
   });
 
   o.beforeEach(() => {
-    sandbox.stub(ConfigLoader, 'load').resolves(config);
+    sandbox.stub(ConfigLoader, 'getDefaultConfig').resolves(config);
     config.objects.clear();
   });
 

diff --git a/packages/lambda-tiler/src/arcgis/__tests__/vector.tiler.server.test.ts b/packages/lambda-tiler/src/arcgis/__tests__/vector.tiler.server.test.ts
@@ -17,7 +17,7 @@ o.spec('arcgis/VectorTileServer', () => {
   });
 
   o.beforeEach(() => {
-    sandbox.stub(ConfigLoader, 'load').resolves(config);
+    sandbox.stub(ConfigLoader, 'getDefaultConfig').resolves(config);
     config.objects.clear();
   });
 

diff --git a/packages/lambda-tiler/src/routes/__tests__/attribution.test.ts b/packages/lambda-tiler/src/routes/__tests__/attribution.test.ts
@@ -284,7 +284,8 @@ o.spec('/v1/attribution', () => {
 
   o.beforeEach(() => {
     LogConfig.get().level = 'silent';
-    sandbox.stub(ConfigLoader, 'load').resolves(config);
+    sandbox.stub(ConfigLoader, 'getDefaultConfig').resolves(config);
+
     config.objects.clear();
 
     config.put(TileSetAerial);

diff --git a/packages/lambda-tiler/src/routes/__tests__/health.test.ts b/packages/lambda-tiler/src/routes/__tests__/health.test.ts
@@ -30,7 +30,7 @@ o.spec('/v1/health', async () => {
   const fakeTileSet = FakeData.tileSetRaster('health');
   o.beforeEach(() => {
     config.objects.clear();
-    sandbox.stub(ConfigLoader, 'load').resolves(config);
+    sandbox.stub(ConfigLoader, 'getDefaultConfig').resolves(config);
     config.put(fakeTileSet);
   });
 

diff --git a/packages/lambda-tiler/src/routes/__tests__/tile.json.test.ts b/packages/lambda-tiler/src/routes/__tests__/tile.json.test.ts
@@ -1,12 +1,14 @@
-import { ConfigProviderMemory } from '@basemaps/config';
+import { base58, ConfigProviderMemory } from '@basemaps/config';
 import { Env } from '@basemaps/shared';
+import { fsa } from '@chunkd/fs';
 import o from 'ospec';
 import sinon from 'sinon';
 import { handler } from '../../index.js';
 import { ConfigLoader } from '../../util/config.loader.js';
 import { CoSources } from '../../util/source.cache.js';
 import { FakeData } from '../../__tests__/config.data.js';
 import { Api, mockRequest, mockUrlRequest } from '../../__tests__/xyz.util.js';
+import { FsMemory } from './memory.fs.js';
 
 o.spec('/v1/tiles/:tileSet/:tileMatrix/tile.json', () => {
   const config = new ConfigProviderMemory();
@@ -17,7 +19,7 @@ o.spec('/v1/tiles/:tileSet/:tileMatrix/tile.json', () => {
   });
 
   o.beforeEach(() => {
-    sandbox.stub(ConfigLoader, 'load').resolves(config);
+    sandbox.stub(ConfigLoader, 'getDefaultConfig').resolves(config);
     config.objects.clear();
     CoSources.cache.clear();
   });
@@ -131,6 +133,24 @@ o.spec('/v1/tiles/:tileSet/:tileMatrix/tile.json', () => {
     });
   });
 
+  o('should load from config bundle', async () => {
+    const memoryFs = new FsMemory();
+    fsa.register('memory://', memoryFs);
+    const fakeTileSet = FakeData.tileSetRaster('🦄 🌈');
+
+    const cfgBundle = new ConfigProviderMemory();
+    cfgBundle.put(fakeTileSet);
+    memoryFs.write('memory://linz-basemaps/bar.json', JSON.stringify(cfgBundle.toJson()));
+
+    const configLocation = base58.encode(Buffer.from('memory://linz-basemaps/bar.json'));
+    const request = mockUrlRequest('/v1/tiles/🦄 🌈/NZTM2000Quad/tile.json', `?config=${configLocation}`, Api.header);
+    const res = await handler.router.handle(request);
+    o(res.status).equals(200);
+
+    const body = JSON.parse(Buffer.from(res.body, 'base64').toString());
+    o(body.tiles[0].includes(`config=${configLocation}`)).equals(true);
+  });
+
   o('should serve convert zoom to tile matrix', async () => {
     const fakeTileSet = FakeData.tileSetVector('fake-vector');
     fakeTileSet.maxZoom = 15;

diff --git a/packages/lambda-tiler/src/routes/__tests__/tile.style.json.test.ts b/packages/lambda-tiler/src/routes/__tests__/tile.style.json.test.ts
@@ -15,7 +15,7 @@ o.spec('/v1/styles', () => {
     process.env[Env.PublicUrlBase] = host;
   });
   o.beforeEach(() => {
-    sandbox.stub(ConfigLoader, 'load').resolves(config);
+    sandbox.stub(ConfigLoader, 'getDefaultConfig').resolves(config);
   });
   o.afterEach(() => {
     sandbox.restore();

diff --git a/packages/lambda-tiler/src/routes/__tests__/wmts.test.ts b/packages/lambda-tiler/src/routes/__tests__/wmts.test.ts
@@ -11,7 +11,7 @@ o.spec('WMTSRouting', () => {
   const config = new ConfigProviderMemory();
 
   o.before(() => {
-    sandbox.stub(ConfigLoader, 'load').resolves(config);
+    sandbox.stub(ConfigLoader, 'getDefaultConfig').resolves(config);
   });
 
   o.afterEach(() => {

diff --git a/packages/lambda-tiler/src/routes/__tests__/xyz.test.ts b/packages/lambda-tiler/src/routes/__tests__/xyz.test.ts
@@ -17,7 +17,7 @@ o.spec('/v1/tiles', () => {
 
   o.beforeEach(() => {
     LogConfig.get().level = 'silent';
-    sandbox.stub(ConfigLoader, 'load').resolves(config);
+    sandbox.stub(ConfigLoader, 'getDefaultConfig').resolves(config);
     config.objects.clear();
 
     for (const tileSetName of TileSetNames) config.put(FakeData.tileSetRaster(tileSetName));

diff --git a/packages/lambda-tiler/src/routes/tile.json.ts b/packages/lambda-tiler/src/routes/tile.json.ts
@@ -18,8 +18,9 @@ export async function tileJsonGet(req: LambdaHttpRequest<TileJsonGet>): Promise<
 
   const apiKey = Validate.apiKey(req);
 
-  req.timer.start('tileset:load');
   const config = await ConfigLoader.load(req);
+
+  req.timer.start('tileset:load');
   const tileSet = await config.TileSet.get(config.TileSet.id(req.params.tileSet));
   req.timer.end('tileset:load');
   if (tileSet == null) return NotFound();
@@ -28,9 +29,15 @@ export async function tileJsonGet(req: LambdaHttpRequest<TileJsonGet>): Promise<
 
   const host = Env.get(Env.PublicUrlBase) ?? '';
 
+  const configLocation = req.query.get('config');
+  const queryParams = new URLSearchParams();
+  queryParams.set('api', apiKey);
+  if (configLocation) queryParams.set('config', configLocation);
+
   const tileUrl =
     [host, 'v1', 'tiles', tileSet.name, tileMatrix.identifier, '{z}', '{x}', '{y}'].join('/') +
-    `.${format[0]}?api=${apiKey}`;
+    `.${format[0]}?` +
+    queryParams.toString();
 
   const tileJson: TileJson = { tiles: [tileUrl], tilejson: '3.0.0' };
   const maxZoom = TileMatrixSet.convertZoomLevel(tileSet.maxZoom ?? 30, GoogleTms, tileMatrix, true);

diff --git a/packages/lambda-tiler/src/util/config.cache.ts b/packages/lambda-tiler/src/util/config.cache.ts
@@ -3,14 +3,19 @@ import { fsa } from '@basemaps/shared';
 import { SwappingLru } from './swapping.lru.js';
 
 class LruConfig {
-  configProvider: Promise<ConfigProviderMemory>;
+  configProvider: Promise<ConfigProviderMemory | null>;
 
   constructor(config: Promise<ConfigBundled>) {
-    this.configProvider = config.then((c) => {
-      const configProvider = ConfigProviderMemory.fromJson(c);
-      configProvider.createVirtualTileSets();
-      return configProvider;
-    });
+    this.configProvider = config
+      .then((c) => {
+        const configProvider = ConfigProviderMemory.fromJson(c);
+        configProvider.createVirtualTileSets();
+        return configProvider;
+      })
+      .catch((e) => {
+        if (e.code === 404) return null;
+        throw e;
+      });
   }
 
   get size(): number {
@@ -28,17 +33,12 @@ export class ConfigCache {
   get(location: string): Promise<ConfigProviderMemory | null> {
     const existing = this.cache.get(location)?.configProvider;
     if (existing != null) return existing;
-    try {
-      const configJson = fsa.readJson<ConfigBundled>(location);
-      const config = new LruConfig(configJson);
-      this.cache.set(location, config);
-      return config.configProvider;
-    } catch (e: any) {
-      if (e.code === 404) return Promise.resolve(null);
-      throw e;
-    }
+    const configJson = fsa.readJson<ConfigBundled>(location);
+    const config = new LruConfig(configJson);
+    this.cache.set(location, config);
+    return config.configProvider;
   }
 }
 
-/** Cache 20 configs(Around 500KB each)*/
+/** Cache 20 configs (Around <30KB -> 5MB each)*/
 export const CachedConfig = new ConfigCache(20);
diff --git a/packages/lambda-tiler/src/util/config.loader.ts b/packages/lambda-tiler/src/util/config.loader.ts
@@ -1,4 +1,4 @@
-import { BasemapsConfigProvider } from '@basemaps/config';
+import { base58, BasemapsConfigProvider } from '@basemaps/config';
 import { LambdaHttpResponse } from '@linzjs/lambda';
 import { parseUri } from '@chunkd/core';
 import { LambdaHttpRequest } from '@linzjs/lambda';
@@ -7,20 +7,28 @@ import { getDefaultConfig } from '@basemaps/shared';
 
 // FIXME load this from process.env COG BUCKETS?
 const SafeBuckets = new Set(['linz-workflow-artifacts', 'linz-basemaps']);
+const SafeProtocols = new Set(['s3', 'memory']);
 
 export class ConfigLoader {
+  /** Exposed for testing */
+  static async getDefaultConfig(): Promise<BasemapsConfigProvider> {
+    return getDefaultConfig();
+  }
   static async load(req: LambdaHttpRequest): Promise<BasemapsConfigProvider> {
     const rawLocation = req.query.get('config');
-    if (rawLocation == null) return getDefaultConfig();
+    if (rawLocation == null) return this.getDefaultConfig();
 
-    const configLocation = rawLocation.startsWith('s3://')
+    const configLocation = rawLocation.includes('://')
       ? rawLocation
-      : Buffer.from(rawLocation, 'base64url').toString();
+      : Buffer.from(base58.decode(rawLocation)).toString();
 
     const r = parseUri(configLocation);
+
     if (r == null) throw new LambdaHttpResponse(400, 'Invalid config location');
-    if (r.protocol !== 's3') throw new LambdaHttpResponse(400, `Invalid configuration location protocol:${r.protocol}`);
-    if (SafeBuckets.has(r.bucket)) {
+    if (!SafeProtocols.has(r.protocol)) {
+      throw new LambdaHttpResponse(400, `Invalid configuration location protocol:${r.protocol}`);
+    }
+    if (!SafeBuckets.has(r.bucket)) {
       throw new LambdaHttpResponse(400, `Bucket: "${r.bucket}" is not a allowed bucket location`);
     }