feat(server): bundle basemaps-server cli so its easier to install (#2218

)

* wip

* refactor: fixup imports

* refactor: add missing file

* docs: update docs for new cli

* feat(scripts): include shebang in cli bundling

* refactor: fix up lint

* refactor: add logs to imports

* refactor: fix loading require.resolve when not in esm

* refactor: bind to all addresses

* fix: load all files including local files

* fix: create a unique id using the hash of the uri

* fix: include .tif files too

* fix: ensure virtual tilesets are created from memory

* test: add tests for prefix

* refactor: use smaller hash

* feat: track more fine grained tile creation times

* fix: serve wmts even if the provider could not be found

package.json

@@ -29,7 +29,7 @@
     "@types/sinon": "^10.0.0",
     "conventional-github-releaser": "^3.1.5",
     "cross-env": "^7.0.3",
-    "esbuild": "^0.13.15",
+    "esbuild": "^0.14.39",
     "lerna": "4.0.0",
     "ospec": "^4.0.1",
     "rimraf": "^3.0.0",

packages/bathymetry/src/index.ts

@@ -1,4 +1,4 @@
-import { BaseCommandLine } from '@basemaps/cli/build/cli/base.cli.js';
+import { BaseCommandLine } from '@basemaps/shared/build/cli/base.js';
 import { makeTempFolder } from '@basemaps/cli/build/cli/folder.js';
 import { GoogleTms, TileMatrixSets } from '@basemaps/geo';
 import { Env, fsa, LogConfig } from '@basemaps/shared';

packages/cli/src/cli/base.cli.ts

@@ -1,60 +0,0 @@
-#!/usr/bin/env node
-import { LogConfig, LoggerFatalError } from '@basemaps/shared';
-import { GitTag } from '@basemaps/shared/build/cli/git.tag.js';
-import { CommandLineParser } from '@rushstack/ts-command-line';
-import 'source-map-support/register.js';
-import * as ulid from 'ulid';
-
-/** Useful traceability information  */
-export const CliInfo: { package: string; version: string; hash: string } = {
-  // Detect unlinked packages looks for this string since its a package name, slightly work around it
-  package: '@' + 'basemaps/cli',
-  version: process.env.GIT_VERSION ?? GitTag().version,
-  hash: process.env.GIT_HASH ?? GitTag().hash,
-};
-
-/** Unique Id for this instance of the cli being run */
-export const CliId = ulid.ulid();
-
-export abstract class BaseCommandLine extends CommandLineParser {
-  verbose = this.defineFlagParameter({
-    parameterLongName: '--verbose',
-    parameterShortName: '-v',
-    description: 'Show extra logging detail',
-  });
-  extraVerbose = this.defineFlagParameter({
-    parameterLongName: '--vv',
-    parameterShortName: '-V',
-    description: 'Show extra extra logging detail',
-  });
-
-  protected onExecute(): Promise<void> {
-    if (this.verbose.value) {
-      LogConfig.get().level = 'debug';
-    } else if (this.extraVerbose.value) {
-      LogConfig.get().level = 'trace';
-    } else {
-      LogConfig.get().level = 'info';
-    }
-
-    const logger = LogConfig.get().child({ id: CliId });
-    logger.info(CliInfo, 'CliStart');
-    LogConfig.set(logger);
-
-    return super.onExecute();
-  }
-  protected onDefineParameters(): void {
-    // Nothing
-  }
-
-  public run(): void {
-    this.executeWithoutErrorHandling().catch((err) => {
-      if (err instanceof LoggerFatalError) {
-        LogConfig.get().fatal(err.obj, err.message);
-      } else {
-        LogConfig.get().fatal({ err }, 'Failed to run command');
-      }
-      process.exit(1);
-    });
-  }
-}

packages/cli/src/cli/cogify/action.cog.ts

@@ -1,4 +1,5 @@
 import { Config, Env, fsa, LogConfig, LoggerFatalError, LogType } from '@basemaps/shared';
+import { CliId } from '@basemaps/shared/build/cli/base.js';
 import {
   CommandLineAction,
   CommandLineFlagParameter,
@@ -14,7 +15,6 @@ import { CogVrt } from '../../cog/cog.vrt.js';
 import { Cutline } from '../../cog/cutline.js';
 import { CogJob } from '../../cog/types.js';
 import { Gdal } from '../../gdal/gdal.js';
-import { CliId } from '../base.cli.js';
 import { makeTempFolder, makeTiffFolder } from '../folder.js';
 import path from 'path';
 import { insertConfigImagery, insertConfigTileSet } from './imagery.config.js';

packages/cli/src/cli/cogify/action.job.ts

@@ -9,7 +9,7 @@ import {
 import { JobCreationContext } from '../../cog/cog.stac.job.js';
 import { CogJobFactory, MaxConcurrencyDefault } from '../../cog/job.factory.js';
 import { GdalCogBuilderDefaults, GdalCogBuilderResampling, GdalResamplingOptions } from '../../gdal/gdal.config.js';
-import { CliId } from '../base.cli.js';
+import { CliId } from '@basemaps/shared/build/cli/base.js';
 
 export class CLiInputData {
   path: CommandLineStringParameter;

packages/cli/src/cli/cogify/index.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
+import { BaseCommandLine } from '@basemaps/shared/build/cli/base.js';
 import 'source-map-support/register.js';
-import { BaseCommandLine } from '../base.cli.js';
 import { ActionCogCreate } from './action.cog.js';
 import { ActionJobCreate } from './action.job.js';
 

packages/cli/src/cli/util.ts

@@ -2,7 +2,7 @@ import { Env, fsa, LogConfig, Projection } from '@basemaps/shared';
 import CloudFormation from 'aws-sdk/clients/cloudformation.js';
 import CloudFront from 'aws-sdk/clients/cloudfront.js';
 import S3 from 'aws-sdk/clients/s3.js';
-import { CliId } from './base.cli.js';
+import { CliId } from '@basemaps/shared/build/cli/base.js';
 import crypto from 'crypto';
 import path from 'path';
 import { gzip } from 'zlib';

packages/cli/src/cog/cog.stac.job.ts

@@ -18,7 +18,7 @@ import {
   CompositeError,
 } from '@basemaps/shared';
 import { MultiPolygon, toFeatureCollection, toFeatureMultiPolygon } from '@linzjs/geojson';
-import { CliInfo } from '../cli/base.cli.js';
+import { CliInfo } from '@basemaps/shared/build/cli/base.js';
 import { GdalCogBuilderDefaults, GdalCogBuilderResampling } from '../gdal/gdal.config.js';
 import { ProjectionLoader } from './projection.loader.js';
 import { CogStac, CogStacItem, CogStacItemExtensions, CogStacKeywords } from './stac.js';

packages/config/src/__test__/prefix.test.ts

@@ -0,0 +1,32 @@
+import o from 'ospec';
+import { Config } from '../base.config.js';
+import { ConfigPrefix } from '../config/prefix.js';
+
+o.spec('ConfigPrefix', () => {
+  o('should prefix values', () => {
+    o(Config.prefix(ConfigPrefix.TileSet, '123')).equals('ts_123');
+    o(Config.prefix(ConfigPrefix.Imagery, '123')).equals('im_123');
+  });
+
+  o('should unprefix values', () => {
+    o(Config.unprefix(ConfigPrefix.TileSet, 'ts_123')).equals('123');
+    o(Config.unprefix(ConfigPrefix.Imagery, 'im_123')).equals('123');
+  });
+
+  o('should not unprefix unknown values', () => {
+    o(Config.unprefix(ConfigPrefix.TileSet, 'im_123')).equals('im_123');
+    o(Config.unprefix(ConfigPrefix.Imagery, 'ts_123')).equals('ts_123');
+  });
+
+  o('should get prefix values', () => {
+    o(Config.getPrefix('ts_123')).equals(ConfigPrefix.TileSet);
+    o(Config.getPrefix('im_123')).equals(ConfigPrefix.Imagery);
+  });
+
+  o('should not return unknown prefixes', () => {
+    o(Config.getPrefix('jj_123')).equals(null);
+    o(Config.getPrefix('123')).equals(null);
+    o(Config.getPrefix('_123')).equals(null);
+    o(Config.getPrefix('123_123')).equals(null);
+  });
+});

packages/config/src/base.config.ts

@@ -1,6 +1,6 @@
 import { Epsg } from '@basemaps/geo';
 import { BaseConfig } from './config/base.js';
-import { ConfigPrefix } from './config/prefix.js';
+import { ConfigPrefix, ConfigPrefixes } from './config/prefix.js';
 import { ConfigLayer, ConfigTileSet, TileSetType } from './config/tile.set.js';
 import {
   ConfigImagery,
@@ -66,6 +66,15 @@ export class ConfigInstance {
     return `${prefix}_${id}`;
   }
 
+  /** Attempt to get the configuration prefix from a id */
+  getPrefix(id: string): ConfigPrefix | null {
+    const joinIndex = id.indexOf('_');
+    if (joinIndex === -1) return null;
+    const prefix = id.slice(0, joinIndex) as ConfigPrefix;
+    if (ConfigPrefixes.has(prefix)) return prefix;
+    return null;
+  }
+
   /**
    * Remove the prefix from a dynamoDb id
    */

packages/config/src/config/prefix.ts

@@ -6,3 +6,5 @@ export enum ConfigPrefix {
   Style = 'st',
   ProcessingJob = 'pj',
 }
+
+export const ConfigPrefixes: Set<ConfigPrefix> = new Set(Object.values(ConfigPrefix));

packages/config/src/json/json.config.ts

@@ -1,9 +1,11 @@
 import { Bounds, GoogleTms, ImageFormat, Nztm2000QuadTms, TileMatrixSet, VectorFormat } from '@basemaps/geo';
 import { fsa } from '@chunkd/fs';
+import { createHash } from 'crypto';
 import { basename } from 'path';
 import ulid from 'ulid';
 import { Config } from '../base.config.js';
 import { parseRgba } from '../color.js';
+import { BaseConfig } from '../config/base.js';
 import { ConfigImagery } from '../config/imagery.js';
 import { ConfigPrefix } from '../config/prefix.js';
 import { ConfigProvider } from '../config/provider.js';
@@ -15,15 +17,19 @@ import { zProviderConfig } from './parse.provider.js';
 import { zStyleJson } from './parse.style.js';
 import { zTileSetConfig } from './parse.tile.set.js';
 
-export function guessIdFromUri(uri: string): string {
+export function guessIdFromUri(uri: string): string | null {
   const parts = uri.split('/');
   const id = parts.pop();
 
-  if (id == null) throw new Error('Could not get id from URI: ' + uri);
-  const date = new Date(ulid.decodeTime(id));
-  if (date.getUTCFullYear() < 2015) throw new Error('Could not get id from URI: ' + uri);
-  if (date.getUTCFullYear() > new Date().getUTCFullYear() + 1) throw new Error('Could not get id from URI: ' + uri);
-  return id;
+  if (id == null) return null;
+  try {
+    const date = new Date(ulid.decodeTime(id));
+    if (date.getUTCFullYear() < 2015) return null;
+    if (date.getUTCFullYear() > new Date().getUTCFullYear() + 1) return null;
+    return id;
+  } catch (e) {
+    return null;
+  }
 }
 
 export class ConfigJson {
@@ -39,34 +45,41 @@ export class ConfigJson {
   }
 
   /** Import configuration from a base path */
-  static async fromPath(basePath: string, log: LogType): Promise<ConfigJson> {
+  static async fromPath(basePath: string, log: LogType): Promise<ConfigProviderMemory> {
     const cfg = new ConfigJson(basePath, log);
 
-    for await (const pvPath of fsa.list(fsa.join(basePath, 'provider'))) {
-      if (!pvPath.endsWith('.json')) continue;
-      const pv = await cfg.provider(await fsa.readJson(pvPath));
-      cfg.mem.put(pv);
-    }
+    for await (const filePath of fsa.list(basePath)) {
+      if (!filePath.endsWith('.json')) continue;
 
-    for await (const tsPath of fsa.list(fsa.join(basePath, 'tileset'))) {
-      if (!tsPath.endsWith('.json')) continue;
-      const ts = await cfg.tileSet(await fsa.readJson(tsPath));
-      cfg.mem.put(ts);
-    }
+      const bc: BaseConfig = (await fsa.readJson(filePath)) as BaseConfig;
+      const prefix = Config.getPrefix(bc.id);
+      if (prefix == null) {
+        log.warn({ path: filePath }, 'Invalid JSON file found');
+        continue;
+      }
 
-    for await (const stylePath of fsa.list(fsa.join(basePath, 'style'))) {
-      if (!stylePath.endsWith('.json')) continue;
-      const ts = await cfg.style(await fsa.readJson(stylePath));
-      cfg.mem.put(ts);
+      log.trace({ path: filePath, type: prefix, config: bc.id }, 'Config:Load');
+
+      switch (prefix) {
+        case ConfigPrefix.TileSet:
+          cfg.mem.put(await cfg.tileSet(bc));
+          break;
+        case ConfigPrefix.Provider:
+          cfg.mem.put(await cfg.provider(bc));
+          break;
+        case ConfigPrefix.Style:
+          cfg.mem.put(await cfg.style(bc));
+          break;
+      }
     }
-
-    return cfg;
+    return cfg.mem;
   }
 
   async provider(obj: unknown): Promise<ConfigProvider> {
     const pv = zProviderConfig.parse(obj);
+    this.logger.info({ config: pv.id }, 'Config:Loaded:Provider');
 
-    const provider: ConfigProvider = {
+    return {
       id: pv.id,
       name: Config.unprefix(ConfigPrefix.Provider, pv.id),
       serviceIdentification: pv.serviceIdentification,
@@ -75,11 +88,12 @@ export class ConfigJson {
       updatedAt: Date.now(),
       version: 1,
     };
-    return provider;
   }
 
   async style(obj: unknown): Promise<ConfigVectorStyle> {
     const st = zStyleJson.parse(obj);
+    this.logger.info({ config: st.id }, 'Config:Loaded:Style');
+
     return {
       id: st.id,
       name: st.name,
@@ -91,15 +105,16 @@ export class ConfigJson {
 
   async tileSet(obj: unknown): Promise<ConfigTileSet> {
     const ts = zTileSetConfig.parse(obj);
+    this.logger.info({ config: ts.id }, 'Config:Loaded:TileSet');
 
     const imageryFetch: Promise<ConfigImagery>[] = [];
     if (ts.type === TileSetType.Raster) {
       for (const layer of ts.layers) {
-        if (layer[2193] != null && layer[2193].startsWith('s3://')) {
+        if (layer[2193] != null) {
           imageryFetch.push(this.loadImagery(layer[2193], Nztm2000QuadTms, layer.name));
         }
 
-        if (layer[3857] != null && layer[3857].startsWith('s3://')) {
+        if (layer[3857] != null) {
           imageryFetch.push(this.loadImagery(layer[3857], GoogleTms, layer.name));
         }
       }
@@ -159,16 +174,18 @@ export class ConfigJson {
   }
 
   async _loadImagery(uri: string, tileMatrix: TileMatrixSet, name: string): Promise<ConfigImagery> {
-    this.logger.trace({ uri }, 'FetchImagery');
-
     // TODO is there a better way of guessing the imagery id & tile matrix?
-    const id = Config.prefix(ConfigPrefix.Imagery, guessIdFromUri(uri));
+    const imageId = guessIdFromUri(uri) ?? createHash('sha256').update(uri).digest('base64url');
+    const id = Config.prefix(ConfigPrefix.Imagery, imageId);
+    this.logger.trace({ uri, imageId: id }, 'FetchImagery');
 
     const fileList = await fsa.toArray(fsa.list(uri));
-    const tiffFiles = fileList.filter((f) => f.endsWith('.tiff'));
+    const tiffFiles = fileList.filter((f) => f.endsWith('.tiff') || f.endsWith('.tif'));
 
     let bounds: Bounds | null = null;
     // Files are stored as `{z}-{x}-{y}.tiff`
+    // TODO the files could actually be smaller than the tile size,
+    // we should really load the tiff at some point to validate the size
     const files = tiffFiles.map((c) => {
       const tileName = basename(c).replace('.tiff', '');
       const [z, x, y] = tileName.split('-').map((f) => Number(f));
@@ -198,7 +215,7 @@ export class ConfigJson {
       return bXyz[2] - aXyz[2];
     });
 
-    this.logger.debug({ uri, files: files.length }, 'FetchImagery:Done');
+    this.logger.debug({ uri, imageId, files: files.length }, 'FetchImagery:Done');
 
     if (bounds == null) throw new Error('Failed to get bounds from URI: ' + uri);
     const now = Date.now();

packages/lambda-tiler/src/routes/tile.wmts.ts

@@ -36,6 +36,7 @@ export async function wmts(req: LambdaHttpRequest): Promise<LambdaHttpResponse>
   const wmtsData = tileWmtsFromPath(action.rest);
   if (wmtsData == null) return NotFound;
   const host = Env.get(Env.PublicUrlBase) ?? '';
+  console.log('WMTS', req);
 
   req.timer.start('tileset:load');
   const tileSets = await wmtsLoadTileSets(wmtsData.name, wmtsData.tileMatrix);
@@ -44,12 +45,11 @@ export async function wmts(req: LambdaHttpRequest): Promise<LambdaHttpResponse>
 
   const providerId = Config.Provider.id('linz');
   const provider = await Config.Provider.get(providerId);
-  if (provider == null) return NotFound;
 
   const apiKey = Router.apiKey(req);
   const xml = new WmtsCapabilities({
     httpBase: host,
-    provider,
+    provider: provider ?? undefined,
     layers: tileSets,
     apiKey,
     formats: getImageFormats(req),

packages/lambda-tiler/src/tile.set.raster.ts

@@ -122,17 +122,15 @@ export class TileSetRaster {
     req.set('layers', layers.length);
     if (TileEtag.isNotModified(req, cacheKey)) return NotModified;
 
-    req.timer.start('tile:compose');
     const res = await TileComposer.compose({
       layers,
       format: xyz.ext,
       background: this.tileSet.background ?? DefaultBackground,
       resizeKernel: this.tileSet.resizeKernel ?? DefaultResizeKernel,
+      metrics: req.timer,
     });
-    req.timer.end('tile:compose');
 
     req.set('layersUsed', res.layers);
-    req.set('allLayersUsed', res.layers === layers.length);
     req.set('bytes', res.buffer.byteLength);
 
     const response = new LambdaHttpResponse(200, 'ok');