feat: add overview archive to imagery config

packages/cli/src/cli/config/action.imagery.config.ts

@@ -1,9 +1,21 @@
-import { base58, ConfigProviderMemory } from '@basemaps/config';
-import { Bounds, Nztm2000QuadTms } from '@basemaps/geo';
+import {
+  base58,
+  ConfigImagery,
+  ConfigJson,
+  ConfigProviderMemory,
+  ConfigTileSet,
+  ConfigTileSetRaster,
+  TileSetType,
+} from '@basemaps/config';
+import { ConfigImageryOverview } from '@basemaps/config/src/config/imagery';
+import { Bounds, ImageFormat, Nztm2000QuadTms, QuadKey, TileMatrixSets } from '@basemaps/geo';
+import { CoSources } from '@basemaps/lambda-tiler/src/util/source.cache';
 import { fsa, LogConfig, Projection } from '@basemaps/shared';
 import { CogTiff } from '@cogeotiff/core';
+import { Cotar } from '@cotar/core';
 import { CommandLineAction, CommandLineFlagParameter, CommandLineStringParameter } from '@rushstack/ts-command-line';
 import { ulid } from 'ulid';
+import { promise } from 'zod';
 
 export class CommandImageryConfig extends CommandLineAction {
   private path: CommandLineStringParameter;
@@ -66,38 +78,39 @@ export class CommandImageryConfig extends CommandLineAction {
       if (gsd == null) gsd = tif.getImage(0).resolution[0];
       if (bounds == null) bounds = imgBounds;
       else bounds = bounds.union(imgBounds);
-      files.push({
-        name: tif.source.uri.replace(path, ''),
-        ...imgBounds,
-      });
+      files.push({ name: tif.source.uri.replace(path, ''), ...imgBounds });
     }
 
+    if (bounds == null) throw new Error('Failed to extract imagery bounds');
+
     const provider = new ConfigProviderMemory();
     const id = ulid();
     let name = path.split('/').at(-2);
     if (name == null) {
       logger.warn({ path, id }, `Unable to extract the imagery name from path, use uild id instead.`);
       name = id;
     }
-    const imagery = {
+    const imagery: ConfigImagery = {
       id: provider.Imagery.id(id),
       name: `${name}-${new Date().getFullYear()}`, // Add a year into name for attribution to extract
       updatedAt: Date.now(),
       projection: Nztm2000QuadTms.projection.code,
       tileMatrix: Nztm2000QuadTms.identifier,
       uri: path,
-      bounds,
+      bounds: bounds.toJson(),
       files,
     };
+    imagery.overviews = await ConfigJson.findImageryOverviews(imagery);
+
     provider.put(imagery);
 
-    const tileSet = {
+    const tileSet: ConfigTileSetRaster = {
       id: 'ts_aerial',
       name: 'aerial',
       title: 'Aerial Imagery Basemap',
       category: 'Basemaps',
-      type: 'raster',
-      format: 'webp',
+      type: TileSetType.Raster,
+      format: ImageFormat.Webp,
       layers: [{ 2193: imagery.id, name: imagery.name, title: imagery.name }],
     };
     provider.put(tileSet);

packages/config/package.json

@@ -29,6 +29,7 @@
   ],
   "dependencies": {
     "@basemaps/geo": "^6.32.1",
+    "@cotar/core": "^5.4.0",
     "base-x": "^4.0.0",
     "zod": "^3.17.3"
   }

packages/config/src/config/imagery.ts

@@ -21,4 +21,23 @@ export interface ConfigImagery extends BaseConfig {
 
   /** list of file basenames and their bounding box */
   files: NamedBounds[];
+
+  /** Separate overview cache */
+  overviews?: ConfigImageryOverview;
+}
+
+export interface ConfigImageryOverview {
+  /** Path to overview archive */
+  path: string;
+  /**
+   * Minium zoom level of overviews
+   * @example 0 means tiles for z0 exist in this overview archive
+   */
+  minZoom: number;
+  /**
+   *  Maximum zoom level of overviews
+   * @example
+   * 15 means tiles for z15 exist in this overview archive
+   */
+  maxZoom: number;
 }

packages/config/src/json/json.config.ts

@@ -1,12 +1,22 @@
-import { Bounds, GoogleTms, ImageFormat, Nztm2000QuadTms, TileMatrixSet, VectorFormat } from '@basemaps/geo';
+import {
+  Bounds,
+  GoogleTms,
+  ImageFormat,
+  Nztm2000QuadTms,
+  QuadKey,
+  TileMatrixSet,
+  TileMatrixSets,
+  VectorFormat,
+} from '@basemaps/geo';
 import { fsa } from '@chunkd/fs';
+import { Cotar } from '@cotar/core';
 import { createHash } from 'crypto';
 import { basename } from 'path';
 import ulid from 'ulid';
 import { ConfigId } from '../base.config.js';
 import { parseRgba } from '../color.js';
 import { BaseConfig } from '../config/base.js';
-import { ConfigImagery } from '../config/imagery.js';
+import { ConfigImagery, ConfigImageryOverview } from '../config/imagery.js';
 import { ConfigPrefix } from '../config/prefix.js';
 import { ConfigProvider } from '../config/provider.js';
 import { ConfigLayer, ConfigTileSet, TileSetType } from '../config/tile.set.js';
@@ -252,9 +262,60 @@ export class ConfigJson {
       bounds,
       files,
     };
+
+    output.overviews = await ConfigJson.findImageryOverviews(output);
     this.mem.put(output);
     // Ensure there is also a tile set for each imagery set
     // this.mem.put(ConfigJson.imageryToTileSet(output));
     return output;
   }
+
+  static async findImageryOverviews(cfg: ConfigImagery): Promise<ConfigImageryOverview | undefined> {
+    if (cfg.overviews) throw new Error('Overviews exist already for config: ' + cfg.id);
+
+    const targetOverviews = fsa.join(cfg.uri, 'overviews.tar.co');
+    const exists = await fsa.exists(targetOverviews);
+    if (!exists) return;
+
+    const smallestTiff = cfg.files.reduce((prev, current) => {
+      if (prev.width < current.width) return prev;
+      return current;
+    }, cfg.files[0]);
+
+    const tileMatrix = TileMatrixSets.tryGet(cfg.projection);
+    if (tileMatrix == null) throw new Error('Missing tileMatrix for imagery:' + cfg.id);
+
+    // Find a maximum tile that could be used in this overview
+    const maxZoomLoc = tileMatrix.sourceToPixels(smallestTiff.x, smallestTiff.y, tileMatrix.maxZoom);
+    const maxTile = tileMatrix.pixelsToTile(maxZoomLoc.x, maxZoomLoc.y, tileMatrix.maxZoom);
+    let qk = QuadKey.fromTile(maxTile);
+
+    const cotar = await Cotar.fromTar(fsa.source(targetOverviews));
+
+    // Query all the quadkeys looking to see what tiles exists
+    const queries: Promise<string | null>[] = [];
+    while (qk.length > 0) {
+      qk = QuadKey.parent(qk);
+      const tile = QuadKey.toTile(qk);
+      queries.push(
+        cotar.index.find(`tiles/${tile.z}/${tile.x}/${tile.y}.webp`).then((c) => {
+          if (c == null) return null;
+          return qk;
+        }),
+      );
+    }
+
+    const result = await Promise.all(queries);
+
+    const overview: ConfigImageryOverview = { path: 'overviews.tar.co', minZoom: tileMatrix.maxZoom, maxZoom: 0 };
+    for (const r of result) {
+      if (r == null) continue;
+      if (r.length < overview.minZoom) overview.minZoom = r.length;
+      if (r.length > overview.maxZoom) overview.maxZoom = r.length;
+    }
+
+    if (overview.minZoom === tileMatrix.maxZoom) return;
+    if (overview.maxZoom === 0) return;
+    return overview;
+  }
 }