fix(cli): Convert overview cli to single thread to fix the import.met…

…a.url not working in cjs (#2576)
linz · Nov 14, 2022 · b92256b · b92256b
1 parent 3a15f37
commit b92256b
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 71 deletions.
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -53,7 +53,6 @@
     "@linzjs/geojson": "^6.32.1",
     "@octokit/core": "^4.0.5",
     "@rushstack/ts-command-line": "^4.3.13",
-    "@wtrpc/core": "^1.0.0",
     "ansi-colors": "^4.1.1",
     "deep-diff": "^1.0.2",
     "flatgeobuf": "^3.23.1",

diff --git a/packages/cli/src/cli/overview/action.create.overview.ts b/packages/cli/src/cli/overview/action.create.overview.ts
@@ -3,9 +3,7 @@ import * as path from 'path';
 import { promises as fs } from 'fs';
 import { CommandLineAction, CommandLineIntegerParameter, CommandLineStringParameter } from '@rushstack/ts-command-line';
 import { GoogleTms, NamedBounds, Nztm2000QuadTms, QuadKey, TileMatrixSet } from '@basemaps/geo';
-import os from 'os';
-import { WorkerRpcPool } from '@wtrpc/core';
-import { JobTiles, RpcContract } from './tile.generator.js';
+import { JobTiles, tile } from './tile.generator.js';
 import { CotarIndexBinary, CotarIndexBuilder, TarReader } from '@cotar/core';
 import { SourceMemory, ChunkSource } from '@chunkd/core';
 import { fsa } from '@chunkd/fs';
@@ -16,12 +14,6 @@ import { CogTiff } from '@cogeotiff/core';
 import { createHash } from 'crypto';
 import { TarBuilder } from '@cotar/tar';
 
-// Create tiles per worker invocation
-const WorkerTaskSize = 500;
-const workerUrl = new URL('./tile.generator.js', import.meta.url);
-const threadCount = os.cpus().length / 8;
-const pool = new WorkerRpcPool<RpcContract>(threadCount, workerUrl);
-
 const DefaultMaxZoom = 15;
 
 export class CommandCreateOverview extends CommandLineAction {
@@ -81,7 +73,13 @@ export class CommandCreateOverview extends CommandLineAction {
     if (tiles.size < 1) throw new Error('Failed to prepare overviews.');
 
     logger.info({ source, path }, 'CreateOverview: GenerateTiles');
-    await this.generateTiles(path, tileMatrix, metadata.bounds, tiles);
+    const jobTiles: JobTiles = {
+      path,
+      files: metadata.bounds,
+      tileMatrix: tileMatrix.identifier,
+      tiles: Array.from(tiles.values()),
+    };
+    await tile(jobTiles, logger);
 
     logger.info({ source, path }, 'CreateOverview: CreatingTarFile');
     await this.createTar(path, logger);
@@ -123,25 +121,6 @@ export class CommandCreateOverview extends CommandLineAction {
     else throw new Error(`Projection code: ${projection} not supported`);
   }
 
-  async generateTiles(
-    path: string,
-    tileMatrix: TileMatrixSet,
-    files: NamedBounds[],
-    tiles: Set<string>,
-  ): Promise<void> {
-    const promises = [];
-    let currentTiles = Array.from(tiles);
-    while (currentTiles.length > 0) {
-      const todo = currentTiles.slice(0, WorkerTaskSize);
-      currentTiles = currentTiles.slice(WorkerTaskSize);
-      const jobTiles: JobTiles = { path, files, tileMatrix: tileMatrix.identifier, tiles: todo };
-      promises.push(pool.run('tile', jobTiles));
-    }
-
-    await Promise.all(promises);
-    await pool.close();
-  }
-
   async createTar(path: string, logger: LogType): Promise<void> {
     const tarFile = 'overviews.tar.co';
     const tarFilePath = fsa.join(path, tarFile);

diff --git a/packages/cli/src/cli/overview/tile.generator.ts b/packages/cli/src/cli/overview/tile.generator.ts
@@ -1,5 +1,3 @@
-import { WorkerRpc } from '@wtrpc/core';
-import { parentPort } from 'node:worker_threads';
 import {
   Bounds,
   GoogleTms,
@@ -11,7 +9,7 @@ import {
   Tile,
   TileMatrixSet,
 } from '@basemaps/geo';
-import { LogConfig } from '@basemaps/shared';
+import { LogType } from '@basemaps/shared';
 import { fsa } from '@chunkd/fs';
 import pLimit from 'p-limit';
 import { CogTiff } from '@cogeotiff/core';
@@ -40,48 +38,42 @@ export interface JobTiles {
   files: NamedBounds[];
 }
 
-const Q = pLimit(2);
-
-let count = 0;
-let skipped = 0;
+const Q = pLimit(20);
 
 export type RpcContract = {
   tile(jobTiles: JobTiles): Promise<void>;
 };
 
-const worker = new WorkerRpc<RpcContract>({
-  async tile(jobTiles: JobTiles): Promise<void> {
-    const logger = LogConfig.get().child({ workerId: worker.id, messageId: worker.messageCount });
-    logger.info({ count, skipped }, 'TaskCount');
+export async function tile(jobTiles: JobTiles, logger: LogType): Promise<void> {
+  let count = 0;
+  let skipped = 0;
+  logger.info({ count, skipped }, 'TaskCount');
 
-    let lastTime = performance.now();
-    const todo = jobTiles.tiles.map((qk) => {
-      return Q(async () => {
-        const tile = QuadKey.toTile(qk);
-        count++;
-        if (count % 100 === 0) {
-          const duration = performance.now() - lastTime;
-          lastTime = Number(performance.now().toFixed(4));
-          logger.info({ count, total: jobTiles.tiles.length, duration }, 'Progress');
-        }
+  let lastTime = performance.now();
+  const todo = jobTiles.tiles.map((qk) => {
+    return Q(async () => {
+      const tile = QuadKey.toTile(qk);
+      count++;
+      if (count % 100 === 0) {
+        const duration = performance.now() - lastTime;
+        lastTime = Number(performance.now().toFixed(4));
+        logger.info({ count, total: jobTiles.tiles.length, duration }, 'Progress');
+      }
 
-        const outputTile = `tiles/${tile.z}/${tile.x}/${tile.y}.webp`;
-        const outputFile = fsa.join(jobTiles.path, outputTile);
-        const exists = await fsa.exists(outputFile);
-        if (exists) {
-          skipped++;
-          return;
-        }
-        const buffer = await getComposedTile(jobTiles, tile);
-        if (buffer != null) await fsa.write(outputFile, buffer);
-      });
+      const outputTile = `tiles/${tile.z}/${tile.x}/${tile.y}.webp`;
+      const outputFile = fsa.join(jobTiles.path, outputTile);
+      const exists = await fsa.exists(outputFile);
+      if (exists) {
+        skipped++;
+        return;
+      }
+      const buffer = await getComposedTile(jobTiles, tile);
+      if (buffer != null) await fsa.write(outputFile, buffer);
     });
+  });
 
-    await Promise.all(todo);
-  },
-});
-
-if (parentPort) worker.bind(parentPort);
+  await Promise.all(todo);
+}
 
 async function getComposedTile(jobTiles: JobTiles, tile: Tile): Promise<Buffer | undefined> {
   const files = jobTiles.files;

diff --git a/yarn.lock b/yarn.lock
@@ -2067,11 +2067,6 @@
     "@typescript-eslint/types" "5.15.0"
     eslint-visitor-keys "^3.0.0"
 
-"@wtrpc/core@^1.0.0":
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/@wtrpc/core/-/core-1.0.0.tgz#8d3afe0871ef1de43ad562b20d841ac53c9da94c"
-  integrity sha512-thoQtOzyXszZLyd6LnD/6x2jYTnWJ7TD20PXBASlmv+IQSPjmrIDBXUnSXfzoWBSUcVxSSu64qh7jm6ODCKnDQ==
-
 "@zxing/text-encoding@0.9.0":
   version "0.9.0"
   resolved "https://registry.yarnpkg.com/@zxing/text-encoding/-/text-encoding-0.9.0.tgz#fb50ffabc6c7c66a0c96b4c03e3d9be74864b70b"