feat(cli): New cli to bundle the assets into cotar file.

packages/cli/package.json

@@ -45,6 +45,8 @@
     "@basemaps/sprites": "^6.29.0",
     "@chunkd/fs": "^8.4.0",
     "@cogeotiff/core": "^7.0.0",
+    "@cotar/core": "^5.4.0",
+    "@cotar/tar": "^5.4.1",
     "@linzjs/geojson": "^6.28.1",
     "@rushstack/ts-command-line": "^4.3.13",
     "ansi-colors": "^4.1.1",

packages/cli/src/cli/config/action.bundle.assets.ts

@@ -0,0 +1,107 @@
+import { CommandLineAction, CommandLineStringParameter } from '@rushstack/ts-command-line';
+import { SourceMemory } from '@chunkd/core';
+import { CotarIndexBinary, CotarIndexBuilder, CotarIndexOptions, TarReader } from '@cotar/core';
+import { LogConfig, LogType } from '@basemaps/shared';
+import { promises as fs } from 'fs';
+import { TarBuilder } from '@cotar/tar';
+import { fsa } from '@chunkd/fs';
+import * as path from 'path';
+
+const Packing = 25; // Packing factor for the hash map
+const MaxSearch = 50; // Max search factor
+
+export class CommandBundleAssets extends CommandLineAction {
+  assets: CommandLineStringParameter;
+  output: CommandLineStringParameter;
+
+  public constructor() {
+    super({
+      actionName: 'bundle-assets',
+      summary: 'Cli tool to create cotar file for the config assets',
+      documentation: 'Create a cotar file from a directory',
+    });
+  }
+
+  protected onDefineParameters(): void {
+    this.assets = this.defineStringParameter({
+      argumentName: 'ASSETS',
+      parameterLongName: '--assets',
+      description: 'Paths to the input assets files, must contain assets/sprites/ and assets/fonts.',
+    });
+
+    this.output = this.defineStringParameter({
+      argumentName: 'OUTPUT',
+      parameterLongName: '--output',
+      description: 'Paths of the output co tar file.',
+    });
+  }
+
+  protected async onExecute(): Promise<void> {
+    const logger = LogConfig.get();
+    const assets = this.assets.value;
+    const output = this.output.value;
+    if (assets == null || output == null) throw new Error('Please provide a input or ouput path');
+    if (!output.endsWith('.tar.co')) throw new Error(`Invalid output, needs to be .tar.co :"${output}"`);
+
+    logger.info({ indput: assets }, 'BundleAssets:Start');
+    const tarFile = await this.buildTar(assets, output, logger);
+    const cotarFile = await this.buildTarCo(tarFile, output, logger);
+    logger.info({ output: cotarFile }, 'BundleAssets:Finish');
+  }
+
+  async buildTar(input: string, output: string, logger: LogType): Promise<string> {
+    const startTime = Date.now();
+    const outputTar = output.split('.co')[0];
+
+    const tarBuilder = new TarBuilder(outputTar);
+
+    const files = await fsa.toArray(fsa.list(input));
+    // Ensure files are put into the same order
+    files.sort((a, b) => a.localeCompare(b));
+    logger.info({ output: outputTar, files: files.length }, 'Tar:Create');
+
+    const basePath = path.resolve(input);
+    for (const file of files) {
+      const filePath = file.replace(basePath, '').slice(1); // Remove the leading '/'
+      await tarBuilder.write(filePath, await fsa.read(file));
+    }
+
+    await tarBuilder.close();
+
+    logger.info({ output: outputTar, stats: tarBuilder.stats, duration: Date.now() - startTime }, 'Tar:Create:Done');
+    return outputTar;
+  }
+
+  async buildTarCo(input: string, output: string, logger: LogType): Promise<string> {
+    logger.info({ output }, 'Cotar:Create');
+
+    const opts: CotarIndexOptions = { packingFactor: 1 + Packing / 100, maxSearch: MaxSearch };
+
+    const indexFile = input + '.index';
+
+    const startTime = Date.now();
+    const buf = await this.toTarIndex(input, indexFile, opts, logger);
+
+    logger.info({ output }, 'Cotar:Craete:WriteTar');
+    await fs.copyFile(input, output);
+    await fs.appendFile(output, buf);
+
+    const duration = Date.now() - startTime;
+    logger.info({ output, duration }, 'Cotar:Created');
+    return output;
+  }
+
+  async toTarIndex(filename: string, indexFileName: string, opts: CotarIndexOptions, logger: LogType): Promise<Buffer> {
+    const fd = await fs.open(filename, 'r');
+    logger.info({ index: indexFileName }, 'Cotar.Index:Start');
+    const startTime = Date.now();
+
+    const { buffer, count } = await CotarIndexBuilder.create(fd, opts);
+
+    logger.info({ count, size: buffer.length, duration: Date.now() - startTime }, 'Cotar.Index:Created');
+    const index = await CotarIndexBinary.create(new SourceMemory('index', buffer));
+    await TarReader.validate(fd, index);
+    await fd.close();
+    return buffer;
+  }
+}

packages/cli/src/cli/index.ts

@@ -4,6 +4,7 @@ import 'source-map-support/register.js';
 import { CommandList } from './aws/action.aws.list.js';
 import { CommandCogCreate } from './cogify/action.cog.js';
 import { CommandJobCreate } from './cogify/action.job.js';
+import { CommandBundleAssets } from './config/action.bundle.assets.js';
 import { CommandBundle } from './config/action.bundle.js';
 import { CommandImport } from './config/action.import.js';
 import { CommandScreenShot } from './screenshot/action.screenshot.js';
@@ -20,6 +21,7 @@ export class BasemapsConfigCommandLine extends BaseCommandLine {
     this.addAction(new CommandJobCreate());
 
     this.addAction(new CommandBundle());
+    this.addAction(new CommandBundleAssets());
     this.addAction(new CommandImport());
 
     this.addAction(new CommandScreenShot());

packages/lambda-tiler/package.json

@@ -30,7 +30,7 @@
     "@basemaps/tiler-sharp": "^6.29.0",
     "@chunkd/fs": "^8.4.0",
     "@cogeotiff/core": "^7.0.0",
-    "@cotar/core": "^5.3.0",
+    "@cotar/core": "^5.4.0",
     "@linzjs/geojson": "^6.28.1",
     "@linzjs/lambda": "^3.0.1",
     "p-limit": "^4.0.0",

yarn.lock

@@ -52,11 +52,6 @@
     source-map-support "^0.5.19"
     ulid "^2.3.0"
 
-"@chunkd/core@^8.0.1":
-  version "8.0.3"
-  resolved "https://registry.yarnpkg.com/@chunkd/core/-/core-8.0.3.tgz#5ab9ef98ea1ccf33933d4de9717ed413a0c99b5a"
-  integrity sha512-BqfSpkOgTIfO5mDwx5x1RkKlARI5ZZYQSsC9KWU9aDbhPew33faOC/EkEW0kSYKbIL4pestI9XjAZ8WAhectkQ==
-
 "@chunkd/core@^8.1.0":
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/@chunkd/core/-/core-8.1.0.tgz#724da02f4059eef853c8b690af6729d77d54702c"
@@ -124,15 +119,22 @@
     "@chunkd/core" "^8.1.0"
     ieee754 "^1.1.13"
 
-"@cotar/core@^5.3.0":
-  version "5.3.0"
-  resolved "https://registry.yarnpkg.com/@cotar/core/-/core-5.3.0.tgz#69423f9ee77a8630325e486256d20c752e2e1193"
-  integrity sha512-4GaAqJsPCM5CwY6Z8p7GO/hA0d5DowuoH7xTje1buu8ggWF1IN2APSvg/TeV2iD7OctxedPTXIbZpMeg5DhOsw==
+"@cotar/core@^5.4.0":
+  version "5.4.0"
+  resolved "https://registry.yarnpkg.com/@cotar/core/-/core-5.4.0.tgz#9d10dee807044fb2f136852a7736a41edce859ff"
+  integrity sha512-hZOsojezrjHu73ZZUx0rz8BCzqmcTmdBBg5OaAg3MziJKIJnWGsCKA1OdL9cT9Csib+iW/DL4UxP/RfMlchZ3w==
   dependencies:
-    "@chunkd/core" "^8.0.1"
+    "@chunkd/core" "^8.4.0"
     "@sindresorhus/fnv1a" "^3.0.0"
     binparse "^2.0.1"
 
+"@cotar/tar@^5.4.1":
+  version "5.4.1"
+  resolved "https://registry.yarnpkg.com/@cotar/tar/-/tar-5.4.1.tgz#f91a79c6050e81a8022a5529af373fabcc1f2280"
+  integrity sha512-2hsVUhjEzezX3RxKEA4ZKisxX4rGPaKy3L2f5PVogpSjuydzpq429+h3L0UPfrKUGBaqeoZkIKdF9j38gMg3OA==
+  dependencies:
+    tar-stream "^2.2.0"
+
 "@emotion/cache@^11.4.0", "@emotion/cache@^11.6.0":
   version "11.6.0"
   resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.6.0.tgz#65fbdbbe4382f1991d8b20853c38e63ecccec9a1"
@@ -7247,6 +7249,17 @@ tar-stream@^2.1.4:
     inherits "^2.0.3"
     readable-stream "^3.1.1"
 
+tar-stream@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"
+  integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==
+  dependencies:
+    bl "^4.0.3"
+    end-of-stream "^1.4.1"
+    fs-constants "^1.0.0"
+    inherits "^2.0.3"
+    readable-stream "^3.1.1"
+
 tar@^6.0.2, tar@^6.1.0:
   version "6.1.0"
   resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.0.tgz#d1724e9bcc04b977b18d5c573b333a2207229a83"