feat(cli): New cli to bundle the assets into cotar file.

packages/cli/package.json

@@ -45,6 +45,8 @@
     "@basemaps/sprites": "^6.29.0",
     "@chunkd/fs": "^8.4.0",
     "@cogeotiff/core": "^7.0.0",
+    "@cotar/core": "^5.4.0",
+    "@cotar/tar": "^5.4.0",
     "@linzjs/geojson": "^6.28.1",
     "@rushstack/ts-command-line": "^4.3.13",
     "ansi-colors": "^4.1.1",

packages/cli/src/cli/config/action.bundle.assets.ts

@@ -0,0 +1,102 @@
+import { CommandLineAction, CommandLineStringParameter } from '@rushstack/ts-command-line';
+import { SourceMemory } from '@chunkd/core';
+import { CotarIndexBinary, CotarIndexBuilder, CotarIndexOptions, TarReader } from '@cotar/core';
+import { LogConfig, LogType } from '@basemaps/shared';
+import { promises as fs } from 'fs';
+import { TarBuilder } from '@cotar/tar';
+import { fsa } from '@chunkd/fs';
+
+const Packing = 25; // Packing factor for the hash map
+const MaxSearch = 50; // Max search factor
+
+export class CommandBundleAssets extends CommandLineAction {
+  path: CommandLineStringParameter;
+  output: CommandLineStringParameter;
+
+  public constructor() {
+    super({
+      actionName: 'bundle-assets',
+      summary: 'Cli tool to create cotar file for the config assets',
+      documentation: 'Create a cotar file from a directory',
+    });
+  }
+
+  protected onDefineParameters(): void {
+    this.path = this.defineStringParameter({
+      argumentName: 'ASSETS',
+      parameterLongName: '--assets',
+      description: 'Paths to the input assets files, must contain assets/sprites/ and assets/fonts.',
+    });
+
+    this.output = this.defineStringParameter({
+      argumentName: 'OUTPUT',
+      parameterLongName: '--output',
+      description: 'Paths of the output co tar file.',
+    });
+  }
+
+  protected async onExecute(): Promise<void> {
+    const logger = LogConfig.get();
+    const path = this.path.value;
+    const output = this.output.value;
+    if (path == null || output == null) throw new Error('Please provide a input or ouput path');
+    if (output.endsWith('.tar')) throw new Error(`Invalid output, needs to be .tar.co :"${output}"`);
+
+    logger.info({ indput: path }, 'BundleAssets:Start');
+    const tarFile = await this.buildTar(path, output, logger);
+    const cotarFile = await this.buildTarCo(tarFile, output, logger);
+    logger.info({ output: cotarFile }, 'BundleAssets:Finish');
+  }
+
+  async buildTar(input: string, output: string, logger: LogType): Promise<string> {
+    const startTime = Date.now();
+    const outputTar = output.split('.co')[0];
+
+    const tarBuilder = new TarBuilder(outputTar);
+
+    const files = await fsa.toArray(fsa.list(input));
+    // Ensure files are put into the same order
+    files.sort((a, b) => a.localeCompare(b));
+    logger.info({ output: outputTar, files: files.length }, 'Tar:Create');
+
+    for (const file of files) await tarBuilder.write(file, await fsa.read(file));
+
+    await tarBuilder.close();
+
+    logger.info({ output: outputTar, stats: tarBuilder.stats, duration: Date.now() - startTime }, 'Tar:Create:Done');
+    return outputTar;
+  }
+
+  async buildTarCo(input: string, output: string, logger: LogType): Promise<string> {
+    logger.info({ output }, 'Cotar:Create');
+
+    const opts: CotarIndexOptions = { packingFactor: 1 + Packing / 100, maxSearch: MaxSearch };
+
+    const indexFile = input + '.index';
+
+    const startTime = Date.now();
+    const buf = await this.toTarIndex(input, indexFile, opts, logger);
+
+    logger.info({ output }, 'Cotar:Craete:WriteTar');
+    await fs.copyFile(input, output);
+    await fs.appendFile(output, buf);
+
+    const duration = Date.now() - startTime;
+    logger.info({ output, duration }, 'Cotar:Created');
+    return output;
+  }
+
+  async toTarIndex(filename: string, indexFileName: string, opts: CotarIndexOptions, logger: LogType): Promise<Buffer> {
+    const fd = await fs.open(filename, 'r');
+    logger.info({ index: indexFileName }, 'Cotar.Index:Start');
+    const startTime = Date.now();
+
+    const { buffer, count } = await CotarIndexBuilder.create(fd, opts);
+
+    logger.info({ count, size: buffer.length, duration: Date.now() - startTime }, 'Cotar.Index:Created');
+    const index = await CotarIndexBinary.create(new SourceMemory('index', buffer));
+    await TarReader.validate(fd, index);
+    await fd.close();
+    return buffer;
+  }
+}

packages/cli/src/cli/index.ts

@@ -4,6 +4,7 @@ import 'source-map-support/register.js';
 import { CommandList } from './aws/action.aws.list.js';
 import { CommandCogCreate } from './cogify/action.cog.js';
 import { CommandJobCreate } from './cogify/action.job.js';
+import { CommandBundleAssets } from './config/action.bundle.assets.js';
 import { CommandBundle } from './config/action.bundle.js';
 import { CommandImport } from './config/action.import.js';
 import { CommandScreenShot } from './screenshot/action.screenshot.js';
@@ -20,6 +21,7 @@ export class BasemapsConfigCommandLine extends BaseCommandLine {
     this.addAction(new CommandJobCreate());
 
     this.addAction(new CommandBundle());
+    this.addAction(new CommandBundleAssets());
     this.addAction(new CommandImport());
 
     this.addAction(new CommandScreenShot());

yarn.lock

@@ -133,6 +133,22 @@
     "@sindresorhus/fnv1a" "^3.0.0"
     binparse "^2.0.1"
 
+"@cotar/core@^5.4.0":
+  version "5.4.0"
+  resolved "https://registry.yarnpkg.com/@cotar/core/-/core-5.4.0.tgz#9d10dee807044fb2f136852a7736a41edce859ff"
+  integrity sha512-hZOsojezrjHu73ZZUx0rz8BCzqmcTmdBBg5OaAg3MziJKIJnWGsCKA1OdL9cT9Csib+iW/DL4UxP/RfMlchZ3w==
+  dependencies:
+    "@chunkd/core" "^8.4.0"
+    "@sindresorhus/fnv1a" "^3.0.0"
+    binparse "^2.0.1"
+
+"@cotar/tar@^5.4.0":
+  version "5.4.0"
+  resolved "https://registry.yarnpkg.com/@cotar/tar/-/tar-5.4.0.tgz#f7508a09e509505fbc6b8dea544658536ba35040"
+  integrity sha512-fmWTPgzqEzmzlPxJs0dksBMeNuiLFsKK22Fy9T2tshWpP7ONJei410wQi2akb5glOqdw90ldZV7rq6a4Kupxfw==
+  dependencies:
+    tar-stream "^2.2.0"
+
 "@emotion/cache@^11.4.0", "@emotion/cache@^11.6.0":
   version "11.6.0"
   resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-11.6.0.tgz#65fbdbbe4382f1991d8b20853c38e63ecccec9a1"
@@ -7247,6 +7263,17 @@ tar-stream@^2.1.4:
     inherits "^2.0.3"
     readable-stream "^3.1.1"
 
+tar-stream@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"
+  integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==
+  dependencies:
+    bl "^4.0.3"
+    end-of-stream "^1.4.1"
+    fs-constants "^1.0.0"
+    inherits "^2.0.3"
+    readable-stream "^3.1.1"
+
 tar@^6.0.2, tar@^6.1.0:
   version "6.1.0"
   resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.0.tgz#d1724e9bcc04b977b18d5c573b333a2207229a83"