diff --git a/packages/cli/src/cli/config/action.bundle.assets.ts b/packages/cli/src/cli/config/action.bundle.assets.ts index a4b86f150..539ee0338 100644 --- a/packages/cli/src/cli/config/action.bundle.assets.ts +++ b/packages/cli/src/cli/config/action.bundle.assets.ts @@ -2,14 +2,26 @@ import { CommandLineAction, CommandLineStringParameter } from '@rushstack/ts-com import { SourceMemory } from '@chunkd/core'; import { CotarIndexBinary, CotarIndexBuilder, CotarIndexOptions, TarReader } from '@cotar/core'; import { LogConfig, LogType } from '@basemaps/shared'; -import { promises as fs } from 'fs'; +import { createReadStream, promises as fs } from 'fs'; import { TarBuilder } from '@cotar/tar'; import { fsa } from '@chunkd/fs'; import * as path from 'path'; +import { Readable } from 'stream'; +import { createHash } from 'crypto'; +import { base58 } from '@basemaps/config'; const Packing = 25; // Packing factor for the hash map const MaxSearch = 50; // Max search factor +export async function hashFile(stream: Readable): Promise { + return new Promise((resolve, reject) => { + const hash = createHash('sha256'); + stream.on('data', (chunk) => hash.update(chunk)); + stream.on('end', () => resolve(base58.encode(hash.digest()))); + stream.on('error', (err) => reject(err)); + }); +} + export class CommandBundleAssets extends CommandLineAction { assets: CommandLineStringParameter; output: CommandLineStringParameter; @@ -46,7 +58,11 @@ export class CommandBundleAssets extends CommandLineAction { logger.info({ indput: assets }, 'BundleAssets:Start'); const tarFile = await this.buildTar(assets, output, logger); const cotarFile = await this.buildTarCo(tarFile, output, logger); - logger.info({ output: cotarFile }, 'BundleAssets:Finish'); + const cotarHash = await hashFile(createReadStream(cotarFile)); + + await fs.rename(cotarFile, cotarFile.replace('.tar.co', `-${cotarHash}.tar.co`)); + + logger.info({ output: cotarFile, hash: cotarHash }, 'BundleAssets:Finish'); } async buildTar(input: string, output: string, logger: LogType): Promise { diff --git a/packages/cli/src/cli/config/action.import.ts b/packages/cli/src/cli/config/action.import.ts index e8bde3e06..aa61bda44 100644 --- a/packages/cli/src/cli/config/action.import.ts +++ b/packages/cli/src/cli/config/action.import.ts @@ -3,6 +3,7 @@ import { BaseConfig, Config, ConfigBundle, ConfigBundled, ConfigProviderMemory } import { CommandLineAction, CommandLineFlagParameter, CommandLineStringParameter } from '@rushstack/ts-command-line'; import { Q, Updater } from './config.update.js'; import { invalidateCache } from '../util.js'; +import fetch from 'node-fetch'; export class CommandImport extends CommandLineAction { private config: CommandLineStringParameter; @@ -49,8 +50,9 @@ export class CommandImport extends CommandLineAction { const config = this.config.value; const backup = this.backup.value; if (config == null) throw new Error('Please provide a config json'); - if (commit && !config.startsWith('s3://')) - throw new Error('To acturally import into dynamo has to use the config file from s3.'); + if (commit && !config.startsWith('s3://')) { + throw new Error('To actually import into dynamo has to use the config file from s3.'); + } const HostPrefix = Env.isProduction() ? '' : 'dev.'; const healthEndpoint = `https://${HostPrefix}basemaps.linz.govt.nz/v1/health`; diff --git a/packages/config/package.json b/packages/config/package.json index e1976efde..fddb21f20 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -29,6 +29,7 @@ ], "dependencies": { "@basemaps/geo": "^6.28.1", + "base-x": "^4.0.0", "zod": "^3.17.3" } } diff --git a/packages/config/src/base58.ts b/packages/config/src/base58.ts new file mode 100644 index 000000000..2d2479a84 --- /dev/null +++ b/packages/config/src/base58.ts @@ -0,0 +1,10 @@ +import baseX from 'base-x'; +import { BinaryLike, createHash } from 'crypto'; + +const Base58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'; +export const base58 = baseX(Base58); + +/** Hash something with sha256 then encode it as a base58 text string */ +export function sha256base58(obj: BinaryLike): string { + return base58.encode(createHash('sha256').update(obj).digest()); +} diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 95920bd05..44987653e 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -27,3 +27,4 @@ export { TileSetNameComponents, TileSetNameParser } from './tile.set.name.js'; export { parseHex, parseRgba } from './color.js'; export { ConfigJson } from './json/json.config.js'; export { standardizeLayerName } from './json/name.convertor.js'; +export { base58, sha256base58 } from './base58.js'; diff --git a/packages/config/src/memory/memory.config.ts b/packages/config/src/memory/memory.config.ts index ce86e0f82..15b53488b 100644 --- a/packages/config/src/memory/memory.config.ts +++ b/packages/config/src/memory/memory.config.ts @@ -1,5 +1,4 @@ import { ImageFormat } from '@basemaps/geo'; -import { createHash } from 'crypto'; import { decodeTime, ulid } from 'ulid'; import { BasemapsConfigObject, BasemapsConfigProvider, Config } from '../base.config.js'; import { BaseConfig } from '../config/base.js'; @@ -11,6 +10,7 @@ import { ConfigTileSet, TileSetType } from '../config/tile.set.js'; import { ConfigVectorStyle } from '../config/vector.style.js'; import { ConfigBundle } from '../config/config.bundle.js'; import { standardizeLayerName } from '../json/name.convertor.js'; +import { sha256base58 } from '../base58.js'; /** bundle the configuration as a single JSON object */ export interface ConfigBundled { @@ -92,7 +92,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider { } } - cfg.hash = createHash('sha256').update(JSON.stringify(cfg)).digest('base64url'); + cfg.hash = sha256base58(JSON.stringify(cfg)); cfg.id = Config.prefix(ConfigPrefix.ConfigBundle, ulid()); return cfg; diff --git a/yarn.lock b/yarn.lock index 10c2e6340..0a7baf1d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1778,7 +1778,7 @@ ansi-escapes@^4.2.1: dependencies: type-fest "^0.21.3" -"ansi-regex@>=5.0.1 <6.0.0", ansi-regex@^5.0.0, ansi-regex@^5.0.1: +ansi-regex@^5.0.0, ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== @@ -1942,6 +1942,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +base-x@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/base-x/-/base-x-4.0.0.tgz#d0e3b7753450c73f8ad2389b5c018a4af7b2224a" + integrity sha512-FuwxlW4H5kh37X/oW59pwTzzTKRzfrrQwhmyspRM7swOEZcHtDZSCt45U6oKgtuFE+WYPblePMVIPR4RZrh/hw== + base64-js@^1.0.2: version "1.3.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" @@ -4055,7 +4060,19 @@ hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.1: dependencies: react-is "^16.7.0" -hosted-git-info@>=2.8.9, hosted-git-info@^2.1.4, hosted-git-info@^4.0.1, hosted-git-info@^5.0.0: +hosted-git-info@^2.1.4: + version "2.8.9" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9" + integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== + +hosted-git-info@^4.0.1: + version "4.1.0" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz#827b82867e9ff1c8d0c4d9d53880397d2c86d224" + integrity sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA== + dependencies: + lru-cache "^6.0.0" + +hosted-git-info@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-5.0.0.tgz#df7a06678b4ebd722139786303db80fdf302ea56" integrity sha512-rRnjWu0Bxj+nIfUOkz0695C0H6tRrN5iYIzYejb0tDEefe2AekHu/U5Kn9pEie5vsJqpNQU02az7TGSH3qpz4Q== @@ -4759,7 +4776,7 @@ lodash.templatesettings@^4.0.0: dependencies: lodash._reinterpolate "^3.0.0" -lodash@>=4.17.21, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.2.1, lodash@^4.7.0: +lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.2.1, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -5422,10 +5439,19 @@ normalize-package-data@^4.0.0: semver "^7.3.5" validate-npm-package-license "^3.0.4" -normalize-url@2.0.1, "normalize-url@>=4.5.1 <5.0.0", normalize-url@^3.3.0: - version "4.5.1" - resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.1.tgz#0dd90cf1288ee1d1313b87081c9a5932ee48518a" - integrity sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA== +normalize-url@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-2.0.1.tgz#835a9da1551fa26f70e92329069a23aa6574d7e6" + integrity sha512-D6MUW4K/VzoJ4rJ01JFKxDrtY1v9wrgzCX5f2qj/lzH1m/lW6MhUZFKerVsnyjOhOsYzI9Kqqak+10l4LvLpMw== + dependencies: + prepend-http "^2.0.0" + query-string "^5.0.1" + sort-keys "^2.0.0" + +normalize-url@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-3.3.0.tgz#b2e1c4dc4f7c6d57743df733a4f5978d18650559" + integrity sha512-U+JJi7duF1o+u2pynbp2zXDW2/PADgC30f0GsHZtRh+HOcXHnw137TrNlyxxRvWW5fjKd3bcLHPxofWuCjaeZg== npm-bundled@^1.1.1: version "1.1.1" @@ -5558,7 +5584,7 @@ number-is-nan@^1.0.0: resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= -object-assign@^4.0.1, object-assign@^4.1.1: +object-assign@^4.0.1, object-assign@^4.1.0, object-assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= @@ -6220,6 +6246,15 @@ q@^1.4.1, q@^1.5.1: resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc= +query-string@^5.0.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/query-string/-/query-string-5.1.1.tgz#a78c012b71c17e05f2e3fa2319dd330682efb3cb" + integrity sha512-gjWOsm2SoGlgLEdAGt7a6slVOk9mGiXmPFMqrEhLQ68rhQuBnpfs3+EmlvqKyxnCo9/PPlF+9MtY02S1aFg+Jw== + dependencies: + decode-uri-component "^0.2.0" + object-assign "^4.1.0" + strict-uri-encode "^1.0.0" + querystring@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" @@ -7076,6 +7111,11 @@ stream-shift@^1.0.0: resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== +strict-uri-encode@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713" + integrity sha512-R3f198pcvnB+5IpnBlRkphuE9n46WyVl8I39W/ZUTZLz4nqSP/oLYUrcnJrw462Ds8he4YKMov2efsTIw1BDGQ== + string-argv@~0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da" @@ -7406,7 +7446,17 @@ treeverse@^2.0.0: resolved "https://registry.yarnpkg.com/treeverse/-/treeverse-2.0.0.tgz#036dcef04bc3fd79a9b79a68d4da03e882d8a9ca" integrity sha512-N5gJCkLu1aXccpOTtqV6ddSEi6ZmGkh3hjmbu1IjcavJK4qyOVQmi0myQKM7z5jVGmD68SJoliaVrMmVObhj6A== -"trim-newlines@>=3.0.1 <4.0.0", trim-newlines@^1.0.0, trim-newlines@^2.0.0, trim-newlines@^3.0.0: +trim-newlines@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613" + integrity sha512-Nm4cF79FhSTzrLKGDMi3I4utBtFv8qKy4sq1enftf2gMdpqI8oVQTAfySkTz5r49giVzDj88SVZXP4CeYQwjaw== + +trim-newlines@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-2.0.0.tgz#b403d0b91be50c331dfc4b82eeceb22c3de16d20" + integrity sha512-MTBWv3jhVjTU7XR3IQHllbiJs8sc75a80OEhB6or/q7pLTWgQ0bMGQXXYQSrSuXe6WiKWDZ5txXY5P59a/coVA== + +trim-newlines@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144" integrity sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw==