Skip to content

Commit

Permalink
feat(config): use base58 hashes
Browse files Browse the repository at this point in the history
  • Loading branch information
@blacha
blacha committed Jul 21, 2022
1 parent 604b748 commit e11628f
Show file tree
Hide file tree
Showing 6 changed files with 91 additions and 12 deletions.
20 changes: 18 additions & 2 deletions packages/cli/src/cli/config/action.bundle.assets.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,26 @@ import { CommandLineAction, CommandLineStringParameter } from '@rushstack/ts-com
import { SourceMemory } from '@chunkd/core';
import { CotarIndexBinary, CotarIndexBuilder, CotarIndexOptions, TarReader } from '@cotar/core';
import { LogConfig, LogType } from '@basemaps/shared';
import { promises as fs } from 'fs';
import { createReadStream, promises as fs } from 'fs';
import { TarBuilder } from '@cotar/tar';
import { fsa } from '@chunkd/fs';
import * as path from 'path';
import { Readable } from 'stream';
import { createHash } from 'crypto';
import { base58 } from '@basemaps/config';

const Packing = 25; // Packing factor for the hash map
const MaxSearch = 50; // Max search factor

export async function hashFile(stream: Readable): Promise<string> {
return new Promise((resolve, reject) => {
const hash = createHash('sha256');
stream.on('data', (chunk) => hash.update(chunk));
stream.on('end', () => resolve(base58.encode(hash.digest())));
stream.on('error', (err) => reject(err));
});
}

export class CommandBundleAssets extends CommandLineAction {
assets: CommandLineStringParameter;
output: CommandLineStringParameter;
Expand Down Expand Up @@ -46,7 +58,11 @@ export class CommandBundleAssets extends CommandLineAction {
logger.info({ indput: assets }, 'BundleAssets:Start');
const tarFile = await this.buildTar(assets, output, logger);
const cotarFile = await this.buildTarCo(tarFile, output, logger);
logger.info({ output: cotarFile }, 'BundleAssets:Finish');
const cotarHash = await hashFile(createReadStream(cotarFile));

await fs.rename(cotarFile, cotarFile.replace('.tar.co', `-${cotarHash}.tar.co`));

logger.info({ output: cotarFile, hash: cotarHash }, 'BundleAssets:Finish');
}

async buildTar(input: string, output: string, logger: LogType): Promise<string> {
Expand Down
1 change: 1 addition & 0 deletions packages/config/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
],
"dependencies": {
"@basemaps/geo": "^6.28.1",
"base-x": "^4.0.0",
"zod": "^3.17.3"
}
}
10 changes: 10 additions & 0 deletions packages/config/src/base58.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import baseX from 'base-x';
import { BinaryLike, createHash } from 'crypto';

const Base58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
export const base58 = baseX(Base58);

/** Hash something with sha256 then encode it as a base58 text string */
export function sha256base58(obj: BinaryLike): string {
return base58.encode(createHash('sha256').update(obj).digest());
}
1 change: 1 addition & 0 deletions packages/config/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,4 @@ export { TileSetNameComponents, TileSetNameParser } from './tile.set.name.js';
export { parseHex, parseRgba } from './color.js';
export { ConfigJson } from './json/json.config.js';
export { standardizeLayerName } from './json/name.convertor.js';
export { base58, sha256base58 } from './base58.js';
3 changes: 2 additions & 1 deletion packages/config/src/memory/memory.config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { ConfigTileSet, TileSetType } from '../config/tile.set.js';
import { ConfigVectorStyle } from '../config/vector.style.js';
import { ConfigBundle } from '../config/config.bundle.js';
import { standardizeLayerName } from '../json/name.convertor.js';
import { sha256base58 } from '../base58.js';

/** bundle the configuration as a single JSON object */
export interface ConfigBundled {
Expand Down Expand Up @@ -92,7 +93,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider {
}
}

cfg.hash = createHash('sha256').update(JSON.stringify(cfg)).digest('base64url');
cfg.hash = sha256base58(JSON.stringify(cfg));
cfg.id = Config.prefix(ConfigPrefix.ConfigBundle, ulid());

return cfg;
Expand Down
68 changes: 59 additions & 9 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1778,7 +1778,7 @@ ansi-escapes@^4.2.1:
dependencies:
type-fest "^0.21.3"

"ansi-regex@>=5.0.1 <6.0.0", ansi-regex@^5.0.0, ansi-regex@^5.0.1:
ansi-regex@^5.0.0, ansi-regex@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304"
integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==
Expand Down Expand Up @@ -1942,6 +1942,11 @@ balanced-match@^1.0.0:
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==

base-x@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/base-x/-/base-x-4.0.0.tgz#d0e3b7753450c73f8ad2389b5c018a4af7b2224a"
integrity sha512-FuwxlW4H5kh37X/oW59pwTzzTKRzfrrQwhmyspRM7swOEZcHtDZSCt45U6oKgtuFE+WYPblePMVIPR4RZrh/hw==

base64-js@^1.0.2:
version "1.3.1"
resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1"
Expand Down Expand Up @@ -4055,7 +4060,19 @@ hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.1:
dependencies:
react-is "^16.7.0"

hosted-git-info@>=2.8.9, hosted-git-info@^2.1.4, hosted-git-info@^4.0.1, hosted-git-info@^5.0.0:
hosted-git-info@^2.1.4:
version "2.8.9"
resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9"
integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==

hosted-git-info@^4.0.1:
version "4.1.0"
resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz#827b82867e9ff1c8d0c4d9d53880397d2c86d224"
integrity sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==
dependencies:
lru-cache "^6.0.0"

hosted-git-info@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-5.0.0.tgz#df7a06678b4ebd722139786303db80fdf302ea56"
integrity sha512-rRnjWu0Bxj+nIfUOkz0695C0H6tRrN5iYIzYejb0tDEefe2AekHu/U5Kn9pEie5vsJqpNQU02az7TGSH3qpz4Q==
Expand Down Expand Up @@ -4759,7 +4776,7 @@ lodash.templatesettings@^4.0.0:
dependencies:
lodash._reinterpolate "^3.0.0"

lodash@>=4.17.21, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.2.1, lodash@^4.7.0:
lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.2.1, lodash@^4.7.0:
version "4.17.21"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
Expand Down Expand Up @@ -5422,10 +5439,19 @@ normalize-package-data@^4.0.0:
semver "^7.3.5"
validate-npm-package-license "^3.0.4"

normalize-url@2.0.1, "normalize-url@>=4.5.1 <5.0.0", normalize-url@^3.3.0:
version "4.5.1"
resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.1.tgz#0dd90cf1288ee1d1313b87081c9a5932ee48518a"
integrity sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==
normalize-url@2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-2.0.1.tgz#835a9da1551fa26f70e92329069a23aa6574d7e6"
integrity sha512-D6MUW4K/VzoJ4rJ01JFKxDrtY1v9wrgzCX5f2qj/lzH1m/lW6MhUZFKerVsnyjOhOsYzI9Kqqak+10l4LvLpMw==
dependencies:
prepend-http "^2.0.0"
query-string "^5.0.1"
sort-keys "^2.0.0"

normalize-url@^3.3.0:
version "3.3.0"
resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-3.3.0.tgz#b2e1c4dc4f7c6d57743df733a4f5978d18650559"
integrity sha512-U+JJi7duF1o+u2pynbp2zXDW2/PADgC30f0GsHZtRh+HOcXHnw137TrNlyxxRvWW5fjKd3bcLHPxofWuCjaeZg==

npm-bundled@^1.1.1:
version "1.1.1"
Expand Down Expand Up @@ -5558,7 +5584,7 @@ number-is-nan@^1.0.0:
resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d"
integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=

object-assign@^4.0.1, object-assign@^4.1.1:
object-assign@^4.0.1, object-assign@^4.1.0, object-assign@^4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=
Expand Down Expand Up @@ -6220,6 +6246,15 @@ q@^1.4.1, q@^1.5.1:
resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=

query-string@^5.0.1:
version "5.1.1"
resolved "https://registry.yarnpkg.com/query-string/-/query-string-5.1.1.tgz#a78c012b71c17e05f2e3fa2319dd330682efb3cb"
integrity sha512-gjWOsm2SoGlgLEdAGt7a6slVOk9mGiXmPFMqrEhLQ68rhQuBnpfs3+EmlvqKyxnCo9/PPlF+9MtY02S1aFg+Jw==
dependencies:
decode-uri-component "^0.2.0"
object-assign "^4.1.0"
strict-uri-encode "^1.0.0"

querystring@0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
Expand Down Expand Up @@ -7076,6 +7111,11 @@ stream-shift@^1.0.0:
resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d"
integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==

strict-uri-encode@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"
integrity sha512-R3f198pcvnB+5IpnBlRkphuE9n46WyVl8I39W/ZUTZLz4nqSP/oLYUrcnJrw462Ds8he4YKMov2efsTIw1BDGQ==

string-argv@~0.3.1:
version "0.3.1"
resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da"
Expand Down Expand Up @@ -7406,7 +7446,17 @@ treeverse@^2.0.0:
resolved "https://registry.yarnpkg.com/treeverse/-/treeverse-2.0.0.tgz#036dcef04bc3fd79a9b79a68d4da03e882d8a9ca"
integrity sha512-N5gJCkLu1aXccpOTtqV6ddSEi6ZmGkh3hjmbu1IjcavJK4qyOVQmi0myQKM7z5jVGmD68SJoliaVrMmVObhj6A==

"trim-newlines@>=3.0.1 <4.0.0", trim-newlines@^1.0.0, trim-newlines@^2.0.0, trim-newlines@^3.0.0:
trim-newlines@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613"
integrity sha512-Nm4cF79FhSTzrLKGDMi3I4utBtFv8qKy4sq1enftf2gMdpqI8oVQTAfySkTz5r49giVzDj88SVZXP4CeYQwjaw==

trim-newlines@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-2.0.0.tgz#b403d0b91be50c331dfc4b82eeceb22c3de16d20"
integrity sha512-MTBWv3jhVjTU7XR3IQHllbiJs8sc75a80OEhB6or/q7pLTWgQ0bMGQXXYQSrSuXe6WiKWDZ5txXY5P59a/coVA==

trim-newlines@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144"
integrity sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw==
Expand Down

0 comments on commit e11628f

Please sign in to comment.