diff --git a/.gitignore b/.gitignore index 8515d58..9568fb9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ dist .nyc_output tmp node_modules +coverage diff --git a/package.json b/package.json index a4fe624..566f565 100644 --- a/package.json +++ b/package.json @@ -25,15 +25,17 @@ "test:web": "playwright-test test/**/*.spec.js --cov && nyc report", "test:node": "c8 --check-coverage --branches 95 --functions 83 --lines 94 mocha test/**/*.spec.js", "test": "npm run test:node", - "coverage": "c8 --reporter=html mocha test/test-*.js && npm_config_yes=true npx st -d coverage -p 8080", + "coverage": "c8 --reporter=html mocha test/**/*.spec.js && npm_config_yes=true npx st -d coverage -p 8080", "typecheck": "tsc --build", "test:convergence": "mocha test/convergence.js" }, "dependencies": { - "multiformats": "^11.0.1", "@ipld/dag-pb": "^4.0.0", + "@multiformats/murmur3": "^2.1.3", + "@perma/map": "^1.0.2", "@web-std/stream": "1.0.1", "actor": "^2.3.1", + "multiformats": "^11.0.1", "protobufjs": "^7.1.2", "rabin-rs": "^2.1.0" }, diff --git a/src/directory/api.ts b/src/directory/api.ts index bf851b4..107a318 100644 --- a/src/directory/api.ts +++ b/src/directory/api.ts @@ -82,7 +82,6 @@ export interface View extends Writer { readonly writer: BlockWriter readonly settings: EncoderSettings - links(): IterableIterator state: State entries(): IterableIterator<[string, EntryLink]> diff --git a/src/lib.js b/src/lib.js index 901304c..6aa50ee 100644 --- a/src/lib.js +++ b/src/lib.js @@ -21,6 +21,11 @@ export { set, remove, } from "./directory.js" +export { + create as createShardedDirectoryWriter, + close as closeShardedDirectory, + fork as forkShardedDirectory, +} from "./sharded-directory.js" /** * @template [Layout=unknown] diff --git a/src/sharded-directory.js b/src/sharded-directory.js new file mode 100644 index 0000000..18b21c4 --- /dev/null +++ b/src/sharded-directory.js @@ -0,0 +1,319 @@ + +import * as PermaMap from "@perma/map" +import * as UnixFSPermaMap from "@perma/map/unixfs" +import * as PB from "@ipld/dag-pb" +import { murmur364 } from "@multiformats/murmur3" +import { Block } from 'multiformats/block' +import * as API from "./directory/api.js" +import * as File from "./file.js" +import * as UnixFS from "./codec.js" +import { set, remove } from "./directory.js" + +export * from "./directory/api.js" +export { set, remove } from "./directory.js" + +export const configure = File.configure +export const defaults = File.defaults + +/** + * @template [Layout=unknown] + * @param {API.Options} config + * @returns {API.View} + */ +export const create = ({ writer, settings = defaults(), metadata = {} }) => + new HAMTDirectoryWriter({ + writer, + metadata, + settings, + entries: new HashMap(), + closed: false, + }) + +/** + * @template {API.State} Writer + * @param {Writer} writer + * @returns {Writer} + */ +const asWritable = writer => { + if (!writer.closed) { + return writer + } else { + throw new Error("Can not change written HAMT directory, but you can .fork() and make changes to it") + } +} + +/** + * @template {unknown} Layout + * @param {{ state: API.State }} view + * @param {API.CloseOptions} options + * @returns {Promise} + */ +export const close = async ( + view, + { closeWriter = false, releaseLock = false } = {} +) => { + const { writer, settings, metadata } = asWritable(view.state) + view.state.closed = true + + const { entries } = view.state + /* c8 ignore next 3 */ + if (!(entries instanceof HashMap)) { + throw new Error(`not a HAMT: ${entries}`) + } + + const hamt = entries.builder.build() + const blocks = iterateBlocks(hamt, hamt.root, settings) + + /** @type {UnixFS.BlockView?} */ + let root = null + for await (const block of blocks) { + root = block + // we make sure that writer has some capacity for this write. If it + // does not we await. + if ((writer.desiredSize || 0) <= 0) { + await writer.ready + } + // once writer has some capacity we write a block, however we do not + // await completion as we don't care when it's taken off the stream. + writer.write(block) + } + /* c8 ignore next */ + if (root == null) throw new Error("no root block yielded") + + if (closeWriter) { + await writer.close() + } else if (releaseLock) { + writer.releaseLock() + } + + return { + cid: root.cid, + dagByteLength: UnixFS.cumulativeDagByteLength(root.bytes, root.value.entries), + } +} + +/** + * @template {unknown} Layout + * @param {UnixFSPermaMap.PersistentHashMap} hamt + * @param {UnixFSPermaMap.BitmapIndexedNode} node + * @param {API.EncoderSettings} settings + * @returns {AsyncIterableIterator>} + */ +const iterateBlocks = async function* (hamt, node, settings) { + /** @type {UnixFS.DirectoryEntryLink[]} */ + const entries = [] + for (const ent of UnixFSPermaMap.iterate(node)) { + if ('key' in ent) { + entries.push(/** @type {UnixFS.DirectoryEntryLink} */ ({ + name: `${ent.prefix ?? ''}${ent.key ?? ''}`, + dagByteLength: ent.value.dagByteLength, + cid: ent.value.cid, + })) + } else { + /** @type {UnixFS.BlockView?} */ + let root = null + for await (const block of iterateBlocks(hamt, ent.node, settings)) { + yield block + root = block + } + /* c8 ignore next */ + if (root == null) throw new Error("no root block yielded") + + entries.push(/** @type {UnixFS.ShardedDirectoryLink} */ ({ + name: ent.prefix, + dagByteLength: UnixFS.cumulativeDagByteLength(root.bytes, root.value.entries), + cid: root.cid + })) + } + } + + const shard = UnixFS.createDirectoryShard( + entries, + UnixFSPermaMap.bitField(node), + UnixFSPermaMap.tableSize(hamt), + murmur364.code + ) + yield await encodeHAMTShardBlock(shard, settings) +} + +/** + * @template {unknown} Layout + * @param {UnixFS.DirectoryShard} shard + * @param {API.EncoderSettings} settings + * @returns {Promise>} + */ +async function encodeHAMTShardBlock (shard, settings) { + const bytes = UnixFS.encodeHAMTShard(shard) + const hash = await settings.hasher.digest(bytes) + const cid = settings.linker.createLink(PB.code, hash) + // @ts-ignore Link is not CID + return new Block({ cid, bytes, value: shard }) +} + +/** + * @template L1, L2 + * @param {API.View} state + * @param {Partial>} options + * @returns {API.View} + */ +export const fork = ( + { state }, + { + writer = state.writer, + metadata = state.metadata, + settings = state.settings, + } = {} +) => + new HAMTDirectoryWriter({ + writer, + metadata, + settings, + entries: new HashMap(UnixFSPermaMap.from(state.entries.entries()).createBuilder()), + closed: false, + }) + +/** + * @template [Layout=unknown] + * @implements {API.View} + */ +class HAMTDirectoryWriter { + /** + * @param {API.State} state + */ + constructor(state) { + this.state = state + } + get writer() { + return this.state.writer + } + get settings() { + return this.state.settings + } + + /** + * @param {string} name + * @param {UnixFS.FileLink | UnixFS.DirectoryLink} link + * @param {API.WriteOptions} [options] + */ + + set(name, link, options) { + return set(this, name, link, options) + } + + /** + * @param {string} name + */ + remove(name) { + return remove(this, name) + } + + /** + * @template L + * @param {Partial>} [options] + * @returns {API.View} + */ + fork(options) { + return fork(this, options) + } + + /** + * @param {API.CloseOptions} [options] + * @returns {Promise} + */ + close(options) { + return close(this, options) + } + + entries() { + return this.state.entries.entries() + } + /** + * @param {string} name + */ + has(name) { + return this.state.entries.has(name) + } + get size() { + return this.state.entries.size + } +} + +/** + * @implements {Map} + */ +class HashMap extends Map { + /** + * @param {UnixFSPermaMap.HashMapBuilder} [builder] + */ + constructor (builder = UnixFSPermaMap.builder()) { + super() + /** @type {UnixFSPermaMap.HashMapBuilder} */ + this.builder = builder + } + + clear() { + this.builder = UnixFSPermaMap.builder() + } + + /** + * @param {string} key + */ + delete(key) { + const { root } = this.builder + this.builder.delete(key) + return this.builder.root !== root + } + + /** + * @param {(value: API.EntryLink, key: string, map: Map) => void} callbackfn + * @param {any} [thisArg] + */ + forEach(callbackfn, thisArg = this) { + for (const [k, v] of this.builder.root.entries()) { + callbackfn.call(thisArg, v, k, this) + } + } + + /** + * @param {string} key + */ + get(key) { + return PermaMap.get(this.builder, key) + } + + /** + * @param {string} key + */ + has(key) { + return PermaMap.has(this.builder, key) + } + + /** + * @param {string} key + * @param {API.EntryLink} value + */ + set(key, value) { + this.builder.set(key, value) + return this + } + + get size () { + return this.builder.size + } + + [Symbol.iterator]() { + return this.builder.root.entries() + } + + entries() { + return this.builder.root.entries() + } + + keys() { + return this.builder.root.keys() + } + + values() { + return this.builder.root.values() + } +} diff --git a/src/unixfs.ts b/src/unixfs.ts index 4020083..a4e88b7 100644 --- a/src/unixfs.ts +++ b/src/unixfs.ts @@ -6,6 +6,7 @@ import type { Link as IPLDLink, Version as LinkVersion, Block as IPLDBlock, + BlockView as IPLDBlockView } from "multiformats" import { Data, type IData } from "../gen/unixfs.js" export type { MultihashHasher, MultibaseEncoder, MultihashDigest, BlockEncoder } @@ -401,3 +402,10 @@ export interface Block< A extends number = number, V extends LinkVersion = LinkVersion > extends IPLDBlock {} + +export interface BlockView< + T = unknown, + C extends number = number, + A extends number = number, + V extends LinkVersion = LinkVersion +> extends IPLDBlockView {} diff --git a/test/directory.spec.js b/test/directory.spec.js index a03c815..e054cab 100644 --- a/test/directory.spec.js +++ b/test/directory.spec.js @@ -563,35 +563,6 @@ describe("test directory", () => { assert.deepEqual([...root.entries()], [["file.txt", fileLink]]) }) - it("can enumerate links", async function () { - const { writable } = new TransformStream() - const writer = writable.getWriter() - const root = UnixFS.createDirectoryWriter({ writer }) - - assert.deepEqual([...root.links()], []) - /** @type {Link.Link} */ - const cid = Link.parse( - "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" - ) - const fileLink = { - cid, - dagByteLength: 45, - contentByteLength: 37, - } - - root.set("file.txt", fileLink) - assert.deepEqual( - [...root.links()], - [ - { - name: "file.txt", - cid: fileLink.cid, - dagByteLength: fileLink.dagByteLength, - }, - ] - ) - }) - it(".has", async function () { const { writable } = new TransformStream() const writer = writable.getWriter() diff --git a/test/sharded-directory.spec.js b/test/sharded-directory.spec.js new file mode 100644 index 0000000..a95436e --- /dev/null +++ b/test/sharded-directory.spec.js @@ -0,0 +1,800 @@ +import * as UnixFS from "../src/lib.js" +import { TransformStream } from "@web-std/stream" +import { assert } from "chai" +import { encodeUTF8, Link, collect, importFile } from "./util.js" + +const createChannel = () => new TransformStream() +describe("test directory", () => { + it("empty dir", async () => { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + const root = UnixFS.createShardedDirectoryWriter({ writer }) + const link = await root.close() + writer.close() + + assert.deepEqual(link, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeifoplefg5piy3pjhlp73q7unqx4hwecxeu7opfqfmg352pkpljt6m" + ), + dagByteLength: 9, + }) + const output = await collect(readable) + + assert.deepEqual( + output.map($ => $.cid), + [ + Link.parse( + "bafybeifoplefg5piy3pjhlp73q7unqx4hwecxeu7opfqfmg352pkpljt6m" + ), + ] + ) + }) + + it("basic file in directory", async () => { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + const blocks = collect(readable) + const root = UnixFS.createShardedDirectoryWriter({ writer }) + const file = UnixFS.createFileWriter(root) + const content = encodeUTF8("this file does not have much content\n") + file.write(content) + const fileLink = await file.close() + + assert.deepEqual(fileLink, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ), + dagByteLength: 45, + contentByteLength: 37, + }) + + root.set("file.txt", fileLink) + const rootLink = await root.close() + + assert.deepEqual(rootLink, { + dagByteLength: 133, + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeibbyshlpvztob4mtwznmnkzoc4upgcf6ghaulujxglzgmglcdubtm" + ), + }) + + writer.close() + + const output = await blocks + + assert.deepEqual( + output.map($ => $.cid), + [ + Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ), + Link.parse( + "bafybeibbyshlpvztob4mtwznmnkzoc4upgcf6ghaulujxglzgmglcdubtm" + ), + ] + ) + }) + + it("many files in directory", async () => { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + const blocks = collect(readable) + const root = UnixFS.createShardedDirectoryWriter({ writer }) + const file = UnixFS.createFileWriter(root) + const content = encodeUTF8("this file does not have much content\n") + file.write(content) + const fileLink = await file.close() + + assert.deepEqual(fileLink, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ), + dagByteLength: 45, + contentByteLength: 37, + }) + + for (let i = 0; i < 100; i++) { + root.set(`file${i}.txt`, fileLink) + } + + const rootLink = await root.close() + + assert.deepEqual(rootLink, { + dagByteLength: 11591, + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeidzpkzefoys5ani6qfvrpxyjiolmy6ng445uceov2a33r5bw43qwe" + ), + }) + + writer.close() + + const output = await blocks + + assert.deepEqual( + output.map($ => $.cid), + [ + Link.parse("bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4"), + Link.parse("bafybeic66itcox6c3pozwsktz552f3pd3eanqr74jpvjezchwrkpqemjru"), + Link.parse("bafybeigyad752jkaj6qrlgtvovw5dzvhcj7pfvo5pjxkdlzec3kn3qqcoy"), + Link.parse("bafybeiflrsirdjonnavtsdg7vb63z7mcnzuymuv6eiwxw2wxqkezhludjm"), + Link.parse("bafybeigw2ilsvwhg3uglrmryyuk7dtu4yudr5naerrzb5e7ibmk7rscu3y"), + Link.parse("bafybeicprkb6dv56v3ezgj4yffbsueamhkkodfsxvwyaty3okfu6tgq3rm"), + Link.parse("bafybeienx5re7fb3s2crypbkkyp5l5zo5xb5bqfxh67ieq2aivgtaw5bqq"), + Link.parse("bafybeiewng4vb4elq23cjybjhehg2z3lshskzstxzgrhllyb7jsz2dckdq"), + Link.parse("bafybeifz4lbafvzkj7njb3cdr7r3ngl5643jhtghl2ntbvoyx5hocvepvy"), + Link.parse("bafybeibperpo4gxoi7x3g7entslorxizzy3imr44hujjqrus4hfs4ekqge"), + Link.parse("bafybeiamtplq4n5kdlhorxmougus3y54r52frrvotkduzy7kfgyrepvylu"), + Link.parse("bafybeieqvwd6ditluxwzrbvq3ffusuykxbljlqyf7gbf7esi6ake4xh27a"), + Link.parse("bafybeigkk3fanqwihj5qautj4yzluxnh3okblouotd2qkreijejdic2fui"), + Link.parse("bafybeiafn56xmx6hqgs4ig4yc24cdnbzyghjml6yhg3hmmemkrwl4irluu"), + Link.parse("bafybeieu5uzq5jbtuhnaazl36pjygv57virwr3tbdgqujhpya5w7dfosz4"), + Link.parse("bafybeid57gn3655jtgnnocwnjznifyltepqoiu3chbawyy2f263hm3qylm"), + Link.parse("bafybeig3iwqy4v44nvgyabirtbel6sbk6pzfuwdpzj4z26vczda2nycyrq"), + Link.parse("bafybeigrpoorhusehwpw2caoe7mw65xaundu227vcxqv6mqfeo65tcwxqm"), + Link.parse("bafybeif3iq6dnq2qixkoqnmyvijplu6x5depgmfgpfncpxkcx5ytajrxxy"), + Link.parse("bafybeidzpkzefoys5ani6qfvrpxyjiolmy6ng445uceov2a33r5bw43qwe"), + ] + ) + }) + + it("nested directory", async () => { + const { readable, writable } = new TransformStream() + const blocks = collect(readable) + const writer = writable.getWriter() + const root = UnixFS.createShardedDirectoryWriter({ writer }) + const nested = UnixFS.createShardedDirectoryWriter(root) + + root.set("nested", await nested.close()) + assert.deepEqual(await root.close(), { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeiesoparpjbe5rwoo6liouikyw2nypo6v3d3n36vb334oddrmp52mq" + ), + dagByteLength: 102, + }) + writer.close() + + const items = await blocks + assert.deepEqual( + items.map(({ cid }) => cid.toString()), + [ + "bafybeifoplefg5piy3pjhlp73q7unqx4hwecxeu7opfqfmg352pkpljt6m", + "bafybeiesoparpjbe5rwoo6liouikyw2nypo6v3d3n36vb334oddrmp52mq", + ] + ) + }) + + it("double nested directory", async () => { + const { readable, writable } = new TransformStream() + const blocks = collect(readable) + const writer = writable.getWriter() + + const root = UnixFS.createShardedDirectoryWriter({ writer }) + const nested = UnixFS.createShardedDirectoryWriter(root) + + root.set("nested", await nested.close()) + const main = UnixFS.createShardedDirectoryWriter({ writer }) + main.set("root", await root.close()) + const link = await main.close() + writer.close() + const items = await blocks + assert.deepEqual( + items.map(({ cid }) => cid.toString()), + [ + "bafybeifoplefg5piy3pjhlp73q7unqx4hwecxeu7opfqfmg352pkpljt6m", + "bafybeiesoparpjbe5rwoo6liouikyw2nypo6v3d3n36vb334oddrmp52mq", + "bafybeifni4qs2xfgtzhk2xw7emp5j7h5ayyw73xizcba2qxry6dc4vqaom", + ] + ) + }) + + it("throws if file already exists", async () => { + const { readable, writable } = new TransformStream() + const blocks = collect(readable) + const writer = writable.getWriter() + + const root = UnixFS.createShardedDirectoryWriter({ writer }) + + const hello = await importFile(root, ["hello"]) + assert.deepEqual(hello, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq" + ), + contentByteLength: 5, + dagByteLength: 13, + }) + + const bye = await importFile(root, ["bye"]) + assert.deepEqual(bye, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeigl43jff4muiw2m6kzqhm7xpz6ti7etiujklpnc6vpblzjvvwqmta" + ), + dagByteLength: 11, + contentByteLength: 3, + }) + + root.set("hello", hello) + assert.throws( + () => root.set("hello", bye), + /Directory already contains entry with name "hello"/ + ) + root.set("bye", bye) + const link = await root.close() + + assert.deepEqual(link, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeihxagpxz7lekn7exw6ob526d6pgvnzc3kgtpkbh7ze73e2oc7oxpa" + ), + dagByteLength: 164, + }) + writer.close() + const items = await blocks + assert.deepEqual( + items.map(item => item.cid.toString()), + [ + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq", + "bafybeigl43jff4muiw2m6kzqhm7xpz6ti7etiujklpnc6vpblzjvvwqmta", + "bafybeihxagpxz7lekn7exw6ob526d6pgvnzc3kgtpkbh7ze73e2oc7oxpa", + ] + ) + }) + + it("can overwrite existing", async () => { + const { readable, writable } = new TransformStream() + const blocks = collect(readable) + const writer = writable.getWriter() + + const root = UnixFS.createShardedDirectoryWriter({ writer }) + + const hello = await importFile(root, ["hello"]) + assert.deepEqual(hello, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq" + ), + contentByteLength: 5, + dagByteLength: 13, + }) + + const bye = await importFile(root, ["bye"]) + assert.deepEqual(bye, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeigl43jff4muiw2m6kzqhm7xpz6ti7etiujklpnc6vpblzjvvwqmta" + ), + dagByteLength: 11, + contentByteLength: 3, + }) + + root.set("hello", hello) + root.set("hello", bye, { overwrite: true }) + const link = await root.close() + + assert.deepEqual(link, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeibzscho4rtevqlxvlen7te535kvrawffcdry42iol2kr5nr3itjgy" + ), + dagByteLength: 99, + }) + writer.close() + const items = await blocks + assert.deepEqual( + items.map(item => item.cid.toString()), + [ + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq", + "bafybeigl43jff4muiw2m6kzqhm7xpz6ti7etiujklpnc6vpblzjvvwqmta", + "bafybeibzscho4rtevqlxvlen7te535kvrawffcdry42iol2kr5nr3itjgy", + ] + ) + }) + + it("can delete entries", async () => { + const { readable, writable } = createChannel() + const writer = writable.getWriter() + const reader = collect(readable) + + const root = UnixFS.createShardedDirectoryWriter({ writer }) + + const hello = await importFile(root, ["hello"]) + assert.deepEqual(hello, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq" + ), + contentByteLength: 5, + dagByteLength: 13, + }) + + root.set("hello", hello) + root.remove("hello") + const link = await root.close() + + assert.deepEqual(link, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeifoplefg5piy3pjhlp73q7unqx4hwecxeu7opfqfmg352pkpljt6m" + ), + dagByteLength: 9, + }) + writer.close() + const blocks = await reader + assert.deepEqual( + blocks.map(block => block.cid.toString()), + [ + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq", + "bafybeifoplefg5piy3pjhlp73q7unqx4hwecxeu7opfqfmg352pkpljt6m", + ] + ) + }) + + it("throws on invalid filenames", async () => { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + const reader = collect(readable) + + const root = UnixFS.createShardedDirectoryWriter({ writer }) + const hello = await importFile(root, ["hello"]) + + assert.throws( + () => root.set("hello/world", hello), + /Directory entry name "hello\/world" contains forbidden "\/" character/ + ) + writer.close() + }) + + it("can not change after close", async () => { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + const reader = collect(readable) + + const root = UnixFS.createShardedDirectoryWriter({ writer }) + + const hello = await importFile(root, ["hello"]) + assert.deepEqual(hello, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq" + ), + contentByteLength: 5, + dagByteLength: 13, + }) + + const bye = await importFile(root, ["bye"]) + assert.deepEqual(bye, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeigl43jff4muiw2m6kzqhm7xpz6ti7etiujklpnc6vpblzjvvwqmta" + ), + dagByteLength: 11, + contentByteLength: 3, + }) + + root.set("hello", hello) + assert.deepEqual(await root.close(), { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeihccqhztoqxfi5mmnv55iofsz7slpzq4gnktf3vzycavqbms5eote" + ), + dagByteLength: 101, + }) + + assert.throws( + () => root.set("bye", bye), + /Can not change written directory, but you can \.fork\(\) and make changes to it/ + ) + + writer.close() + const blocks = await reader + assert.deepEqual( + blocks.map(block => block.cid.toString()), + [ + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq", + "bafybeigl43jff4muiw2m6kzqhm7xpz6ti7etiujklpnc6vpblzjvvwqmta", + "bafybeihccqhztoqxfi5mmnv55iofsz7slpzq4gnktf3vzycavqbms5eote", + ] + ) + + try { + await root.close() + assert.fail() + } catch (/** @type {any} */ err) { + assert.equal(err.message, "Can not change written HAMT directory, but you can .fork() and make changes to it") + } + }) + + it("can fork and edit", async () => { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + const reader = collect(readable) + + const root = UnixFS.createShardedDirectoryWriter({ writer }) + + const hello = await importFile(root, ["hello"]) + assert.deepEqual(hello, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq" + ), + contentByteLength: 5, + dagByteLength: 13, + }) + + const bye = await importFile(root, ["bye"]) + assert.deepEqual(bye, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeigl43jff4muiw2m6kzqhm7xpz6ti7etiujklpnc6vpblzjvvwqmta" + ), + dagByteLength: 11, + contentByteLength: 3, + }) + + root.set("hello", hello) + assert.deepEqual(await root.close(), { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeihccqhztoqxfi5mmnv55iofsz7slpzq4gnktf3vzycavqbms5eote" + ), + dagByteLength: 101, + }) + + const fork = root.fork() + fork.set("bye", bye) + assert.deepEqual(await fork.close(), { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeihxagpxz7lekn7exw6ob526d6pgvnzc3kgtpkbh7ze73e2oc7oxpa" + ), + dagByteLength: 164, + }) + + writer.close() + const blocks = await reader + assert.deepEqual( + blocks.map(block => block.cid.toString()), + [ + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq", + "bafybeigl43jff4muiw2m6kzqhm7xpz6ti7etiujklpnc6vpblzjvvwqmta", + "bafybeihccqhztoqxfi5mmnv55iofsz7slpzq4gnktf3vzycavqbms5eote", + "bafybeihxagpxz7lekn7exw6ob526d6pgvnzc3kgtpkbh7ze73e2oc7oxpa", + ] + ) + }) + + it("can autoclose", async () => { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + const reader = collect(readable) + + const root = UnixFS.createShardedDirectoryWriter({ writer }) + const file = UnixFS.createFileWriter(root) + file.write(new TextEncoder().encode("hello")) + root.set("hello", await file.close()) + assert.deepEqual(await root.close({ closeWriter: true }), { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeihccqhztoqxfi5mmnv55iofsz7slpzq4gnktf3vzycavqbms5eote" + ), + dagByteLength: 101, + }) + + const blocks = await reader + assert.deepEqual( + blocks.map(block => block.cid.toString()), + [ + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq", + "bafybeihccqhztoqxfi5mmnv55iofsz7slpzq4gnktf3vzycavqbms5eote", + ] + ) + }) + + it("fork into other stream", async () => { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + const reader = collect(readable) + + const root = UnixFS.createShardedDirectoryWriter({ writer }) + + const hello = await importFile(root, ["hello"]) + assert.deepEqual(hello, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq" + ), + contentByteLength: 5, + dagByteLength: 13, + }) + + const bye = await importFile(root, ["bye"]) + assert.deepEqual(bye, { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeigl43jff4muiw2m6kzqhm7xpz6ti7etiujklpnc6vpblzjvvwqmta" + ), + dagByteLength: 11, + contentByteLength: 3, + }) + + root.set("hello", hello) + assert.deepEqual(await root.close(), { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeihccqhztoqxfi5mmnv55iofsz7slpzq4gnktf3vzycavqbms5eote" + ), + dagByteLength: 101, + }) + + const patch = new TransformStream() + const patchWriter = patch.writable.getWriter() + const patchReader = collect(patch.readable) + + const fork = root.fork({ writer: patchWriter }) + fork.set("bye", bye) + assert.deepEqual(await fork.close(), { + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeihxagpxz7lekn7exw6ob526d6pgvnzc3kgtpkbh7ze73e2oc7oxpa" + ), + dagByteLength: 164, + }) + + writer.close() + const blocks = await reader + assert.deepEqual( + blocks.map(block => block.cid.toString()), + [ + "bafybeid3weurg3gvyoi7nisadzolomlvoxoppe2sesktnpvdve3256n5tq", + "bafybeigl43jff4muiw2m6kzqhm7xpz6ti7etiujklpnc6vpblzjvvwqmta", + "bafybeihccqhztoqxfi5mmnv55iofsz7slpzq4gnktf3vzycavqbms5eote", + ] + ) + + patchWriter.close() + const delta = await patchReader + assert.deepEqual( + delta.map(block => block.cid.toString()), + ["bafybeihxagpxz7lekn7exw6ob526d6pgvnzc3kgtpkbh7ze73e2oc7oxpa"] + ) + }) + + it("can close writer", async function () { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + const blocks = collect(readable) + const root = UnixFS.createShardedDirectoryWriter({ writer }) + const file = UnixFS.createFileWriter(root) + + file.write(encodeUTF8("this file does not have much content\n")) + assert.equal(writable.locked, true) + root.set("file.txt", await file.close()) + const link = await root.close({ releaseLock: true, closeWriter: true }) + + await blocks + + assert.deepEqual(link, { + dagByteLength: 133, + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeibbyshlpvztob4mtwznmnkzoc4upgcf6ghaulujxglzgmglcdubtm" + ), + }) + }) + + it("can release writer lock", async function () { + const { readable, writable } = new TransformStream() + const writer = writable.getWriter() + const blocks = collect(readable) + const root = UnixFS.createShardedDirectoryWriter({ writer }) + const file = UnixFS.createFileWriter(root) + + file.write(encodeUTF8("this file does not have much content\n")) + assert.equal(writable.locked, true) + root.set("file.txt", await file.close()) + const link = await root.close({ releaseLock: true }) + assert.equal(writable.locked, false) + + writable.close() + await blocks + + assert.deepEqual(link, { + dagByteLength: 133, + /** @type {Link.Link} */ + cid: Link.parse( + "bafybeibbyshlpvztob4mtwznmnkzoc4upgcf6ghaulujxglzgmglcdubtm" + ), + }) + }) + + it("can enumerate entries", async function () { + const { writable } = new TransformStream() + const writer = writable.getWriter() + const root = UnixFS.createShardedDirectoryWriter({ writer }) + + assert.deepEqual([...root.entries()], []) + /** @type {Link.Link} */ + const cid = Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ) + const fileLink = { + cid, + dagByteLength: 45, + contentByteLength: 37, + } + + root.set("file.txt", fileLink) + assert.deepEqual([...root.entries()], [["file.txt", fileLink]]) + }) + + it(".has", async function () { + const { writable } = new TransformStream() + const writer = writable.getWriter() + const root = UnixFS.createShardedDirectoryWriter({ writer }) + assert.equal(root.has("file.txt"), false) + /** @type {Link.Link} */ + const cid = Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ) + + root.set("file.txt", { + cid, + dagByteLength: 45, + contentByteLength: 37, + }) + assert.equal(root.has("file.txt"), true) + + root.remove("file.txt") + assert.equal(root.has("file.txt"), false) + }) + + it(".size", async function () { + const { writable } = new TransformStream() + const writer = writable.getWriter() + const root = UnixFS.createShardedDirectoryWriter({ writer }) + assert.equal(root.size, 0) + /** @type {Link.Link} */ + const cid = Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ) + + root.set("file.txt", { + cid, + dagByteLength: 45, + contentByteLength: 37, + }) + assert.equal(root.size, 1) + + root.remove("file.txt") + assert.equal(root.size, 0) + }) + + it("writer state .clear", async function () { + const { writable } = new TransformStream() + const writer = writable.getWriter() + const root = UnixFS.createShardedDirectoryWriter({ writer }) + assert.equal(root.size, 0) + /** @type {Link.Link} */ + const cid = Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ) + + const fileLink = { + cid, + dagByteLength: 45, + contentByteLength: 37, + } + root.set("file.txt", fileLink) + assert.equal(root.size, 1) + + root.state.entries.clear() + assert.equal(root.size, 0) + }) + + it("writer state .forEach", async function () { + const { writable } = new TransformStream() + const writer = writable.getWriter() + const root = UnixFS.createShardedDirectoryWriter({ writer }) + assert.equal(root.size, 0) + /** @type {Link.Link} */ + const cid = Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ) + + const fileLink = { + cid, + dagByteLength: 45, + contentByteLength: 37, + } + root.set("file.txt", fileLink) + assert.equal(root.size, 1) + root.state.entries.forEach(entry => assert.deepEqual(entry, fileLink)) + }) + + it("writer state .get", async function () { + const { writable } = new TransformStream() + const writer = writable.getWriter() + const root = UnixFS.createShardedDirectoryWriter({ writer }) + assert.equal(root.size, 0) + /** @type {Link.Link} */ + const cid = Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ) + + const fileLink = { + cid, + dagByteLength: 45, + contentByteLength: 37, + } + root.set("file.txt", fileLink) + assert.equal(root.size, 1) + assert.deepEqual(root.state.entries.get("file.txt"), fileLink) + }) + + it("writer state .[Symbol.iterator]", async function () { + const { writable } = new TransformStream() + const writer = writable.getWriter() + const root = UnixFS.createShardedDirectoryWriter({ writer }) + assert.equal(root.size, 0) + /** @type {Link.Link} */ + const cid = Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ) + + const fileLink = { + cid, + dagByteLength: 45, + contentByteLength: 37, + } + root.set("file.txt", fileLink) + assert.equal(root.size, 1) + assert.deepEqual([...root.state.entries], [["file.txt", fileLink]]) + }) + + it("writer state .keys", async function () { + const { writable } = new TransformStream() + const writer = writable.getWriter() + const root = UnixFS.createShardedDirectoryWriter({ writer }) + assert.equal(root.size, 0) + /** @type {Link.Link} */ + const cid = Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ) + + const fileLink = { + cid, + dagByteLength: 45, + contentByteLength: 37, + } + root.set("file.txt", fileLink) + assert.equal(root.size, 1) + assert.deepEqual([...root.state.entries.keys()], ["file.txt"]) + }) + + it("writer state .values", async function () { + const { writable } = new TransformStream() + const writer = writable.getWriter() + const root = UnixFS.createShardedDirectoryWriter({ writer }) + assert.equal(root.size, 0) + /** @type {Link.Link} */ + const cid = Link.parse( + "bafybeidequ5soq6smzafv4lb76i5dkvl5fzgvrxz4bmlc2k4dkikklv2j4" + ) + + const fileLink = { + cid, + dagByteLength: 45, + contentByteLength: 37, + } + root.set("file.txt", fileLink) + assert.equal(root.size, 1) + assert.deepEqual([...root.state.entries.values()], [fileLink]) + }) +}) diff --git a/yarn.lock b/yarn.lock index 1eddff7..792db75 100644 --- a/yarn.lock +++ b/yarn.lock @@ -280,6 +280,14 @@ "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" +"@multiformats/murmur3@^2.1.3": + version "2.1.3" + resolved "https://registry.yarnpkg.com/@multiformats/murmur3/-/murmur3-2.1.3.tgz#eb690132bcc898d74257287d133cdf693f81f888" + integrity sha512-YvLK1IrLnRckPsvXhOkZjaIGNonsEdD1dL3NPSaLilV/WjVYeBgnNZXTUsaPzFXGrIFM7motx+yCmmqzXO6gtQ== + dependencies: + multiformats "^11.0.0" + murmurhash3js-revisited "^3.0.0" + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz" @@ -301,6 +309,13 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" +"@perma/map@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@perma/map/-/map-1.0.2.tgz#838043c5f4c2ec30f1f9dfef7862ab50e2947d5b" + integrity sha512-hujwGOY6yTYnpf5YAtpD5MJAI1kcsVPqyN0lxG8Sampf/InO3jmX/MlJCHCGFPpPqB5JyO5WNnL+tUs1Umqe0A== + dependencies: + murmurhash3js-revisited "^3.0.0" + "@polka/url@^0.5.0": version "0.5.0" resolved "https://registry.npmjs.org/@polka/url/-/url-0.5.0.tgz" @@ -1981,6 +1996,11 @@ multiformats@^11.0.0, multiformats@^11.0.1: resolved "https://registry.npmjs.org/multiformats/-/multiformats-11.0.1.tgz" integrity sha512-atWruyH34YiknSdL5yeIir00EDlJRpHzELYQxG7Iy29eCyL+VrZHpPrX5yqlik3jnuqpLpRKVZ0SGVb9UzKaSA== +murmurhash3js-revisited@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/murmurhash3js-revisited/-/murmurhash3js-revisited-3.0.0.tgz#6bd36e25de8f73394222adc6e41fa3fac08a5869" + integrity sha512-/sF3ee6zvScXMb1XFJ8gDsSnY+X8PbOyjIuBhtgis10W2Jx4ZjIhikUCIF9c4gpJxVnQIsPAFrSwTCuAjicP6g== + nanoid@3.3.3: version "3.3.3" resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz"