feat!: auto-shard based on node size

packages/ipfs-unixfs-importer/README.md

@@ -116,7 +116,7 @@ The input's file paths and directory structure will be preserved in the [`dag-pb
 `options` is an JavaScript option that might include the following keys:
 
 - `wrapWithDirectory` (boolean, defaults to false): if true, a wrapping node will be created
-- `shardSplitThreshold` (positive integer, defaults to 1000): the number of directory entries above which we decide to use a sharding directory builder (instead of the default flat one)
+- `shardSplitThreshold` (positive integer, defaults to 256KiB): if the serialized node is larger than this it will be converted to a HAMT sharded directory
 - `chunker` (string, defaults to `"fixed"`): the chunking strategy. Supports:
   - `fixed`
   - `rabin`

packages/ipfs-unixfs-importer/src/dir-flat.js

@@ -1,6 +1,6 @@
 import { encode, prepare } from '@ipld/dag-pb'
 import { UnixFS } from 'ipfs-unixfs'
-import Dir from './dir.js'
+import { Dir, CID_V0, CID_V1 } from './dir.js'
 import persist from './utils/persist.js'
 
 /**
@@ -32,6 +32,7 @@ class DirFlat extends Dir {
   async put (name, value) {
     this.cid = undefined
     this.size = undefined
+    this.nodeSize = undefined
 
     this._children[name] = value
   }
@@ -68,6 +69,43 @@ class DirFlat extends Dir {
     }
   }
 
+  calculateNodeSize () {
+    if (this.nodeSize !== undefined) {
+      return this.nodeSize
+    }
+
+    const links = []
+
+    for (const name of Object.keys(this._children)) {
+      const child = this._children[name]
+      let size
+
+      if (child instanceof Dir) {
+        size = child.calculateNodeSize()
+      } else {
+        size = child.size
+      }
+
+      if (child.size != null && child.cid) {
+        links.push({
+          Name: name,
+          Tsize: size,
+          Hash: this.options.cidVersion === 0 ? CID_V0 : CID_V1
+        })
+      }
+    }
+
+    const unixfs = new UnixFS({
+      type: 'directory',
+      mtime: this.mtime,
+      mode: this.mode
+    })
+
+    this.nodeSize = encode(prepare({ Data: unixfs.marshal(), Links: links })).length
+
+    return this.nodeSize
+  }
+
   /**
    * @param {Blockstore} block
    * @returns {AsyncIterable<ImportResult>}

packages/ipfs-unixfs-importer/src/dir-sharded.js

@@ -1,6 +1,6 @@
 import { encode, prepare } from '@ipld/dag-pb'
 import { UnixFS } from 'ipfs-unixfs'
-import Dir from './dir.js'
+import { Dir, CID_V0, CID_V1 } from './dir.js'
 import persist from './utils/persist.js'
 import { createHAMT, Bucket } from 'hamt-sharding'
 
@@ -35,6 +35,10 @@ class DirSharded extends Dir {
    * @param {InProgressImportResult | Dir} value
    */
   async put (name, value) {
+    this.cid = undefined
+    this.size = undefined
+    this.nodeSize = undefined
+
     await this._bucket.put(name, value)
   }
 
@@ -66,6 +70,16 @@ class DirSharded extends Dir {
     }
   }
 
+  calculateNodeSize () {
+    if (this.nodeSize !== undefined) {
+      return this.nodeSize
+    }
+
+    this.nodeSize = calculateSize(this._bucket, this, this.options)
+
+    return this.nodeSize
+  }
+
   /**
    * @param {Blockstore} blockstore
    * @returns {AsyncIterable<ImportResult>}
@@ -85,7 +99,7 @@ export default DirSharded
 /**
  * @param {Bucket<?>} bucket
  * @param {Blockstore} blockstore
- * @param {*} shardRoot
+ * @param {DirSharded | null} shardRoot
  * @param {ImporterOptions} options
  * @returns {AsyncIterable<ImportResult>}
  */
@@ -183,3 +197,76 @@ async function * flush (bucket, blockstore, shardRoot, options) {
     size
   }
 }
+
+/**
+ * @param {Bucket<?>} bucket
+ * @param {DirSharded | null} shardRoot
+ * @param {ImporterOptions} options
+ */
+function calculateSize (bucket, shardRoot, options) {
+  const children = bucket._children
+  const links = []
+
+  for (let i = 0; i < children.length; i++) {
+    const child = children.get(i)
+
+    if (!child) {
+      continue
+    }
+
+    const labelPrefix = i.toString(16).toUpperCase().padStart(2, '0')
+
+    if (child instanceof Bucket) {
+      const size = calculateSize(child, null, options)
+
+      links.push({
+        Name: labelPrefix,
+        Tsize: size,
+        Hash: options.cidVersion === 0 ? CID_V0 : CID_V1
+      })
+    } else if (typeof child.value.flush === 'function') {
+      const dir = child.value
+      const size = dir.nodeSize()
+
+      links.push({
+        Name: labelPrefix + child.key,
+        Tsize: size,
+        Hash: options.cidVersion === 0 ? CID_V0 : CID_V1
+      })
+    } else {
+      const value = child.value
+
+      if (!value.cid) {
+        continue
+      }
+
+      const label = labelPrefix + child.key
+      const size = value.size
+
+      links.push({
+        Name: label,
+        Tsize: size,
+        Hash: value.cid
+      })
+    }
+  }
+
+  // go-ipfs uses little endian, that's why we have to
+  // reverse the bit field before storing it
+  const data = Uint8Array.from(children.bitField().reverse())
+  const dir = new UnixFS({
+    type: 'hamt-sharded-directory',
+    data,
+    fanout: bucket.tableSize(),
+    hashType: options.hamtHashCode,
+    mtime: shardRoot && shardRoot.mtime,
+    mode: shardRoot && shardRoot.mode
+  })
+
+  const buffer = encode(prepare({
+    Data: dir.marshal(),
+    Links: links
+  }))
+
+  return buffer.length
+}

packages/ipfs-unixfs-importer/src/dir.js

@@ -1,9 +1,10 @@
+import { CID } from 'multiformats/cid'
+
 /**
  * @typedef {import('./types').ImporterOptions} ImporterOptions
  * @typedef {import('./types').ImportResult} ImportResult
  * @typedef {import('./types').InProgressImportResult} InProgressImportResult
  * @typedef {import('interface-blockstore').Blockstore} Blockstore
- * @typedef {import('multiformats/cid').CID} CID
  *
  * @typedef {object} DirProps
  * @property {boolean} root
@@ -17,7 +18,7 @@
  * @property {number} [mode]
  * @property {import('ipfs-unixfs').Mtime} [mtime]
  */
-class Dir {
+export class Dir {
   /**
    * @param {DirProps} props
    * @param {ImporterOptions} options
@@ -40,6 +41,8 @@ class Dir {
     this.cid = undefined
     /** @type {number | undefined} */
     this.size = undefined
+    /** @type {number | undefined} */
+    this.nodeSize = undefined
   }
 
   /**
@@ -66,6 +69,18 @@ class Dir {
    * @returns {AsyncIterable<ImportResult>}
    */
   async * flush (blockstore) { }
+
+  /**
+   * @returns {number}
+   */
+  calculateNodeSize () {
+    return 0
+  }
 }
 
-export default Dir
+// we use these to calculate the node size to use as a check for whether a directory
+// should be sharded or not. Since CIDs have a constant length and We're only
+// interested in the data length and not the actual content identifier we can use
+// any old CID instead of having to hash the data which is expensive.
+export const CID_V0 = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn')
+export const CID_V1 = CID.parse('zdj7WbTaiJT1fgatdet9Ei9iDB5hdCxkbVyhyh8YTUnXMiwYi')

packages/ipfs-unixfs-importer/src/flat-to-shard.js

@@ -2,7 +2,7 @@ import DirSharded from './dir-sharded.js'
 import DirFlat from './dir-flat.js'
 
 /**
- * @typedef {import('./dir').default} Dir
+ * @typedef {import('./dir').Dir} Dir
  * @typedef {import('./types').ImporterOptions} ImporterOptions
  */
 
@@ -16,7 +16,7 @@ import DirFlat from './dir-flat.js'
 async function flatToShard (child, dir, threshold, options) {
   let newDir = dir
 
-  if (dir instanceof DirFlat && dir.directChildrenCount() >= threshold) {
+  if (dir instanceof DirFlat && dir.calculateNodeSize() > threshold) {
     newDir = await convertToShard(dir, options)
   }
 

packages/ipfs-unixfs-importer/src/options.js

@@ -33,7 +33,8 @@ const defaultOptions = {
   leafType: 'file', // 'raw'
   cidVersion: 0,
   progress: () => () => {},
-  shardSplitThreshold: 1000,
+  // https://github.com/ipfs/go-ipfs/pull/8114/files#diff-eec963b47a6e1080d9d8023b4e438e6e3591b4154f7379a7e728401d2055374aR319
+  shardSplitThreshold: 262144,
   fileImportConcurrency: 50,
   blockWriteConcurrency: 10,
   minChunkSize: 262144,

packages/ipfs-unixfs-importer/src/tree-builder.js

@@ -1,6 +1,6 @@
 import DirFlat from './dir-flat.js'
 import flatToShard from './flat-to-shard.js'
-import Dir from './dir.js'
+import { Dir } from './dir.js'
 import toPathComponents from './utils/to-path-components.js'
 
 /**

packages/ipfs-unixfs/src/index.js

@@ -25,7 +25,7 @@ const DEFAULT_FILE_MODE = parseInt('0644', 8)
 const DEFAULT_DIRECTORY_MODE = parseInt('0755', 8)
 
 /**
- * @param {string | number | undefined} [mode]
+ * @param {string | number | null | undefined} [mode]
  */
 export function parseMode (mode) {
   if (mode == null) {
@@ -161,7 +161,7 @@ class UnixFS {
    * @param {number} [options.hashType]
    * @param {number} [options.fanout]
    * @param {MtimeLike | null} [options.mtime]
-   * @param {number | string} [options.mode]
+   * @param {number | string | null} [options.mode]
    */
   constructor (options = {
     type: 'file'