feat: enable custom formats for dag put and get (#3347)

Ensures we can use custom/extra IPLD formats with `ipfs.dag.get` and `ipfs.dag.put` over HTTP as well as directly into core.

Adds an example to show how to do it.

The basic idea is you configure your node with the extra formats and pass that node into the http server, so instead of having the IPLD format logic in the http method endpoint and in core it's just in core.  The http client works the same as it did before.

Co-authored-by: Vasco Santos <vasco.santos@moxy.studio>
Co-authored-by: Janko Simonovic <simonovic86@gmail.com>

examples/custom-ipld-formats/README.md

@@ -0,0 +1,31 @@
+# Custom IPLD formats
+
+This example shows you how to configure an IPFS daemon with the ability to load extra IPLD formats so you can use them in your applications.
+
+## Before you start
+
+First clone this repo, install dependencies in the project root and build the project.
+
+```console
+$ git clone https://github.com/ipfs/js-ipfs.git
+$ cd js-ipfs
+$ npm install
+$ npm run build
+```
+
+## Running the example
+
+Running this example should result in metrics being logged out to the console every few seconds.
+
+```
+> npm start
+```
+
+## Play with the configuration!
+
+By default, IPFS is only configured to support a few common IPLD formats. Your application may require extra or more esoteric formats, in which case you can configure your node to support them using `options.ipld.formats` passed to the client or an in-process node or even a daemon if you start it with a wrapper.
+
+See the following files for different configuration:
+
+* [./in-process-node.js](./in-process-node.js) for running an in-process node as part of your confiugration
+* [./daemon-node.js](./daemon-node.js) for running a node as a separate daemon process

examples/custom-ipld-formats/daemon-node.js

@@ -0,0 +1,96 @@
+// ordinarily we'd open a PR against the multicodec module to get our
+// codec number added but since we're just testing we shim our new
+// codec into the base-table.json file - this has to be done
+// before requiring other modules as the int table will become read-only
+const codecName = 'dag-test'
+const codecNumber = 392091
+
+const baseTable = require('multicodec/src/base-table.json')
+baseTable[codecName] = codecNumber
+
+// now require modules as usual
+const IPFSDaemon = require('ipfs-cli/src/daemon')
+const multihashing = require('multihashing-async')
+const multihash = multihashing.multihash
+const multicodec = require('multicodec')
+const CID = require('cids')
+const ipfsHttpClient = require('ipfs-http-client')
+const uint8ArrayToString = require('uint8arrays/to-string')
+
+async function main () {
+  // see https://github.com/ipld/interface-ipld-format for the interface definition
+  const format = {
+    codec: codecNumber,
+    defaultHashAlg: multicodec.SHA2_256,
+    util: {
+      serialize (data) {
+        return Buffer.from(JSON.stringify(data))
+      },
+      deserialize (buf) {
+        return JSON.parse(uint8ArrayToString(buf))
+      },
+      async cid (buf) {
+        const multihash = await multihashing(buf, format.defaultHashAlg)
+
+        return new CID(1, format.codec, multihash)
+      }
+    },
+    resolver: {
+      resolve: (buf, path) => {
+        return {
+          value: format.util.deserialize(buf),
+          remainderPath: path
+        }
+      }
+    }
+  }
+
+  // start an IPFS Daemon
+  const daemon = new IPFSDaemon({
+    ipld: {
+      formats: [
+        format
+      ]
+    }
+  })
+  await daemon.start()
+
+  // in another process:
+  const client = ipfsHttpClient({
+    url: `http://localhost:${daemon._httpApi._apiServers[0].info.port}`,
+    ipld: {
+      formats: [
+        format
+      ]
+    }
+  })
+
+  const data = {
+    hello: 'world'
+  }
+
+  const cid = await client.dag.put(data, {
+    format: codecName,
+    hashAlg: multihash.codes[format.defaultHashAlg]
+  })
+
+  console.info(`Put ${JSON.stringify(data)} = CID(${cid})`)
+
+  const {
+    value
+  } = await client.dag.get(cid)
+
+  console.info(`Get CID(${cid}) = ${JSON.stringify(value)}`)
+
+  await daemon.stop()
+}
+
+main()
+  .catch(err => {
+    console.error(err)
+    process.exit(1)
+  })
+  .then(() => {
+    // https://github.com/libp2p/js-libp2p/issues/779
+    process.exit(0)
+  })

examples/custom-ipld-formats/in-process-node.js

@@ -0,0 +1,73 @@
+// ordinarily we'd open a PR against the multicodec module to get our
+// codec number added but since we're just testing we shim our new
+// codec into the base-table.json file - this has to be done
+// before requiring other modules as the int table will become read-only
+const codecName = 'dag-test'
+const codecNumber = 392091
+
+const baseTable = require('multicodec/src/base-table.json')
+baseTable[codecName] = codecNumber
+
+// now require modules as usual
+const IPFS = require('ipfs-core')
+const multihashing = require('multihashing-async')
+const multicodec = require('multicodec')
+const CID = require('cids')
+
+async function main () {
+  // see https://github.com/ipld/interface-ipld-format for the interface definition
+  const format = {
+    codec: codecNumber,
+    defaultHashAlg: multicodec.SHA2_256,
+    util: {
+      serialize (data) {
+        return Buffer.from(JSON.stringify(data))
+      },
+      deserialize (buf) {
+        return JSON.parse(buf.toString('utf8'))
+      },
+      async cid (buf) {
+        const multihash = await multihashing(buf, format.defaultHashAlg)
+
+        return new CID(1, format.codec, multihash)
+      }
+    }
+  }
+
+  const node = await IPFS.create({
+    ipld: {
+      formats: [
+        format
+      ]
+    }
+  })
+
+  const data = {
+    hello: 'world'
+  }
+
+  const cid = await node.dag.put(data, {
+    format: codecName,
+    hashAlg: format.defaultHashAlg
+  })
+
+  console.info(`Put ${JSON.stringify(data)} = CID(${cid})`)
+
+  const {
+    value
+  } = await node.dag.get(cid)
+
+  console.info(`Get CID(${cid}) = ${JSON.stringify(value)}`)
+
+  await node.stop()
+}
+
+main()
+  .catch(err => {
+    console.error(err)
+    process.exit(1)
+  })
+  .then(() => {
+    // https://github.com/libp2p/js-libp2p/issues/779
+    process.exit(0)
+  })

examples/custom-ipld-formats/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "example-custom-ipld-formats",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {
+    "test": "test-ipfs-example"
+  },
+  "license": "MIT",
+  "devDependencies": {
+    "execa": "^4.0.3",
+    "test-ipfs-example": "^2.0.3"
+  },
+  "dependencies": {
+    "cids": "^1.0.0",
+    "ipfs-cli": "0.0.1",
+    "ipfs-core": "0.0.1",
+    "ipfs-http-client": "^47.0.0",
+    "multicodec": "^2.0.1",
+    "multihashing-async": "^2.0.1",
+    "uint8arrays": "^1.1.0"
+  }
+}

examples/custom-ipld-formats/test.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const path = require('path')
+const {
+  waitForOutput
+} = require('test-ipfs-example/utils')
+
+const testInProcessNode = async () => {
+  await waitForOutput(
+    'Put {"hello":"world"} = CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq)\n' +
+    'Get CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq) = {"hello":"world"}', path.resolve(__dirname, 'in-process-node.js'))
+}
+
+const testDaemonNode = async () => {
+  await waitForOutput(
+    'Put {"hello":"world"} = CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq)\n' +
+    'Get CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq) = {"hello":"world"}', path.resolve(__dirname, 'daemon-node.js'))
+}
+
+async function test () {
+  console.info('Testing in-process node')
+  await testInProcessNode()
+
+  console.info('Testing daemon node')
+  await testDaemonNode()
+}
+
+module.exports = test

packages/ipfs-cli/src/daemon.js

@@ -28,6 +28,11 @@ class Daemon {
     }
   }
 
+  /**
+   * Starts the IPFS HTTP server
+   *
+   * @returns {Promise<Daemon>}
+   */
   async start () {
     log('starting')
 

packages/ipfs-core/package.json

@@ -77,7 +77,7 @@
     "ipfs-unixfs-exporter": "^3.0.4",
     "ipfs-unixfs-importer": "^3.0.4",
     "ipfs-utils": "^4.0.0",
-    "ipld": "^0.27.1",
+    "ipld": "^0.27.2",
     "ipld-bitcoin": "^0.4.0",
     "ipld-block": "^0.10.1",
     "ipld-dag-cbor": "^0.17.0",

packages/ipfs-core/src/components/start.js

@@ -294,6 +294,7 @@ function createApi ({
     id: Components.id({ peerId, libp2p }),
     init: async () => { throw new AlreadyInitializedError() }, // eslint-disable-line require-await
     isOnline,
+    ipld,
     key: {
       export: Components.key.export({ keychain }),
       gen: Components.key.gen({ keychain }),

packages/ipfs-http-client/src/dag/get.js

@@ -1,40 +1,29 @@
 'use strict'
 
-const dagPB = require('ipld-dag-pb')
-const dagCBOR = require('ipld-dag-cbor')
-const raw = require('ipld-raw')
 const configure = require('../lib/configure')
+const multicodec = require('multicodec')
+const loadFormat = require('../lib/ipld-formats')
 
-const resolvers = {
-  'dag-cbor': dagCBOR.resolver,
-  'dag-pb': dagPB.resolver,
-  raw: raw.resolver
-}
-
-module.exports = configure((api, options) => {
-  const getBlock = require('../block/get')(options)
-  const dagResolve = require('./resolve')(options)
+module.exports = configure((api, opts) => {
+  const getBlock = require('../block/get')(opts)
+  const dagResolve = require('./resolve')(opts)
+  const load = loadFormat(opts.ipld)
 
   /**
    * @type {import('..').Implements<import('ipfs-core/src/components/dag/get')>}
    */
   const get = async (cid, options = {}) => {
     const resolved = await dagResolve(cid, options)
     const block = await getBlock(resolved.cid, options)
-    const dagResolver = resolvers[resolved.cid.codec]
 
-    if (!dagResolver) {
-      throw Object.assign(
-        new Error(`Missing IPLD format "${resolved.cid.codec}"`),
-        { missingMulticodec: resolved.cid.codec }
-      )
-    }
+    const codecName = multicodec.getName(resolved.cid.code)
+    const format = await load(codecName)
 
-    if (resolved.cid.codec === 'raw' && !resolved.remainderPath) {
+    if (resolved.cid.code === multicodec.RAW && !resolved.remainderPath) {
       resolved.remainderPath = '/'
     }
 
-    return dagResolver.resolve(block.data, resolved.remainderPath)
+    return format.resolver.resolve(block.data, resolved.remainderPath)
   }
 
   return get