From e3a5daf9dd30127703a66d3c308a6192409bf3f1 Mon Sep 17 00:00:00 2001 From: Janko Simonovic Date: Wed, 30 Sep 2020 16:05:36 +0200 Subject: [PATCH 01/14] feat: enable custom formats for server/client DAG (put|get) operations --- packages/ipfs-http-client/src/dag/get.js | 27 ++++++++++++------- .../ipfs-http-server/src/api/resources/dag.js | 15 ++++++++--- packages/ipfs-http-server/src/index.js | 1 + 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/packages/ipfs-http-client/src/dag/get.js b/packages/ipfs-http-client/src/dag/get.js index 1094b97377..0a7082f233 100644 --- a/packages/ipfs-http-client/src/dag/get.js +++ b/packages/ipfs-http-client/src/dag/get.js @@ -4,23 +4,32 @@ const dagPB = require('ipld-dag-pb') const dagCBOR = require('ipld-dag-cbor') const raw = require('ipld-raw') const configure = require('../lib/configure') - -const resolvers = { - 'dag-cbor': dagCBOR.resolver, - 'dag-pb': dagPB.resolver, - raw: raw.resolver -} +const multicodec = require('multicodec') module.exports = configure((api, options) => { const getBlock = require('../block/get')(options) const dagResolve = require('./resolve')(options) + const formats = { + [multicodec.DAG_PB]: dagPB, + [multicodec.DAG_CBOR]: dagCBOR, + [multicodec.RAW]: raw + } + + const ipldOptions = (options && options.ipld) || {} + const configuredFormats = (ipldOptions && ipldOptions.formats) || [] + configuredFormats.forEach(format => { + formats[format.codec] = format + }) + return async (cid, options = {}) => { const resolved = await dagResolve(cid, options) const block = await getBlock(resolved.cid, options) - const dagResolver = resolvers[resolved.cid.codec] - if (!dagResolver) { + const codec = multicodec[resolved.cid.codec.toUpperCase().replace(/-/g, '_')] + const format = formats[codec] + + if (!format) { throw Object.assign( new Error(`Missing IPLD format "${resolved.cid.codec}"`), { missingMulticodec: resolved.cid.codec } @@ -31,6 +40,6 @@ module.exports = configure((api, options) => { resolved.remainderPath = '/' } - return dagResolver.resolve(block.data, resolved.remainderPath) + return format.resolver.resolve(block.data, resolved.remainderPath) } }) diff --git a/packages/ipfs-http-server/src/api/resources/dag.js b/packages/ipfs-http-server/src/api/resources/dag.js index f0d49e4cf5..6612a83a44 100644 --- a/packages/ipfs-http-server/src/api/resources/dag.js +++ b/packages/ipfs-http-server/src/api/resources/dag.js @@ -203,11 +203,20 @@ exports.put = { } else { const codec = multicodec[format.toUpperCase().replace(/-/g, '_')] - if (!IpldFormats[codec]) { - throw new Error(`Missing IPLD format "${codec}"`) + let ipldFormat = IpldFormats[codec] + if (!ipldFormat) { + // look at the passed config + const { opts } = request.server.app + if (opts) { + const { ipld } = opts + ipldFormat = ipld.formats.find((f) => f.codec === codec) + } } - node = await IpldFormats[codec].util.deserialize(data) + if (!ipldFormat) { + throw new Error(`Missing IPLD format "${codec}"`) + } + node = await ipldFormat.util.deserialize(data) } return { diff --git a/packages/ipfs-http-server/src/index.js b/packages/ipfs-http-server/src/index.js index af446891b8..11d3f7e5b8 100644 --- a/packages/ipfs-http-server/src/index.js +++ b/packages/ipfs-http-server/src/index.js @@ -95,6 +95,7 @@ class HttpApi { compression: false }) server.app.ipfs = ipfs + server.app.opts = opts await server.register({ plugin: Pino, From 97f7fc5f98a5bd65a85b0884f0f1dde497d6874a Mon Sep 17 00:00:00 2001 From: Janko Simonovic Date: Wed, 30 Sep 2020 16:11:10 +0200 Subject: [PATCH 02/14] fix: fix possible undefined --- packages/ipfs-http-server/src/api/resources/dag.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-http-server/src/api/resources/dag.js b/packages/ipfs-http-server/src/api/resources/dag.js index 6612a83a44..86cdcf560e 100644 --- a/packages/ipfs-http-server/src/api/resources/dag.js +++ b/packages/ipfs-http-server/src/api/resources/dag.js @@ -209,7 +209,9 @@ exports.put = { const { opts } = request.server.app if (opts) { const { ipld } = opts - ipldFormat = ipld.formats.find((f) => f.codec === codec) + if (ipld) { + ipldFormat = ipld.formats.find((f) => f.codec === codec) + } } } From d246a4d7aa2343b450e61227bb7c5182fe540305 Mon Sep 17 00:00:00 2001 From: Janko Simonovic Date: Wed, 30 Sep 2020 16:44:25 +0200 Subject: [PATCH 03/14] chore: refactor format fetch --- packages/ipfs-http-server/src/api/resources/dag.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/ipfs-http-server/src/api/resources/dag.js b/packages/ipfs-http-server/src/api/resources/dag.js index 86cdcf560e..b36c2c0e79 100644 --- a/packages/ipfs-http-server/src/api/resources/dag.js +++ b/packages/ipfs-http-server/src/api/resources/dag.js @@ -206,12 +206,9 @@ exports.put = { let ipldFormat = IpldFormats[codec] if (!ipldFormat) { // look at the passed config - const { opts } = request.server.app - if (opts) { - const { ipld } = opts - if (ipld) { - ipldFormat = ipld.formats.find((f) => f.codec === codec) - } + const ipldOpts = (request.server.app.opts && request.server.app.opts.ipld) || {} + if (ipldOpts.formats) { + ipldFormat = ipldOpts.formats.find((f) => f.codec === codec) } } From a8f08257ddce159ceeae79783d7f80a269f14f20 Mon Sep 17 00:00:00 2001 From: Janko Simonovic Date: Wed, 30 Sep 2020 19:48:06 +0200 Subject: [PATCH 04/14] fix: pass opts to the server --- packages/ipfs-http-server/src/index.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/ipfs-http-server/src/index.js b/packages/ipfs-http-server/src/index.js index 11d3f7e5b8..ca3bf3e3a5 100644 --- a/packages/ipfs-http-server/src/index.js +++ b/packages/ipfs-http-server/src/index.js @@ -25,7 +25,7 @@ function hapiInfoToMultiaddr (info) { return toMultiaddr(uri) } -async function serverCreator (serverAddrs, createServer, ipfs, cors) { +async function serverCreator (serverAddrs, createServer, ipfs, cors, opts) { serverAddrs = serverAddrs || [] // just in case the address is just string serverAddrs = Array.isArray(serverAddrs) ? serverAddrs : [serverAddrs] @@ -33,7 +33,7 @@ async function serverCreator (serverAddrs, createServer, ipfs, cors) { const servers = [] for (const address of serverAddrs) { const addrParts = address.split('/') - const server = await createServer(addrParts[2], addrParts[4], ipfs, cors) + const server = await createServer(addrParts[2], addrParts[4], ipfs, cors, opts) await server.start() server.info.ma = hapiInfoToMultiaddr(server.info) servers.push(server) @@ -49,7 +49,12 @@ class HttpApi { this._log.error = debug(LOG_ERROR) } - async start () { + /** + * Starts the IPFS HTTP server + * @param {object} [opts.ipld.formats] - IPLD custom formats + * @return {Promise} + */ + async start (opts = {}) { this._log('starting') const ipfs = this._ipfs @@ -63,13 +68,13 @@ class HttpApi { this._apiServers = await serverCreator(apiAddrs, this._createApiServer, ipfs, { origin: config.API.HTTPHeaders['Access-Control-Allow-Origin'] || [], credentials: Boolean(config.API.HTTPHeaders['Access-Control-Allow-Credentials']) - }) + }, opts) this._log('started') return this } - async _createApiServer (host, port, ipfs, cors) { + async _createApiServer (host, port, ipfs, cors, opts) { cors = { ...cors, additionalHeaders: ['X-Stream-Output', 'X-Chunked-Output', 'X-Content-Length'], From 1258569c1603fa84c5e49baf6b354a2ac454f886 Mon Sep 17 00:00:00 2001 From: Janko Simonovic Date: Wed, 30 Sep 2020 19:51:57 +0200 Subject: [PATCH 05/14] feat: pass opts --- packages/ipfs-cli/src/daemon.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/ipfs-cli/src/daemon.js b/packages/ipfs-cli/src/daemon.js index 59ef5fbb9b..c020da975d 100644 --- a/packages/ipfs-cli/src/daemon.js +++ b/packages/ipfs-cli/src/daemon.js @@ -27,7 +27,12 @@ class Daemon { } } - async start () { + /** + * Starts the IPFS HTTP server + * @param {object} [opts.ipld.formats] - IPLD custom formats + * @return {Promise} + */ + async start (opts = {}) { log('starting') const repo = typeof this._options.repo === 'string' || this._options.repo == null @@ -40,7 +45,7 @@ class Daemon { // start HTTP servers (if API or Gateway is enabled in options) const httpApi = new HttpApi(ipfs, ipfsOpts) - this._httpApi = await httpApi.start() + this._httpApi = await httpApi.start(opts) const httpGateway = new HttpGateway(ipfs, ipfsOpts) this._httpGateway = await httpGateway.start() From 64d37b9c7df940e8ccd7fb4b59f19925d9e7842a Mon Sep 17 00:00:00 2001 From: Janko Simonovic Date: Wed, 30 Sep 2020 21:03:16 +0200 Subject: [PATCH 06/14] chore: add documentation --- packages/ipfs-http-server/src/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ipfs-http-server/src/index.js b/packages/ipfs-http-server/src/index.js index ca3bf3e3a5..15b4e0d5eb 100644 --- a/packages/ipfs-http-server/src/index.js +++ b/packages/ipfs-http-server/src/index.js @@ -51,6 +51,7 @@ class HttpApi { /** * Starts the IPFS HTTP server + * @param {object} [opts] - specify advanced configuration * @param {object} [opts.ipld.formats] - IPLD custom formats * @return {Promise} */ From f8b938984dd7a59a6cf2d0556a258c142338242c Mon Sep 17 00:00:00 2001 From: Janko Simonovic Date: Wed, 30 Sep 2020 22:04:14 +0200 Subject: [PATCH 07/14] fix: fix lint --- packages/ipfs-cli/src/daemon.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ipfs-cli/src/daemon.js b/packages/ipfs-cli/src/daemon.js index c020da975d..63e1507dc8 100644 --- a/packages/ipfs-cli/src/daemon.js +++ b/packages/ipfs-cli/src/daemon.js @@ -29,6 +29,7 @@ class Daemon { /** * Starts the IPFS HTTP server + * @param {object} [opts] - specify advanced configuration * @param {object} [opts.ipld.formats] - IPLD custom formats * @return {Promise} */ From 9c17509bea688bb677a6398803c72cbeb8fe03e9 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 26 Oct 2020 11:42:56 +0000 Subject: [PATCH 08/14] feat: enable custom formats for server/client DAG get and put --- packages/ipfs-cli/src/daemon.js | 3 +- packages/ipfs-http-client/src/dag/get.js | 36 +++-------- packages/ipfs-http-client/src/dag/put.js | 36 ++--------- .../ipfs-http-client/src/lib/ipld-formats.js | 61 +++++++++++++++++++ packages/ipfs-http-client/test/dag.spec.js | 2 +- packages/ipfs-http-server/package.json | 6 -- .../ipfs-http-server/src/api/resources/dag.js | 57 ++--------------- packages/ipfs-http-server/src/index.js | 12 ++-- packages/ipfs-http-server/test/inject/dag.js | 29 +++++++++ 9 files changed, 114 insertions(+), 128 deletions(-) create mode 100644 packages/ipfs-http-client/src/lib/ipld-formats.js diff --git a/packages/ipfs-cli/src/daemon.js b/packages/ipfs-cli/src/daemon.js index 38080569d4..b5a24a2348 100644 --- a/packages/ipfs-cli/src/daemon.js +++ b/packages/ipfs-cli/src/daemon.js @@ -30,9 +30,10 @@ class Daemon { /** * Starts the IPFS HTTP server + * * @param {object} [opts] - specify advanced configuration * @param {object} [opts.ipld.formats] - IPLD custom formats - * @return {Promise} + * @returns {Promise} */ async start (opts = {}) { log('starting') diff --git a/packages/ipfs-http-client/src/dag/get.js b/packages/ipfs-http-client/src/dag/get.js index 9b727cc6bd..c4b8ee15f3 100644 --- a/packages/ipfs-http-client/src/dag/get.js +++ b/packages/ipfs-http-client/src/dag/get.js @@ -1,26 +1,13 @@ 'use strict' -const dagPB = require('ipld-dag-pb') -const dagCBOR = require('ipld-dag-cbor') -const raw = require('ipld-raw') const configure = require('../lib/configure') const multicodec = require('multicodec') +const loadFormat = require('../lib/ipld-formats') -module.exports = configure((api, options) => { - const getBlock = require('../block/get')(options) - const dagResolve = require('./resolve')(options) - - const formats = { - [multicodec.DAG_PB]: dagPB, - [multicodec.DAG_CBOR]: dagCBOR, - [multicodec.RAW]: raw - } - - const ipldOptions = (options && options.ipld) || {} - const configuredFormats = (ipldOptions && ipldOptions.formats) || [] - configuredFormats.forEach(format => { - formats[format.codec] = format - }) +module.exports = configure((api, opts) => { + const getBlock = require('../block/get')(opts) + const dagResolve = require('./resolve')(opts) + const load = loadFormat(opts) /** * @type {import('..').Implements} @@ -29,17 +16,10 @@ module.exports = configure((api, options) => { const resolved = await dagResolve(cid, options) const block = await getBlock(resolved.cid, options) - const codec = multicodec[resolved.cid.codec.toUpperCase().replace(/-/g, '_')] - const format = formats[codec] - - if (!format) { - throw Object.assign( - new Error(`Missing IPLD format "${resolved.cid.codec}"`), - { missingMulticodec: resolved.cid.codec } - ) - } + const codecName = multicodec.getName(resolved.cid.code) + const format = await load(codecName) - if (resolved.cid.codec === 'raw' && !resolved.remainderPath) { + if (resolved.cid.code === multicodec.RAW && !resolved.remainderPath) { resolved.remainderPath = '/' } diff --git a/packages/ipfs-http-client/src/dag/put.js b/packages/ipfs-http-client/src/dag/put.js index dad5ccc231..720903877c 100644 --- a/packages/ipfs-http-client/src/dag/put.js +++ b/packages/ipfs-http-client/src/dag/put.js @@ -1,8 +1,5 @@ 'use strict' -const dagCBOR = require('ipld-dag-cbor') -const dagPB = require('ipld-dag-pb') -const ipldRaw = require('ipld-raw') const CID = require('cids') const multihash = require('multihashes') const configure = require('../lib/configure') @@ -11,19 +8,10 @@ const toUrlSearchParams = require('../lib/to-url-search-params') const { anySignal } = require('any-signal') const AbortController = require('native-abort-controller') const multicodec = require('multicodec') +const loadFormat = require('../lib/ipld-formats') module.exports = configure((api, opts) => { - const formats = { - [multicodec.DAG_PB]: dagPB, - [multicodec.DAG_CBOR]: dagCBOR, - [multicodec.RAW]: ipldRaw - } - - const ipldOptions = (opts && opts.ipld) || {} - const configuredFormats = (ipldOptions && ipldOptions.formats) || [] - configuredFormats.forEach(format => { - formats[format.codec] = format - }) + const load = loadFormat(opts) /** * @type {import('..').Implements} @@ -39,7 +27,7 @@ module.exports = configure((api, opts) => { const cid = new CID(options.cid) options = { ...options, - format: cid.codec, + format: multicodec.getName(cid.code), hashAlg: multihash.decode(cid.multihash).name } delete options.cid @@ -52,23 +40,7 @@ module.exports = configure((api, opts) => { ...options } - const number = multicodec.getNumber(settings.format) - let format = formats[number] - - if (!format) { - if (opts && opts.ipld && opts.ipld.loadFormat) { - // @ts-ignore - loadFormat expect string but it could be a number - format = await opts.ipld.loadFormat(settings.format) - } - - if (!format) { - throw new Error('Format unsupported - please add support using the options.ipld.formats or options.ipld.loadFormat options') - } - } - - if (!format.util || !format.util.serialize) { - throw new Error('Format does not support utils.serialize function') - } + const format = await load(settings.format) const serialized = format.util.serialize(dagNode) diff --git a/packages/ipfs-http-client/src/lib/ipld-formats.js b/packages/ipfs-http-client/src/lib/ipld-formats.js new file mode 100644 index 0000000000..5076aea2ad --- /dev/null +++ b/packages/ipfs-http-client/src/lib/ipld-formats.js @@ -0,0 +1,61 @@ +'use strict' + +const dagPB = require('ipld-dag-pb') +const dagCBOR = require('ipld-dag-cbor') +const raw = require('ipld-raw') +const multicodec = require('multicodec') + +/** + * @typedef {import('cids')} CID + */ + +/** + * Return an object containing supported IPLD Formats + * + * @param {object} [options] - Options passed to the http client constructor + * @param {object} [options.ipld] - IPLD options passed to the http client constructor + * @param {Array} [options.ipld.formats] - A list of IPLD Formats to use + * @param {Function} [options.ipld.loadFormat] - An async function that can load a format when passed a codec name + * @returns {Function} + */ +module.exports = (options) => { + const formats = { + [multicodec.DAG_PB]: dagPB, + [multicodec.DAG_CBOR]: dagCBOR, + [multicodec.RAW]: raw + } + + const ipldOptions = (options && options.ipld) || {} + const configuredFormats = (ipldOptions && ipldOptions.formats) || [] + configuredFormats.forEach(format => { + formats[format.codec] = format + }) + + const loadExtraFormat = options && options.ipld && options.ipld.loadFormat + + /** + * Attempts to load an IPLD format for the passed CID + * + * @param {string} codec - The code to load the format for + * @returns {Promise} - An IPLD format + */ + const loadFormat = async (codec) => { + const number = multicodec.getNumber(codec) + let format = formats[number] + + if (!format && loadExtraFormat) { + format = await loadExtraFormat(codec) + } + + if (!format) { + throw Object.assign( + new Error(`Missing IPLD format "${codec}"`), + { missingMulticodec: codec } + ) + } + + return format + } + + return loadFormat +} diff --git a/packages/ipfs-http-client/test/dag.spec.js b/packages/ipfs-http-client/test/dag.spec.js index a5f861aeb9..fb3bdc438b 100644 --- a/packages/ipfs-http-client/test/dag.spec.js +++ b/packages/ipfs-http-client/test/dag.spec.js @@ -73,7 +73,7 @@ describe('.dag', function () { it('should error when putting node with esoteric format', () => { const node = uint8ArrayFromString('some data') - return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/Format unsupported/) + return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/Missing IPLD format/) }) it('should attempt to load an unsupported format', async () => { diff --git a/packages/ipfs-http-server/package.json b/packages/ipfs-http-server/package.json index 9fdd0a6897..1548d70788 100644 --- a/packages/ipfs-http-server/package.json +++ b/packages/ipfs-http-server/package.json @@ -40,13 +40,7 @@ "ipfs-core-utils": "^0.4.0", "ipfs-http-gateway": "0.0.1", "ipfs-unixfs": "^2.0.3", - "ipld-bitcoin": "^0.4.0", - "ipld-dag-cbor": "^0.17.0", "ipld-dag-pb": "^0.20.0", - "ipld-ethereum": "^5.0.1", - "ipld-git": "^0.6.1", - "ipld-raw": "^6.0.0", - "ipld-zcash": "^0.5.0", "it-all": "^1.0.4", "it-drain": "^1.0.3", "it-first": "^1.0.4", diff --git a/packages/ipfs-http-server/src/api/resources/dag.js b/packages/ipfs-http-server/src/api/resources/dag.js index b29a45fb65..5b6f6d5643 100644 --- a/packages/ipfs-http-server/src/api/resources/dag.js +++ b/packages/ipfs-http-server/src/api/resources/dag.js @@ -3,7 +3,6 @@ const multipart = require('../../utils/multipart-request-parser') const mh = require('multihashing-async').multihash const Joi = require('../../utils/joi') -const multicodec = require('multicodec') const Boom = require('@hapi/boom') const { cidToString @@ -11,48 +10,6 @@ const { const all = require('it-all') const uint8ArrayToString = require('uint8arrays/to-string') -const IpldFormats = { - get [multicodec.RAW] () { - return require('ipld-raw') - }, - get [multicodec.DAG_PB] () { - return require('ipld-dag-pb') - }, - get [multicodec.DAG_CBOR] () { - return require('ipld-dag-cbor') - }, - get [multicodec.BITCOIN_BLOCK] () { - return require('ipld-bitcoin') - }, - get [multicodec.ETH_ACCOUNT_SNAPSHOT] () { - return require('ipld-ethereum').ethAccountSnapshot - }, - get [multicodec.ETH_BLOCK] () { - return require('ipld-ethereum').ethBlock - }, - get [multicodec.ETH_BLOCK_LIST] () { - return require('ipld-ethereum').ethBlockList - }, - get [multicodec.ETH_STATE_TRIE] () { - return require('ipld-ethereum').ethStateTrie - }, - get [multicodec.ETH_STORAGE_TRIE] () { - return require('ipld-ethereum').ethStorageTrie - }, - get [multicodec.ETH_TX] () { - return require('ipld-ethereum').ethTx - }, - get [multicodec.ETH_TX_TRIE] () { - return require('ipld-ethereum').ethTxTrie - }, - get [multicodec.GIT_RAW] () { - return require('ipld-git') - }, - get [multicodec.ZCASH_BLOCK] () { - return require('ipld-zcash') - } -} - const encodeBufferKeys = (obj, encoding) => { if (!obj) { return obj @@ -198,20 +155,14 @@ exports.put = { throw Boom.badRequest('Failed to parse the JSON: ' + err) } } else { - const codec = multicodec[format.toUpperCase().replace(/-/g, '_')] + // the node is an uncommon format which the client should have + // serialized so deserialize it before continuing + const ipldFormat = await request.server.app.ipfs.ipld._getFormat(format) - let ipldFormat = IpldFormats[codec] if (!ipldFormat) { - // look at the passed config - const ipldOpts = (request.server.app.opts && request.server.app.opts.ipld) || {} - if (ipldOpts.formats) { - ipldFormat = ipldOpts.formats.find((f) => f.codec === codec) - } + throw new Error(`Missing IPLD format "${format}"`) } - if (!ipldFormat) { - throw new Error(`Missing IPLD format "${codec}"`) - } node = await ipldFormat.util.deserialize(data) } diff --git a/packages/ipfs-http-server/src/index.js b/packages/ipfs-http-server/src/index.js index 50c5e91fa4..363cc34312 100644 --- a/packages/ipfs-http-server/src/index.js +++ b/packages/ipfs-http-server/src/index.js @@ -50,11 +50,10 @@ class HttpApi { /** * Starts the IPFS HTTP server - * @param {object} [opts] - specify advanced configuration - * @param {object} [opts.ipld.formats] - IPLD custom formats - * @return {Promise} + * + * @returns {Promise} */ - async start (opts = {}) { + async start () { this._log('starting') const ipfs = this._ipfs @@ -68,13 +67,13 @@ class HttpApi { this._apiServers = await serverCreator(apiAddrs, this._createApiServer, ipfs, { origin: config.API.HTTPHeaders['Access-Control-Allow-Origin'] || [], credentials: Boolean(config.API.HTTPHeaders['Access-Control-Allow-Credentials']) - }, opts) + }) this._log('started') return this } - async _createApiServer (host, port, ipfs, cors, opts) { + async _createApiServer (host, port, ipfs, cors) { cors = { ...cors, additionalHeaders: ['X-Stream-Output', 'X-Chunked-Output', 'X-Content-Length'], @@ -100,7 +99,6 @@ class HttpApi { compression: false }) server.app.ipfs = ipfs - server.app.opts = opts await server.register({ plugin: Pino, diff --git a/packages/ipfs-http-server/test/inject/dag.js b/packages/ipfs-http-server/test/inject/dag.js index f62870069e..5c6151a634 100644 --- a/packages/ipfs-http-server/test/inject/dag.js +++ b/packages/ipfs-http-server/test/inject/dag.js @@ -37,6 +37,9 @@ describe('/dag', () => { get: sinon.stub(), put: sinon.stub(), resolve: sinon.stub() + }, + ipld: { + _getFormat: sinon.stub() } } }) @@ -292,6 +295,32 @@ describe('/dag', () => { expect(res).to.have.deep.nested.property('result.Cid', { '/': cid.toString() }) }) + it('should attempt to load an unsupported format', async () => { + const data = Buffer.from('some data') + const codec = 'git-raw' + const format = { + util: { + deserialize: (buf) => buf + } + } + ipfs.ipld._getFormat.withArgs(codec).returns(format) + + ipfs.dag.put.withArgs(data, { + ...defaultOptions, + format: codec + }).returns(cid) + + const res = await http({ + method: 'POST', + url: '/api/v0/dag/put?format=git-raw&input-enc=raw', + ...await toHeadersAndPayload(data) + }, { ipfs }) + + expect(ipfs.ipld._getFormat.calledWith(codec)).to.be.true() + expect(res).to.have.property('statusCode', 200) + expect(res).to.have.deep.nested.property('result.Cid', { '/': cid.toString() }) + }) + it('accepts a timeout', async () => { const node = { foo: 'bar' From 0aea17636880ff87d10c4d13b583dc61adc9a75d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 26 Oct 2020 11:45:52 +0000 Subject: [PATCH 09/14] chore: remove extra options --- packages/ipfs-cli/src/daemon.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/ipfs-cli/src/daemon.js b/packages/ipfs-cli/src/daemon.js index b5a24a2348..9aef331312 100644 --- a/packages/ipfs-cli/src/daemon.js +++ b/packages/ipfs-cli/src/daemon.js @@ -31,11 +31,9 @@ class Daemon { /** * Starts the IPFS HTTP server * - * @param {object} [opts] - specify advanced configuration - * @param {object} [opts.ipld.formats] - IPLD custom formats - * @returns {Promise} + * @returns {Promise} */ - async start (opts = {}) { + async start () { log('starting') const repo = typeof this._options.repo === 'string' || this._options.repo == null @@ -48,7 +46,7 @@ class Daemon { // start HTTP servers (if API or Gateway is enabled in options) const httpApi = new HttpApi(ipfs, ipfsOpts) - this._httpApi = await httpApi.start(opts) + this._httpApi = await httpApi.start() const httpGateway = new HttpGateway(ipfs, ipfsOpts) this._httpGateway = await httpGateway.start() From 03a9e65b9410972fe12f8a15c91d5aadd22c0911 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 26 Oct 2020 14:25:35 +0000 Subject: [PATCH 10/14] chore: add example showing how to do custom ipld formats --- examples/custom-ipld-formats/README.md | 31 ++++++ examples/custom-ipld-formats/daemon-node.js | 96 +++++++++++++++++++ .../custom-ipld-formats/in-process-node.js | 73 ++++++++++++++ examples/custom-ipld-formats/package.json | 22 +++++ examples/custom-ipld-formats/test.js | 28 ++++++ packages/ipfs-core/src/components/start.js | 1 + packages/ipfs-http-client/src/dag/put.js | 1 - 7 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 examples/custom-ipld-formats/README.md create mode 100644 examples/custom-ipld-formats/daemon-node.js create mode 100644 examples/custom-ipld-formats/in-process-node.js create mode 100644 examples/custom-ipld-formats/package.json create mode 100644 examples/custom-ipld-formats/test.js diff --git a/examples/custom-ipld-formats/README.md b/examples/custom-ipld-formats/README.md new file mode 100644 index 0000000000..14ebefeff5 --- /dev/null +++ b/examples/custom-ipld-formats/README.md @@ -0,0 +1,31 @@ +# Custom IPLD formats + +This example shows you how to configure and IPFS daemon with the ability to load extra IPLD formats so you can use them in your applications. + +## Before you start + +First clone this repo, install dependencies in the project root and build the project. + +```console +$ git clone https://github.com/ipfs/js-ipfs.git +$ cd js-ipfs +$ npm install +$ npm run build +``` + +## Running the example + +Running this example should result in metrics being logged out to the console every few seconds. + +``` +> npm start +``` + +## Play with the configuration! + +By default, IPFS is only configured to support a few common IPLD formats. Your application may require extra or more esoteric formats, in which case you can configure your node to support them using `options.ipld.formats` passed to the client or an in-process node or even a daemon if you start it with a wrapper. + +See the following files for different configuration: + +* [./in-process-node.js](./in-process-node.js) for running an in-process node as part of your confiugration +* [./daemon-node.js](./daemon-node.js) for running a node as a separate daemon process diff --git a/examples/custom-ipld-formats/daemon-node.js b/examples/custom-ipld-formats/daemon-node.js new file mode 100644 index 0000000000..939e6cd410 --- /dev/null +++ b/examples/custom-ipld-formats/daemon-node.js @@ -0,0 +1,96 @@ +// ordinarily we'd open a PR against the multicodec module to get our +// codec number added but since we're just testing we shim our new +// codec into the base-table.json file - this has to be done +// before requiring other modules as the int table will become read-only +const codecName = 'dag-test' +const codecNumber = 392091 + +const baseTable = require('multicodec/src/base-table.json') +baseTable[codecName] = codecNumber + +// now require modules as usual +const IPFSDaemon = require('ipfs-cli/src/daemon') +const multihashing = require('multihashing-async') +const multihash = multihashing.multihash +const multicodec = require('multicodec') +const CID = require('cids') +const ipfsHttpClient = require('ipfs-http-client') +const uint8ArrayToString = require('uint8arrays/to-string') + +async function main () { + // see https://github.com/ipld/interface-ipld-format for the interface definition + const format = { + codec: codecNumber, + defaultHashAlg: multicodec.SHA2_256, + util: { + serialize (data) { + return Buffer.from(JSON.stringify(data)) + }, + deserialize (buf) { + return JSON.parse(uint8ArrayToString(buf)) + }, + async cid (buf) { + const multihash = await multihashing(buf, format.defaultHashAlg) + + return new CID(1, format.codec, multihash) + } + }, + resolver: { + resolve: (buf, path) => { + return { + value: format.util.deserialize(buf), + remainderPath: path + } + } + } + } + + // start an IPFS Daemon + const daemon = new IPFSDaemon({ + ipld: { + formats: [ + format + ] + } + }) + await daemon.start() + + // in another process: + const client = ipfsHttpClient({ + url: `http://localhost:${daemon._httpApi._apiServers[0].info.port}`, + ipld: { + formats: [ + format + ] + } + }) + + const data = { + hello: 'world' + } + + const cid = await client.dag.put(data, { + format: codecName, + hashAlg: multihash.codes[format.defaultHashAlg] + }) + + console.info(`Put ${JSON.stringify(data)} = CID(${cid})`) + + const { + value + } = await client.dag.get(cid) + + console.info(`Get CID(${cid}) = ${JSON.stringify(value)}`) + + await daemon.stop() +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) + .then(() => { + // https://github.com/libp2p/js-libp2p/issues/779 + process.exit(0) + }) diff --git a/examples/custom-ipld-formats/in-process-node.js b/examples/custom-ipld-formats/in-process-node.js new file mode 100644 index 0000000000..9fa19214ff --- /dev/null +++ b/examples/custom-ipld-formats/in-process-node.js @@ -0,0 +1,73 @@ +// ordinarily we'd open a PR against the multicodec module to get our +// codec number added but since we're just testing we shim our new +// codec into the base-table.json file - this has to be done +// before requiring other modules as the int table will become read-only +const codecName = 'dag-test' +const codecNumber = 392091 + +const baseTable = require('multicodec/src/base-table.json') +baseTable[codecName] = codecNumber + +// now require modules as usual +const IPFS = require('ipfs-core') +const multihashing = require('multihashing-async') +const multicodec = require('multicodec') +const CID = require('cids') + +async function main () { + // see https://github.com/ipld/interface-ipld-format for the interface definition + const format = { + codec: codecNumber, + defaultHashAlg: multicodec.SHA2_256, + util: { + serialize (data) { + return Buffer.from(JSON.stringify(data)) + }, + deserialize (buf) { + return JSON.parse(buf.toString('utf8')) + }, + async cid (buf) { + const multihash = await multihashing(buf, format.defaultHashAlg) + + return new CID(1, format.codec, multihash) + } + } + } + + const node = await IPFS.create({ + ipld: { + formats: [ + format + ] + } + }) + + const data = { + hello: 'world' + } + + const cid = await node.dag.put(data, { + format: codecName, + hashAlg: format.defaultHashAlg + }) + + console.info(`Put ${JSON.stringify(data)} = CID(${cid})`) + + const { + value + } = await node.dag.get(cid) + + console.info(`Get CID(${cid}) = ${JSON.stringify(value)}`) + + await node.stop() +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) + .then(() => { + // https://github.com/libp2p/js-libp2p/issues/779 + process.exit(0) + }) diff --git a/examples/custom-ipld-formats/package.json b/examples/custom-ipld-formats/package.json new file mode 100644 index 0000000000..8d07a86782 --- /dev/null +++ b/examples/custom-ipld-formats/package.json @@ -0,0 +1,22 @@ +{ + "name": "example-custom-ipld-formats", + "version": "1.0.0", + "private": true, + "scripts": { + "test": "test-ipfs-example" + }, + "license": "MIT", + "devDependencies": { + "execa": "^4.0.3", + "test-ipfs-example": "^2.0.3" + }, + "dependencies": { + "cids": "^1.0.0", + "ipfs-cli": "0.0.1", + "ipfs-core": "0.0.1", + "ipfs-http-client": "^47.0.0", + "multicodec": "^2.0.1", + "multihashing-async": "^2.0.1", + "uint8arrays": "^1.1.0" + } +} diff --git a/examples/custom-ipld-formats/test.js b/examples/custom-ipld-formats/test.js new file mode 100644 index 0000000000..0ae52257f0 --- /dev/null +++ b/examples/custom-ipld-formats/test.js @@ -0,0 +1,28 @@ +'use strict' + +const path = require('path') +const { + waitForOutput +} = require('test-ipfs-example/utils') + +const testInProcessNode = async () => { + await waitForOutput( + 'Put {"hello":"world"} = CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq)\n' + + 'Get CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq) = {"hello":"world"}', path.resolve(__dirname, 'in-process-node.js')) +} + +const testDaemonNode = async () => { + await waitForOutput( + 'Put {"hello":"world"} = CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq)\n' + + 'Get CID(bagn7ofysecj2eolrvekol2wl6cuneukuzwrqtq6by4x3xgiu2r6gb46lnakyq) = {"hello":"world"}', path.resolve(__dirname, 'daemon-node.js')) +} + +async function test () { + console.info('Testing in-process node') + await testInProcessNode() + + console.info('Testing daemon node') + await testDaemonNode() +} + +module.exports = test diff --git a/packages/ipfs-core/src/components/start.js b/packages/ipfs-core/src/components/start.js index ec72580c83..1f1f286eb2 100644 --- a/packages/ipfs-core/src/components/start.js +++ b/packages/ipfs-core/src/components/start.js @@ -294,6 +294,7 @@ function createApi ({ id: Components.id({ peerId, libp2p }), init: async () => { throw new AlreadyInitializedError() }, // eslint-disable-line require-await isOnline, + ipld, key: { export: Components.key.export({ keychain }), gen: Components.key.gen({ keychain }), diff --git a/packages/ipfs-http-client/src/dag/put.js b/packages/ipfs-http-client/src/dag/put.js index 720903877c..ae062efe9b 100644 --- a/packages/ipfs-http-client/src/dag/put.js +++ b/packages/ipfs-http-client/src/dag/put.js @@ -41,7 +41,6 @@ module.exports = configure((api, opts) => { } const format = await load(settings.format) - const serialized = format.util.serialize(dagNode) // allow aborting requests on body errors From 2eba17ad135d540c54012fc1fdcf61c60ff44844 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 26 Oct 2020 15:09:01 +0000 Subject: [PATCH 11/14] chore: remove redundant options arg --- packages/ipfs-http-server/src/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ipfs-http-server/src/index.js b/packages/ipfs-http-server/src/index.js index 363cc34312..5439be23d2 100644 --- a/packages/ipfs-http-server/src/index.js +++ b/packages/ipfs-http-server/src/index.js @@ -24,7 +24,7 @@ function hapiInfoToMultiaddr (info) { return toMultiaddr(uri) } -async function serverCreator (serverAddrs, createServer, ipfs, cors, opts) { +async function serverCreator (serverAddrs, createServer, ipfs, cors) { serverAddrs = serverAddrs || [] // just in case the address is just string serverAddrs = Array.isArray(serverAddrs) ? serverAddrs : [serverAddrs] @@ -32,7 +32,7 @@ async function serverCreator (serverAddrs, createServer, ipfs, cors, opts) { const servers = [] for (const address of serverAddrs) { const addrParts = address.split('/') - const server = await createServer(addrParts[2], addrParts[4], ipfs, cors, opts) + const server = await createServer(addrParts[2], addrParts[4], ipfs, cors) await server.start() server.info.ma = hapiInfoToMultiaddr(server.info) servers.push(server) From a8567667e9d404ba2cb3332c663cc75ad27b9503 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Mon, 26 Oct 2020 16:13:34 +0000 Subject: [PATCH 12/14] docs: update examples/custom-ipld-formats/README.md Co-authored-by: Vasco Santos --- examples/custom-ipld-formats/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/custom-ipld-formats/README.md b/examples/custom-ipld-formats/README.md index 14ebefeff5..e74bca8f98 100644 --- a/examples/custom-ipld-formats/README.md +++ b/examples/custom-ipld-formats/README.md @@ -1,6 +1,6 @@ # Custom IPLD formats -This example shows you how to configure and IPFS daemon with the ability to load extra IPLD formats so you can use them in your applications. +This example shows you how to configure an IPFS daemon with the ability to load extra IPLD formats so you can use them in your applications. ## Before you start From 26fdb2093b1fca55cf0ca1277a6cd0dc7083f049 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 26 Oct 2020 16:14:07 +0000 Subject: [PATCH 13/14] chore: destructure ipld-formats args --- packages/ipfs-http-client/src/dag/get.js | 2 +- packages/ipfs-http-client/src/dag/put.js | 2 +- .../ipfs-http-client/src/lib/ipld-formats.js | 34 ++++++++----------- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/packages/ipfs-http-client/src/dag/get.js b/packages/ipfs-http-client/src/dag/get.js index c4b8ee15f3..e63f058e6b 100644 --- a/packages/ipfs-http-client/src/dag/get.js +++ b/packages/ipfs-http-client/src/dag/get.js @@ -7,7 +7,7 @@ const loadFormat = require('../lib/ipld-formats') module.exports = configure((api, opts) => { const getBlock = require('../block/get')(opts) const dagResolve = require('./resolve')(opts) - const load = loadFormat(opts) + const load = loadFormat(opts.ipld) /** * @type {import('..').Implements} diff --git a/packages/ipfs-http-client/src/dag/put.js b/packages/ipfs-http-client/src/dag/put.js index ae062efe9b..2a22176591 100644 --- a/packages/ipfs-http-client/src/dag/put.js +++ b/packages/ipfs-http-client/src/dag/put.js @@ -11,7 +11,7 @@ const multicodec = require('multicodec') const loadFormat = require('../lib/ipld-formats') module.exports = configure((api, opts) => { - const load = loadFormat(opts) + const load = loadFormat(opts.ipld) /** * @type {import('..').Implements} diff --git a/packages/ipfs-http-client/src/lib/ipld-formats.js b/packages/ipfs-http-client/src/lib/ipld-formats.js index 5076aea2ad..1ef4c4e8c6 100644 --- a/packages/ipfs-http-client/src/lib/ipld-formats.js +++ b/packages/ipfs-http-client/src/lib/ipld-formats.js @@ -5,6 +5,8 @@ const dagCBOR = require('ipld-dag-cbor') const raw = require('ipld-raw') const multicodec = require('multicodec') +const noop = () => {} + /** * @typedef {import('cids')} CID */ @@ -12,40 +14,34 @@ const multicodec = require('multicodec') /** * Return an object containing supported IPLD Formats * - * @param {object} [options] - Options passed to the http client constructor - * @param {object} [options.ipld] - IPLD options passed to the http client constructor - * @param {Array} [options.ipld.formats] - A list of IPLD Formats to use - * @param {Function} [options.ipld.loadFormat] - An async function that can load a format when passed a codec name + * @param {object} [options] - IPLD options passed to the http client constructor + * @param {Array} [options.formats] - A list of IPLD Formats to use + * @param {Function} [options.loadFormat] - An async function that can load a format when passed a codec number * @returns {Function} */ -module.exports = (options) => { - const formats = { +module.exports = ({ formats = [], loadFormat = noop } = {}) => { + formats = formats || [] + loadFormat = loadFormat || noop + + const configuredFormats = { [multicodec.DAG_PB]: dagPB, [multicodec.DAG_CBOR]: dagCBOR, [multicodec.RAW]: raw } - const ipldOptions = (options && options.ipld) || {} - const configuredFormats = (ipldOptions && ipldOptions.formats) || [] - configuredFormats.forEach(format => { - formats[format.codec] = format + formats.forEach(format => { + configuredFormats[format.codec] = format }) - const loadExtraFormat = options && options.ipld && options.ipld.loadFormat - /** * Attempts to load an IPLD format for the passed CID * * @param {string} codec - The code to load the format for * @returns {Promise} - An IPLD format */ - const loadFormat = async (codec) => { + const loadResolver = async (codec) => { const number = multicodec.getNumber(codec) - let format = formats[number] - - if (!format && loadExtraFormat) { - format = await loadExtraFormat(codec) - } + const format = configuredFormats[number] || await loadFormat(codec) if (!format) { throw Object.assign( @@ -57,5 +53,5 @@ module.exports = (options) => { return format } - return loadFormat + return loadResolver } From 00c06d5df196b5a2507b934be1791e43755fa689 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 27 Oct 2020 10:51:16 +0000 Subject: [PATCH 14/14] chore: upgrade ipld --- packages/ipfs-core/package.json | 2 +- packages/ipfs-http-server/src/api/resources/dag.js | 2 +- packages/ipfs-http-server/test/inject/dag.js | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index b461c8e81f..8cd8384960 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -77,7 +77,7 @@ "ipfs-unixfs-exporter": "^3.0.4", "ipfs-unixfs-importer": "^3.0.4", "ipfs-utils": "^4.0.0", - "ipld": "^0.27.1", + "ipld": "^0.27.2", "ipld-bitcoin": "^0.4.0", "ipld-block": "^0.10.1", "ipld-dag-cbor": "^0.17.0", diff --git a/packages/ipfs-http-server/src/api/resources/dag.js b/packages/ipfs-http-server/src/api/resources/dag.js index 5b6f6d5643..b76e19a33c 100644 --- a/packages/ipfs-http-server/src/api/resources/dag.js +++ b/packages/ipfs-http-server/src/api/resources/dag.js @@ -157,7 +157,7 @@ exports.put = { } else { // the node is an uncommon format which the client should have // serialized so deserialize it before continuing - const ipldFormat = await request.server.app.ipfs.ipld._getFormat(format) + const ipldFormat = await request.server.app.ipfs.ipld.getFormat(format) if (!ipldFormat) { throw new Error(`Missing IPLD format "${format}"`) diff --git a/packages/ipfs-http-server/test/inject/dag.js b/packages/ipfs-http-server/test/inject/dag.js index 5c6151a634..ab1a273d48 100644 --- a/packages/ipfs-http-server/test/inject/dag.js +++ b/packages/ipfs-http-server/test/inject/dag.js @@ -39,7 +39,7 @@ describe('/dag', () => { resolve: sinon.stub() }, ipld: { - _getFormat: sinon.stub() + getFormat: sinon.stub() } } }) @@ -303,7 +303,7 @@ describe('/dag', () => { deserialize: (buf) => buf } } - ipfs.ipld._getFormat.withArgs(codec).returns(format) + ipfs.ipld.getFormat.withArgs(codec).returns(format) ipfs.dag.put.withArgs(data, { ...defaultOptions, @@ -316,7 +316,7 @@ describe('/dag', () => { ...await toHeadersAndPayload(data) }, { ipfs }) - expect(ipfs.ipld._getFormat.calledWith(codec)).to.be.true() + expect(ipfs.ipld.getFormat.calledWith(codec)).to.be.true() expect(res).to.have.property('statusCode', 200) expect(res).to.have.deep.nested.property('result.Cid', { '/': cid.toString() }) })