From 450c581c73ccfcd0a03a379e931d1ef51e7add91 Mon Sep 17 00:00:00 2001 From: David Dias Date: Thu, 9 Mar 2017 16:06:32 +0000 Subject: [PATCH 1/8] feat: dag.put --- package.json | 1 + src/dag/dag.js | 79 ++++++++++++++++++++++++++++++++++++++ test/interface/dag.spec.js | 34 ++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 src/dag/dag.js create mode 100644 test/interface/dag.spec.js diff --git a/package.json b/package.json index 0fabfe21c..9e86be21d 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "glob": "^7.1.2", "ipfs-block": "~0.6.1", "ipfs-unixfs": "~0.1.14", + "ipld-dag-cbor": "^0.12.0", "ipld-dag-pb": "~0.13.1", "is-ipfs": "^0.3.2", "is-stream": "^1.1.0", diff --git a/src/dag/dag.js b/src/dag/dag.js new file mode 100644 index 000000000..ab81f041c --- /dev/null +++ b/src/dag/dag.js @@ -0,0 +1,79 @@ +'use strict' + +const dagPB = require('ipld-dag-pb') +const dagCBOR = require('ipld-dag-cbor') +const promisify = require('promisify-es6') +const CID = require('cids') +const multihash = require('multihashes') + +function noop () {} + +module.exports = (send) => { + const api = { + put: promisify((dagNode, options, callback) => { + if (typeof options === 'function') { + return setImmediate(() => callback(new Error('no options were passed'))) + } + + callback = callback || noop + + let hashAlg = options.hashAlg || 'sha2-256' + let format + let inputEnc + + if (options.cid && CID.isCID(options.cid)) { + format = options.cid.codec + hashAlg = multihash.decode(options.cid.multihash).name + prepare() + } else if (options.format) { + format = options.format + prepare() + } else { + callback(new Error('Invalid arguments')) + } + + function prepare () { + if (format === 'dag-cbor') { + // TODO change this once + // https://github.com/ipfs/go-ipfs/issues/3771 is finished + format = 'cbor' + + inputEnc = 'cbor' + dagCBOR.util.serialize(dagNode, finalize) + } + if (format === 'dag-pb') { + // TODO change this once + // https://github.com/ipfs/go-ipfs/issues/3771 is finished + format = 'protobuf' + + inputEnc = 'protobuf' + dagPB.util.serialize(dagNode, finalize) + } + } + + function finalize (err, serialized) { + if (err) { return callback(err) } + + send({ + path: 'dag/put', + qs: { + hashAlg: hashAlg, // not implemented in go yet https://github.com/ipfs/go-ipfs/issues/3771 + format: format, + inputenc: inputEnc + }, + files: serialized + }, (err, result) => { + if (err) { + return callback(err) + } + // TODO handle the result + }) + } + }), + get: promisify((cid, path, options, callback) => { + // TODO + }) + } + + return api +} diff --git a/test/interface/dag.spec.js b/test/interface/dag.spec.js new file mode 100644 index 000000000..6c68680e7 --- /dev/null +++ b/test/interface/dag.spec.js @@ -0,0 +1,34 @@ +/* eslint-env mocha */ + +'use strict' + +const test = require('interface-ipfs-core') +const parallel = require('async/parallel') + +const IPFSApi = require('../../src') + +const DaemonFactory = require('ipfsd-ctl') +const df = DaemonFactory.create() + +const nodes = [] +const common = { + setup: function (callback) { + callback(null, { + spawnNode: (cb) => { + df.spawn((err, _ipfsd) => { + if (err) { + return cb(err) + } + + nodes.push(_ipfsd) + cb(null, IPFSApi(_ipfsd.apiAddr)) + }) + } + }) + }, + teardown: function (callback) { + parallel(nodes.map((node) => (cb) => node.stop(cb)), callback) + } +} + +test.dag(common) From 7a8b030176cc73c98db96b2d8e45e5e240653c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antonio=20Tenorio-Forn=C3=A9s?= Date: Sat, 24 Jun 2017 13:19:10 +0200 Subject: [PATCH 2/8] Feat(dag): DAG api (#568) * Implement dag.get * Handle dag put response, enable CIDs for dag get and fix dag get request param --- src/dag/dag.js | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/dag/dag.js b/src/dag/dag.js index ab81f041c..6e86f62e8 100644 --- a/src/dag/dag.js +++ b/src/dag/dag.js @@ -66,12 +66,53 @@ module.exports = (send) => { if (err) { return callback(err) } - // TODO handle the result + if (result.Cid) { + return callback(null, new CID(result.Cid['/'])) + } else { + return callback(result) + } }) } }), get: promisify((cid, path, options, callback) => { - // TODO + if (typeof path === 'function') { + callback = path + path = undefined + } + + if (typeof options === 'function') { + callback = options + options = {} + } + + options = options || {} + + if (CID.isCID(cid)) { + cid = cid.toBaseEncodedString() + } + + if (typeof cid === 'string') { + const split = cid.split('/') + cid = split[0] + split.shift() + + if (split.length > 0) { + path = split.join('/') + } else { + path = '/' + } + } + + send({ + path: 'dag/get', + args: cid + '/' + path, + qs: options + }, (err, result) => { + if (err) { + return callback(err) + } + callback(undefined, {value: result}) + }) }) } From 2e0c4db9b7cf27a9ab4ae4b73d65f88ebb492fc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 11 Aug 2017 23:37:13 +0200 Subject: [PATCH 3/8] feat(dag): update option names to reflect go-ipfs API --- src/dag/dag.js | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/dag/dag.js b/src/dag/dag.js index 6e86f62e8..4acf0ae36 100644 --- a/src/dag/dag.js +++ b/src/dag/dag.js @@ -17,7 +17,7 @@ module.exports = (send) => { callback = callback || noop - let hashAlg = options.hashAlg || 'sha2-256' + let hashAlg = options.hash || 'sha2-256' let format let inputEnc @@ -33,20 +33,12 @@ module.exports = (send) => { } function prepare () { - if (format === 'dag-cbor') { - // TODO change this once - // https://github.com/ipfs/go-ipfs/issues/3771 is finished - format = 'cbor' + inputEnc = 'raw' - inputEnc = 'cbor' + if (format === 'dag-cbor') { dagCBOR.util.serialize(dagNode, finalize) } if (format === 'dag-pb') { - // TODO change this once - // https://github.com/ipfs/go-ipfs/issues/3771 is finished - format = 'protobuf' - - inputEnc = 'protobuf' dagPB.util.serialize(dagNode, finalize) } } @@ -57,9 +49,9 @@ module.exports = (send) => { send({ path: 'dag/put', qs: { - hashAlg: hashAlg, // not implemented in go yet https://github.com/ipfs/go-ipfs/issues/3771 + hash: hashAlg, format: format, - inputenc: inputEnc + 'input-enc': inputEnc }, files: serialized }, (err, result) => { From af9fc47f1646bab5f9811eefef3fc6b6c3abd528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 13 Aug 2017 18:38:45 +0200 Subject: [PATCH 4/8] feat(dag): proper get implementation --- src/dag/dag.js | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/dag/dag.js b/src/dag/dag.js index 4acf0ae36..666437ed6 100644 --- a/src/dag/dag.js +++ b/src/dag/dag.js @@ -5,6 +5,7 @@ const dagCBOR = require('ipld-dag-cbor') const promisify = require('promisify-es6') const CID = require('cids') const multihash = require('multihashes') +const block = require('./block') function noop () {} @@ -96,14 +97,28 @@ module.exports = (send) => { } send({ - path: 'dag/get', + path: 'dag/resolve', args: cid + '/' + path, qs: options - }, (err, result) => { + }, (err, resolved) => { if (err) { return callback(err) } - callback(undefined, {value: result}) + + let resolvedCid = new CID(resolved['Cid']['/']) + + block(send).get(resolvedCid, (err, blk) => { + if (err) { + return callback(err) + } + + if (resolvedCid.codec === 'dag-cbor') { + dagCBOR.resolver.resolve(blk, resolved['RemPath'], callback) + } + if (resolvedCid.codec === 'dag-pb') { + dagCBOR.resolver.resolve(blk, resolved['RemPath'], callback) + } + }) }) }) } From fce3138cf24a48e0cff6c7c566f346e385bce74e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 13 Aug 2017 20:40:10 +0200 Subject: [PATCH 5/8] feat(dag): rebase, use waterfall for put --- src/dag/dag.js | 127 ------------------------------------- src/dag/get.js | 61 ++++++++++++++++++ src/dag/index.js | 12 ++++ src/dag/put.js | 68 ++++++++++++++++++++ src/utils/load-commands.js | 1 + 5 files changed, 142 insertions(+), 127 deletions(-) delete mode 100644 src/dag/dag.js create mode 100644 src/dag/get.js create mode 100644 src/dag/index.js create mode 100644 src/dag/put.js diff --git a/src/dag/dag.js b/src/dag/dag.js deleted file mode 100644 index 666437ed6..000000000 --- a/src/dag/dag.js +++ /dev/null @@ -1,127 +0,0 @@ -'use strict' - -const dagPB = require('ipld-dag-pb') -const dagCBOR = require('ipld-dag-cbor') -const promisify = require('promisify-es6') -const CID = require('cids') -const multihash = require('multihashes') -const block = require('./block') - -function noop () {} - -module.exports = (send) => { - const api = { - put: promisify((dagNode, options, callback) => { - if (typeof options === 'function') { - return setImmediate(() => callback(new Error('no options were passed'))) - } - - callback = callback || noop - - let hashAlg = options.hash || 'sha2-256' - let format - let inputEnc - - if (options.cid && CID.isCID(options.cid)) { - format = options.cid.codec - hashAlg = multihash.decode(options.cid.multihash).name - prepare() - } else if (options.format) { - format = options.format - prepare() - } else { - callback(new Error('Invalid arguments')) - } - - function prepare () { - inputEnc = 'raw' - - if (format === 'dag-cbor') { - dagCBOR.util.serialize(dagNode, finalize) - } - if (format === 'dag-pb') { - dagPB.util.serialize(dagNode, finalize) - } - } - - function finalize (err, serialized) { - if (err) { return callback(err) } - - send({ - path: 'dag/put', - qs: { - hash: hashAlg, - format: format, - 'input-enc': inputEnc - }, - files: serialized - }, (err, result) => { - if (err) { - return callback(err) - } - if (result.Cid) { - return callback(null, new CID(result.Cid['/'])) - } else { - return callback(result) - } - }) - } - }), - get: promisify((cid, path, options, callback) => { - if (typeof path === 'function') { - callback = path - path = undefined - } - - if (typeof options === 'function') { - callback = options - options = {} - } - - options = options || {} - - if (CID.isCID(cid)) { - cid = cid.toBaseEncodedString() - } - - if (typeof cid === 'string') { - const split = cid.split('/') - cid = split[0] - split.shift() - - if (split.length > 0) { - path = split.join('/') - } else { - path = '/' - } - } - - send({ - path: 'dag/resolve', - args: cid + '/' + path, - qs: options - }, (err, resolved) => { - if (err) { - return callback(err) - } - - let resolvedCid = new CID(resolved['Cid']['/']) - - block(send).get(resolvedCid, (err, blk) => { - if (err) { - return callback(err) - } - - if (resolvedCid.codec === 'dag-cbor') { - dagCBOR.resolver.resolve(blk, resolved['RemPath'], callback) - } - if (resolvedCid.codec === 'dag-pb') { - dagCBOR.resolver.resolve(blk, resolved['RemPath'], callback) - } - }) - }) - }) - } - - return api -} diff --git a/src/dag/get.js b/src/dag/get.js new file mode 100644 index 000000000..af3fa55e7 --- /dev/null +++ b/src/dag/get.js @@ -0,0 +1,61 @@ +'use strict' + +const dagPB = require('ipld-dag-pb') +const dagCBOR = require('ipld-dag-cbor') +const promisify = require('promisify-es6') +const CID = require('cids') +const waterfall = require('async/waterfall') +const block = require('../block') + +module.exports = (send) => { + return promisify((cid, path, options, callback) => { + if (typeof path === 'function') { + callback = path + path = undefined + } + + if (typeof options === 'function') { + callback = options + options = {} + } + + options = options || {} + + if (CID.isCID(cid)) { + cid = cid.toBaseEncodedString() + } + + if (typeof cid === 'string') { + const split = cid.split('/') + cid = split[0] + split.shift() + + if (split.length > 0) { + path = split.join('/') + } else { + path = '/' + } + } + + waterfall([ + cb => { + send({ + path: 'dag/resolve', + args: cid + '/' + path, + qs: options + }, cb) + }, + (resolved, cb) => { + block(send).get(new CID(resolved['Cid']['/']), (err, blk) => cb(err, blk, resolved['RemPath'])) + }, + (blk, path, cb) => { + if (blk.cid.codec === 'dag-cbor') { + dagCBOR.resolver.resolve(blk, path, cb) + } + if (blk.cid.codec === 'dag-pb') { + dagPB.resolver.resolve(blk, path, cb) + } + } + ], callback) + }) +} diff --git a/src/dag/index.js b/src/dag/index.js new file mode 100644 index 000000000..bb6b1333c --- /dev/null +++ b/src/dag/index.js @@ -0,0 +1,12 @@ +'use strict' + +const moduleConfig = require('../utils/module-config') + +module.exports = (arg) => { + const send = moduleConfig(arg) + + return { + get: require('./get')(send), + put: require('./put')(send) + } +} diff --git a/src/dag/put.js b/src/dag/put.js new file mode 100644 index 000000000..ee67261b6 --- /dev/null +++ b/src/dag/put.js @@ -0,0 +1,68 @@ +'use strict' + +const dagPB = require('ipld-dag-pb') +const dagCBOR = require('ipld-dag-cbor') +const promisify = require('promisify-es6') +const CID = require('cids') +const multihash = require('multihashes') + +function noop () {} + +module.exports = (send) => { + return promisify((dagNode, options, callback) => { + if (typeof options === 'function') { + return setImmediate(() => callback(new Error('no options were passed'))) + } + + callback = callback || noop + + let hashAlg = options.hash || 'sha2-256' + let format + let inputEnc + + if (options.cid && CID.isCID(options.cid)) { + format = options.cid.codec + hashAlg = multihash.decode(options.cid.multihash).name + prepare() + } else if (options.format) { + format = options.format + prepare() + } else { + callback(new Error('Invalid arguments')) + } + + function prepare () { + inputEnc = 'raw' + + if (format === 'dag-cbor') { + dagCBOR.util.serialize(dagNode, finalize) + } + if (format === 'dag-pb') { + dagPB.util.serialize(dagNode, finalize) + } + } + + function finalize (err, serialized) { + if (err) { return callback(err) } + + send({ + path: 'dag/put', + qs: { + hash: hashAlg, + format: format, + 'input-enc': inputEnc + }, + files: serialized + }, (err, result) => { + if (err) { + return callback(err) + } + if (result['Cid']) { + return callback(null, new CID(result['Cid']['/'])) + } else { + return callback(result) + } + }) + } + }) +} diff --git a/src/utils/load-commands.js b/src/utils/load-commands.js index e86982e18..2a19a7b6d 100644 --- a/src/utils/load-commands.js +++ b/src/utils/load-commands.js @@ -21,6 +21,7 @@ function requireCommands () { bootstrap: require('../bootstrap'), commands: require('../commands'), config: require('../config'), + dag: require('../dag'), dht: require('../dht'), diag: require('../diag'), id: require('../id'), From b7e95c2d75406ce4a2ad9537b71832aba4f0402a Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 22 Feb 2018 14:20:49 +0100 Subject: [PATCH 6/8] fix(dag): path logic for DAG get was wrong --- src/dag/get.js | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/dag/get.js b/src/dag/get.js index af3fa55e7..989d5373b 100644 --- a/src/dag/get.js +++ b/src/dag/get.js @@ -20,23 +20,12 @@ module.exports = (send) => { } options = options || {} + path = path || '' if (CID.isCID(cid)) { cid = cid.toBaseEncodedString() } - if (typeof cid === 'string') { - const split = cid.split('/') - cid = split[0] - split.shift() - - if (split.length > 0) { - path = split.join('/') - } else { - path = '/' - } - } - waterfall([ cb => { send({ From d98f8e1730ac140893ee6d447d04804f7b6ff77d Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 22 Feb 2018 14:21:55 +0100 Subject: [PATCH 7/8] fix(dag): js-ipld format resolver take the raw block The js-ipld formats API changed, they now take a raw data block rather than an IPFS block. --- src/dag/get.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/dag/get.js b/src/dag/get.js index 989d5373b..ead6a732c 100644 --- a/src/dag/get.js +++ b/src/dag/get.js @@ -35,14 +35,16 @@ module.exports = (send) => { }, cb) }, (resolved, cb) => { - block(send).get(new CID(resolved['Cid']['/']), (err, blk) => cb(err, blk, resolved['RemPath'])) + block(send).get(new CID(resolved['Cid']['/']), (err, ipfsBlock) => { + cb(err, ipfsBlock, resolved['RemPath']) + }) }, - (blk, path, cb) => { - if (blk.cid.codec === 'dag-cbor') { - dagCBOR.resolver.resolve(blk, path, cb) + (ipfsBlock, path, cb) => { + if (ipfsBlock.cid.codec === 'dag-cbor') { + dagCBOR.resolver.resolve(ipfsBlock.data, path, cb) } - if (blk.cid.codec === 'dag-pb') { - dagPB.resolver.resolve(blk, path, cb) + if (ipfsBlock.cid.codec === 'dag-pb') { + dagPB.resolver.resolve(ipfsBlock.data, path, cb) } } ], callback) From 49dd9a7b0e83ae092fa9338b0cd69b029ed967bb Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Fri, 23 Feb 2018 13:22:41 +0100 Subject: [PATCH 8/8] fix(dag): use SendOneFile for dag put There was a refactoring, now the usual way of doing a HTTP request is through SendOneFile. --- src/dag/put.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/dag/put.js b/src/dag/put.js index ee67261b6..541c0d512 100644 --- a/src/dag/put.js +++ b/src/dag/put.js @@ -5,10 +5,13 @@ const dagCBOR = require('ipld-dag-cbor') const promisify = require('promisify-es6') const CID = require('cids') const multihash = require('multihashes') +const SendOneFile = require('../utils/send-one-file') function noop () {} module.exports = (send) => { + const sendOneFile = SendOneFile(send, 'dag/put') + return promisify((dagNode, options, callback) => { if (typeof options === 'function') { return setImmediate(() => callback(new Error('no options were passed'))) @@ -44,16 +47,14 @@ module.exports = (send) => { function finalize (err, serialized) { if (err) { return callback(err) } - - send({ - path: 'dag/put', + const sendOptions = { qs: { hash: hashAlg, format: format, 'input-enc': inputEnc - }, - files: serialized - }, (err, result) => { + } + } + sendOneFile(serialized, sendOptions, (err, result) => { if (err) { return callback(err) }